// MoonBit -> Yjs export: runs scripted edits on MoonBit replicas and prints a
// JSON report (scripts/verify_moonbit.mjs checks it with Yjs):
//
// moon run cmd/export > fixtures/moonbit_out.json
// node scripts/verify_moonbit.mjs fixtures/moonbit_out.json
///|
fn b64(b : Bytes) -> Json {
Json::string(@base64.encode(b))
}
///|
/// `Any` as JSON. Values JSON cannot express are tagged:
/// {"$undefined":true}, {"$bigint":"12"}, {"$bytes":""}.
fn any_to_json(v : @lib0.Any) -> Json {
match v {
Undefined => Json::object({ "$undefined": Json::boolean(true) })
Null => Json::null()
Bool(b) => Json::boolean(b)
Number(d) => Json::number(d)
BigInt(n) => Json::object({ "$bigint": Json::string(n.to_string()) })
String(s) => Json::string(s)
Array(items) => Json::array(items.map(any_to_json))
Object(entries) => {
let m = Map([])
for e in entries {
m[e.0] = any_to_json(e.1)
}
Json::object(m)
}
Bytes(b) => Json::object({ "$bytes": b64(b) })
}
}
///|
/// Runs operations on MoonBit replicas and records them so that the verifier
/// can replay exactly the same operations on Yjs.
priv struct Script {
name : String
clients : Array[UInt32]
docs : Array[@doc.Doc]
logs : Array[Array[Bytes]]
/// the same local updates in format v2
logs_v2 : Array[Array[Bytes]]
ops : Array[Json]
texts : Array[String]
maps : Array[String]
arrays : Array[String]
}
///|
fn Script::new(
name : String,
clients : Array[UInt32],
texts~ : Array[String] = ["t"],
maps~ : Array[String] = [],
arrays~ : Array[String] = [],
) -> Script {
let docs = clients.map(id => @doc.Doc::new(id))
let logs : Array[Array[Bytes]] = docs.map(_ => [])
let logs_v2 : Array[Array[Bytes]] = docs.map(_ => [])
for i, d in docs {
d.on_update((u, is_local) => if is_local { logs[i].push(u) })
d.on_update_v2((u, is_local) => if is_local { logs_v2[i].push(u) })
}
{ name, clients, docs, logs, logs_v2, ops: [], texts, maps, arrays, }
}
///|
fn Script::insert(
self : Script,
doc : Int32,
pos : Int32,
text : String,
name~ : String = "t",
) -> Unit {
@types.Text::new(self.docs[doc], name).insert(pos, text)
self.ops.push({
"op": "insert",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"text": Json(text),
})
}
///|
fn Script::delete(
self : Script,
doc : Int32,
pos : Int32,
len : Int32,
name~ : String = "t",
) -> Unit {
@types.Text::new(self.docs[doc], name).delete(pos, len)
self.ops.push({
"op": "delete",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"len": Json(len),
})
}
///|
fn Script::set(
self : Script,
doc : Int32,
key : String,
value : @lib0.Any,
name~ : String = "m",
) -> Unit {
@types.YMap::new(self.docs[doc], name).set(key, value)
self.ops.push({
"op": "set",
"doc": Json(doc),
"name": Json(name),
"key": Json(key),
"value": any_to_json(value),
})
}
///|
fn Script::map_delete(
self : Script,
doc : Int32,
key : String,
name~ : String = "m",
) -> Unit {
@types.YMap::new(self.docs[doc], name).delete(key)
self.ops.push({
"op": "mapDelete",
"doc": Json(doc),
"name": Json(name),
"key": Json(key),
})
}
///|
fn Script::array_insert(
self : Script,
doc : Int32,
pos : Int32,
values : Array[@lib0.Any],
name~ : String = "a",
) -> Unit {
@types.YArray::new(self.docs[doc], name).insert(pos, values)
self.ops.push({
"op": "arrayInsert",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"values": Json::array(values.map(any_to_json)),
})
}
///|
fn Script::array_push(
self : Script,
doc : Int32,
values : Array[@lib0.Any],
name~ : String = "a",
) -> Unit {
@types.YArray::new(self.docs[doc], name).push(values)
self.ops.push({
"op": "arrayPush",
"doc": Json(doc),
"name": Json(name),
"values": Json::array(values.map(any_to_json)),
})
}
///|
fn Script::array_delete(
self : Script,
doc : Int32,
pos : Int32,
len : Int32,
name~ : String = "a",
) -> Unit {
@types.YArray::new(self.docs[doc], name).delete(pos, len)
self.ops.push({
"op": "arrayDelete",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"len": Json(len),
})
}
///|
/// Sends `from`'s missing data to `to` as a diff against `to`'s state vector.
fn Script::sync(self : Script, from : Int32, to : Int32) -> Unit {
let diff = self.docs[from].encode_state_as_update(
sv=self.docs[to].state_vector(),
)
self.docs[to].apply_update(diff) catch {
e => abort(e.to_string())
}
self.ops.push({ "op": "sync", "from": Json(from), "to": Json(to) })
}
///|
fn Script::sync_all(self : Script) -> Unit {
for to in 0.. Unit {
let update = @lib0.from_hex(update_hex) catch { e => abort(e.to_string()) }
self.docs[doc].apply_update(update) catch {
e => abort(e.to_string())
}
self.ops.push({ "op": "apply", "doc": Json(doc), "update": b64(update) })
}
///|
fn Script::report(self : Script) -> Json {
let docs = self.docs.mapi((i, d) => {
let texts = Map([])
for name in self.texts {
texts[name] = Json::string(@types.Text::new(d, name).to_string())
}
let deltas = Map([])
for name in self.texts {
deltas[name] = @types.delta_to_json(
@types.Text::new(d, name).to_delta() catch {
e => abort("to_delta failed: \{e}")
},
)
}
let maps = Map([])
for name in self.maps {
let m = Map([])
for e in @types.YMap::new(d, name).entries() {
m[e.0] = any_to_json(e.1)
}
maps[name] = Json::object(m)
}
let arrays = Map([])
for name in self.arrays {
arrays[name] = Json::array(
@types.YArray::new(d, name).to_array().map(any_to_json),
)
}
let json : Json = {
"arrays": Json::object(arrays),
"client": Json::number(self.clients[i].to_double()),
"state": b64(d.encode_state_as_update()),
"stateVector": b64(d.encode_state_vector()),
"localUpdates": Json::array(self.logs[i].map(b64)),
"stateV2": b64(d.encode_state_as_update_v2()),
"localUpdatesV2": Json::array(self.logs_v2[i].map(b64)),
// merged and diffed without a document
"merged": b64(
@update.merge_updates(self.logs[i]) catch {
e => abort(e.to_string())
},
),
"mergedV2": b64(
@update.merge_updates_v2(self.logs_v2[i]) catch {
e => abort(e.to_string())
},
),
"diffs": Json::array(
self.docs
.mapi((j, other) => (j, other))
.filter(p => p.0 != i)
.map(p => {
let sv = p.1.encode_state_vector()
let state = d.encode_state_as_update()
let state_v2 = d.encode_state_as_update_v2()
{
"againstReplica": Json::number(p.0.to_double()),
"stateVector": b64(sv),
"v1": b64(
@update.diff_update(state, sv) catch {
e => abort(e.to_string())
},
),
"v2": b64(
@update.diff_update_v2(state_v2, sv) catch {
e => abort(e.to_string())
},
),
}
}),
),
"texts": Json::object(texts),
"deltas": Json::object(deltas),
"maps": Json::object(maps),
}
json
})
{
"name": Json(self.name),
"clients": Json::array(self.clients.map(c => Json::number(c.to_double()))),
"ops": Json::array(self.ops),
"docs": Json::array(docs),
}
}
///|
fn text_editing() -> Script {
let s = Script::new("text_editing", [12345])
s.insert(0, 0, "Hello world")
s.insert(0, 5, ",")
s.insert(0, 12, "!")
s.delete(0, 0, 1)
s.insert(0, 0, "h")
s.insert(0, 6, " δΈζ ")
s.insert(0, text_len(s.docs[0]), "ππ")
// delete inside a surrogate pair: both halves become U+FFFD, like Yjs
s.delete(0, text_len(s.docs[0]) - 3, 1)
s.insert(0, 3, "π")
s.delete(0, 2, 5)
s
}
///|
fn text_len(d : @doc.Doc) -> Int32 {
@types.Text::new(d, "t").length()
}
///|
fn map_values() -> Script {
let s = Script::new("map_values", [3000000001], texts=[], maps=["m"])
s.set(0, "int", @lib0.Any::int(42))
s.set(0, "negative", @lib0.Any::int(-7))
s.set(0, "float", Number(1.5))
s.set(0, "double", Number(0.1))
s.set(0, "big", BigInt(1099511627776L))
s.set(0, "text", String("δΈζ π"))
s.set(0, "flags", Array([Bool(true), Bool(false), Null, Undefined]))
s.set(
0,
"object",
Object([("a", @lib0.Any::int(1)), ("nested", Object([("b", Array([]))]))]),
)
s.set(0, "bytes", Bytes(b"\x01\x02\xff"))
s.set(0, "int", @lib0.Any::int(43))
s.map_delete(0, "negative")
s.set(0, "negative", @lib0.Any::int(-8))
s.map_delete(0, "double")
s
}
///|
fn concurrent_two() -> Script {
let s = Script::new("concurrent_two", [1, 2], maps=["m"])
s.insert(0, 0, "base text")
s.sync(0, 1)
// both insert at the same position without seeing each other
s.insert(0, 4, "[A]")
s.insert(1, 4, "[B]")
s.insert(0, 0, "a0")
s.insert(1, 0, "b0")
s.delete(1, 6, 3)
s.set(0, "who", String("first"))
s.set(1, "who", String("second"))
s.sync(1, 0)
s.insert(0, 2, "")
s.sync_all()
s
}
///|
fn concurrent_three() -> Script {
let s = Script::new("concurrent_three", [7, 4000000000, 3], maps=["m"])
s.insert(0, 0, "abc")
s.sync_all()
s.insert(0, 1, "X")
s.insert(1, 1, "Y")
s.insert(2, 1, "Z")
s.delete(2, 0, 2)
s.sync(2, 1)
s.insert(1, 2, "π")
s.delete(0, 3, 1)
s.set(2, "k", @lib0.Any::int(3))
s.set(0, "k", @lib0.Any::int(1))
s.sync(1, 0)
s.insert(0, 0, ">")
s.sync_all()
s.insert(1, 3, "after")
s.sync_all()
s
}
///|
fn preserve_unknown() -> Script {
// A Yjs document with bold/italic formats, an embed, XML elements, a hook
// and a sub-document. MoonBit edits it; the verifier checks that Yjs still
// sees all of that content.
let s = Script::new("preserve_unknown", [55], texts=["rich"], maps=["m"])
s.apply_yjs(0, yjs_rich_text_update)
s.apply_yjs(0, yjs_nested_types_update)
s.insert(0, 2, "-in-bold-", name="rich")
// deletes the end of the bold run right before the embed: Yjs cleans up
// the formatting gap, and so must we
s.delete(0, 12, 2, name="rich")
s.delete(0, 4, 3, name="rich")
s.insert(0, text_len_of(s.docs[0], "rich"), " end", name="rich")
s.set(0, "plain", String("value"))
s
}
///|
fn text_len_of(d : @doc.Doc, name : String) -> Int32 {
@types.Text::new(d, name).length()
}
///|
fn arrays() -> Script {
let s = Script::new("arrays", [21, 3000000002], texts=[], arrays=["a"])
s.array_push(0, [@lib0.Any::int(1), String("two"), Bytes(b""), Null])
s.sync_all()
// concurrent inserts at the same index, deletes and pushes
s.array_insert(0, 1, [String("A"), Object([("k", Bool(true))])])
s.array_insert(1, 1, [String("B")])
s.array_delete(1, 2, 1)
s.array_push(0, [Array([@lib0.Any::int(5)])])
s.array_push(1, [String("end")])
s.sync(1, 0)
s.array_delete(0, 0, 2)
s.array_insert(0, 0, [Bytes(b"ΓΏ "), @lib0.Any::int(-3)])
s.sync_all()
s
}
///|
fn attrs_json(attrs : Array[(String, @lib0.Any)]) -> Json {
let m = Map([])
for a in attrs {
m[a.0] = any_to_json(a.1)
}
Json::object(m)
}
///|
/// Inserts text with exactly the given formatting (`ytext.insert(i, s, attrs)`).
fn Script::insert_attrs(
self : Script,
doc : Int32,
pos : Int32,
text : String,
attributes : Array[(String, @lib0.Any)],
name~ : String = "t",
) -> Unit {
@types.Text::new(self.docs[doc], name).insert_with_attributes(
pos, text, attributes,
)
self.ops.push({
"op": "insert",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"text": Json(text),
"attributes": attrs_json(attributes),
})
}
///|
fn Script::format(
self : Script,
doc : Int32,
pos : Int32,
len : Int32,
attributes : Array[(String, @lib0.Any)],
name~ : String = "t",
) -> Unit {
@types.Text::new(self.docs[doc], name).format(pos, len, attributes)
self.ops.push({
"op": "format",
"doc": Json(doc),
"name": Json(name),
"pos": Json(pos),
"len": Json(len),
"attributes": attrs_json(attributes),
})
}
///|
fn Script::embed(
self : Script,
doc : Int32,
pos : Int32,
embed : @lib0.Any,
attributes? : Array[(String, @lib0.Any)],
name~ : String = "t",
) -> Unit {
let text = @types.Text::new(self.docs[doc], name)
match attributes {
Some(a) => text.insert_embed(pos, embed, attributes=a)
None => text.insert_embed(pos, embed)
}
let op : Map[String, Json] = Map([])
op["op"] = Json("embed")
op["doc"] = Json(doc)
op["name"] = Json(name)
op["pos"] = Json(pos)
op["embed"] = any_to_json(embed)
if attributes is Some(a) {
op["attributes"] = attrs_json(a)
}
self.ops.push(Json::object(op))
}
///|
fn Script::apply_delta(
self : Script,
doc : Int32,
delta : Array[@types.DeltaOp],
name~ : String = "t",
) -> Unit {
@types.Text::new(self.docs[doc], name).apply_delta(delta)
self.ops.push({
"op": "applyDelta",
"doc": Json(doc),
"name": Json(name),
"delta": @types.delta_to_json(delta),
})
}
///|
/// Rich text: formatting, embeds, deltas, and two replicas formatting the same
/// range at the same time (which makes both clean up afterwards).
fn rich_text() -> Script {
let s = Script::new("rich_text", [31, 3000000003], texts=["rich"])
let bold = [("bold", @lib0.Any::Bool(true))]
s.insert_attrs(0, 0, "Hello world", bold, name="rich")
s.sync_all()
s.format(0, 0, 5, [("italic", Bool(true))], name="rich")
s.format(1, 3, 6, [("color", String("red"))], name="rich")
s.insert(1, 4, "!", name="rich")
s.embed(
0,
6,
Object([("image", String("x.png")), ("width", @lib0.Any::int(100))]),
attributes=[("link", Object([("href", String("https://example.com"))]))],
name="rich",
)
s.sync_all()
// both replicas underline the same range: redundant formatting is cleaned up
s.format(0, 0, 4, [("underline", Bool(true))], name="rich")
s.format(1, 0, 4, [("underline", Bool(true))], name="rich")
s.sync_all()
s.apply_delta(
1,
[
Retain(2, []),
Delete(3),
Insert(String("Γ©π"), [
("link", Object([("href", String("https://x.org"))])),
]),
Retain(1, [("bold", Null)]),
],
name="rich",
)
s.sync_all()
s.delete(0, 1, 4, name="rich")
s.insert_attrs(0, 2, "δΈ", [("size", @lib0.Any::int(14))], name="rich")
s.sync_all()
s
}
///|
fn main {
let cases = [
text_editing(),
map_values(),
concurrent_two(),
concurrent_three(),
preserve_unknown(),
arrays(),
rich_text(),
]
let report : Json = {
"generator": "moon_ycrdt cmd/export",
"cases": Json::array(cases.map(c => c.report())),
}
println(report.stringify(indent=2))
}