Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(collaboration): upgrade Loro to fix shallow snapshots (#6760)

teo9b05a7e5785e
diff --git a/Cargo.lock b/Cargo.lock
index 0fc9f6971a..7e68a35566 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9896,9 +9896,9 @@ dependencies = [
[[package]]
name = "loro"
-version = "1.13.7"
+version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bec673c3332ede11cc58d5d88de55a01ba17cf28b0bd0feb66ca1c66fb757ad"
+checksum = "8e86dea63b66576db244cbc3a40bdb17517263c93f318e2fbe08aca3be1192cb"
dependencies = [
"enum-as-inner 0.6.1",
"generic-btree",
@@ -9912,9 +9912,9 @@ dependencies = [
[[package]]
name = "loro-common"
-version = "1.13.1"
+version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b51042936156a1d537df8a38938ec8ed24b45de14530af933c1d02de39c9e056"
+checksum = "2146b353a56b0a743f8643725cb9f42dfe209665939b59db901ecea732a3ab29"
dependencies = [
"arbitrary",
"enum-as-inner 0.6.1",
@@ -9942,9 +9942,9 @@ dependencies = [
[[package]]
name = "loro-internal"
-version = "1.13.7"
+version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b635801310e2047f74be15f58988f2f78e7874e77aa08643fe9e6eec356f6848"
+checksum = "8d5806b65ee68c3ae7891959dd968898860dab74fbf6498b312de9b548dce5ec"
dependencies = [
"append-only-bytes",
"arref",
@@ -9989,9 +9989,9 @@ dependencies = [
[[package]]
name = "loro-kv-store"
-version = "1.13.7"
+version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b42b8d6d2c95abf67d7f50a60171759d693206796d7f8f03f3fe68e531c88794"
+checksum = "841e206d3749fd14f23e64985be230dad2b2199da1d381c44c3dd0d2b9583fcc"
dependencies = [
"bytes",
"ensure-cov",
diff --git a/bun.lock b/bun.lock
index 827d60f0f6..aa3c360af2 100644
--- a/bun.lock
+++ b/bun.lock
@@ -498,7 +498,7 @@
"undici": ">=6.24.0",
},
"catalog": {
- "loro-crdt": "1.13.7",
+ "loro-crdt": "1.16.3",
},
"packages": {
"@0no-co/graphql.web": ["@0no-co/[email protected]", "", { "peerDependencies": { "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0" }, "optionalPeers": ["graphql"] }, "sha512-Q1+pRlLhE31GOY/2c9BAEnFTNxO7Awtc6fhhEDlxyCBQ2N0IhD32cPVvPChrK9mwBNSgRdW/sF1kd2e0ojHj1Q=="],
@@ -2805,7 +2805,7 @@
"loose-envify": ["[email protected]", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
- "loro-crdt": ["[email protected]", "", {}, "sha512-tOvBUBSVseebXx8L5TMEdVYwgTOVZMMUtA9tQAs2vpPNRx/hClsLGhjmo9D3J4LopCVrQ6cuj3ZdBeyxQheWAg=="],
+ "loro-crdt": ["[email protected]", "", {}, "sha512-0FP3TsPTCIVLg7fbjh7cJTt3evpbFD/KRfnd4k9R3ZuDTzl83nuiYK/8LRxQ35xKzcHAcvMowoL9pecNDw1bkw=="],
"lru-cache": ["[email protected]", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="],
diff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.json
index d55ad49a93..c0b39f0876 100644
--- a/nix-support/node_modules-hashes.json
+++ b/nix-support/node_modules-hashes.json
@@ -1,6 +1,6 @@
{
"nodeModules": {
- "aarch64-darwin": "sha256-ocV7mJdXeLh+LKFwv5nDtNqYS0btPkEj0XxPiEWta84=",
- "x86_64-linux": "sha256-8Hv/bFFhoVNM6FugB655Z/VpLYK2Mle0VxzWIsZbaBI="
+ "aarch64-darwin": "sha256-UdoatrQ33Socze4bAzdZS/NvIC1Uc1Z+A7n9QXEFunA=",
+ "x86_64-linux": "sha256-aQvn+NvaP9aUvoZyz4fPhS9/3xcyP4/XPlV9etbUMOM="
}
}
diff --git a/package.json b/package.json
index bcdab69d8e..83681aa51c 100644
--- a/package.json
+++ b/package.json
@@ -78,7 +78,7 @@
"packages/spreadsheet"
],
"catalog": {
- "loro-crdt": "1.13.7"
+ "loro-crdt": "1.16.3"
}
}
}
diff --git a/packages/collaboration/src/collab/snapshot.test.ts b/packages/collaboration/src/collab/snapshot.test.ts
new file mode 100644
index 0000000000..28c9e1e64f
--- /dev/null
+++ b/packages/collaboration/src/collab/snapshot.test.ts
@@ -0,0 +1,104 @@
+import { LoroDoc } from 'loro-crdt';
+import { describe, expect, it } from 'vitest';
+
+describe('collaboration snapshots', () => {
+ it.each([1, 2, 3, 4, 5, 6, 7, 8, 9])('roundtrips %i branches', (heads) => {
+ const doc = new LoroDoc();
+ // Three independent histories reproduce Loro 1.13.7 choosing one head as
+ // the shallow root and dropping the history required by the other two.
+ for (let index = 1; index <= heads; index++) {
+ const peer = String(index);
+ const branch = new LoroDoc();
+ branch.setPeerId(peer);
+ branch.getMap('values').set(peer, peer);
+ doc.import(branch.export({ mode: 'update' }));
+ branch.free();
+ }
+
+ const restored = new LoroDoc();
+ restored.import(
+ doc.export({
+ mode: 'shallow-snapshot',
+ frontiers: doc.oplogFrontiers(),
+ })
+ );
+ expect(restored.toJSON()).toEqual(doc.toJSON());
+ expect(restored.version().toJSON()).toEqual(doc.version().toJSON());
+ expect(restored.oplogFrontiers()).toEqual(doc.oplogFrontiers());
+
+ const beforeEdit = doc.version();
+ restored.getMap('values').set('after-reconnect', 'preserved');
+ doc.import(restored.export({ mode: 'update', from: beforeEdit }));
+ expect(doc.toJSON()).toEqual(restored.toJSON());
+ const beforeRemoteEdit = restored.version();
+ doc.getMap('values').set('remote-edit', 'preserved');
+ restored.import(doc.export({ mode: 'update', from: beforeRemoteEdit }));
+ expect(restored.toJSON()).toEqual(doc.toJSON());
+ expect(restored.version().toJSON()).toEqual(doc.version().toJSON());
+ restored.free();
+ doc.free();
+ });
+
+ it.each([2, 3, 5])('compacts shared history with %i heads', (heads) => {
+ const doc = new LoroDoc();
+ doc.setPeerId('100');
+ const text = doc.getText('content');
+ text.insert(0, 'discarded history '.repeat(10_000));
+ doc.commit();
+ text.delete(0, text.length);
+ text.insert(0, 'Saved content');
+ doc.commit();
+ const common = doc.export({ mode: 'snapshot' });
+ const root = doc.oplogFrontiers();
+ for (let peer = 1; peer <= heads; peer++) {
+ const branch = new LoroDoc();
+ branch.import(common);
+ branch.setPeerId(String(peer));
+ branch.getMap('values').set(String(peer), peer);
+ doc.import(
+ branch.export({ mode: 'update', from: branch.frontiersToVV(root) })
+ );
+ branch.free();
+ }
+ expect(doc.oplogFrontiers()).toHaveLength(heads);
+
+ const full = doc.export({ mode: 'snapshot' });
+ const compact = doc.export({
+ mode: 'shallow-snapshot',
+ frontiers: doc.oplogFrontiers(),
+ });
+ const restored = new LoroDoc();
+ restored.import(compact);
+ expect(restored.isShallow()).toBe(true);
+ expect(restored.shallowSinceFrontiers()).toEqual(root);
+ expect(restored.toJSON()).toEqual(doc.toJSON());
+ expect(restored.version().toJSON()).toEqual(doc.version().toJSON());
+ expect(compact.length).toBeLessThan(full.length / 2);
+
+ restored.getMap('values').set('after-reconnect', true);
+ doc.import(restored.export({ mode: 'update', from: doc.version() }));
+ expect(doc.toJSON()).toEqual(restored.toJSON());
+ restored.free();
+ doc.free();
+ });
+
+ it('still compacts a document with one head', () => {
+ const doc = new LoroDoc();
+ doc.getText('content').insert(0, 'before');
+ doc.commit();
+ doc.getText('content').insert(6, ' after');
+ doc.commit();
+
+ const restored = new LoroDoc();
+ restored.import(
+ doc.export({
+ mode: 'shallow-snapshot',
+ frontiers: doc.oplogFrontiers(),
+ })
+ );
+ expect(restored.isShallow()).toBe(true);
+ expect(restored.toJSON()).toEqual(doc.toJSON());
+ restored.free();
+ doc.free();
+ });
+});
diff --git a/services/sync-service/Cargo.toml b/services/sync-service/Cargo.toml
index fd2ed39c13..a9579d4c5d 100644
--- a/services/sync-service/Cargo.toml
+++ b/services/sync-service/Cargo.toml
@@ -43,7 +43,7 @@ serde.workspace = true
getrandom = { workspace = true, features = ["wasm_js"] }
wasm-bindgen.workspace = true
uuid = { workspace = true, features = ["js"] }
-loro = "1.13"
+loro = "1.16.2"
macro_sync_service_jwt = { path = "../../crates/macro_sync_service_jwt" }
matchit = "0.8.4"
worker-rs-otel = { path = "../../crates/worker-rs-otel" }
diff --git a/services/sync-service/src/state.rs b/services/sync-service/src/state.rs
index cef4c599e8..9e4933bd68 100644
--- a/services/sync-service/src/state.rs
+++ b/services/sync-service/src/state.rs
@@ -9,6 +9,9 @@ use worker::Result;
use crate::error::ResultExt;
+#[cfg(test)]
+mod test;
+
const FROM_CLIENT_TAG: &str = "from_client";
const FROM_SERVICE_TAG: &str = "from_service";
const FRONTIERS_ID_SEPERATOR: &str = "|";
diff --git a/services/sync-service/src/state/test.rs b/services/sync-service/src/state/test.rs
new file mode 100644
index 0000000000..9c7872d1d9
--- /dev/null
+++ b/services/sync-service/src/state/test.rs
@@ -0,0 +1,97 @@
+use super::*;
+
+#[test]
+fn concurrent_branches_survive_session_snapshot_roundtrip() {
+ for heads in 1..=9 {
+ let state = DocumentState::new();
+ for peer in 1..=heads {
+ let branch = LoroDoc::new();
+ branch.set_peer_id(peer).unwrap();
+ branch
+ .get_map("values")
+ .insert(&peer.to_string(), peer as i64)
+ .unwrap();
+ state
+ .loro_doc
+ .import(&branch.export(ExportMode::all_updates()).unwrap())
+ .unwrap();
+ }
+
+ let snapshot = state.export_shallow_snapshot().unwrap();
+ let restored = LoroDoc::new();
+ restored.import(&snapshot).unwrap();
+ assert_eq!(restored.get_deep_value(), state.loro_doc.get_deep_value());
+ assert_eq!(restored.oplog_vv(), state.loro_doc.oplog_vv());
+ assert_eq!(restored.oplog_frontiers(), state.loro_doc.oplog_frontiers());
+
+ restored
+ .get_map("values")
+ .insert("after-reconnect", true)
+ .unwrap();
+ state
+ .loro_doc
+ .import(
+ &restored
+ .export(ExportMode::updates(&state.loro_doc.oplog_vv()))
+ .unwrap(),
+ )
+ .unwrap();
+ state
+ .loro_doc
+ .get_map("values")
+ .insert("remote-edit", true)
+ .unwrap();
+ restored
+ .import(
+ &state
+ .loro_doc
+ .export(ExportMode::updates(&restored.oplog_vv()))
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(restored.get_deep_value(), state.loro_doc.get_deep_value());
+ assert_eq!(restored.oplog_vv(), state.loro_doc.oplog_vv());
+ }
+}
+
+#[test]
+fn shared_history_still_compacts_with_concurrent_heads() {
+ for heads in [2, 3, 5] {
+ let state = DocumentState::new();
+ let text = state.loro_doc.get_text("content");
+ text.insert(0, &"discarded history ".repeat(10_000))
+ .unwrap();
+ state.loro_doc.commit();
+ text.delete(0, text.len_unicode()).unwrap();
+ text.insert(0, "Saved content").unwrap();
+ state.loro_doc.commit();
+ let root = state.loro_doc.oplog_frontiers();
+ let common = state.export_snapshot(None).unwrap();
+ for peer in 1..=heads {
+ let branch = LoroDoc::from_snapshot(&common).unwrap();
+ branch.set_peer_id(peer).unwrap();
+ branch
+ .get_map("values")
+ .insert(&peer.to_string(), peer as i64)
+ .unwrap();
+ state
+ .loro_doc
+ .import(
+ &branch
+ .export(ExportMode::updates(&state.loro_doc.oplog_vv()))
+ .unwrap(),
+ )
+ .unwrap();
+ }
+ assert_eq!(state.loro_doc.oplog_frontiers().len(), heads as usize);
+
+ let full = state.export_snapshot(None).unwrap();
+ let compact = state.export_shallow_snapshot().unwrap();
+ let restored = LoroDoc::from_snapshot(&compact).unwrap();
+ assert!(restored.is_shallow());
+ assert_eq!(restored.shallow_since_frontiers(), root);
+ assert_eq!(restored.get_deep_value(), state.loro_doc.get_deep_value());
+ assert_eq!(restored.oplog_vv(), state.loro_doc.oplog_vv());
+ assert!(compact.len() < full.len() / 2);
+ }
+}