Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(sync-service): find touched nodes from imported op spans (#8254)

teo29608099d3b0
diff --git a/services/sync-service/AGENTS.md b/services/sync-service/AGENTS.md
index dee28ce67a..93900d2d48 100644
--- a/services/sync-service/AGENTS.md
+++ b/services/sync-service/AGENTS.md
@@ -56,6 +56,11 @@ updates over WebSocket.
- Durable Object in-memory state (including the loro document) evicts after
10s of inactivity even with open connections; the alarm-based heartbeat in
`src/keepalive.rs` is what keeps it warm, not WebSocket traffic.
+- The Durable Object handles one request or WebSocket message at a time
+ (`mutation_barrier` in `src/durable_object.rs`). Keep per-update work
+ proportional to the update; `LoroDoc::diff`, `fork`, and checkouts cost time
+ that grows with document history. A backlog past `DEFAULT_TIMEOUT_MS` (4.5 s)
+ makes the front worker fail connects, which clients see as close 1006.
- `migration-test-hooks` is test-only fault injection — never enable it in a
deployed worker (see the feature's doc comment in `Cargo.toml`).
- Never run `just deploy-dev` / `just deploy-prod` without explicit approval.
diff --git a/services/sync-service/README.md b/services/sync-service/README.md
index 638c5d808c..ee876152bf 100644
--- a/services/sync-service/README.md
+++ b/services/sync-service/README.md
@@ -58,7 +58,8 @@ A consuming client can fetch this schema during build time and generate the corr
## Features
-By default `bebop-owned-all`, `alarm-keep-alive` and `create-default-state` are all enabled.
+By default `bebop-owned-all`, `do-sqlite-snapshot-storage` and `search-service` are enabled. `just worker-build`
+adds `create-default-state`; `alarm-keep-alive` is opt-in.
- `bebop-owned-all`: Enables all bebop owned types
- `create-default-state`: As of writing this there is no mechanism to initialize a document's state
diff --git a/services/sync-service/src/state.rs b/services/sync-service/src/state.rs
index 1af0c60299..997eacb6e3 100644
--- a/services/sync-service/src/state.rs
+++ b/services/sync-service/src/state.rs
@@ -1,7 +1,7 @@
-use std::{borrow::Cow, sync::Mutex};
+use std::{borrow::Cow, collections::HashSet, sync::Mutex};
use loro::{
- Container, ContainerID, ExportMode, Frontiers, LoroDoc, LoroValue, ToJson, VersionVector,
+ Container, ContainerID, ExportMode, Frontiers, ID, LoroDoc, LoroValue, ToJson, VersionVector,
};
use tracing::debug;
use web_time::Instant;
@@ -107,11 +107,11 @@ impl DocumentState {
/// Import an update and report whether it changed the operation log plus
/// the deduplicated Lexical node IDs used by blame tracking.
pub fn import(&self, update: &[u8]) -> Result<ImportedUpdate> {
- let before = self.loro_doc.oplog_frontiers();
+ let before = self.loro_doc.oplog_vv();
self.loro_doc
.import_with(update, FROM_CLIENT_TAG)
.context("failed to import update")?;
- let after = self.loro_doc.oplog_frontiers();
+ let after = self.loro_doc.oplog_vv();
*self
.last_update
@@ -153,15 +153,24 @@ impl DocumentState {
}))
}
- /// Diff the two frontiers and return the Lexical node IDs whose backing
- /// containers were modified, deduplicated. Empty `Vec` on diff failure.
- fn touched_lexical_ids(&self, before: &Frontiers, after: &Frontiers) -> Vec<String> {
- let Ok(diff) = self.loro_doc.diff(before, after) else {
- return Vec::new();
- };
- let mut touched: Vec<String> = diff
+ /// The deduplicated Lexical node IDs whose backing containers were written
+ /// by operations in `after` but not in `before`. This runs on every update,
+ /// so it reads only those operations: diffing the two versions checks the
+ /// document out and costs time that grows with its history.
+ fn touched_lexical_ids(&self, before: &VersionVector, after: &VersionVector) -> Vec<String> {
+ let mut containers = HashSet::new();
+ for span in after.sub_iter(before) {
+ let Ok(len) = usize::try_from(span.counter.end - span.counter.start) else {
+ continue;
+ };
+ containers.extend(
+ self.loro_doc
+ .get_changed_containers_in(ID::new(span.peer, span.counter.start), len),
+ );
+ }
+ let mut touched: Vec<String> = containers
.iter()
- .filter_map(|(cid, _)| self.find_lexical_id(cid))
+ .filter_map(|cid| self.find_lexical_id(cid))
.collect();
touched.sort();
touched.dedup();
diff --git a/services/sync-service/src/state/test.rs b/services/sync-service/src/state/test.rs
index d3a65d1baf..355d7bd3d3 100644
--- a/services/sync-service/src/state/test.rs
+++ b/services/sync-service/src/state/test.rs
@@ -1,5 +1,67 @@
+use loro::{LoroMap, LoroMovableList, LoroText};
+
use super::*;
+#[test]
+fn imported_updates_report_the_lexical_nodes_they_touch() {
+ let state = DocumentState::new();
+ let writer = LoroDoc::new();
+ writer.set_peer_id(1).unwrap();
+ let children = writer
+ .get_map("root")
+ .insert_container("children", LoroMovableList::new())
+ .unwrap();
+ let mut texts = Vec::new();
+ for id in ["intro", "body", "outro"] {
+ let node = children.push_container(LoroMap::new()).unwrap();
+ node.insert_container("$", LoroMap::new())
+ .unwrap()
+ .insert("id", id)
+ .unwrap();
+ texts.push(node.insert_container("text", LoroText::new()).unwrap());
+ }
+ writer.commit();
+ let created = state
+ .import(&writer.export(ExportMode::all_updates()).unwrap())
+ .unwrap();
+ assert!(created.changed);
+ assert_eq!(created.touched_nodes, ["body", "intro", "outro"]);
+
+ let seen = writer.oplog_vv();
+ for (peer, text) in [(2, &texts[0]), (3, &texts[2]), (2, &texts[0])] {
+ writer.set_peer_id(peer).unwrap();
+ text.insert(0, "typed").unwrap();
+ writer.commit();
+ }
+ let edited = state
+ .import(&writer.export(ExportMode::updates(&seen)).unwrap())
+ .unwrap();
+ assert!(edited.changed);
+ assert_eq!(edited.touched_nodes, ["intro", "outro"]);
+
+ let seen = writer.oplog_vv();
+ writer.set_peer_id(4).unwrap();
+ texts[1].insert(0, "first").unwrap();
+ writer.commit();
+ let missing = writer.oplog_vv();
+ let first = writer.export(ExportMode::updates(&seen)).unwrap();
+ let child = children.insert_container(0, LoroMap::new()).unwrap();
+ child
+ .insert_container("$", LoroMap::new())
+ .unwrap()
+ .insert("id", "heading")
+ .unwrap();
+ writer.commit();
+ let early = state
+ .import(&writer.export(ExportMode::updates(&missing)).unwrap())
+ .unwrap();
+ assert!(!early.changed);
+ assert!(early.touched_nodes.is_empty());
+ let unblocking = state.import(&first).unwrap();
+ assert!(unblocking.changed);
+ assert_eq!(unblocking.touched_nodes, ["body", "heading"]);
+}
+
#[test]
fn concurrent_branches_survive_session_snapshot_roundtrip() {
for heads in 1..=9 {
diff --git a/services/sync-service/tests/multi_writer_history.test.ts b/services/sync-service/tests/multi_writer_history.test.ts
new file mode 100644
index 0000000000..862be34471
--- /dev/null
+++ b/services/sync-service/tests/multi_writer_history.test.ts
@@ -0,0 +1,73 @@
+import { LoroMap, LoroMovableList, LoroText } from "loro-crdt";
+import { Miniflare } from "miniflare";
+import { afterAll, beforeAll, expect, test } from "vitest";
+import { FromPeer, FromRemote } from "../bebop/generated/schema";
+import { createTestUser, setupMiniflare } from "./utils";
+
+let mf: Miniflare;
+
+beforeAll(async () => {
+ mf = await setupMiniflare();
+});
+
+afterAll(async () => {
+ await mf.dispose();
+});
+
+// Every request to the Durable Object waits behind queued messages, and the
+// front worker fails a connect it can't answer within `DEFAULT_TIMEOUT_MS`.
+const CONNECT_BUDGET_MS = 4_500;
+
+test("keystrokes into sections typed by rotating AI writers are acknowledged within the connect budget", async () => {
+ const documentId = crypto.randomUUID();
+ const ai = await createTestUser(mf, documentId, { userId: "ai" });
+ const children = ai.doc.getMap("root").setContainer("children", new LoroMovableList());
+ for (let paragraph = 0; paragraph < 400; paragraph++) {
+ const node = children.insertContainer(paragraph, new LoroMap());
+ node.setContainer("$", new LoroMap()).set("id", `paragraph-${paragraph}`);
+ node.setContainer("text", new LoroText()).insert(0, "lorem ipsum ".repeat(30));
+ }
+ ai.doc.commit();
+ const sections: LoroText[] = [];
+ for (const writer of [0, 1, 2]) {
+ ai.doc.setPeerId(BigInt(100 + writer));
+ const node = children.insertContainer(writer * 7 + 1, new LoroMap());
+ node.setContainer("$", new LoroMap()).set("id", `section-${writer}`);
+ sections.push(node.setContainer("text", new LoroText()));
+ ai.doc.commit();
+ }
+ for (let keystroke = 0; keystroke < 15_000; keystroke++) {
+ ai.doc.setPeerId(BigInt(100 + (keystroke % 3)));
+ const section = sections[keystroke % 3];
+ section.insert(section.length, "abc");
+ ai.doc.commit();
+ }
+ ai.connection.send(
+ FromPeer.fromPeerUpdate({ updates: [ai.doc.export({ mode: "update" })], id: "history" }).encode(),
+ );
+ for (;;) {
+ const message = FromRemote.decode(new Uint8Array((await ai.connection.waitForNextMessage(120_000)) as ArrayBuffer));
+ if (message.isRemoteUpdateAck() && message.value.id === "history") break;
+ }
+
+ const started = performance.now();
+ for (let keystroke = 0; keystroke < 300; keystroke++) {
+ const version = ai.doc.oplogVersion();
+ ai.doc.setPeerId(BigInt(100 + (keystroke % 3)));
+ const section = sections[keystroke % 3];
+ section.insert(section.length, "xyz");
+ ai.doc.commit();
+ ai.connection.send(
+ FromPeer.fromPeerUpdate({
+ updates: [ai.doc.export({ mode: "update", from: version })],
+ id: `keystroke-${keystroke}`,
+ }).encode(),
+ );
+ }
+ for (;;) {
+ const message = FromRemote.decode(new Uint8Array((await ai.connection.waitForNextMessage(120_000)) as ArrayBuffer));
+ if (message.isRemoteUpdateAck() && message.value.id === "keystroke-299") break;
+ }
+
+ expect(performance.now() - started).toBeLessThan(CONNECT_BUDGET_MS);
+}, 180_000);