Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(sync-service): name every document history author (#9014)
Wolf Mermelstein158a17189595
diff --git a/apps/web/src/features/block-md/history/HistoryContext.tsx b/apps/web/src/features/block-md/history/HistoryContext.tsxindex 502ddec2f5..033c342522 100644--- a/apps/web/src/features/block-md/history/HistoryContext.tsx+++ b/apps/web/src/features/block-md/history/HistoryContext.tsx@@ -157,8 +157,11 @@ export function HistoryProvider(props: {return peers ? [...new Set(peers.values())] : [];});const userEntries = mapArray(uniqueUserIds, (userId) => {+ const macroId = tryMacroId(userId);const displayName = () =>- getDisplayName(tryMacroId(userId), { emailFallback: 'local-part' });+ macroId+ ? getDisplayName(macroId, { emailFallback: 'local-part' })+ : userLabel(userId);return { userId, displayName, color: userColor(userId) };});const userById = (userId: string): HistoryUser =>diff --git a/apps/web/src/features/block-md/history/utils.ts b/apps/web/src/features/block-md/history/utils.tsindex 9848d92975..1603225abd 100644--- a/apps/web/src/features/block-md/history/utils.ts+++ b/apps/web/src/features/block-md/history/utils.ts@@ -7,8 +7,15 @@ export function userColor(userId: string): string {return `var(--color-${color}, var(--color-pink))`;}+/**+ * Author sync-service records for edits no user made, such as the imported+ * content a document starts from. Mirrors `SYSTEM_AUTHOR_ID` in sync-service.+ */+export const SYSTEM_AUTHOR_ID = 'system';+export function userLabel(userId: string): string {if (userId === 'unknown') return 'Unknown';+ if (userId === SYSTEM_AUTHOR_ID) return MACRO_AGENT_NAME;if (isMacroAgentId(userId)) return `${MACRO_AGENT_NAME} (AI)`;const id = tryMacroId(userId);return id ? macroIdToEmail(id) : userId;diff --git a/services/sync-service/AGENTS.md b/services/sync-service/AGENTS.mdindex fe51144eac..4a235ff152 100644--- a/services/sync-service/AGENTS.md+++ b/services/sync-service/AGENTS.md@@ -64,6 +64,11 @@ legacy reserved AI peer range; Canvas workers use random full-width peer IDs.proportional to the update; `LoroDoc::diff`, `fork`, and checkouts cost timethat grows with document history. A backlog past `DEFAULT_TIMEOUT_MS` (4.5 s)makes the front worker fail connects, which clients see as close 1006.+- Document history names each Loro peer from D1 `peer_user_map`, so every+ path that writes operations maps the peers it writes with: WebSocket and+ HTTP `/update` writes to their user, `/initialize` snapshots to the system+ author (`SYSTEM_AUTHOR_ID`), and `/copy` copies the source's mappings first.+ A new write path does the same (`tests/peer_authors.test.ts`).- `migration-test-hooks` is test-only fault injection — never enable it in adeployed worker (see the feature's doc comment in `Cargo.toml`).- Never run `just deploy-dev` / `just deploy-prod` without explicit approval.diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rsindex ba4c0c87fc..d906ca9462 100644--- a/services/sync-service/src/cf_worker.rs+++ b/services/sync-service/src/cf_worker.rs@@ -6,7 +6,11 @@ use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Resultuse crate::{auth::is_internal,- constants::header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},+ constants::{+ USER_PEER_D1_BINDING,+ header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},+ },+ d1::copy_user_mappings,durable_object::{CopyDocumentRequest, GetSnapshotRequest, response, status_codes},error::ResultExt,generated::schema::InitializeFromSnapshotRequest,@@ -179,6 +183,11 @@ pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Resubuf};+ // The copy keeps the source's history, so it keeps its authors too. Copied+ // before initializing, which maps every still-unmapped peer to the system.+ let db = env.d1(USER_PEER_D1_BINDING)?;+ copy_user_mappings(&db, document_id, &new_document_id).await?;+let initialize_path = format!("/document/{}/initialize", new_document_id);let ss_res = do_helper(&env, init_body, &initialize_path, &req, &new_document_id).await?;Ok(ss_res)diff --git a/services/sync-service/src/constants.rs b/services/sync-service/src/constants.rsindex 4e534e11ed..5a0671b8b4 100644--- a/services/sync-service/src/constants.rs+++ b/services/sync-service/src/constants.rs@@ -6,3 +6,9 @@ pub mod header_names {}pub const USER_PEER_D1_BINDING: &str = "USER_PEER_MAPPING";++/// Author of operations written without a user: snapshots that services+/// initialize documents from (markdown and Notion imports, uploads, copies of+/// unattributed history). Keep in sync with the web history UI+/// (`SYSTEM_AUTHOR_ID`).+pub const SYSTEM_AUTHOR_ID: &str = "system";diff --git a/services/sync-service/src/d1.rs b/services/sync-service/src/d1.rsindex 838562a934..c52cf0ce2a 100644--- a/services/sync-service/src/d1.rs+++ b/services/sync-service/src/d1.rs@@ -41,6 +41,46 @@ pub async fn insert_user_mapping(Ok(())}+/// Map peers that wrote operations to `user_id`, keeping any mapping a peer+/// already has: a peer that registered explicitly stays with its owner.+pub async fn insert_missing_user_mappings(+ db: &D1Database,+ document_id: &str,+ user_id: &str,+ peer_ids: &[u64],+) -> worker::Result<()> {+ for chunk in peer_ids.chunks(BATCH_CHUNK_SIZE) {+ let statements = chunk+ .iter()+ .map(|peer_id| {+ db.prepare(+ "INSERT OR IGNORE INTO peer_user_map (document_id, peer_id, user_id) VALUES (?, ?, ?);",+ )+ .bind(&[document_id.into(), peer_id.to_string().into(), user_id.into()])+ })+ .collect::<worker::Result<Vec<_>>>()?;+ db.batch(statements).await?;+ }+ Ok(())+}++/// Give a copied document the source document's peer mappings. Existing+/// mappings on the target are kept.+pub async fn copy_user_mappings(+ db: &D1Database,+ source_document_id: &str,+ target_document_id: &str,+) -> worker::Result<()> {+ db.prepare(+ "INSERT OR IGNORE INTO peer_user_map (document_id, peer_id, user_id) \+ SELECT ?, peer_id, user_id FROM peer_user_map WHERE document_id = ?;",+ )+ .bind(&[target_document_id.into(), source_document_id.into()])?+ .run()+ .await?;+ Ok(())+}+pub async fn get_user_id_from_peer_id(db: D1Database,document_id: &str,diff --git a/services/sync-service/src/domain/document.rs b/services/sync-service/src/domain/document.rsindex cdb0a42130..e10f6c1116 100644--- a/services/sync-service/src/domain/document.rs+++ b/services/sync-service/src/domain/document.rs@@ -174,6 +174,22 @@ fn export_state(doc: &LoroDoc, mode: ExportMode<'_>) -> Result<(Vec<u8>, Vec<u8>Ok((snapshot, revision))}+/// Peers that authored operations carried by an update or snapshot, so they+/// can be mapped to the writer. Read from the blob, not the import result, so a+/// retried write that changes nothing still registers its peers.+pub fn writing_peers(blob: &[u8]) -> Result<Vec<u64>, DocumentError> {+ let meta = LoroDoc::decode_import_blob_meta(blob, false)+ .map_err(|_| DocumentError::Invalid("Invalid Loro update."))?;+ let mut peers: Vec<u64> = meta+ .partial_end_vv+ .iter()+ .filter(|(peer, end)| **end > meta.partial_start_vv.get(peer).copied().unwrap_or(0))+ .map(|(peer, _)| *peer)+ .collect();+ peers.sort_unstable();+ Ok(peers)+}+/// Synchronous preflight. The caller must apply the returned delta before its/// next await, so websocket writes cannot interleave with this version check.pub fn prepare_update(diff --git a/services/sync-service/src/domain/document/test.rs b/services/sync-service/src/domain/document/test.rsindex d9830b2c9c..0c73cd685d 100644--- a/services/sync-service/src/domain/document/test.rs+++ b/services/sync-service/src/domain/document/test.rs@@ -313,3 +313,29 @@ fn session_attribution_uses_the_verified_human_or_agent_identity() {);assert!(DocumentAttribution::from_session_claims(None, Some("x".repeat(1025))).is_err());}++#[test]+fn writing_peers_are_the_peers_with_operations_in_the_blob() {+ let doc = LoroDoc::new();+ doc.set_peer_id(11).unwrap();+ doc.get_text("content").insert(0, "imported").unwrap();+ doc.commit();+ doc.set_peer_id(22).unwrap();+ doc.get_text("content").insert(0, "edited ").unwrap();+ doc.commit();+ let snapshot = doc.export(ExportMode::Snapshot).unwrap();+ assert_eq!(writing_peers(&snapshot).unwrap(), [11, 22]);++ let seen = doc.oplog_vv();+ doc.set_peer_id(33).unwrap();+ doc.get_text("content").insert(0, "more ").unwrap();+ doc.commit();+ let update = doc+ .export(ExportMode::Updates {+ from: Cow::Owned(seen),+ })+ .unwrap();+ assert_eq!(writing_peers(&update).unwrap(), [33]);++ assert!(writing_peers(b"not a loro blob").is_err());+}diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rsindex c4da68fce4..192e688e91 100644--- a/services/sync-service/src/durable_object.rs+++ b/services/sync-service/src/durable_object.rs@@ -19,9 +19,11 @@ use worker::{use crate::{auth::{AccessLevel, TokenFrom, WriteAccess, decode_jwt, socket_access},- constants::USER_PEER_D1_BINDING,- d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},- domain::document::DocumentAttribution,+ constants::{SYSTEM_AUTHOR_ID, USER_PEER_D1_BINDING},+ d1::{+ PeerWithUserId, get_user_id_from_peer_id, insert_missing_user_mappings, insert_user_mapping,+ },+ domain::document::{DocumentAttribution, writing_peers},dss_internal::InteractionReason,error::ResultExt,generated::schema::InitializeFromSnapshotRequest,@@ -145,6 +147,9 @@ pub struct DocumentSyncSession {pending_blame: Arc<Mutex<Vec<crate::d1::BlameEvent>>>,/// Distinct editors since the last snapshot notification.pending_editors: Arc<Mutex<BTreeSet<DocumentAttribution>>>,+ /// Writing peers already recorded in D1 since this object woke, so+ /// updates skip remapping them.+ mapped_writing_peers: Mutex<BTreeSet<u64>>,surface_lifecycle: Mutex<Option<SurfaceLifecycle>>,source_migration: Mutex<Option<surface_migration::SourceState>>,// Serialize whole callbacks, including persistence awaits. A freeze must@@ -503,6 +508,13 @@ impl DocumentSyncSession {debug!(document_id = document_id, "Initializing snapshot");let body_raw = req.bytes().await?;let body = InitializeFromSnapshotRequest::deserialize(&body_raw).with_context(|| format!("Failed to deserialize InitializeFromSnapshotRequest with document_id: [{document_id}]"))?;+ // Services initialize documents (imports, uploads, copies), so the+ // snapshot's peers belong to the system author unless already mapped.+ // Mapped before storing: a failed store is retried, a stored one is not.+ let writers = writing_peers(&body.snapshot)+ .map_err(|error| Error::from(format!("invalid initial snapshot: {error}")))?;+ let db = self.env.d1(USER_PEER_D1_BINDING)?;+ insert_missing_user_mappings(&db, document_id, SYSTEM_AUTHOR_ID, &writers).await?;storage.store_snapshot(&body.snapshot).await?;*self.document_id@@ -1007,6 +1019,7 @@ impl DurableObject for DocumentSyncSession {inbound: Arc::new(Mutex::new(HashMap::new())),pending_blame: Arc::new(Mutex::new(Vec::new())),pending_editors: Arc::new(Mutex::new(Default::default())),+ mapped_writing_peers: Mutex::new(Default::default()),surface_lifecycle: Mutex::new(None),source_migration: Mutex::new(None),mutation_barrier: futures::lock::Mutex::new(()),diff --git a/services/sync-service/src/durable_object/document_api.rs b/services/sync-service/src/durable_object/document_api.rsindex d8e6a99fe6..28a6dc1d42 100644--- a/services/sync-service/src/durable_object/document_api.rs+++ b/services/sync-service/src/durable_object/document_api.rs@@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize};use super::document_effects::WorkerDocumentEffects;use super::*;use crate::{+ constants::SYSTEM_AUTHOR_ID,+ d1::insert_missing_user_mappings,domain::document::{self, DocumentError, MAX_BINARY_BYTES, MAX_REVISION_BYTES},storage::DocumentUpdateStorage,};@@ -123,6 +125,12 @@ impl DocumentSyncSession {let storage = self.session_storage().await?;// Only agent actors are stored with the operation; humans are still editors.let is_agent = claims.actor.is_some();+ // Agents write for the user they act on behalf of; tokens without a user+ // are service writes.+ let author = claims+ .user_id+ .clone()+ .unwrap_or_else(|| SYSTEM_AUTHOR_ID.to_owned());let editor = match document::DocumentAttribution::from_session_claims(claims.actor,claims.user_id,@@ -148,6 +156,13 @@ impl DocumentSyncSession {Ok(prepared) => prepared,Err(error) => return error_response(error),};+ // Taken from the request, so a retry of an applied update still maps them.+ let writers = match document::writing_peers(&update) {+ Ok(writers) => writers,+ Err(error) => return error_response(error),+ };+ let db = self.env.d1(USER_PEER_D1_BINDING)?;+ insert_missing_user_mappings(&db, document_id, &author, &writers).await?;Response::from_json(&UpdateResponse {revision: STANDARD.encode(prepared.revision),applied: prepared.applied,diff --git a/services/sync-service/src/durable_object/peer_ids.rs b/services/sync-service/src/durable_object/peer_ids.rsindex 2c08f3bf3e..9fdf6e21c9 100644--- a/services/sync-service/src/durable_object/peer_ids.rs+++ b/services/sync-service/src/durable_object/peer_ids.rs@@ -1,5 +1,9 @@-use super::WebSocketMetadata;-use crate::ai_peer::is_ai_peer;+use worker::{Error, Result};++use super::{WebSocketMetadata, Wsm};+use crate::{+ ai_peer::is_ai_peer, constants::USER_PEER_D1_BINDING, d1::insert_missing_user_mappings,+};impl WebSocketMetadata {pub(super) fn active_peer_ids(&self, include_ai: bool) -> Vec<u64> {@@ -14,5 +18,47 @@ impl WebSocketMetadata {}}+impl Wsm<'_> {+ /// Map peers that wrote through this socket without registering first (a+ /// client's peer id can change, and offline edits replay under an old one).+ /// Unlike registration this keeps existing mappings and leaves the+ /// socket's own peers, and so presence, unchanged.+ pub async fn add_writing_peerids(&mut self, peerids: &[u64], document_id: &str) -> Result<()> {+ self.maybe_update_ws_meta_map().await?;+ let ws_id = self.get_ws_id()?.to_string();+ let user_id = {+ let wmm = self.dss.ws_meta_map.lock("Wsm::add_writing_peerids get");+ let meta = wmm.get(&ws_id).ok_or(Error::from("missing ws metadata"))?;+ let Some(user_id) = meta.user_id.clone() else {+ return Ok(());+ };+ user_id+ };+ let new_peers: Vec<u64> = {+ let mapped = self+ .dss+ .mapped_writing_peers+ .lock("Wsm::add_writing_peerids mapped");+ peerids+ .iter()+ .copied()+ .filter(|peer| !mapped.contains(peer))+ .collect()+ };+ if new_peers.is_empty() {+ return Ok(());+ }+ let db = self.dss.env.d1(USER_PEER_D1_BINDING)?;+ insert_missing_user_mappings(&db, document_id, &user_id, &new_peers).await?;+ // Remembered only once stored, so a failed write is retried with the+ // client's resent update.+ self.dss+ .mapped_writing_peers+ .lock("Wsm::add_writing_peerids extend")+ .extend(new_peers);+ Ok(())+ }+}+#[cfg(test)]mod test;diff --git a/services/sync-service/src/socket/protocol.rs b/services/sync-service/src/socket/protocol.rsindex 0b23a21d82..b4a55fe2a4 100644--- a/services/sync-service/src/socket/protocol.rs+++ b/services/sync-service/src/socket/protocol.rs@@ -5,6 +5,7 @@ use worker::Result;use crate::{auth::WriteAccess,+ domain::document::writing_peers,durable_object::{DocumentSyncSession, Wsm},error::ResultExt,generated::schema::{FromPeer, FromRemote},@@ -254,6 +255,20 @@ pub async fn process_message(}}+ // Map every peer the update was written with, not only the one the+ // client registered, so history can name its author. Before the+ // ack: a failure here makes the client resend and retry it.+ let mut writers = Vec::new();+ for update in &updates {+ writers.extend(+ writing_peers(update)+ .map_err(|error| worker::Error::from(error.to_string()))?,+ );+ }+ Wsm::new(dss, sender.websocket())+ .add_writing_peerids(&writers, document_id)+ .await?;+// ACK only after the update is durably stored.sender.send(FromRemote::RemoteUpdateAck { id })diff --git a/services/sync-service/tests/peer_authors.test.ts b/services/sync-service/tests/peer_authors.test.tsnew file mode 100644index 0000000000..afc77115da--- /dev/null+++ b/services/sync-service/tests/peer_authors.test.ts@@ -0,0 +1,159 @@+import { LoroDoc } from "loro-crdt";+import type { Miniflare } from "miniflare";+import { afterAll, beforeAll, describe, expect, test } from "vitest";+import {+ FromPeer,+ InitializeFromSnapshotRequest,+} from "../bebop/generated/schema";+import {+ copyDocument,+ createTestUser,+ getTokenForDocument,+ setupMiniflare,+} from "./utils";++// Document history names the author of every peer through /metadata, so each+// path that writes operations must map the peers it writes with.++let mf: Miniflare;++beforeAll(async () => {+ mf = await setupMiniflare();+}, 60_000);++afterAll(async () => {+ await mf?.dispose();+});++async function initialize(documentId: string, snapshot: Uint8Array) {+ const response = await mf.dispatchFetch(+ `http://localhost:8787/document/${documentId}/initialize`,+ {+ method: "POST",+ headers: {+ "Content-Type": "application/octet-stream",+ Authorization: `Bearer ${getTokenForDocument(documentId, "importer", "owner")}`,+ },+ body: InitializeFromSnapshotRequest.encode({ snapshot }),+ },+ );+ expect(response.status).toBe(200);+}++async function authors(documentId: string): Promise<Record<string, string>> {+ const response = await mf.dispatchFetch(+ `http://localhost:8787/document/${documentId}/metadata`,+ {+ headers: {+ Authorization: `Bearer ${getTokenForDocument(documentId, "reader", "view")}`,+ },+ },+ );+ expect(response.status).toBe(200);+ const body = (await response.json()) as {+ peers: { peer_id: string; user_id: string }[];+ };+ return Object.fromEntries(body.peers.map((p) => [p.peer_id, p.user_id]));+}++describe("peer authors", () => {+ test("an initialized snapshot's peers belong to the system author", async () => {+ const doc = new LoroDoc();+ doc.setPeerId(1001n);+ doc.getText("content").insert(0, "imported markdown");+ doc.commit();++ await initialize("initialized-doc", doc.export({ mode: "snapshot" }));++ expect(await authors("initialized-doc")).toEqual({ "1001": "system" });+ });++ test("a websocket write maps peers the client never registered", async () => {+ const seed = new LoroDoc();+ seed.setPeerId(2001n);+ seed.getText("content").insert(0, "start");+ seed.commit();+ await initialize("websocket-doc", seed.export({ mode: "snapshot" }));++ const user = await createTestUser(mf, "websocket-doc", { userId: "editor-user" });+ const registeredPeer = user.doc.peerIdStr;+ // A client's peer id can change after it registered; its edits arrive+ // under the new id without another registration.+ const from = user.doc.version();+ user.doc.setPeerId(2002n);+ user.doc.getText("content").insert(0, "typed ");+ user.doc.commit();+ user.connection.send(+ FromPeer.fromPeerUpdate({+ updates: [user.doc.export({ mode: "update", from })],+ id: crypto.randomUUID(),+ }).encode(),+ );+ const ack = await user.readNextMessage();+ expect(ack.isRemoteUpdateAck()).toBe(true);++ expect(await authors("websocket-doc")).toEqual({+ "2001": "system",+ "2002": "editor-user",+ [registeredPeer]: "editor-user",+ });+ user.getWebSocket().close();+ });++ test("an HTTP update maps its peers to the token's user", async () => {+ const doc = new LoroDoc();+ doc.setPeerId(3001n);+ doc.getText("content").insert(0, "start");+ doc.commit();+ await initialize("http-doc", doc.export({ mode: "snapshot" }));++ const from = doc.version();+ const expectedRevision = Buffer.from(from.encode()).toString("base64");+ doc.setPeerId(3002n);+ doc.getText("content").insert(0, "agent ");+ doc.commit();+ const response = await mf.dispatchFetch(+ "http://localhost:8787/document/http-doc/update",+ {+ method: "POST",+ headers: {+ Authorization: `Bearer ${getTokenForDocument("http-doc", "editor-user", "edit")}`,+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ expectedRevision,+ update: Buffer.from(doc.export({ mode: "update", from })).toString(+ "base64",+ ),+ }),+ },+ );+ expect(response.status).toBe(200);++ expect(await authors("http-doc")).toEqual({+ "3001": "system",+ "3002": "editor-user",+ });+ });++ test("a copy keeps the source document's authors", async () => {+ const seed = new LoroDoc();+ seed.setPeerId(4001n);+ seed.getText("content").insert(0, "start");+ seed.commit();+ await initialize("copy-source", seed.export({ mode: "snapshot" }));+ const user = await createTestUser(mf, "copy-source", { userId: "editor-user" });+ user.makeChange(" edited");+ const ack = await user.readNextMessage();+ expect(ack.isRemoteUpdateAck()).toBe(true);+ user.getWebSocket().close();++ const response = await copyDocument(mf, "copy-source", "copy-target");+ expect(response.status).toBe(200);++ expect(await authors("copy-target")).toEqual({+ "4001": "system",+ [user.doc.peerIdStr]: "editor-user",+ });+ });+});