Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(sync-service): name every document history author (#9014)

Wolf Mermelstein158a17189595
diff --git a/apps/web/src/features/block-md/history/HistoryContext.tsx b/apps/web/src/features/block-md/history/HistoryContext.tsx
index 502ddec2f5..033c342522 100644
--- a/apps/web/src/features/block-md/history/HistoryContext.tsx
+++ b/apps/web/src/features/block-md/history/HistoryContext.tsx
@@ -157,8 +157,11 @@ export function HistoryProvider(props: {
return peers ? [...new Set(peers.values())] : [];
});
const userEntries = mapArray(uniqueUserIds, (userId) => {
+ const macroId = tryMacroId(userId);
const displayName = () =>
- getDisplayName(tryMacroId(userId), { emailFallback: 'local-part' });
+ macroId
+ ? getDisplayName(macroId, { emailFallback: 'local-part' })
+ : userLabel(userId);
return { userId, displayName, color: userColor(userId) };
});
const userById = (userId: string): HistoryUser =>
diff --git a/apps/web/src/features/block-md/history/utils.ts b/apps/web/src/features/block-md/history/utils.ts
index 9848d92975..1603225abd 100644
--- a/apps/web/src/features/block-md/history/utils.ts
+++ b/apps/web/src/features/block-md/history/utils.ts
@@ -7,8 +7,15 @@ export function userColor(userId: string): string {
return `var(--color-${color}, var(--color-pink))`;
}
+/**
+ * Author sync-service records for edits no user made, such as the imported
+ * content a document starts from. Mirrors `SYSTEM_AUTHOR_ID` in sync-service.
+ */
+export const SYSTEM_AUTHOR_ID = 'system';
+
export function userLabel(userId: string): string {
if (userId === 'unknown') return 'Unknown';
+ if (userId === SYSTEM_AUTHOR_ID) return MACRO_AGENT_NAME;
if (isMacroAgentId(userId)) return `${MACRO_AGENT_NAME} (AI)`;
const id = tryMacroId(userId);
return id ? macroIdToEmail(id) : userId;
diff --git a/services/sync-service/AGENTS.md b/services/sync-service/AGENTS.md
index fe51144eac..4a235ff152 100644
--- a/services/sync-service/AGENTS.md
+++ b/services/sync-service/AGENTS.md
@@ -64,6 +64,11 @@ legacy reserved AI peer range; Canvas workers use random full-width peer IDs.
proportional to the update; `LoroDoc::diff`, `fork`, and checkouts cost time
that grows with document history. A backlog past `DEFAULT_TIMEOUT_MS` (4.5 s)
makes the front worker fail connects, which clients see as close 1006.
+- Document history names each Loro peer from D1 `peer_user_map`, so every
+ path that writes operations maps the peers it writes with: WebSocket and
+ HTTP `/update` writes to their user, `/initialize` snapshots to the system
+ author (`SYSTEM_AUTHOR_ID`), and `/copy` copies the source's mappings first.
+ A new write path does the same (`tests/peer_authors.test.ts`).
- `migration-test-hooks` is test-only fault injection — never enable it in a
deployed worker (see the feature's doc comment in `Cargo.toml`).
- Never run `just deploy-dev` / `just deploy-prod` without explicit approval.
diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rs
index ba4c0c87fc..d906ca9462 100644
--- a/services/sync-service/src/cf_worker.rs
+++ b/services/sync-service/src/cf_worker.rs
@@ -6,7 +6,11 @@ use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Result
use crate::{
auth::is_internal,
- constants::header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},
+ constants::{
+ USER_PEER_D1_BINDING,
+ header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},
+ },
+ d1::copy_user_mappings,
durable_object::{CopyDocumentRequest, GetSnapshotRequest, response, status_codes},
error::ResultExt,
generated::schema::InitializeFromSnapshotRequest,
@@ -179,6 +183,11 @@ pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Resu
buf
};
+ // The copy keeps the source's history, so it keeps its authors too. Copied
+ // before initializing, which maps every still-unmapped peer to the system.
+ let db = env.d1(USER_PEER_D1_BINDING)?;
+ copy_user_mappings(&db, document_id, &new_document_id).await?;
+
let initialize_path = format!("/document/{}/initialize", new_document_id);
let ss_res = do_helper(&env, init_body, &initialize_path, &req, &new_document_id).await?;
Ok(ss_res)
diff --git a/services/sync-service/src/constants.rs b/services/sync-service/src/constants.rs
index 4e534e11ed..5a0671b8b4 100644
--- a/services/sync-service/src/constants.rs
+++ b/services/sync-service/src/constants.rs
@@ -6,3 +6,9 @@ pub mod header_names {
}
pub const USER_PEER_D1_BINDING: &str = "USER_PEER_MAPPING";
+
+/// Author of operations written without a user: snapshots that services
+/// initialize documents from (markdown and Notion imports, uploads, copies of
+/// unattributed history). Keep in sync with the web history UI
+/// (`SYSTEM_AUTHOR_ID`).
+pub const SYSTEM_AUTHOR_ID: &str = "system";
diff --git a/services/sync-service/src/d1.rs b/services/sync-service/src/d1.rs
index 838562a934..c52cf0ce2a 100644
--- a/services/sync-service/src/d1.rs
+++ b/services/sync-service/src/d1.rs
@@ -41,6 +41,46 @@ pub async fn insert_user_mapping(
Ok(())
}
+/// Map peers that wrote operations to `user_id`, keeping any mapping a peer
+/// already has: a peer that registered explicitly stays with its owner.
+pub async fn insert_missing_user_mappings(
+ db: &D1Database,
+ document_id: &str,
+ user_id: &str,
+ peer_ids: &[u64],
+) -> worker::Result<()> {
+ for chunk in peer_ids.chunks(BATCH_CHUNK_SIZE) {
+ let statements = chunk
+ .iter()
+ .map(|peer_id| {
+ db.prepare(
+ "INSERT OR IGNORE INTO peer_user_map (document_id, peer_id, user_id) VALUES (?, ?, ?);",
+ )
+ .bind(&[document_id.into(), peer_id.to_string().into(), user_id.into()])
+ })
+ .collect::<worker::Result<Vec<_>>>()?;
+ db.batch(statements).await?;
+ }
+ Ok(())
+}
+
+/// Give a copied document the source document's peer mappings. Existing
+/// mappings on the target are kept.
+pub async fn copy_user_mappings(
+ db: &D1Database,
+ source_document_id: &str,
+ target_document_id: &str,
+) -> worker::Result<()> {
+ db.prepare(
+ "INSERT OR IGNORE INTO peer_user_map (document_id, peer_id, user_id) \
+ SELECT ?, peer_id, user_id FROM peer_user_map WHERE document_id = ?;",
+ )
+ .bind(&[target_document_id.into(), source_document_id.into()])?
+ .run()
+ .await?;
+ Ok(())
+}
+
pub async fn get_user_id_from_peer_id(
db: D1Database,
document_id: &str,
diff --git a/services/sync-service/src/domain/document.rs b/services/sync-service/src/domain/document.rs
index cdb0a42130..e10f6c1116 100644
--- a/services/sync-service/src/domain/document.rs
+++ b/services/sync-service/src/domain/document.rs
@@ -174,6 +174,22 @@ fn export_state(doc: &LoroDoc, mode: ExportMode<'_>) -> Result<(Vec<u8>, Vec<u8>
Ok((snapshot, revision))
}
+/// Peers that authored operations carried by an update or snapshot, so they
+/// can be mapped to the writer. Read from the blob, not the import result, so a
+/// retried write that changes nothing still registers its peers.
+pub fn writing_peers(blob: &[u8]) -> Result<Vec<u64>, DocumentError> {
+ let meta = LoroDoc::decode_import_blob_meta(blob, false)
+ .map_err(|_| DocumentError::Invalid("Invalid Loro update."))?;
+ let mut peers: Vec<u64> = meta
+ .partial_end_vv
+ .iter()
+ .filter(|(peer, end)| **end > meta.partial_start_vv.get(peer).copied().unwrap_or(0))
+ .map(|(peer, _)| *peer)
+ .collect();
+ peers.sort_unstable();
+ Ok(peers)
+}
+
/// Synchronous preflight. The caller must apply the returned delta before its
/// next await, so websocket writes cannot interleave with this version check.
pub fn prepare_update(
diff --git a/services/sync-service/src/domain/document/test.rs b/services/sync-service/src/domain/document/test.rs
index d9830b2c9c..0c73cd685d 100644
--- a/services/sync-service/src/domain/document/test.rs
+++ b/services/sync-service/src/domain/document/test.rs
@@ -313,3 +313,29 @@ fn session_attribution_uses_the_verified_human_or_agent_identity() {
);
assert!(DocumentAttribution::from_session_claims(None, Some("x".repeat(1025))).is_err());
}
+
+#[test]
+fn writing_peers_are_the_peers_with_operations_in_the_blob() {
+ let doc = LoroDoc::new();
+ doc.set_peer_id(11).unwrap();
+ doc.get_text("content").insert(0, "imported").unwrap();
+ doc.commit();
+ doc.set_peer_id(22).unwrap();
+ doc.get_text("content").insert(0, "edited ").unwrap();
+ doc.commit();
+ let snapshot = doc.export(ExportMode::Snapshot).unwrap();
+ assert_eq!(writing_peers(&snapshot).unwrap(), [11, 22]);
+
+ let seen = doc.oplog_vv();
+ doc.set_peer_id(33).unwrap();
+ doc.get_text("content").insert(0, "more ").unwrap();
+ doc.commit();
+ let update = doc
+ .export(ExportMode::Updates {
+ from: Cow::Owned(seen),
+ })
+ .unwrap();
+ assert_eq!(writing_peers(&update).unwrap(), [33]);
+
+ assert!(writing_peers(b"not a loro blob").is_err());
+}
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index c4da68fce4..192e688e91 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -19,9 +19,11 @@ use worker::{
use crate::{
auth::{AccessLevel, TokenFrom, WriteAccess, decode_jwt, socket_access},
- constants::USER_PEER_D1_BINDING,
- d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},
- domain::document::DocumentAttribution,
+ constants::{SYSTEM_AUTHOR_ID, USER_PEER_D1_BINDING},
+ d1::{
+ PeerWithUserId, get_user_id_from_peer_id, insert_missing_user_mappings, insert_user_mapping,
+ },
+ domain::document::{DocumentAttribution, writing_peers},
dss_internal::InteractionReason,
error::ResultExt,
generated::schema::InitializeFromSnapshotRequest,
@@ -145,6 +147,9 @@ pub struct DocumentSyncSession {
pending_blame: Arc<Mutex<Vec<crate::d1::BlameEvent>>>,
/// Distinct editors since the last snapshot notification.
pending_editors: Arc<Mutex<BTreeSet<DocumentAttribution>>>,
+ /// Writing peers already recorded in D1 since this object woke, so
+ /// updates skip remapping them.
+ mapped_writing_peers: Mutex<BTreeSet<u64>>,
surface_lifecycle: Mutex<Option<SurfaceLifecycle>>,
source_migration: Mutex<Option<surface_migration::SourceState>>,
// Serialize whole callbacks, including persistence awaits. A freeze must
@@ -503,6 +508,13 @@ impl DocumentSyncSession {
debug!(document_id = document_id, "Initializing snapshot");
let body_raw = req.bytes().await?;
let body = InitializeFromSnapshotRequest::deserialize(&body_raw).with_context(|| format!("Failed to deserialize InitializeFromSnapshotRequest with document_id: [{document_id}]"))?;
+ // Services initialize documents (imports, uploads, copies), so the
+ // snapshot's peers belong to the system author unless already mapped.
+ // Mapped before storing: a failed store is retried, a stored one is not.
+ let writers = writing_peers(&body.snapshot)
+ .map_err(|error| Error::from(format!("invalid initial snapshot: {error}")))?;
+ let db = self.env.d1(USER_PEER_D1_BINDING)?;
+ insert_missing_user_mappings(&db, document_id, SYSTEM_AUTHOR_ID, &writers).await?;
storage.store_snapshot(&body.snapshot).await?;
*self
.document_id
@@ -1007,6 +1019,7 @@ impl DurableObject for DocumentSyncSession {
inbound: Arc::new(Mutex::new(HashMap::new())),
pending_blame: Arc::new(Mutex::new(Vec::new())),
pending_editors: Arc::new(Mutex::new(Default::default())),
+ mapped_writing_peers: Mutex::new(Default::default()),
surface_lifecycle: Mutex::new(None),
source_migration: Mutex::new(None),
mutation_barrier: futures::lock::Mutex::new(()),
diff --git a/services/sync-service/src/durable_object/document_api.rs b/services/sync-service/src/durable_object/document_api.rs
index d8e6a99fe6..28a6dc1d42 100644
--- a/services/sync-service/src/durable_object/document_api.rs
+++ b/services/sync-service/src/durable_object/document_api.rs
@@ -6,6 +6,8 @@ use serde::{Deserialize, Serialize};
use super::document_effects::WorkerDocumentEffects;
use super::*;
use crate::{
+ constants::SYSTEM_AUTHOR_ID,
+ d1::insert_missing_user_mappings,
domain::document::{self, DocumentError, MAX_BINARY_BYTES, MAX_REVISION_BYTES},
storage::DocumentUpdateStorage,
};
@@ -123,6 +125,12 @@ impl DocumentSyncSession {
let storage = self.session_storage().await?;
// Only agent actors are stored with the operation; humans are still editors.
let is_agent = claims.actor.is_some();
+ // Agents write for the user they act on behalf of; tokens without a user
+ // are service writes.
+ let author = claims
+ .user_id
+ .clone()
+ .unwrap_or_else(|| SYSTEM_AUTHOR_ID.to_owned());
let editor = match document::DocumentAttribution::from_session_claims(
claims.actor,
claims.user_id,
@@ -148,6 +156,13 @@ impl DocumentSyncSession {
Ok(prepared) => prepared,
Err(error) => return error_response(error),
};
+ // Taken from the request, so a retry of an applied update still maps them.
+ let writers = match document::writing_peers(&update) {
+ Ok(writers) => writers,
+ Err(error) => return error_response(error),
+ };
+ let db = self.env.d1(USER_PEER_D1_BINDING)?;
+ insert_missing_user_mappings(&db, document_id, &author, &writers).await?;
Response::from_json(&UpdateResponse {
revision: STANDARD.encode(prepared.revision),
applied: prepared.applied,
diff --git a/services/sync-service/src/durable_object/peer_ids.rs b/services/sync-service/src/durable_object/peer_ids.rs
index 2c08f3bf3e..9fdf6e21c9 100644
--- a/services/sync-service/src/durable_object/peer_ids.rs
+++ b/services/sync-service/src/durable_object/peer_ids.rs
@@ -1,5 +1,9 @@
-use super::WebSocketMetadata;
-use crate::ai_peer::is_ai_peer;
+use worker::{Error, Result};
+
+use super::{WebSocketMetadata, Wsm};
+use crate::{
+ ai_peer::is_ai_peer, constants::USER_PEER_D1_BINDING, d1::insert_missing_user_mappings,
+};
impl WebSocketMetadata {
pub(super) fn active_peer_ids(&self, include_ai: bool) -> Vec<u64> {
@@ -14,5 +18,47 @@ impl WebSocketMetadata {
}
}
+impl Wsm<'_> {
+ /// Map peers that wrote through this socket without registering first (a
+ /// client's peer id can change, and offline edits replay under an old one).
+ /// Unlike registration this keeps existing mappings and leaves the
+ /// socket's own peers, and so presence, unchanged.
+ pub async fn add_writing_peerids(&mut self, peerids: &[u64], document_id: &str) -> Result<()> {
+ self.maybe_update_ws_meta_map().await?;
+ let ws_id = self.get_ws_id()?.to_string();
+ let user_id = {
+ let wmm = self.dss.ws_meta_map.lock("Wsm::add_writing_peerids get");
+ let meta = wmm.get(&ws_id).ok_or(Error::from("missing ws metadata"))?;
+ let Some(user_id) = meta.user_id.clone() else {
+ return Ok(());
+ };
+ user_id
+ };
+ let new_peers: Vec<u64> = {
+ let mapped = self
+ .dss
+ .mapped_writing_peers
+ .lock("Wsm::add_writing_peerids mapped");
+ peerids
+ .iter()
+ .copied()
+ .filter(|peer| !mapped.contains(peer))
+ .collect()
+ };
+ if new_peers.is_empty() {
+ return Ok(());
+ }
+ let db = self.dss.env.d1(USER_PEER_D1_BINDING)?;
+ insert_missing_user_mappings(&db, document_id, &user_id, &new_peers).await?;
+ // Remembered only once stored, so a failed write is retried with the
+ // client's resent update.
+ self.dss
+ .mapped_writing_peers
+ .lock("Wsm::add_writing_peerids extend")
+ .extend(new_peers);
+ Ok(())
+ }
+}
+
#[cfg(test)]
mod test;
diff --git a/services/sync-service/src/socket/protocol.rs b/services/sync-service/src/socket/protocol.rs
index 0b23a21d82..b4a55fe2a4 100644
--- a/services/sync-service/src/socket/protocol.rs
+++ b/services/sync-service/src/socket/protocol.rs
@@ -5,6 +5,7 @@ use worker::Result;
use crate::{
auth::WriteAccess,
+ domain::document::writing_peers,
durable_object::{DocumentSyncSession, Wsm},
error::ResultExt,
generated::schema::{FromPeer, FromRemote},
@@ -254,6 +255,20 @@ pub async fn process_message(
}
}
+ // Map every peer the update was written with, not only the one the
+ // client registered, so history can name its author. Before the
+ // ack: a failure here makes the client resend and retry it.
+ let mut writers = Vec::new();
+ for update in &updates {
+ writers.extend(
+ writing_peers(update)
+ .map_err(|error| worker::Error::from(error.to_string()))?,
+ );
+ }
+ Wsm::new(dss, sender.websocket())
+ .add_writing_peerids(&writers, document_id)
+ .await?;
+
// ACK only after the update is durably stored.
sender
.send(FromRemote::RemoteUpdateAck { id })
diff --git a/services/sync-service/tests/peer_authors.test.ts b/services/sync-service/tests/peer_authors.test.ts
new file mode 100644
index 0000000000..afc77115da
--- /dev/null
+++ b/services/sync-service/tests/peer_authors.test.ts
@@ -0,0 +1,159 @@
+import { LoroDoc } from "loro-crdt";
+import type { Miniflare } from "miniflare";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import {
+ FromPeer,
+ InitializeFromSnapshotRequest,
+} from "../bebop/generated/schema";
+import {
+ copyDocument,
+ createTestUser,
+ getTokenForDocument,
+ setupMiniflare,
+} from "./utils";
+
+// Document history names the author of every peer through /metadata, so each
+// path that writes operations must map the peers it writes with.
+
+let mf: Miniflare;
+
+beforeAll(async () => {
+ mf = await setupMiniflare();
+}, 60_000);
+
+afterAll(async () => {
+ await mf?.dispose();
+});
+
+async function initialize(documentId: string, snapshot: Uint8Array) {
+ const response = await mf.dispatchFetch(
+ `http://localhost:8787/document/${documentId}/initialize`,
+ {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/octet-stream",
+ Authorization: `Bearer ${getTokenForDocument(documentId, "importer", "owner")}`,
+ },
+ body: InitializeFromSnapshotRequest.encode({ snapshot }),
+ },
+ );
+ expect(response.status).toBe(200);
+}
+
+async function authors(documentId: string): Promise<Record<string, string>> {
+ const response = await mf.dispatchFetch(
+ `http://localhost:8787/document/${documentId}/metadata`,
+ {
+ headers: {
+ Authorization: `Bearer ${getTokenForDocument(documentId, "reader", "view")}`,
+ },
+ },
+ );
+ expect(response.status).toBe(200);
+ const body = (await response.json()) as {
+ peers: { peer_id: string; user_id: string }[];
+ };
+ return Object.fromEntries(body.peers.map((p) => [p.peer_id, p.user_id]));
+}
+
+describe("peer authors", () => {
+ test("an initialized snapshot's peers belong to the system author", async () => {
+ const doc = new LoroDoc();
+ doc.setPeerId(1001n);
+ doc.getText("content").insert(0, "imported markdown");
+ doc.commit();
+
+ await initialize("initialized-doc", doc.export({ mode: "snapshot" }));
+
+ expect(await authors("initialized-doc")).toEqual({ "1001": "system" });
+ });
+
+ test("a websocket write maps peers the client never registered", async () => {
+ const seed = new LoroDoc();
+ seed.setPeerId(2001n);
+ seed.getText("content").insert(0, "start");
+ seed.commit();
+ await initialize("websocket-doc", seed.export({ mode: "snapshot" }));
+
+ const user = await createTestUser(mf, "websocket-doc", { userId: "editor-user" });
+ const registeredPeer = user.doc.peerIdStr;
+ // A client's peer id can change after it registered; its edits arrive
+ // under the new id without another registration.
+ const from = user.doc.version();
+ user.doc.setPeerId(2002n);
+ user.doc.getText("content").insert(0, "typed ");
+ user.doc.commit();
+ user.connection.send(
+ FromPeer.fromPeerUpdate({
+ updates: [user.doc.export({ mode: "update", from })],
+ id: crypto.randomUUID(),
+ }).encode(),
+ );
+ const ack = await user.readNextMessage();
+ expect(ack.isRemoteUpdateAck()).toBe(true);
+
+ expect(await authors("websocket-doc")).toEqual({
+ "2001": "system",
+ "2002": "editor-user",
+ [registeredPeer]: "editor-user",
+ });
+ user.getWebSocket().close();
+ });
+
+ test("an HTTP update maps its peers to the token's user", async () => {
+ const doc = new LoroDoc();
+ doc.setPeerId(3001n);
+ doc.getText("content").insert(0, "start");
+ doc.commit();
+ await initialize("http-doc", doc.export({ mode: "snapshot" }));
+
+ const from = doc.version();
+ const expectedRevision = Buffer.from(from.encode()).toString("base64");
+ doc.setPeerId(3002n);
+ doc.getText("content").insert(0, "agent ");
+ doc.commit();
+ const response = await mf.dispatchFetch(
+ "http://localhost:8787/document/http-doc/update",
+ {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${getTokenForDocument("http-doc", "editor-user", "edit")}`,
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ expectedRevision,
+ update: Buffer.from(doc.export({ mode: "update", from })).toString(
+ "base64",
+ ),
+ }),
+ },
+ );
+ expect(response.status).toBe(200);
+
+ expect(await authors("http-doc")).toEqual({
+ "3001": "system",
+ "3002": "editor-user",
+ });
+ });
+
+ test("a copy keeps the source document's authors", async () => {
+ const seed = new LoroDoc();
+ seed.setPeerId(4001n);
+ seed.getText("content").insert(0, "start");
+ seed.commit();
+ await initialize("copy-source", seed.export({ mode: "snapshot" }));
+ const user = await createTestUser(mf, "copy-source", { userId: "editor-user" });
+ user.makeChange(" edited");
+ const ack = await user.readNextMessage();
+ expect(ack.isRemoteUpdateAck()).toBe(true);
+ user.getWebSocket().close();
+
+ const response = await copyDocument(mf, "copy-source", "copy-target");
+ expect(response.status).toBe(200);
+
+ expect(await authors("copy-target")).toEqual({
+ "4001": "system",
+ [user.doc.peerIdStr]: "editor-user",
+ });
+ });
+});