Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

refactor(sync): decouple spreadsheet business logic (#6671)

teo303d1c0b9a6e
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts
index ed43634d1c..3d7ab4cde2 100644
--- a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts
@@ -20,7 +20,7 @@ before attribution, and on human-only collab sessions. */
document_id: string;
/** Version marker for the sync snapshot, when the caller supplies one. */
document_version_id?: DocumentSyncContentUpdatedMetadataDocumentVersionId;
- /** File type of the sync document (markdown today). */
+ /** File type of the sync document, resolved by the document backend. */
file_type: FileType;
on_behalf_of?: DocumentSyncContentUpdatedMetadataOnBehalfOf;
}
diff --git a/apps/web/src/lib/service-clients/service-storage/openapi.json b/apps/web/src/lib/service-clients/service-storage/openapi.json
index b68126c5dc..772bc8a825 100644
--- a/apps/web/src/lib/service-clients/service-storage/openapi.json
+++ b/apps/web/src/lib/service-clients/service-storage/openapi.json
@@ -21195,7 +21195,7 @@
},
"file_type": {
"$ref": "#/components/schemas/FileType",
- "description": "File type of the sync document (markdown today)."
+ "description": "File type of the sync document, resolved by the document backend."
},
"on_behalf_of": {
"oneOf": [
diff --git a/bun.lock b/bun.lock
index 045a73ff4c..827d60f0f6 100644
--- a/bun.lock
+++ b/bun.lock
@@ -333,6 +333,7 @@
"@loro-mirror/core": "workspace:*",
"loro-crdt": "catalog:",
"ssf": "0.11.2",
+ "ts-pattern": "^5.9.0",
},
"devDependencies": {
"typescript": "^5.9.3",
diff --git a/crates/documents/src/domain/events.rs b/crates/documents/src/domain/events.rs
index 17b985c169..2053c6c1dd 100644
--- a/crates/documents/src/domain/events.rs
+++ b/crates/documents/src/domain/events.rs
@@ -126,7 +126,7 @@ pub struct DocumentContentUploadedMetadata {
pub struct DocumentSyncContentUpdatedMetadata {
/// The id of the live-collab document whose content changed.
pub document_id: String,
- /// File type of the sync document (markdown today).
+ /// File type of the sync document, resolved by the document backend.
pub file_type: FileType,
/// Version marker for the sync snapshot, when the caller supplies one.
pub document_version_id: Option<String>,
diff --git a/crates/documents/src/domain/ports.rs b/crates/documents/src/domain/ports.rs
index 107e8083af..b59d55ba92 100644
--- a/crates/documents/src/domain/ports.rs
+++ b/crates/documents/src/domain/ports.rs
@@ -404,7 +404,7 @@ pub async fn task_property_edit_receipt<A: EntityAccessService>(
.await
}
-/// Use case for relaying document content-upload events.
+/// Use cases for relaying document content-change events.
pub trait DocumentContentEventService: Send + Sync + 'static {
/// Load the document owner and publish a content-uploaded event.
fn publish_content_uploaded(
@@ -413,6 +413,15 @@ pub trait DocumentContentEventService: Send + Sync + 'static {
file_type: FileType,
document_version_id: Option<String>,
) -> impl Future<Output = Result<(), DocumentError>> + Send;
+
+ /// Resolve the stored file type and publish a sync-content event. Sync callers
+ /// supply document identity and attribution without interpreting the content.
+ fn publish_sync_content_updated(
+ &self,
+ document_id: &str,
+ actor: Option<String>,
+ on_behalf_of: Option<String>,
+ ) -> impl Future<Output = Result<(), DocumentError>> + Send;
}
/// Service interface for document operations.
diff --git a/crates/documents/src/domain/service.rs b/crates/documents/src/domain/service.rs
index c250fabf59..5dfa0fb6ae 100644
--- a/crates/documents/src/domain/service.rs
+++ b/crates/documents/src/domain/service.rs
@@ -1,5 +1,7 @@
//! Document service implementation.
+mod content_events;
+
#[cfg(test)]
mod tests;
@@ -53,9 +55,8 @@ use crate::domain::models::{
use super::branch_name::{build_task_branch_name, user_branch_prefix};
use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};
use super::events::{
- DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,
- DocumentDeletedMetadata, DocumentInteractionMetadata, DocumentMacroEvent,
- DocumentUpdatedMetadata, InteractionReason,
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentInteractionMetadata, DocumentMacroEvent, DocumentUpdatedMetadata, InteractionReason,
};
use super::models::{
CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,
@@ -880,45 +881,6 @@ impl<
}
}
-impl<
- R: DocumentRepo,
- U: PresignedUploadUrlPort,
- T: TaskPropertiesPort,
- C: ConnectionService,
- Eam: EntityAccessManagementService,
- F: ForeignEntityService,
- B: MacroEventBroker,
- S: DocumentSyncPort,
-> DocumentContentEventService for DocumentServiceImpl<R, U, T, C, Eam, F, B, S>
-{
- #[tracing::instrument(err, skip(self))]
- async fn publish_content_uploaded(
- &self,
- document_id: &str,
- file_type: FileType,
- document_version_id: Option<String>,
- ) -> Result<(), DocumentError> {
- let document = self
- .repo
- .get_basic_document(document_id)
- .await
- .map_err(|error| map_basic_document_error(document_id, error.into()))?;
-
- self.macro_event_broker
- .send_event(&DocumentMacroEvent::content_uploaded(
- document_id,
- DocumentContentUploadedMetadata {
- document_id: document_id.to_string(),
- owner: document.owner,
- file_type,
- document_version_id,
- },
- ))
- .map(|_| ())
- .map_err(|error| DocumentError::Internal(error.into()))
- }
-}
-
impl<
R: DocumentRepo,
U: PresignedUploadUrlPort,
diff --git a/crates/documents/src/domain/service/content_events.rs b/crates/documents/src/domain/service/content_events.rs
new file mode 100644
index 0000000000..ecac378c37
--- /dev/null
+++ b/crates/documents/src/domain/service/content_events.rs
@@ -0,0 +1,76 @@
+//! Resolve document metadata before publishing content-change events.
+
+use super::*;
+use crate::domain::events::{DocumentContentUploadedMetadata, DocumentSyncContentUpdatedMetadata};
+
+impl<
+ R: DocumentRepo,
+ U: PresignedUploadUrlPort,
+ T: TaskPropertiesPort,
+ C: ConnectionService,
+ Eam: EntityAccessManagementService,
+ F: ForeignEntityService,
+ B: MacroEventBroker,
+ S: DocumentSyncPort,
+> DocumentContentEventService for DocumentServiceImpl<R, U, T, C, Eam, F, B, S>
+{
+ #[tracing::instrument(err, skip(self))]
+ async fn publish_content_uploaded(
+ &self,
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ ) -> Result<(), DocumentError> {
+ let document = self
+ .repo
+ .get_basic_document(document_id)
+ .await
+ .map_err(|error| map_basic_document_error(document_id, error.into()))?;
+
+ self.macro_event_broker
+ .send_event(&DocumentMacroEvent::content_uploaded(
+ document_id,
+ DocumentContentUploadedMetadata {
+ document_id: document_id.to_string(),
+ owner: document.owner,
+ file_type,
+ document_version_id,
+ },
+ ))
+ .map(|_| ())
+ .map_err(|error| DocumentError::Internal(error.into()))
+ }
+
+ #[tracing::instrument(err, skip(self))]
+ async fn publish_sync_content_updated(
+ &self,
+ document_id: &str,
+ actor: Option<String>,
+ on_behalf_of: Option<String>,
+ ) -> Result<(), DocumentError> {
+ let document = self
+ .repo
+ .get_basic_document(document_id)
+ .await
+ .map_err(|error| map_basic_document_error(document_id, error.into()))?;
+ let file_type = document
+ .file_type
+ .as_deref()
+ .and_then(|value| FileType::from_str(value).ok())
+ .ok_or_else(|| DocumentError::BadRequest("Unknown document file type".to_string()))?;
+
+ self.macro_event_broker
+ .send_event(&DocumentMacroEvent::sync_content_updated(
+ document_id,
+ DocumentSyncContentUpdatedMetadata::from_extract(
+ document_id.to_string(),
+ file_type,
+ None,
+ actor,
+ on_behalf_of,
+ ),
+ ))
+ .map(|_| ())
+ .map_err(|error| DocumentError::Internal(error.into()))
+ }
+}
diff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rs
index d76202078f..65793e6d13 100644
--- a/crates/documents/src/domain/service/tests.rs
+++ b/crates/documents/src/domain/service/tests.rs
@@ -16,6 +16,8 @@ use crate::domain::ports::{DocumentContentEventService, MockDocumentRepo};
use super::*;
use activity::{Actor, Attribution};
+mod sync_content;
+
fn make_test_metadata() -> DocumentMetadata {
DocumentMetadata {
document_id: "doc-1".to_string(),
diff --git a/crates/documents/src/domain/service/tests/sync_content.rs b/crates/documents/src/domain/service/tests/sync_content.rs
new file mode 100644
index 0000000000..2868e65418
--- /dev/null
+++ b/crates/documents/src/domain/service/tests/sync_content.rs
@@ -0,0 +1,65 @@
+use super::*;
+
+#[tokio::test]
+async fn notifications_resolve_the_stored_type_and_preserve_attribution() {
+ for file_type in ["md", "spreadsheet"] {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document()
+ .withf(|id| id == "doc-1")
+ .return_once(move |_| {
+ let mut document = task_document_context("doc-1");
+ document.file_type = Some(file_type.to_string());
+ Box::pin(std::future::ready(Ok(document)))
+ });
+ let (service, broker) = make_test_service_with_event_broker(repo);
+ let actor = "bot|00000000-0000-0000-0000-00000000a1a1";
+ let user = "macro|[email protected]";
+
+ service
+ .publish_sync_content_updated("doc-1", Some(actor.into()), Some(user.into()))
+ .await
+ .unwrap();
+
+ let published = broker.published();
+ let events = published.lock().unwrap();
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0].key, "doc-1");
+ assert_eq!(
+ events[0].payload["event_type"],
+ "document.sync_content_updated"
+ );
+ assert_eq!(events[0].payload["metadata"]["file_type"], file_type);
+ assert_eq!(events[0].payload["metadata"]["actor"], actor);
+ assert_eq!(events[0].payload["metadata"]["on_behalf_of"], user);
+ }
+}
+
+#[tokio::test]
+async fn missing_documents_do_not_publish_events() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document().return_once(|_| {
+ Box::pin(std::future::ready(Err(anyhow!(
+ "no rows returned by a query that expected to return at least one row"
+ ))))
+ });
+ let (service, broker) = make_test_service_with_event_broker(repo);
+ let result = service
+ .publish_sync_content_updated("missing", None, None)
+ .await;
+ assert!(matches!(result, Err(DocumentError::NotFound(_))));
+ assert!(broker.published().lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn notifications_surface_broker_failures() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document()
+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));
+ let (service, broker) =
+ make_test_service_with_configured_event_broker(repo, TestEventBroker::failing());
+ let result = service
+ .publish_sync_content_updated("doc-1", None, None)
+ .await;
+ assert!(matches!(result, Err(DocumentError::Internal(_))));
+ assert!(broker.published().lock().unwrap().is_empty());
+}
diff --git a/crates/documents/src/inbound/axum_router.rs b/crates/documents/src/inbound/axum_router.rs
index 9c16249841..77c8489a41 100644
--- a/crates/documents/src/inbound/axum_router.rs
+++ b/crates/documents/src/inbound/axum_router.rs
@@ -39,6 +39,7 @@ pub mod get_location;
pub mod get_short_id;
pub mod put_interaction;
pub mod put_snapshot;
+pub mod sync_content_updated;
pub mod system_skills;
pub mod task_duplicates;
pub mod team_share;
diff --git a/crates/documents/src/inbound/axum_router/sync_content_updated.rs b/crates/documents/src/inbound/axum_router/sync_content_updated.rs
new file mode 100644
index 0000000000..5f10684c35
--- /dev/null
+++ b/crates/documents/src/inbound/axum_router/sync_content_updated.rs
@@ -0,0 +1,42 @@
+//! Handler for sync-content notifications from the synchronization service.
+
+use axum::{
+ Json,
+ extract::{Path, State},
+ http::StatusCode,
+};
+use entity_access::domain::ports::EntityAccessService;
+use macro_authorization::{InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService};
+use serde::Deserialize;
+
+use super::{DocumentRouterState, Params};
+use crate::domain::{models::DocumentError, ports::DocumentContentEventService};
+
+/// Attribution from the sync service's verified document token.
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SyncContentUpdatedRequest {
+ /// Actor that performed the edit, if any.
+ pub actor: Option<String>,
+ /// User represented by the actor, if any.
+ pub on_behalf_of: Option<String>,
+}
+
+/// Publish an event using the document's stored metadata.
+#[tracing::instrument(err, skip(state, _internal_authorization))]
+pub async fn sync_content_updated_handler<
+ T: DocumentContentEventService,
+ Svc: EntityAccessService,
+ Auth: MacroAuthorizationService,
+>(
+ State(state): State<DocumentRouterState<T, Svc, Auth>>,
+ _internal_authorization: MacroAuthorizationExtractor<Auth, InternalOnly>,
+ Path(Params { document_id }): Path<Params>,
+ Json(request): Json<SyncContentUpdatedRequest>,
+) -> Result<StatusCode, DocumentError> {
+ state
+ .service
+ .publish_sync_content_updated(&document_id, request.actor, request.on_behalf_of)
+ .await?;
+ Ok(StatusCode::OK)
+}
diff --git a/crates/documents/src/inbound/axum_router/tests.rs b/crates/documents/src/inbound/axum_router/tests.rs
index 750fb9fbe9..79064cca87 100644
--- a/crates/documents/src/inbound/axum_router/tests.rs
+++ b/crates/documents/src/inbound/axum_router/tests.rs
@@ -43,6 +43,8 @@ use tower::ServiceExt;
use uuid::Uuid;
use super::{DocumentRouterState, content_uploaded::content_uploaded_handler, documents_router};
+
+mod sync_content;
use crate::{
domain::{
content::DocumentContent,
@@ -101,6 +103,13 @@ struct ContentUploadedCall {
document_version_id: Option<String>,
}
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct SyncContentCall {
+ document_id: String,
+ actor: Option<String>,
+ on_behalf_of: Option<String>,
+}
+
#[derive(Clone, Debug, Eq, PartialEq)]
struct TeamSlugCall {
team_id: String,
@@ -121,6 +130,7 @@ struct FakeDocumentService {
import_calls: Mutex<Vec<ImportEmailAttachmentCall>>,
upload_snapshot_calls: Mutex<Vec<UploadSnapshotCall>>,
content_uploaded_calls: Mutex<Vec<ContentUploadedCall>>,
+ sync_content_calls: Mutex<Vec<SyncContentCall>>,
internal_get_calls: Mutex<Vec<String>>,
team_slug_calls: Mutex<Vec<TeamSlugCall>>,
team_slug_result: Mutex<Option<TeamSlugResult>>,
@@ -439,6 +449,23 @@ impl DocumentService for FakeDocumentService {
}
impl DocumentContentEventService for FakeDocumentService {
+ async fn publish_sync_content_updated(
+ &self,
+ document_id: &str,
+ actor: Option<String>,
+ on_behalf_of: Option<String>,
+ ) -> Result<(), DocumentError> {
+ self.sync_content_calls
+ .lock()
+ .unwrap()
+ .push(SyncContentCall {
+ document_id: document_id.to_string(),
+ actor,
+ on_behalf_of,
+ });
+ Ok(())
+ }
+
async fn publish_content_uploaded(
&self,
document_id: &str,
@@ -868,6 +895,16 @@ fn test_router() -> (
>,
),
)
+ .route(
+ "/{document_id}/sync-content-updated",
+ axum::routing::post(
+ super::sync_content_updated::sync_content_updated_handler::<
+ FakeDocumentService,
+ FakeEntityAccessService,
+ FakeAuthorizationService,
+ >,
+ ),
+ )
.with_state(state);
(
diff --git a/crates/documents/src/inbound/axum_router/tests/sync_content.rs b/crates/documents/src/inbound/axum_router/tests/sync_content.rs
new file mode 100644
index 0000000000..43f3f49ec4
--- /dev/null
+++ b/crates/documents/src/inbound/axum_router/tests/sync_content.rs
@@ -0,0 +1,43 @@
+use super::*;
+
+#[tokio::test]
+async fn sync_notifications_require_internal_auth_and_forward_only_identity_and_attribution() {
+ let (router, service, _, _) = test_router();
+ let body = || Body::from(r#"{"actor":"bot|agent","on_behalf_of":"macro|[email protected]"}"#);
+ let user_request = Request::post("/doc-1/sync-content-updated")
+ .header("authorization", format!("Bearer {JWT_TOKEN}"))
+ .header("content-type", "application/json")
+ .body(body())
+ .unwrap();
+ assert_eq!(
+ send_status(&router, user_request).await,
+ StatusCode::FORBIDDEN
+ );
+ assert!(service.sync_content_calls.lock().unwrap().is_empty());
+
+ let internal_request = Request::post("/doc-1/sync-content-updated")
+ .header(INTERNAL_API_KEY_HEADER, STANDARD_INTERNAL_KEY)
+ .header("content-type", "application/json")
+ .body(body())
+ .unwrap();
+ assert_eq!(send_status(&router, internal_request).await, StatusCode::OK);
+ assert_eq!(
+ *service.sync_content_calls.lock().unwrap(),
+ [SyncContentCall {
+ document_id: "doc-1".into(),
+ actor: Some("bot|agent".into()),
+ on_behalf_of: Some("macro|[email protected]".into()),
+ }]
+ );
+
+ let forged_type = Request::post("/doc-1/sync-content-updated")
+ .header(INTERNAL_API_KEY_HEADER, STANDARD_INTERNAL_KEY)
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"file_type":"md"}"#))
+ .unwrap();
+ assert_eq!(
+ send_status(&router, forged_type).await,
+ StatusCode::UNPROCESSABLE_ENTITY
+ );
+ assert_eq!(service.sync_content_calls.lock().unwrap().len(), 1);
+}
diff --git a/packages/sdk/generated/storage/types.gen.ts b/packages/sdk/generated/storage/types.gen.ts
index d050ef8e7b..278913bf01 100644
--- a/packages/sdk/generated/storage/types.gen.ts
+++ b/packages/sdk/generated/storage/types.gen.ts
@@ -4876,7 +4876,7 @@ export type DocumentSyncContentUpdatedMetadata = {
*/
document_version_id?: string | null;
/**
- * File type of the sync document (markdown today).
+ * File type of the sync document, resolved by the document backend.
*/
file_type: FileType;
on_behalf_of?: null | MacroUserIdStr;
diff --git a/packages/sdk/specs/storage.json b/packages/sdk/specs/storage.json
index b68126c5dc..772bc8a825 100644
--- a/packages/sdk/specs/storage.json
+++ b/packages/sdk/specs/storage.json
@@ -21195,7 +21195,7 @@
},
"file_type": {
"$ref": "#/components/schemas/FileType",
- "description": "File type of the sync document (markdown today)."
+ "description": "File type of the sync document, resolved by the document backend."
},
"on_behalf_of": {
"oneOf": [
diff --git a/packages/spreadsheet/package.json b/packages/spreadsheet/package.json
index fe038971c2..69b9f64a48 100644
--- a/packages/spreadsheet/package.json
+++ b/packages/spreadsheet/package.json
@@ -10,7 +10,8 @@
"@ironcalc/wasm": "0.8.4",
"@loro-mirror/core": "workspace:*",
"loro-crdt": "catalog:",
- "ssf": "0.11.2"
+ "ssf": "0.11.2",
+ "ts-pattern": "^5.9.0"
},
"scripts": {
"type-check": "tsc --noEmit --project tsconfig.json",
diff --git a/packages/spreadsheet/src/ai-workbook.ts b/packages/spreadsheet/src/ai-workbook.ts
index 61fbde77cc..17ad2bff9e 100644
--- a/packages/spreadsheet/src/ai-workbook.ts
+++ b/packages/spreadsheet/src/ai-workbook.ts
@@ -17,6 +17,7 @@ import type {
WorkbookCalculation,
} from './calculation';
import { fillCopies } from './cell-copy';
+import { validateSpreadsheetDocument } from './document-validation';
import {
type CellSelection,
selectionAddresses,
@@ -546,6 +547,7 @@ export function prepareSpreadsheetEdit(
});
}
fork.commit({ origin: 'spreadsheet-ai-edit' });
+ validateSpreadsheetDocument(fork);
const workbook = readSpreadsheetWorkbook(fork);
const results = calculate(workbook, calculator);
const sheets = summaries(workbook, results);
diff --git a/packages/spreadsheet/src/document-validation.test.ts b/packages/spreadsheet/src/document-validation.test.ts
new file mode 100644
index 0000000000..4e12516756
--- /dev/null
+++ b/packages/spreadsheet/src/document-validation.test.ts
@@ -0,0 +1,116 @@
+import { readFileSync } from 'node:fs';
+import { LoroCounter, LoroDoc, LoroMap, LoroText } from 'loro-crdt';
+import { afterEach, describe, expect, it } from 'vitest';
+import { validateSpreadsheetDocument } from './document-validation';
+
+const docs: LoroDoc[] = [];
+function workbook() {
+ const doc = new LoroDoc();
+ docs.push(doc);
+ doc.import(
+ readFileSync(
+ new URL(
+ '../../../static_assets/spreadsheet-golden.1.bin',
+ import.meta.url
+ )
+ )
+ );
+ return doc;
+}
+afterEach(() => {
+ for (const doc of docs.splice(0)) doc.free();
+});
+
+describe('spreadsheet document validation outside sync', () => {
+ it('accepts native snapshots and imported financial styles and metadata', () => {
+ const doc = workbook();
+ for (const [root, value] of [
+ ['spreadsheetNumberFormats', '#,##0.00;[Red](#,##0.00);"—"'],
+ ['spreadsheetFontNames', 'Calibri'],
+ ['spreadsheetBorderBottomStyles', 'double'],
+ ['spreadsheetBorderBottomColors', '#123456'],
+ ])
+ doc.getMap(root).set('A1', value);
+ doc.getMap('spreadsheetSheetMetadata').set(
+ 'sheet1',
+ JSON.stringify({
+ merges: ['A1:C1'],
+ rowHeights: { 0: 32 },
+ hiddenRows: [5],
+ hiddenColumns: [5],
+ freeze: { rows: 3, columns: 1 },
+ definedNames: [{ name: 'Rate', formula: '0.1', local: true }],
+ })
+ );
+ expect(() => validateSpreadsheetDocument(doc)).not.toThrow();
+ });
+
+ it.each([
+ ['root', 'content', 'bad'],
+ ['spreadsheetUnknown', 'A1', 'bad'],
+ ['spreadsheetValues', 'AA1', 'bad'],
+ ['spreadsheetValues', 'A1001', 'bad'],
+ ['spreadsheetValues', 'A1', 'x'.repeat(10_001)],
+ ['spreadsheetValues', 'bad!sheet!A1', 'bad'],
+ ['spreadsheetFontSize', 'A1', 'large'],
+ ['spreadsheetFormats', 'A1', 'unsupported'],
+ ['spreadsheetSheetNames', 'sheet1', ' Untrimmed '],
+ ['spreadsheetSheetOrder', 'bad!id', 1],
+ [
+ 'spreadsheetSheetRetentions',
+ 'sheet1!peer',
+ '{"name":"Sheet1","order":0,"revision":-1}',
+ ],
+ ['spreadsheetColumnWidths', '0', 641],
+ ['spreadsheetRowAdditions', 'peer', 1001],
+ ['spreadsheetMeta', 'formatVersion', 2],
+ ])('rejects invalid %s entries', (root, key, value) => {
+ const doc = workbook();
+ doc.getMap(root as string).set(key as string, value);
+ expect(() => validateSpreadsheetDocument(doc)).toThrow(
+ 'native spreadsheet maps'
+ );
+ });
+
+ it.each([
+ '{"merges":["A1:AA1001"]}',
+ '{"rowHeights":{"1000":32}}',
+ '{"hiddenColumns":[26]}',
+ '{"freeze":{"rows":1001,"columns":1}}',
+ '{"definedNames":[{"name":"Rate","formula":false}]}',
+ '{"unexpected":true}',
+ ])('rejects invalid workbook metadata %s', (metadata) => {
+ const doc = workbook();
+ doc.getMap('spreadsheetSheetMetadata').set('sheet1', metadata);
+ expect(() => validateSpreadsheetDocument(doc)).toThrow();
+ });
+
+ it('rejects nested containers even when their deep values look like valid scalars', () => {
+ const text = workbook();
+ text
+ .getMap('spreadsheetValues')
+ .setContainer('A1', new LoroText())
+ .insert(0, 'looks valid');
+ expect(text.toJSON().spreadsheetValues).toEqual({ A1: 'looks valid' });
+ expect(() => validateSpreadsheetDocument(text)).toThrow();
+ const counter = workbook();
+ counter
+ .getMap('spreadsheetFontSize')
+ .setContainer('A1', new LoroCounter())
+ .increment(10);
+ expect(() => validateSpreadsheetDocument(counter)).toThrow();
+ const map = workbook();
+ map.getMap('spreadsheetValues').setContainer('A1', new LoroMap());
+ expect(() => validateSpreadsheetDocument(map)).toThrow();
+ });
+
+ it('rejects other content types and wrong root container types', () => {
+ const markdown = new LoroDoc();
+ docs.push(markdown);
+ markdown.getText('root').insert(0, 'markdown');
+ expect(() => validateSpreadsheetDocument(markdown)).toThrow();
+ const doc = workbook();
+ doc.getText('spreadsheetValues').insert(0, 'not a map');
+ expect(() => validateSpreadsheetDocument(doc)).toThrow();
+ });
+});
diff --git a/packages/spreadsheet/src/document-validation.ts b/packages/spreadsheet/src/document-validation.ts
new file mode 100644
index 0000000000..82cbce9dfe
--- /dev/null
+++ b/packages/spreadsheet/src/document-validation.ts
@@ -0,0 +1,159 @@
+import { type LoroDoc, LoroMap } from 'loro-crdt';
+import { match } from 'ts-pattern';
+import {
+ isSpreadsheetStyleEntry,
+ MAX_COLUMN_WIDTH,
+ MIN_COLUMN_WIDTH,
+ parseCellAddress,
+ SPREADSHEET_COLUMNS,
+ SPREADSHEET_FORMAT_VERSION,
+ SPREADSHEET_MAX_CELL_LENGTH,
+ SPREADSHEET_MAX_ROWS,
+} from './spreadsheet-document';
+import { SPREADSHEET_LORO_SCHEMA } from './spreadsheet-schema';
+import {
+ isSpreadsheetSheetId,
+ validateSpreadsheetSheetName,
+} from './spreadsheet-sheet-registry';
+import { parseWorkbookMetadata } from './workbook-metadata';
+
+const MAX_DOCUMENT_ENTRIES = 1_000_000;
+const textEncoder = new TextEncoder();
+const knownRoots = new Set([
+ 'spreadsheetMeta',
+ ...Object.keys(SPREADSHEET_LORO_SCHEMA.definition),
+]);
+
+function sheetName(value: unknown): value is string {
+ if (typeof value !== 'string') return false;
+ try {
+ return validateSpreadsheetSheetName(value) === value;
+ } catch {
+ return false;
+ }
+}
+
+function sheetKey(key: string): string | undefined {
+ const parts = key.split('!');
+ if (parts.length === 1) return key;
+ if (parts.length === 2 && isSpreadsheetSheetId(parts[0])) return parts[1];
+}
+
+function integer(value: unknown, min: number, max: number): boolean {
+ return (
+ typeof value === 'number' &&
+ Number.isInteger(value) &&
+ value >= min &&
+ value <= max
+ );
+}
+
+function retainedSheet(key: string, value: unknown): boolean {
+ const separator = key.indexOf('!');
+ if (
+ separator < 0 ||
+ !isSpreadsheetSheetId(key.slice(0, separator)) ||
+ !key.slice(separator + 1).length ||
+ textEncoder.encode(key.slice(separator + 1)).length > 64 ||
+ typeof value !== 'string' ||
+ textEncoder.encode(value).length > 512
+ )
+ return false;
+ try {
+ const identity = JSON.parse(value);
+ return (
+ !!identity &&
+ sheetName(identity.name) &&
+ Number.isFinite(identity.order) &&
+ Number.isFinite(identity.revision) &&
+ identity.revision >= 0
+ );
+ } catch {
+ return false;
+ }
+}
+
+function validEntry(root: string, key: string, value: unknown): boolean {
+ return match(root)
+ .with(
+ 'spreadsheetMeta',
+ () => key === 'formatVersion' && value === SPREADSHEET_FORMAT_VERSION
+ )
+ .with(
+ 'spreadsheetSheetMetadata',
+ () => isSpreadsheetSheetId(key) && !!parseWorkbookMetadata(value)
+ )
+ .with(
+ 'spreadsheetSheetNames',
+ () => isSpreadsheetSheetId(key) && sheetName(value)
+ )
+ .with(
+ 'spreadsheetSheetOrder',
+ () => isSpreadsheetSheetId(key) && Number.isFinite(value)
+ )
+ .with(
+ 'spreadsheetDeletedSheets',
+ () => isSpreadsheetSheetId(key) && typeof value === 'boolean'
+ )
+ .with(
+ 'spreadsheetSheetRevivals',
+ () => textEncoder.encode(key).length <= 200 && isSpreadsheetSheetId(value)
+ )
+ .with('spreadsheetSheetRetentions', () => retainedSheet(key, value))
+ .otherwise(() => validCellEntry(root, key, value));
+}
+
+function validCellEntry(root: string, key: string, value: unknown): boolean {
+ const field = sheetKey(key);
+ if (field === undefined) return false;
+ if (root === 'spreadsheetColumnWidths')
+ return (
+ /^\d+$/.test(field) &&
+ Number(field) < SPREADSHEET_COLUMNS &&
+ integer(value, MIN_COLUMN_WIDTH, MAX_COLUMN_WIDTH)
+ );
+ if (root === 'spreadsheetRowAdditions')
+ return (
+ field.length > 0 &&
+ textEncoder.encode(field).length <= 64 &&
+ integer(value, 1, SPREADSHEET_MAX_ROWS)
+ );
+ if (!parseCellAddress(field)) return false;
+ if (root === 'spreadsheetValues')
+ return (
+ typeof value === 'string' && value.length <= SPREADSHEET_MAX_CELL_LENGTH
+ );
+ return isSpreadsheetStyleEntry(root, value);
+}
+
+/** Validate native workbook state at the spreadsheet business boundary.
+ * Sync stores arbitrary CRDT documents; it does not enforce this schema.
+ */
+export function validateSpreadsheetDocument(doc: LoroDoc): void {
+ const invalid = () =>
+ new Error(
+ 'Only native spreadsheet maps with supported values may be edited.'
+ );
+ let entries = 0;
+ let formatVersion: unknown;
+ // Inspect actual containers and entries, not deep JSON, which turns nested
+ // text/counter containers into scalars indistinguishable from cell values.
+ for (const [name, id] of Object.entries(doc.getShallowValue())) {
+ const container = doc.getContainerById(id);
+ if (!knownRoots.has(name) || !(container instanceof LoroMap))
+ throw invalid();
+ entries += container.size;
+ if (entries > MAX_DOCUMENT_ENTRIES)
+ throw new Error('The spreadsheet exceeds the supported entry limit.');
+ for (const [key, value] of container.entries()) {
+ if (
+ !['string', 'boolean', 'number'].includes(typeof value) ||
+ !validEntry(name, key, value)
+ )
+ throw invalid();
+ if (name === 'spreadsheetMeta' && key === 'formatVersion')
+ formatVersion = value;
+ }
+ }
+ if (formatVersion !== SPREADSHEET_FORMAT_VERSION) throw invalid();
+}
diff --git a/packages/spreadsheet/src/spreadsheet-document.ts b/packages/spreadsheet/src/spreadsheet-document.ts
index dafba9639f..f5fb52d56e 100644
--- a/packages/spreadsheet/src/spreadsheet-document.ts
+++ b/packages/spreadsheet/src/spreadsheet-document.ts
@@ -197,6 +197,14 @@ const styleFields: {
},
};
const styleKeys = Object.keys(styleFields) as (keyof SpreadsheetCellStyle)[];
+const styleFieldsByMap = new Map(
+ Object.values(styleFields).map((field) => [field.map, field])
+);
+
+/** Validate a persisted style using the same rules as local cell edits. */
+export function isSpreadsheetStyleEntry(map: string, value: unknown): boolean {
+ return styleFieldsByMap.get(map)?.valid(value) ?? false;
+}
/** Clipboard styles and local patches use the same validation as remote data. */
export function isSpreadsheetCellStyle(
diff --git a/services/ai-editing-worker/src/spreadsheet/README.md b/services/ai-editing-worker/src/spreadsheet/README.md
index aa6851496c..f91bd4322b 100644
--- a/services/ai-editing-worker/src/spreadsheet/README.md
+++ b/services/ai-editing-worker/src/spreadsheet/README.md
@@ -7,12 +7,19 @@ by the Rust document tool types. This endpoint needs `SYNC_WS_BASE`; it does not
call an LLM or require model API keys.
The worker loads a coherent Loro snapshot and revision using
-`GET /document/:id/spreadsheet-snapshot`. Reads and scratch calculations are pure.
+`GET /document/:id/state`. Reads and scratch calculations are pure.
Edits operate on a disposable fork and send one delta with the caller's expected
-revision to `POST /document/:id/spreadsheet-update`. The Durable Object validates
+revision to `POST /document/:id/update`. The Durable Object validates
the signed document grant, compares the live version and persists/broadcasts the
delta atomically. A stale revision returns 409 and requires a fresh read.
+Sync treats the content as an arbitrary Loro document. This worker validates the
+loaded native workbook before reads/calculations/edits, and the spreadsheet edit
+use case validates its resulting fork before exporting a delta. The shared
+`packages/spreadsheet/src/document-validation.ts` reuses cell style, sheet name
+and metadata validators, including actual container-type checks. A concurrent
+change after validation is rejected by Sync's revision comparison.
+
Authorization is checked at both Rust's entity-access boundary and the sync
endpoint. Viewers may read and calculate; mutations require edit or owner access.
Comment-only grants do not permit spreadsheet writes. Neither an internal key
diff --git a/services/ai-editing-worker/src/spreadsheet/integration.e2e.ts b/services/ai-editing-worker/src/spreadsheet/integration.e2e.ts
index 70c9fb56b6..87c53fa364 100644
--- a/services/ai-editing-worker/src/spreadsheet/integration.e2e.ts
+++ b/services/ai-editing-worker/src/spreadsheet/integration.e2e.ts
@@ -10,7 +10,7 @@ import type {
} from '@macro-inc/spreadsheet/ai-types';
import { LoroDoc } from 'loro-crdt';
import { Miniflare, type WebSocket } from 'miniflare';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
import {
FromPeer,
FromRemote,
@@ -26,6 +26,8 @@ let mf: Miniflare;
// Miniflare's conditional Fetcher replacement conflicts with Bun's global
// Request types. Keep the exercised service-binding surface explicit here.
let sync: { fetch(input: string, init: RequestInit): Promise<Response> };
+const notifications: { path: string; auth: string | null; body: unknown }[] =
+ [];
function token(
id: string,
@@ -95,10 +97,21 @@ beforeAll(async () => {
INTERNAL_API_SECRET: 'local',
SPS_API_SECRET_KEY: 'local',
SPS_URL: 'http://discard.test',
+ DSS_URL: 'https://dss.test',
+ DSS_INTERNAL_AUTH_KEY: 'local',
local: true,
},
// All telemetry/indexing outbound calls remain inside this fixture.
- outboundService: async () => new Response(null, { status: 204 }),
+ outboundService: async (request) => {
+ const path = new URL(request.url).pathname;
+ if (path.endsWith('/sync-content-updated'))
+ notifications.push({
+ path,
+ auth: request.headers.get('x-document-storage-service-auth-key'),
+ body: await request.json(),
+ });
+ return new Response(null, { status: 200 });
+ },
},
],
});
@@ -199,6 +212,72 @@ async function success(
}
describe('real spreadsheet editing worker and sync Durable Object', () => {
+ it('publishes content changes to the document backend without assigning a file type', async () => {
+ const id = await create();
+ const initial = await success(id, { action: 'read' });
+ await success(id, {
+ action: 'edit',
+ expectedRevision: initial.revision,
+ operations: [{ type: 'add_sheet', name: 'Notification' }],
+ });
+ await vi.waitFor(() =>
+ expect(notifications).toContainEqual({
+ path: `/internal/documents/${id}/sync-content-updated`,
+ auth: 'local',
+ body: {
+ actor: 'spreadsheet-test-agent',
+ on_behalf_of: 'macro|[email protected]',
+ },
+ })
+ );
+ });
+
+ it('validates spreadsheet state in the editing backend after generic sync accepts it', async () => {
+ const id = await create();
+ const response = await sync.fetch(
+ `https://sync.test/document/${id}/state`,
+ {
+ headers: { Authorization: `Bearer ${token(id)}` },
+ }
+ );
+ expect(response.status).toBe(200);
+ const source = (await response.json()) as {
+ snapshot: string;
+ revision: string;
+ };
+ const doc = new LoroDoc();
+ try {
+ doc.import(Buffer.from(source.snapshot, 'base64'));
+ const from = doc.version();
+ doc.getMap('spreadsheetValues').set('AA1', 'outside the supported grid');
+ const committed = await sync.fetch(
+ `https://sync.test/document/${id}/update`,
+ {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${token(id)}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ expectedRevision: source.revision,
+ update: Buffer.from(doc.export({ mode: 'update', from })).toString(
+ 'base64'
+ ),
+ }),
+ }
+ );
+ expect(committed.status).toBe(200);
+ const rejected = await request(id, { action: 'read' });
+ expect(rejected.status).toBe(400);
+ expect(await rejected.json()).toEqual({
+ error:
+ 'Only native spreadsheet maps with supported values may be edited.',
+ });
+ } finally {
+ doc.free();
+ }
+ });
+
it('edits with no browser, broadcasts to an open collaborator, and refuses to overwrite their newer edit', async () => {
const id = await create();
const initial = await success(id, { action: 'read' });
diff --git a/services/ai-editing-worker/src/spreadsheet/runtime.test.ts b/services/ai-editing-worker/src/spreadsheet/runtime.test.ts
index c835e9f917..a478d6b7db 100644
--- a/services/ai-editing-worker/src/spreadsheet/runtime.test.ts
+++ b/services/ai-editing-worker/src/spreadsheet/runtime.test.ts
@@ -54,6 +54,28 @@ function store() {
const signal = () => new AbortController().signal;
describe('deterministic spreadsheet worker runtime', () => {
+ it.each(['read', 'calculate', 'edit'] as const)(
+ 'rejects invalid persisted workbook state before %s or persistence',
+ async (action) => {
+ const { doc, storage, revision } = store();
+ doc.getMap('spreadsheetFontSize').set('A1', 'invalid');
+ const request =
+ action === 'read'
+ ? { action }
+ : action === 'calculate'
+ ? { action, formulas: [{ formula: '=1+1' }] }
+ : {
+ action,
+ expectedRevision: revision(),
+ operations: [{ type: 'add_sheet' as const, name: 'New' }],
+ };
+ await expect(
+ runSpreadsheetRequest(request, storage, calculator, signal())
+ ).rejects.toThrow('native spreadsheet maps');
+ expect(storage.commit).not.toHaveBeenCalled();
+ }
+ );
+
it('AI can write and reread native mention cells without an open editor', async () => {
const { doc, storage, revision } = store();
const value =
@@ -248,7 +270,10 @@ describe('deterministic spreadsheet worker runtime', () => {
storage.commit(revision(), new Uint8Array([1, 2]), signal())
).rejects.toMatchObject({ status: 409 });
expect(fetcher.mock.calls[0][0]).toBe(
- 'https://sync.example/document/document-id/spreadsheet-snapshot'
+ 'https://sync.example/document/document-id/state'
+ );
+ expect(fetcher.mock.calls[1][0]).toBe(
+ 'https://sync.example/document/document-id/update'
);
expect(fetcher.mock.calls[1][1]?.headers).toEqual({
Authorization: 'Bearer scoped-token',
diff --git a/services/ai-editing-worker/src/spreadsheet/runtime.ts b/services/ai-editing-worker/src/spreadsheet/runtime.ts
index 386e588a61..96d1ee7f72 100644
--- a/services/ai-editing-worker/src/spreadsheet/runtime.ts
+++ b/services/ai-editing-worker/src/spreadsheet/runtime.ts
@@ -8,6 +8,7 @@ import {
readSpreadsheetForAi,
} from '@macro-inc/spreadsheet/ai-workbook';
import type { SpreadsheetCalculator } from '@macro-inc/spreadsheet/calculation';
+import { validateSpreadsheetDocument } from '@macro-inc/spreadsheet/document-validation';
import { LoroDoc } from 'loro-crdt';
import { nextAiPeerId } from '../ai-editing/awareness/ai-peer';
@@ -160,12 +161,12 @@ export function createSpreadsheetStorage(
else if (base.protocol === 'ws:') base.protocol = 'http:';
if (base.protocol !== 'http:' && base.protocol !== 'https:')
throw new Error('Invalid sync service URL.');
- const endpoint = `${base.toString().replace(/\/$/, '')}/document/${encodeURIComponent(documentId)}/spreadsheet`;
+ const endpoint = `${base.toString().replace(/\/$/, '')}/document/${encodeURIComponent(documentId)}`;
const headers = { Authorization: `Bearer ${documentToken}` };
return {
async load(signal) {
const data = await responseJson(
- await fetcher(`${endpoint}-snapshot`, { headers, signal }),
+ await fetcher(`${endpoint}/state`, { headers, signal }),
'read'
);
if (typeof data.revision !== 'string' || !data.revision)
@@ -177,7 +178,7 @@ export function createSpreadsheetStorage(
},
async commit(expectedRevision, update, signal) {
const data = await responseJson(
- await fetcher(`${endpoint}-update`, {
+ await fetcher(`${endpoint}/update`, {
method: 'POST',
headers: { ...headers, 'Content-Type': 'application/json' },
body: JSON.stringify({ expectedRevision, update: encode(update) }),
@@ -222,6 +223,7 @@ export async function runSpreadsheetRequest(
const doc = new LoroDoc();
try {
doc.import(snapshot);
+ validateSpreadsheetDocument(doc);
if (request.action === 'read') {
const response = readSpreadsheetForAi(doc, revision, request, calculator);
checkDeadline();
diff --git a/services/document_storage_service/src/api/internal.rs b/services/document_storage_service/src/api/internal.rs
index 29b7cadbf1..9572360603 100644
--- a/services/document_storage_service/src/api/internal.rs
+++ b/services/document_storage_service/src/api/internal.rs
@@ -140,6 +140,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {
>,
),
)
+ .route(
+ "/documents/{document_id}/sync-content-updated",
+ post(
+ documents_hex::inbound::axum_router::sync_content_updated::sync_content_updated_handler::<
+ DocumentService,
+ EntityAccessService,
+ AuthorizationService,
+ >,
+ ),
+ )
.route(
"/documents/{document_id}/interaction",
put(
diff --git a/services/sync-service/README.md b/services/sync-service/README.md
index b3ef4c9455..eb8ac6c3b9 100644
--- a/services/sync-service/README.md
+++ b/services/sync-service/README.md
@@ -119,14 +119,14 @@ just test-no-alarm
when making a new deploy you need to make a cloudflare KV store
and D1 database
-### Atomic spreadsheet updates
+### Atomic document updates
-`GET /document/:id/spreadsheet-snapshot` returns JSON `{snapshot, revision}`.
+`GET /document/:id/state` returns JSON `{snapshot, revision}`.
Both fields use standard base64: `snapshot` is a full Loro snapshot and `revision`
is an encoded Loro version vector from that same state. Requests require a signed
Bearer document permission token. An internal API key does not replace this token.
-`POST /document/:id/spreadsheet-update` accepts JSON
+`POST /document/:id/update` accepts JSON
`{expectedRevision, update}`, where `update` is a base64 Loro update exported from
the snapshot revision. Edit or owner access is required; viewers can read only.
The service validates a disposable fork, compares the current revision, and
@@ -135,9 +135,22 @@ operation log and broadcast pipeline. It returns `{revision, applied}` only afte
persistence. A stale revision returns 409 without applying changes. Retrying a
fully applied delta succeeds with `applied: false`, even after intervening edits.
-Only native spreadsheets (`spreadsheetMeta.formatVersion = 1`) and their known
-scalar maps are accepted. Unsupported roots, nested data, invalid cells/styles,
-missing dependencies, and snapshot bodies are rejected. Binary updates and read
-snapshots are limited to 4 MiB, revisions to 64 KiB, and HTTP bodies are bounded
-while streaming. Signed actor/user attribution is bounded and stored beside the
-existing operation log; request bodies cannot specify attribution.
+Sync accepts arbitrary Loro document schemas. It validates the update encoding,
+operation/byte limits and causal dependencies, but does not interpret cells,
+styles, sheet names or document content types. Spreadsheet validation and
+operations live in `packages/spreadsheet` and the backend AI editing worker.
+Binary updates and read snapshots are limited to 4 MiB, revisions to 64 KiB,
+and HTTP bodies are bounded while streaming. Signed actor/user attribution is
+bounded and stored beside the existing operation log; request bodies cannot
+specify attribution.
+
+The existing binary `/snapshot` endpoint is unchanged.
+
+Content notifications go to DSS's internal
+`POST /internal/documents/:id/sync-content-updated` endpoint. The documents domain
+loads the stored file type and publishes `document.sync_content_updated`, which
+lets search choose its supported extractor. Sync no longer infers Markdown from
+CRDT roots or labels updates as Markdown. The `search-service` feature continues
+to control these notifications for compatibility with existing build commands.
+
+Deploy DSS first, then roll out Sync and the AI editing worker together.
diff --git a/services/sync-service/src/auth.rs b/services/sync-service/src/auth.rs
index e7072a32e3..2aaf059bd7 100644
--- a/services/sync-service/src/auth.rs
+++ b/services/sync-service/src/auth.rs
@@ -133,30 +133,32 @@ pub fn decode_jwt(
Ok(claims)
}
-/// Dedicated access boundary for spreadsheet HTTP requests. Internal service
+/// Dedicated access boundary for document HTTP requests. Internal service
/// credentials do not substitute for a signed, document-scoped user grant.
-pub fn spreadsheet_access(
+pub fn document_access(
req: &worker::Request,
env: &worker::Env,
document_id: &str,
-) -> Result<(crate::spreadsheet::SpreadsheetAccess, AuthToken), crate::spreadsheet::SpreadsheetError>
-{
- use crate::spreadsheet::SpreadsheetError;
+) -> Result<
+ (crate::domain::document::DocumentAccess, AuthToken),
+ crate::domain::document::DocumentError,
+> {
+ use crate::domain::document::DocumentError;
let header = req
.headers()
.get(header_names::AUTHORIZATION)
- .map_err(|_| SpreadsheetError::Unauthorized)?
- .ok_or(SpreadsheetError::Unauthorized)?;
+ .map_err(|_| DocumentError::Unauthorized)?
+ .ok_or(DocumentError::Unauthorized)?;
let token = header
.strip_prefix("Bearer ")
- .ok_or(SpreadsheetError::Unauthorized)?;
+ .ok_or(DocumentError::Unauthorized)?;
let claims = macro_sync_service_jwt::decode::<AuthToken>(
token,
&Secrets::from(env).document_permissions_secret,
)
- .map_err(|_| SpreadsheetError::Unauthorized)?;
- let access = crate::spreadsheet::SpreadsheetAccess::authorize(
+ .map_err(|_| DocumentError::Unauthorized)?;
+ let access = crate::domain::document::DocumentAccess::authorize(
document_id,
&claims.document_id,
claims.access_level >= AccessLevel::Edit,
diff --git a/services/sync-service/src/domain.rs b/services/sync-service/src/domain.rs
new file mode 100644
index 0000000000..54f38d0b07
--- /dev/null
+++ b/services/sync-service/src/domain.rs
@@ -0,0 +1,3 @@
+//! Document synchronization policy, independent of content and storage.
+
+pub mod document;
diff --git a/services/sync-service/src/domain/document.rs b/services/sync-service/src/domain/document.rs
new file mode 100644
index 0000000000..d86d21f5a1
--- /dev/null
+++ b/services/sync-service/src/domain/document.rs
@@ -0,0 +1,211 @@
+//! Content-agnostic document snapshots and atomic CRDT updates.
+//!
+//! This module has no HTTP or storage dependencies. The durable-object adapter
+//! supplies verified JWT claims and persists accepted updates through its oplog.
+use std::borrow::Cow;
+
+use loro::{ExportMode, LoroDoc, VersionVector};
+
+pub const MAX_BINARY_BYTES: usize = 4 * 1024 * 1024;
+pub const MAX_REVISION_BYTES: usize = 64 * 1024;
+const MAX_UPDATE_OPERATIONS: u64 = 100_000;
+
+#[derive(Debug, thiserror::Error)]
+pub enum DocumentError {
+ #[error("The document token does not grant access to this document.")]
+ Unauthorized,
+ #[error("Editing this document requires edit access.")]
+ Forbidden,
+ #[error("The document changed. Read a fresh snapshot before editing.")]
+ Conflict,
+ #[error("The document request exceeds the size limit.")]
+ TooLarge,
+ #[error("{0}")]
+ Invalid(&'static str),
+ #[error("The document update could not be persisted.")]
+ Persistence,
+ #[error(
+ "The document was saved, but its update notifications could not be completed. Read the document before retrying."
+ )]
+ Notification,
+}
+
+/// Capability minted only after the adapter verifies a document permission JWT.
+pub struct DocumentAccess {
+ writable: bool,
+}
+
+impl DocumentAccess {
+ pub fn authorize(
+ requested_document: &str,
+ token_document: &str,
+ writable: bool,
+ ) -> Result<Self, DocumentError> {
+ if requested_document != token_document {
+ return Err(DocumentError::Unauthorized);
+ }
+ Ok(Self { writable })
+ }
+
+ pub fn require_edit(&self) -> Result<(), DocumentError> {
+ if !self.writable {
+ return Err(DocumentError::Forbidden);
+ }
+ Ok(())
+ }
+}
+
+pub struct PreparedUpdate {
+ /// Only new, validated operations, ready for the existing persistence path.
+ pub update: Vec<u8>,
+ pub revision: Vec<u8>,
+ pub applied: bool,
+}
+
+/// Signed-token attribution stored alongside the existing operation log.
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
+pub struct DocumentAttribution {
+ pub actor: String,
+ pub on_behalf_of: Option<String>,
+}
+
+impl DocumentAttribution {
+ pub fn from_signed_claims(
+ actor: String,
+ on_behalf_of: Option<String>,
+ ) -> Result<Self, DocumentError> {
+ if actor.len() > 256 || on_behalf_of.as_ref().is_some_and(|user| user.len() > 1024) {
+ return Err(DocumentError::Invalid("Invalid signed attribution."));
+ }
+ Ok(Self {
+ actor,
+ on_behalf_of,
+ })
+ }
+}
+
+/// Persistence port: applying the update must happen before the first await.
+pub trait DocumentUpdatePort {
+ fn document(&self) -> &LoroDoc;
+ async fn apply_and_persist(&self, update: &[u8]) -> Result<(), DocumentError>;
+}
+
+/// Side effects after a durable write; the use case owns their order and whether
+/// a newly applied edit needs document publication. Implementations own only the
+/// notification transport and scheduling mechanisms.
+pub trait DocumentUpdateEffects {
+ fn broadcast(&self, update: &[u8]) -> Result<(), DocumentError>;
+ fn publish_changed_document(&self) -> Result<(), DocumentError>;
+ async fn keep_alive(&self) -> Result<(), DocumentError>;
+}
+
+pub async fn update(
+ access: &DocumentAccess,
+ port: &impl DocumentUpdatePort,
+ effects: &impl DocumentUpdateEffects,
+ expected_revision: &[u8],
+ update: &[u8],
+) -> Result<PreparedUpdate, DocumentError> {
+ let prepared = prepare_update(access, port.document(), expected_revision, update)?;
+ // Persist retries too: a prior request may have applied in memory but lost
+ // its storage write or response. Duplicate Loro imports are idempotent.
+ port.apply_and_persist(&prepared.update).await?;
+ effects.broadcast(&prepared.update)?;
+ if prepared.applied {
+ effects.publish_changed_document()?;
+ }
+ effects.keep_alive().await?;
+ Ok(prepared)
+}
+
+pub fn snapshot(
+ _access: &DocumentAccess,
+ doc: &LoroDoc,
+) -> Result<(Vec<u8>, Vec<u8>), DocumentError> {
+ let snapshot = doc
+ .export(ExportMode::Snapshot)
+ .map_err(|_| DocumentError::Invalid("The document snapshot could not be exported."))?;
+ if snapshot.len() > MAX_BINARY_BYTES {
+ return Err(DocumentError::TooLarge);
+ }
+ let revision = doc.oplog_vv().encode();
+ if revision.len() > MAX_REVISION_BYTES {
+ return Err(DocumentError::TooLarge);
+ }
+ Ok((snapshot, revision))
+}
+
+/// Synchronous preflight. The caller must apply the returned delta before its
+/// next await, so websocket writes cannot interleave with this version check.
+pub fn prepare_update(
+ access: &DocumentAccess,
+ doc: &LoroDoc,
+ expected_revision: &[u8],
+ update: &[u8],
+) -> Result<PreparedUpdate, DocumentError> {
+ access.require_edit()?;
+ if update.len() > MAX_BINARY_BYTES || expected_revision.len() > MAX_REVISION_BYTES {
+ return Err(DocumentError::TooLarge);
+ }
+ let expected = VersionVector::decode(expected_revision)
+ .map_err(|_| DocumentError::Invalid("Invalid document revision."))?;
+ let meta = LoroDoc::decode_import_blob_meta(update, true)
+ .map_err(|_| DocumentError::Invalid("Invalid Loro update."))?;
+ if meta.mode.is_snapshot() {
+ return Err(DocumentError::Invalid(
+ "Send a Loro update, not a snapshot.",
+ ));
+ }
+ let operations: u64 = meta
+ .partial_end_vv
+ .iter()
+ .map(|(peer, end)| {
+ end.checked_sub(meta.partial_start_vv.get(peer).copied().unwrap_or(0))
+ .and_then(|count| u64::try_from(count).ok())
+ .unwrap_or(u64::MAX)
+ })
+ .try_fold(0_u64, |total, count| total.checked_add(count))
+ .ok_or(DocumentError::TooLarge)?;
+ if operations > MAX_UPDATE_OPERATIONS || meta.change_num > 10_000 {
+ return Err(DocumentError::TooLarge);
+ }
+ let current = doc.oplog_vv();
+ let preview = doc.fork();
+ let imported = preview
+ .import(update)
+ .map_err(|_| DocumentError::Invalid("Invalid Loro update."))?;
+ if imported.pending.is_some() || preview.state_vv() != preview.oplog_vv() {
+ return Err(DocumentError::Invalid(
+ "The update has missing dependencies.",
+ ));
+ }
+ let revision = preview.oplog_vv();
+ // Recognizing an already applied delta makes a lost HTTP response safely
+ // retryable, even if other users have edited after the first application.
+ if revision == current {
+ return Ok(PreparedUpdate {
+ update: update.to_vec(),
+ revision: current.encode(),
+ applied: false,
+ });
+ }
+ if current != expected {
+ return Err(DocumentError::Conflict);
+ }
+ let update = preview
+ .export(ExportMode::Updates {
+ from: Cow::Borrowed(&current),
+ })
+ .map_err(|_| DocumentError::Invalid("The validated update could not be exported."))?;
+ if update.len() > MAX_BINARY_BYTES {
+ return Err(DocumentError::TooLarge);
+ }
+ Ok(PreparedUpdate {
+ update,
+ revision: revision.encode(),
+ applied: true,
+ })
+}
+
+#[cfg(test)]
+mod test;
diff --git a/services/sync-service/src/spreadsheet/test.rs b/services/sync-service/src/domain/document/test.rs
similarity index 52%
rename from services/sync-service/src/spreadsheet/test.rs
rename to services/sync-service/src/domain/document/test.rs
index 9c42ea08f2..00a2719567 100644
--- a/services/sync-service/src/spreadsheet/test.rs
+++ b/services/sync-service/src/domain/document/test.rs
@@ -3,14 +3,12 @@ use std::cell::{Cell, RefCell};
fn document() -> LoroDoc {
let doc = LoroDoc::new();
- doc.import(include_bytes!(
- "../../../../static_assets/spreadsheet-golden.1.bin"
- ))
- .unwrap();
+ doc.get_map("metadata").insert("title", "Document").unwrap();
+ doc.commit();
doc
}
-fn access() -> SpreadsheetAccess {
- SpreadsheetAccess::authorize("doc", "doc", true).unwrap()
+fn access() -> DocumentAccess {
+ DocumentAccess::authorize("doc", "doc", true).unwrap()
}
fn delta(doc: &LoroDoc, root: &str, key: &str, value: &str) -> Vec<u8> {
let fork = doc.fork();
@@ -24,10 +22,10 @@ fn delta(doc: &LoroDoc, root: &str, key: &str, value: &str) -> Vec<u8> {
#[test]
fn permission_policy_requires_matching_document_and_edit_grant() {
assert!(matches!(
- SpreadsheetAccess::authorize("other", "doc", true),
- Err(SpreadsheetError::Unauthorized)
+ DocumentAccess::authorize("other", "doc", true),
+ Err(DocumentError::Unauthorized)
));
- let reader = SpreadsheetAccess::authorize("doc", "doc", false).unwrap();
+ let reader = DocumentAccess::authorize("doc", "doc", false).unwrap();
let doc = document();
assert!(snapshot(&reader, &doc).is_ok());
assert!(matches!(
@@ -35,9 +33,9 @@ fn permission_policy_requires_matching_document_and_edit_grant() {
&reader,
&doc,
&doc.oplog_vv().encode(),
- &delta(&doc, "spreadsheetValues", "A1", "x")
+ &delta(&doc, "properties", "A1", "x")
),
- Err(SpreadsheetError::Forbidden)
+ Err(DocumentError::Forbidden)
));
}
@@ -45,30 +43,26 @@ fn permission_policy_requires_matching_document_and_edit_grant() {
fn preflight_is_isolated_and_stale_revision_never_changes_live_state() {
let doc = document();
let revision = doc.oplog_vv().encode();
- let update = delta(&doc, "spreadsheetValues", "A1", "42");
+ let update = delta(&doc, "properties", "A1", "42");
let prepared = prepare_update(&access(), &doc, &revision, &update).unwrap();
assert!(prepared.applied);
- assert!(doc.get_map("spreadsheetValues").get("A1").is_none());
- doc.get_map("spreadsheetValues")
- .insert("B1", "peer")
- .unwrap();
+ assert!(doc.get_map("properties").get("A1").is_none());
+ doc.get_map("properties").insert("B1", "peer").unwrap();
doc.commit();
assert!(matches!(
prepare_update(&access(), &doc, &revision, &update),
- Err(SpreadsheetError::Conflict)
+ Err(DocumentError::Conflict)
));
- assert!(doc.get_map("spreadsheetValues").get("A1").is_none());
+ assert!(doc.get_map("properties").get("A1").is_none());
}
#[test]
fn applied_delta_retry_is_idempotent_even_after_another_edit() {
let doc = document();
let revision = doc.oplog_vv().encode();
- let update = delta(&doc, "spreadsheetValues", "A1", "42");
+ let update = delta(&doc, "properties", "A1", "42");
doc.import(&update).unwrap();
- doc.get_map("spreadsheetValues")
- .insert("B1", "peer")
- .unwrap();
+ doc.get_map("properties").insert("B1", "peer").unwrap();
doc.commit();
let retried = prepare_update(&access(), &doc, &revision, &update).unwrap();
assert!(!retried.applied);
@@ -76,64 +70,85 @@ fn applied_delta_retry_is_idempotent_even_after_another_edit() {
}
#[test]
-fn rejects_other_document_types_roots_invalid_cells_and_snapshots() {
+fn snapshots_and_updates_accept_text_maps_and_nested_containers() {
+ let doc = document();
+ doc.get_text("content").insert(0, "text").unwrap();
+ doc.commit();
+ let (bytes, revision) = snapshot(&access(), &doc).unwrap();
+ let copy = LoroDoc::new();
+ copy.import(&bytes).unwrap();
+ assert_eq!(copy.oplog_vv().encode(), revision);
+ assert_eq!(copy.get_deep_value(), doc.get_deep_value());
+
+ let fork = doc.fork();
+ fork.get_map("arbitrary")
+ .insert_container("nested", loro::LoroText::new())
+ .unwrap()
+ .insert(0, "nested text")
+ .unwrap();
+ let update = fork
+ .export(ExportMode::Updates {
+ from: Cow::Owned(doc.oplog_vv()),
+ })
+ .unwrap();
+ let prepared = prepare_update(&access(), &doc, &revision, &update).unwrap();
+ assert!(prepared.applied);
+ doc.import(&prepared.update).unwrap();
+ assert_eq!(doc.get_deep_value(), fork.get_deep_value());
+}
+
+#[test]
+fn rejects_snapshots_malformed_updates_and_missing_dependencies_without_mutation() {
let doc = document();
let revision = doc.oplog_vv().encode();
- for (root, key, value) in [
- ("root", "content", "bad"),
- ("spreadsheetUnknown", "A1", "bad"),
- ("spreadsheetValues", "AA1", "bad"),
- ("spreadsheetValues", "A1001", "bad"),
- ("spreadsheetFontSize", "A1", "large"),
+ let fork = doc.fork();
+ fork.get_map("properties")
+ .insert("first", "missing")
+ .unwrap();
+ fork.commit();
+ let from = fork.oplog_vv();
+ fork.get_map("properties")
+ .insert("second", "pending")
+ .unwrap();
+ let pending = fork
+ .export(ExportMode::Updates {
+ from: Cow::Owned(from),
+ })
+ .unwrap();
+ for update in [
+ doc.export(ExportMode::Snapshot).unwrap(),
+ vec![1, 2, 3],
+ pending,
] {
- let update = delta(&doc, root, key, value);
assert!(matches!(
prepare_update(&access(), &doc, &revision, &update),
- Err(SpreadsheetError::Invalid(_))
+ Err(DocumentError::Invalid(_))
));
+ assert_eq!(doc.oplog_vv().encode(), revision);
}
- let markdown = LoroDoc::new();
- markdown.get_text("content").insert(0, "text").unwrap();
- assert!(snapshot(&access(), &markdown).is_err());
- assert!(
- prepare_update(
- &access(),
- &doc,
- &revision,
- &doc.export(ExportMode::Snapshot).unwrap()
- )
- .is_err()
- );
- assert_eq!(doc.oplog_vv().encode(), revision);
}
#[test]
-fn enforces_binary_and_schema_bounds() {
+fn enforces_binary_revision_and_operation_bounds() {
let doc = document();
+ let revision = doc.oplog_vv().encode();
+ let update = delta(&doc, "properties", "field", "value");
+ for (revision, update) in [
+ (revision.clone(), vec![0; MAX_BINARY_BYTES + 1]),
+ (vec![0; MAX_REVISION_BYTES + 1], update.clone()),
+ ] {
+ assert!(matches!(
+ prepare_update(&access(), &doc, &revision, &update),
+ Err(DocumentError::TooLarge)
+ ));
+ }
assert!(matches!(
- prepare_update(
- &access(),
- &doc,
- &doc.oplog_vv().encode(),
- &vec![0; MAX_BINARY_BYTES + 1]
- ),
- Err(SpreadsheetError::TooLarge)
- ));
- let update = delta(&doc, "spreadsheetValues", "A1", &"x".repeat(10_001));
- assert!(matches!(
- prepare_update(&access(), &doc, &doc.oplog_vv().encode(), &update),
- Err(SpreadsheetError::Invalid(_))
+ prepare_update(&access(), &doc, &[255], &update),
+ Err(DocumentError::Invalid(_))
));
-}
-
-#[test]
-fn rejects_nested_containers_even_when_their_deep_value_looks_scalar() {
- let doc = document();
let fork = doc.fork();
- fork.get_map("spreadsheetValues")
- .insert_container("A1", loro::LoroText::new())
- .unwrap()
- .insert(0, "looks like a string")
+ fork.get_text("content")
+ .insert(0, &"x".repeat(MAX_UPDATE_OPERATIONS as usize + 1))
.unwrap();
let update = fork
.export(ExportMode::Updates {
@@ -141,9 +156,10 @@ fn rejects_nested_containers_even_when_their_deep_value_looks_scalar() {
})
.unwrap();
assert!(matches!(
- prepare_update(&access(), &doc, &doc.oplog_vv().encode(), &update),
- Err(SpreadsheetError::Invalid(_))
+ prepare_update(&access(), &doc, &revision, &update),
+ Err(DocumentError::TooLarge)
));
+ assert_eq!(doc.oplog_vv().encode(), revision);
}
struct UpdateHarness {
@@ -164,23 +180,23 @@ impl UpdateHarness {
}
}
-impl SpreadsheetUpdatePort for UpdateHarness {
+impl DocumentUpdatePort for UpdateHarness {
fn document(&self) -> &LoroDoc {
&self.doc
}
- async fn apply_and_persist(&self, update: &[u8]) -> Result<(), SpreadsheetError> {
+ async fn apply_and_persist(&self, update: &[u8]) -> Result<(), DocumentError> {
self.calls.borrow_mut().push("persist");
self.doc.import(update).unwrap();
if self.fail_persist {
- return Err(SpreadsheetError::Persistence);
+ return Err(DocumentError::Persistence);
}
Ok(())
}
}
-impl SpreadsheetUpdateEffects for UpdateHarness {
- fn broadcast(&self, update: &[u8]) -> Result<(), SpreadsheetError> {
+impl DocumentUpdateEffects for UpdateHarness {
+ fn broadcast(&self, update: &[u8]) -> Result<(), DocumentError> {
self.calls.borrow_mut().push("broadcast");
let preview = self.doc.fork();
preview.import(update).unwrap();
@@ -188,15 +204,15 @@ impl SpreadsheetUpdateEffects for UpdateHarness {
Ok(())
}
- fn publish_changed_document(&self) -> Result<(), SpreadsheetError> {
+ fn publish_changed_document(&self) -> Result<(), DocumentError> {
self.calls.borrow_mut().push("publish");
if self.fail_publish.get() {
- return Err(SpreadsheetError::Notification);
+ return Err(DocumentError::Notification);
}
Ok(())
}
- async fn keep_alive(&self) -> Result<(), SpreadsheetError> {
+ async fn keep_alive(&self) -> Result<(), DocumentError> {
self.calls.borrow_mut().push("keep_alive");
Ok(())
}
@@ -206,7 +222,7 @@ impl SpreadsheetUpdateEffects for UpdateHarness {
fn update_use_case_persists_before_broadcast_and_publishes_new_edits() {
let port = UpdateHarness::new();
let revision = port.doc.oplog_vv().encode();
- let delta = delta(&port.doc, "spreadsheetValues", "A1", "42");
+ let delta = delta(&port.doc, "properties", "A1", "42");
let prepared =
futures::executor::block_on(update(&access(), &port, &port, &revision, &delta)).unwrap();
assert!(prepared.applied);
@@ -226,15 +242,15 @@ fn update_use_case_persists_before_broadcast_and_publishes_new_edits() {
fn rejected_or_unpersisted_update_never_notifies() {
let mut port = UpdateHarness::new();
let revision = port.doc.oplog_vv().encode();
- let delta = delta(&port.doc, "spreadsheetValues", "A1", "42");
- let viewer = SpreadsheetAccess::authorize("doc", "doc", false).unwrap();
+ let delta = delta(&port.doc, "properties", "A1", "42");
+ let viewer = DocumentAccess::authorize("doc", "doc", false).unwrap();
let result = futures::executor::block_on(update(&viewer, &port, &port, &revision, &delta));
- assert!(matches!(result, Err(SpreadsheetError::Forbidden)));
+ assert!(matches!(result, Err(DocumentError::Forbidden)));
assert!(port.calls.borrow().is_empty());
port.fail_persist = true;
let result = futures::executor::block_on(update(&access(), &port, &port, &revision, &delta));
- assert!(matches!(result, Err(SpreadsheetError::Persistence)));
+ assert!(matches!(result, Err(DocumentError::Persistence)));
assert_eq!(*port.calls.borrow(), ["persist"]);
}
@@ -242,12 +258,12 @@ fn rejected_or_unpersisted_update_never_notifies() {
fn notification_failure_preserves_durable_update_and_allows_idempotent_retry() {
let port = UpdateHarness::new();
let revision = port.doc.oplog_vv().encode();
- let delta = delta(&port.doc, "spreadsheetValues", "A1", "42");
+ let delta = delta(&port.doc, "properties", "A1", "42");
port.fail_publish.set(true);
let result = futures::executor::block_on(update(&access(), &port, &port, &revision, &delta));
- assert!(matches!(result, Err(SpreadsheetError::Notification)));
+ assert!(matches!(result, Err(DocumentError::Notification)));
assert_eq!(*port.calls.borrow(), ["persist", "broadcast", "publish"]);
- assert!(port.doc.get_map("spreadsheetValues").get("A1").is_some());
+ assert!(port.doc.get_map("properties").get("A1").is_some());
port.calls.borrow_mut().clear();
port.fail_publish.set(false);
@@ -256,42 +272,3 @@ fn notification_failure_preserves_durable_update_and_allows_idempotent_retry() {
assert!(!retried.applied);
assert_eq!(*port.calls.borrow(), ["persist", "broadcast", "keep_alive"]);
}
-
-#[test]
-fn imported_financial_styles_and_metadata_are_validated_before_commit() {
- let doc = document();
- let fork = doc.fork();
- for (root, value) in [
- ("spreadsheetNumberFormats", "#,##0.00;[Red](#,##0.00);\"—\""),
- ("spreadsheetFontNames", "Calibri"),
- ("spreadsheetBorderBottomStyles", "double"),
- ("spreadsheetBorderBottomColors", "#123456"),
- ] {
- fork.get_map(root).insert("A1", value).unwrap();
- }
- fork.get_map("spreadsheetSheetMetadata").insert("sheet1", r#"{"merges":["A1:C1"],"rowHeights":{"0":32},"hiddenRows":[5],"hiddenColumns":[5],"freeze":{"rows":3,"columns":1},"definedNames":[{"name":"Rate","formula":"0.1","local":true}]}"#).unwrap();
- let update = fork
- .export(ExportMode::Updates {
- from: Cow::Owned(doc.oplog_vv()),
- })
- .unwrap();
- assert!(prepare_update(&access(), &doc, &doc.oplog_vv().encode(), &update).is_ok());
- for invalid in [
- r#"{"merges":["A1:AA1001"]}"#,
- r#"{"rowHeights":{"1000":32}}"#,
- r#"{"hiddenColumns":[26]}"#,
- r#"{"freeze":{"rows":1001,"columns":1}}"#,
- r#"{"definedNames":[{"name":"Rate","formula":false}]}"#,
- r#"{"unexpected":true}"#,
- ] {
- assert!(
- prepare_update(
- &access(),
- &doc,
- &doc.oplog_vv().encode(),
- &delta(&doc, "spreadsheetSheetMetadata", "sheet1", invalid)
- )
- .is_err()
- );
- }
-}
diff --git a/services/sync-service/src/dss_internal.rs b/services/sync-service/src/dss_internal.rs
index 9bb8beb735..6e7ed62334 100644
--- a/services/sync-service/src/dss_internal.rs
+++ b/services/sync-service/src/dss_internal.rs
@@ -3,6 +3,7 @@ use tracing::error;
use worker::{Env, Fetch, Method, Request, RequestInit};
use crate::constants::header_names::MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY;
+use crate::domain::document::DocumentAttribution;
/// Why a document interaction was reported to DSS.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
@@ -22,8 +23,7 @@ struct InteractionRequest {
}
pub trait DssInternal {
- /// Uploads the raw snapshot bytes for storage, and publishes a
- /// `document.edited` event.
+ /// Uploads the raw snapshot bytes for storage.
async fn publish_shallow_snapshot(
&self,
document_id: &str,
@@ -36,6 +36,13 @@ pub trait DssInternal {
document_id: &str,
reason: InteractionReason,
) -> worker::Result<()>;
+
+ /// Let the document backend resolve content type and notify its consumers.
+ async fn publish_sync_content_updated(
+ &self,
+ document_id: &str,
+ attribution: Option<DocumentAttribution>,
+ ) -> worker::Result<()>;
}
pub struct DssInternalClient<'a> {
@@ -64,6 +71,42 @@ impl<'a> DssInternalClient<'a> {
}
impl DssInternal for DssInternalClient<'_> {
+ async fn publish_sync_content_updated(
+ &self,
+ document_id: &str,
+ attribution: Option<DocumentAttribution>,
+ ) -> worker::Result<()> {
+ let url = format!(
+ "{}/internal/documents/{document_id}/sync-content-updated",
+ self.dss_url()?,
+ );
+ let body = serde_json::json!({
+ "actor": attribution.as_ref().map(|value| &value.actor),
+ "on_behalf_of": attribution.as_ref().and_then(|value| value.on_behalf_of.as_ref()),
+ });
+ let mut request = Request::new_with_init(
+ &url,
+ RequestInit::new()
+ .with_method(Method::Post)
+ .with_body(Some(body.to_string().into())),
+ )?;
+ request.headers_mut()?.set(
+ MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY,
+ &self.internal_auth_key()?,
+ )?;
+ request
+ .headers_mut()?
+ .set("Content-Type", "application/json")?;
+ let response = Fetch::Request(request).send().await?;
+ if response.status_code() != 200 {
+ return Err(worker::Error::from(format!(
+ "DSS sync-content notification returned {}",
+ response.status_code()
+ )));
+ }
+ Ok(())
+ }
+
async fn publish_shallow_snapshot(
&self,
document_id: &str,
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index 1be9c00b0e..d5eba50472 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -21,6 +21,7 @@ use crate::{
auth::{AccessLevel, TokenFrom, decode_jwt},
constants::USER_PEER_D1_BINDING,
d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},
+ domain::document::DocumentAttribution,
dss_internal::{DssInternal, DssInternalClient, InteractionReason},
error::ResultExt,
generated::schema::InitializeFromSnapshotRequest,
@@ -46,8 +47,8 @@ pub mod status_codes {
const DOCUMENT_ID_KEY: &str = "DOCUMENT_ID";
-mod spreadsheet_api;
-mod spreadsheet_effects;
+mod document_api;
+mod document_effects;
mod path {
pub const CONNECT: &str = "connect";
@@ -55,8 +56,8 @@ mod path {
pub const INITIALIZE: &str = "initialize";
pub const RAW: &str = "raw";
pub const SNAPSHOT: &str = "snapshot";
- pub const SPREADSHEET_SNAPSHOT: &str = "spreadsheet-snapshot";
- pub const SPREADSHEET_UPDATE: &str = "spreadsheet-update";
+ pub const STATE: &str = "state";
+ pub const UPDATE: &str = "update";
pub const ACTIVE_PEERS_MARKER: &str = "active_peers";
pub const PEER: &str = "peer";
pub const METADATA: &str = "metadata";
@@ -112,14 +113,6 @@ pub struct WebSocketMetadata {
pub type WsMetaMap = BTreeMap<String, WebSocketMetadata>;
-/// Who a published snapshot's edits are attributed to: the non-human peer that
-/// wrote them and, when its token carried one, the user it acted for.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct EditAttribution {
- pub actor: String,
- pub on_behalf_of: Option<String>,
-}
-
/// Attribution from currently-connected websocket metadata only.
///
/// Closed sockets must not contribute an `actor`: the isolate stays warm while
@@ -127,9 +120,9 @@ pub struct EditAttribution {
/// attributing later human edits.
fn edit_attribution_from_meta<'a>(
metas: impl IntoIterator<Item = &'a WebSocketMetadata>,
-) -> Option<EditAttribution> {
+) -> Option<DocumentAttribution> {
metas.into_iter().find_map(|meta| {
- meta.actor.clone().map(|actor| EditAttribution {
+ meta.actor.clone().map(|actor| DocumentAttribution {
actor,
on_behalf_of: meta.user_id.clone(),
})
@@ -235,7 +228,7 @@ mod u64_serde_strings {
#[cfg(test)]
mod actor_attribution_test {
use super::{
- AccessLevel, CloseFlush, EditAttribution, WebSocketMetadata, edit_attribution_from_meta,
+ AccessLevel, CloseFlush, DocumentAttribution, WebSocketMetadata, edit_attribution_from_meta,
};
fn meta(actor: Option<&str>, user_id: Option<&str>) -> WebSocketMetadata {
@@ -264,7 +257,7 @@ mod actor_attribution_test {
);
assert_eq!(
edit_attribution_from_meta([&stale, &human]),
- Some(EditAttribution {
+ Some(DocumentAttribution {
actor: "bot|stale".to_string(),
on_behalf_of: Some("macro|[email protected]".to_string()),
})
@@ -409,9 +402,8 @@ pub fn get_ws_id(state: &State, ws: &WebSocket) -> Result<String> {
async fn report_new_doc_state(
document_id: &str,
snapshot: &[u8],
- has_markdown_content: bool,
env: &Env,
- attribution: Option<EditAttribution>,
+ attribution: Option<DocumentAttribution>,
) {
if let Err(err) = DssInternalClient::new(env)
.publish_shallow_snapshot(document_id, snapshot)
@@ -420,10 +412,11 @@ async fn report_new_doc_state(
warn!(error=?err, "failed to push snapshot to DSS");
}
#[cfg(feature = "search-service")]
- if has_markdown_content
- && let Err(err) = crate::sps::update(document_id, env, attribution).await
+ if let Err(err) = DssInternalClient::new(env)
+ .publish_sync_content_updated(document_id, attribution)
+ .await
{
- warn!(error=?err, "failed to update search index");
+ warn!(error=?err, "failed to publish document content change");
}
}
@@ -457,7 +450,7 @@ impl DocumentSyncSession {
self.state.get_websockets()
}
- fn edit_attribution(&self) -> Option<EditAttribution> {
+ fn edit_attribution(&self) -> Option<DocumentAttribution> {
let connected: BTreeSet<String> = self
.state
.get_websockets()
@@ -537,8 +530,10 @@ impl DocumentSyncSession {
(path::CONNECT, Some(document_id)) => {
return self.connect_handler(req, document_id).await;
}
- (path::SPREADSHEET_SNAPSHOT | path::SPREADSHEET_UPDATE, Some(document_id)) => {
- return self.spreadsheet_handler(req, document_id).await;
+ (operation @ (path::STATE | path::UPDATE), Some(document_id)) => {
+ return self
+ .document_handler(req, document_id, operation == path::UPDATE)
+ .await;
}
// EXIST, PEER, and WAKEUP don't require auth
@@ -656,16 +651,8 @@ impl DocumentSyncSession {
let document_id_owned = document_id.to_string();
let env = self.env.clone();
let attribution = self.edit_attribution();
- let has_markdown_content = state.has_markdown_content();
self.state.wait_until(async move {
- report_new_doc_state(
- &document_id_owned,
- &snapshot,
- has_markdown_content,
- &env,
- attribution,
- )
- .await;
+ report_new_doc_state(&document_id_owned, &snapshot, &env, attribution).await;
});
}
@@ -1058,16 +1045,10 @@ pub static ROUTER: LazyLock<Router<&str>> = LazyLock::new(|| {
.insert("/document/{document_id}/snapshot", path::SNAPSHOT)
.unwrap();
router
- .insert(
- "/document/{document_id}/spreadsheet-snapshot",
- path::SPREADSHEET_SNAPSHOT,
- )
+ .insert("/document/{document_id}/state", path::STATE)
.unwrap();
router
- .insert(
- "/document/{document_id}/spreadsheet-update",
- path::SPREADSHEET_UPDATE,
- )
+ .insert("/document/{document_id}/update", path::UPDATE)
.unwrap();
router
.insert("/document/{document_id}/peer/{peer_id}", path::PEER)
@@ -1286,7 +1267,7 @@ impl DurableObject for DocumentSyncSession {
if let Some(document_id) = document_id
&& let Ok(snapshot) = doc_state.export_shallow_snapshot()
{
- report_new_doc_state(&document_id, &snapshot, doc_state.has_markdown_content(), &env, attribution).await;
+ report_new_doc_state(&document_id, &snapshot, &env, attribution).await;
report_interaction(&document_id, &env, InteractionReason::Edited).await;
}
});
@@ -1354,16 +1335,8 @@ impl DurableObject for DocumentSyncSession {
}
let attribution = self.edit_attribution();
let env = self.env.clone();
- let has_markdown_content = state.has_markdown_content();
self.state.wait_until(async move {
- report_new_doc_state(
- &document_id,
- &snapshot,
- has_markdown_content,
- &env,
- attribution,
- )
- .await;
+ report_new_doc_state(&document_id, &snapshot, &env, attribution).await;
report_interaction(&document_id, &env, flush.interaction_reason()).await;
});
}
diff --git a/services/sync-service/src/durable_object/spreadsheet_api.rs b/services/sync-service/src/durable_object/document_api.rs
similarity index 70%
rename from services/sync-service/src/durable_object/spreadsheet_api.rs
rename to services/sync-service/src/durable_object/document_api.rs
index 15e7209110..b0160d2194 100644
--- a/services/sync-service/src/durable_object/spreadsheet_api.rs
+++ b/services/sync-service/src/durable_object/document_api.rs
@@ -1,13 +1,13 @@
-//! HTTP adapter for the native spreadsheet snapshot/update service.
+//! HTTP adapter for the document snapshot/update service.
use base64::{Engine, engine::general_purpose::STANDARD};
use futures::StreamExt;
use serde::{Deserialize, Serialize};
-use super::spreadsheet_effects::WorkerSpreadsheetEffects;
+use super::document_effects::WorkerDocumentEffects;
use super::*;
use crate::{
- spreadsheet::{self, MAX_BINARY_BYTES, MAX_REVISION_BYTES, SpreadsheetError},
- storage::SpreadsheetUpdateStorage,
+ domain::document::{self, DocumentError, MAX_BINARY_BYTES, MAX_REVISION_BYTES},
+ storage::DocumentUpdateStorage,
};
const MAX_BODY_BYTES: usize = (MAX_BINARY_BYTES + MAX_REVISION_BYTES).div_ceil(3) * 4 + 1024;
@@ -31,15 +31,15 @@ struct UpdateResponse {
applied: bool,
}
-fn error_response(error: SpreadsheetError) -> Result<Response> {
+fn error_response(error: DocumentError) -> Result<Response> {
let status = match error {
- SpreadsheetError::Unauthorized => 401,
- SpreadsheetError::Forbidden => 403,
- SpreadsheetError::Conflict => 409,
- SpreadsheetError::TooLarge => 413,
- SpreadsheetError::Invalid(_) => 400,
- SpreadsheetError::Persistence => 503,
- SpreadsheetError::Notification => 500,
+ DocumentError::Unauthorized => 401,
+ DocumentError::Forbidden => 403,
+ DocumentError::Conflict => 409,
+ DocumentError::TooLarge => 413,
+ DocumentError::Invalid(_) => 400,
+ DocumentError::Persistence => 503,
+ DocumentError::Notification => 500,
};
Ok(
Response::from_json(&serde_json::json!({ "error": error.to_string() }))?
@@ -48,17 +48,17 @@ fn error_response(error: SpreadsheetError) -> Result<Response> {
}
impl DocumentSyncSession {
- pub(super) async fn spreadsheet_handler(
+ pub(super) async fn document_handler(
&self,
mut req: Request,
document_id: &str,
+ is_update: bool,
) -> Result<Response> {
// A shared internal key never bypasses the user/document permission JWT.
- let (access, claims) = match crate::auth::spreadsheet_access(&req, &self.env, document_id) {
+ let (access, claims) = match crate::auth::document_access(&req, &self.env, document_id) {
Ok(access) => access,
Err(error) => return error_response(error),
};
- let is_update = req.path().ends_with("/spreadsheet-update");
if req.method() != if is_update { Method::Post } else { Method::Get } {
return Ok(response(405));
}
@@ -70,7 +70,7 @@ impl DocumentSyncSession {
}
if !is_update {
let state = self.document_state().await?;
- return match spreadsheet::snapshot(&access, &state.loro_doc) {
+ return match document::snapshot(&access, &state.loro_doc) {
Ok((snapshot, revision)) => Response::from_json(&SnapshotResponse {
snapshot: STANDARD.encode(snapshot),
revision: STANDARD.encode(revision),
@@ -84,33 +84,31 @@ impl DocumentSyncSession {
.and_then(|length| length.parse::<usize>().ok())
.is_some_and(|length| length > MAX_BODY_BYTES)
{
- return error_response(SpreadsheetError::TooLarge);
+ return error_response(DocumentError::TooLarge);
}
let mut bytes = Vec::new();
let mut stream = req.stream()?;
while let Some(chunk) = stream.next().await {
let chunk = chunk?;
if bytes.len().saturating_add(chunk.len()) > MAX_BODY_BYTES {
- return error_response(SpreadsheetError::TooLarge);
+ return error_response(DocumentError::TooLarge);
}
bytes.extend_from_slice(&chunk);
}
let body: UpdateRequest = match serde_json::from_slice(&bytes) {
Ok(body) => body,
- Err(_) => return error_response(SpreadsheetError::Invalid("Invalid update request.")),
+ Err(_) => return error_response(DocumentError::Invalid("Invalid update request.")),
};
if body.update.len() > MAX_BINARY_BYTES.div_ceil(3) * 4
|| body.expected_revision.len() > MAX_REVISION_BYTES.div_ceil(3) * 4
{
- return error_response(SpreadsheetError::TooLarge);
+ return error_response(DocumentError::TooLarge);
}
let (Ok(update), Ok(revision)) = (
STANDARD.decode(body.update),
STANDARD.decode(body.expected_revision),
) else {
- return error_response(SpreadsheetError::Invalid(
- "Invalid base64 update or revision.",
- ));
+ return error_response(DocumentError::Invalid("Invalid base64 update or revision."));
};
let state = self.document_state().await?;
let storage = self.session_storage().await?;
@@ -118,7 +116,7 @@ impl DocumentSyncSession {
.actor
.as_ref()
.map(|actor| {
- spreadsheet::SpreadsheetAttribution::from_signed_claims(
+ document::DocumentAttribution::from_signed_claims(
actor.clone(),
claims.user_id.clone(),
)
@@ -128,12 +126,12 @@ impl DocumentSyncSession {
Ok(attribution) => attribution,
Err(error) => return error_response(error),
};
- let port = SpreadsheetUpdateStorage {
+ let port = DocumentUpdateStorage {
document_state: &state,
storage: &storage,
attribution: attribution.as_ref(),
};
- let effects = WorkerSpreadsheetEffects {
+ let effects = WorkerDocumentEffects {
session: self,
document_state: &state,
document_id,
@@ -141,8 +139,7 @@ impl DocumentSyncSession {
};
// The service synchronously compares + validates + imports before its
// first storage await, just like a websocket update in this isolate.
- let prepared = match spreadsheet::update(&access, &port, &effects, &revision, &update).await
- {
+ let prepared = match document::update(&access, &port, &effects, &revision, &update).await {
Ok(prepared) => prepared,
Err(error) => return error_response(error),
};
diff --git a/services/sync-service/src/durable_object/spreadsheet_effects.rs b/services/sync-service/src/durable_object/document_effects.rs
similarity index 51%
rename from services/sync-service/src/durable_object/spreadsheet_effects.rs
rename to services/sync-service/src/durable_object/document_effects.rs
index 6061f81f45..59685b2b41 100644
--- a/services/sync-service/src/durable_object/spreadsheet_effects.rs
+++ b/services/sync-service/src/durable_object/document_effects.rs
@@ -1,31 +1,29 @@
-//! Worker notification adapter for the spreadsheet update use case.
+//! Worker notification adapter for the document update use case.
use bebop::{Record, SubRecord};
use tracing::{error, warn};
-use super::{
- DocumentSyncSession, EditAttribution, bump_alarm, report_interaction, report_new_doc_state,
-};
+use super::{DocumentSyncSession, bump_alarm, report_interaction, report_new_doc_state};
use crate::{
+ domain::document::{DocumentAttribution, DocumentError, DocumentUpdateEffects},
dss_internal::InteractionReason,
- spreadsheet::{SpreadsheetAttribution, SpreadsheetError, SpreadsheetUpdateEffects},
state::DocumentState,
};
-pub(super) struct WorkerSpreadsheetEffects<'a> {
+pub(super) struct WorkerDocumentEffects<'a> {
pub session: &'a DocumentSyncSession,
pub document_state: &'a DocumentState,
pub document_id: &'a str,
- pub attribution: Option<&'a SpreadsheetAttribution>,
+ pub attribution: Option<&'a DocumentAttribution>,
}
-fn notification_error(error: impl std::fmt::Debug) -> SpreadsheetError {
- error!(error = ?error, "failed to notify spreadsheet update");
- SpreadsheetError::Notification
+fn notification_error(error: impl std::fmt::Debug) -> DocumentError {
+ error!(error = ?error, "failed to notify document update");
+ DocumentError::Notification
}
-impl SpreadsheetUpdateEffects for WorkerSpreadsheetEffects<'_> {
- fn broadcast(&self, update: &[u8]) -> Result<(), SpreadsheetError> {
+impl DocumentUpdateEffects for WorkerDocumentEffects<'_> {
+ fn broadcast(&self, update: &[u8]) -> Result<(), DocumentError> {
let message = crate::generated::schema::FromRemote::RemoteUpdate {
update: bebop::SliceWrapper::Raw(update),
};
@@ -35,17 +33,14 @@ impl SpreadsheetUpdateEffects for WorkerSpreadsheetEffects<'_> {
.map_err(notification_error)?;
for socket in self.session.get_websockets() {
if let Err(error) = socket.send_with_bytes(&message_bytes) {
- warn!(error = ?error, "failed to broadcast spreadsheet update; continuing");
+ warn!(error = ?error, "failed to broadcast document update; continuing");
}
}
Ok(())
}
- fn publish_changed_document(&self) -> Result<(), SpreadsheetError> {
- let attribution = self.attribution.as_ref().map(|claims| EditAttribution {
- actor: claims.actor.clone(),
- on_behalf_of: claims.on_behalf_of.clone(),
- });
+ fn publish_changed_document(&self) -> Result<(), DocumentError> {
+ let attribution = self.attribution.cloned();
let snapshot = self
.document_state
.export_shallow_snapshot()
@@ -53,13 +48,13 @@ impl SpreadsheetUpdateEffects for WorkerSpreadsheetEffects<'_> {
let env = self.session.env.clone();
let document_id = self.document_id.to_owned();
self.session.state.wait_until(async move {
- report_new_doc_state(&document_id, &snapshot, false, &env, attribution).await;
+ report_new_doc_state(&document_id, &snapshot, &env, attribution).await;
report_interaction(&document_id, &env, InteractionReason::Edited).await;
});
Ok(())
}
- async fn keep_alive(&self) -> Result<(), SpreadsheetError> {
+ async fn keep_alive(&self) -> Result<(), DocumentError> {
bump_alarm(&self.session.state)
.await
.map_err(notification_error)
diff --git a/services/sync-service/src/lib.rs b/services/sync-service/src/lib.rs
index 4683030584..33fe9e9a52 100644
--- a/services/sync-service/src/lib.rs
+++ b/services/sync-service/src/lib.rs
@@ -3,6 +3,7 @@ mod auth;
mod cf_worker;
mod constants;
mod d1;
+mod domain;
mod dss_internal;
mod durable_object;
mod error;
@@ -11,9 +12,6 @@ pub mod keepalive;
mod metrics;
mod mutex;
mod secrets;
-mod spreadsheet;
-#[cfg(feature = "search-service")]
-mod sps;
mod state;
mod storage;
mod tags;
diff --git a/services/sync-service/src/spreadsheet.rs b/services/sync-service/src/spreadsheet.rs
deleted file mode 100644
index 35c94c5b19..0000000000
--- a/services/sync-service/src/spreadsheet.rs
+++ /dev/null
@@ -1,573 +0,0 @@
-//! Native spreadsheet access policy and bounded CRDT update validation.
-//!
-//! This module has no HTTP or storage dependencies. The durable-object adapter
-//! supplies verified JWT claims and persists accepted updates through its oplog.
-use std::borrow::Cow;
-
-use loro::{ContainerID, ContainerType, ExportMode, LoroDoc, LoroValue, VersionVector};
-use serde_json::Value;
-
-pub const MAX_BINARY_BYTES: usize = 4 * 1024 * 1024;
-pub const MAX_REVISION_BYTES: usize = 64 * 1024;
-const MAX_UPDATE_OPERATIONS: u64 = 100_000;
-
-#[derive(Debug, thiserror::Error)]
-pub enum SpreadsheetError {
- #[error("The document token does not grant access to this document.")]
- Unauthorized,
- #[error("Editing this spreadsheet requires edit access.")]
- Forbidden,
- #[error("The spreadsheet changed. Read a fresh snapshot before editing.")]
- Conflict,
- #[error("The spreadsheet request exceeds the size limit.")]
- TooLarge,
- #[error("{0}")]
- Invalid(&'static str),
- #[error("The spreadsheet update could not be persisted.")]
- Persistence,
- #[error(
- "The spreadsheet was saved, but its update notifications could not be completed. Read the workbook before retrying."
- )]
- Notification,
-}
-
-/// Capability minted only after the adapter verifies a document permission JWT.
-pub struct SpreadsheetAccess {
- writable: bool,
-}
-
-impl SpreadsheetAccess {
- pub fn authorize(
- requested_document: &str,
- token_document: &str,
- writable: bool,
- ) -> Result<Self, SpreadsheetError> {
- if requested_document != token_document {
- return Err(SpreadsheetError::Unauthorized);
- }
- Ok(Self { writable })
- }
-
- pub fn require_edit(&self) -> Result<(), SpreadsheetError> {
- if !self.writable {
- return Err(SpreadsheetError::Forbidden);
- }
- Ok(())
- }
-}
-
-pub struct PreparedUpdate {
- /// Only new, validated operations, ready for the existing persistence path.
- pub update: Vec<u8>,
- pub revision: Vec<u8>,
- pub applied: bool,
-}
-
-/// Signed-token attribution stored alongside the existing operation log.
-#[derive(serde::Serialize)]
-pub struct SpreadsheetAttribution {
- pub actor: String,
- pub on_behalf_of: Option<String>,
-}
-
-impl SpreadsheetAttribution {
- pub fn from_signed_claims(
- actor: String,
- on_behalf_of: Option<String>,
- ) -> Result<Self, SpreadsheetError> {
- if actor.len() > 256 || on_behalf_of.as_ref().is_some_and(|user| user.len() > 1024) {
- return Err(SpreadsheetError::Invalid("Invalid signed attribution."));
- }
- Ok(Self {
- actor,
- on_behalf_of,
- })
- }
-}
-
-/// Persistence port: applying the update must happen before the first await.
-pub trait SpreadsheetUpdatePort {
- fn document(&self) -> &LoroDoc;
- async fn apply_and_persist(&self, update: &[u8]) -> Result<(), SpreadsheetError>;
-}
-
-/// Side effects after a durable write; the use case owns their order and whether
-/// a newly applied edit needs document publication. Implementations own only the
-/// notification transport and scheduling mechanisms.
-pub trait SpreadsheetUpdateEffects {
- fn broadcast(&self, update: &[u8]) -> Result<(), SpreadsheetError>;
- fn publish_changed_document(&self) -> Result<(), SpreadsheetError>;
- async fn keep_alive(&self) -> Result<(), SpreadsheetError>;
-}
-
-pub async fn update(
- access: &SpreadsheetAccess,
- port: &impl SpreadsheetUpdatePort,
- effects: &impl SpreadsheetUpdateEffects,
- expected_revision: &[u8],
- update: &[u8],
-) -> Result<PreparedUpdate, SpreadsheetError> {
- let prepared = prepare_update(access, port.document(), expected_revision, update)?;
- // Persist retries too: a prior request may have applied in memory but lost
- // its storage write or response. Duplicate Loro imports are idempotent.
- port.apply_and_persist(&prepared.update).await?;
- effects.broadcast(&prepared.update)?;
- if prepared.applied {
- effects.publish_changed_document()?;
- }
- effects.keep_alive().await?;
- Ok(prepared)
-}
-
-pub fn snapshot(
- _access: &SpreadsheetAccess,
- doc: &LoroDoc,
-) -> Result<(Vec<u8>, Vec<u8>), SpreadsheetError> {
- validate_document(doc)?;
- let snapshot = doc.export(ExportMode::Snapshot).map_err(|_| {
- SpreadsheetError::Invalid("The spreadsheet snapshot could not be exported.")
- })?;
- if snapshot.len() > MAX_BINARY_BYTES {
- return Err(SpreadsheetError::TooLarge);
- }
- let revision = doc.oplog_vv().encode();
- if revision.len() > MAX_REVISION_BYTES {
- return Err(SpreadsheetError::TooLarge);
- }
- Ok((snapshot, revision))
-}
-
-/// Synchronous preflight. The caller must apply the returned delta before its
-/// next await, so websocket writes cannot interleave with this version check.
-pub fn prepare_update(
- access: &SpreadsheetAccess,
- doc: &LoroDoc,
- expected_revision: &[u8],
- update: &[u8],
-) -> Result<PreparedUpdate, SpreadsheetError> {
- access.require_edit()?;
- if update.len() > MAX_BINARY_BYTES || expected_revision.len() > MAX_REVISION_BYTES {
- return Err(SpreadsheetError::TooLarge);
- }
- let expected = VersionVector::decode(expected_revision)
- .map_err(|_| SpreadsheetError::Invalid("Invalid spreadsheet revision."))?;
- validate_document(doc)?;
- let meta = LoroDoc::decode_import_blob_meta(update, true)
- .map_err(|_| SpreadsheetError::Invalid("Invalid Loro update."))?;
- if meta.mode.is_snapshot() {
- return Err(SpreadsheetError::Invalid(
- "Send a Loro update, not a snapshot.",
- ));
- }
- let operations: u64 = meta
- .partial_end_vv
- .iter()
- .map(|(peer, end)| {
- end.checked_sub(meta.partial_start_vv.get(peer).copied().unwrap_or(0))
- .and_then(|count| u64::try_from(count).ok())
- .unwrap_or(u64::MAX)
- })
- .try_fold(0_u64, |total, count| total.checked_add(count))
- .ok_or(SpreadsheetError::TooLarge)?;
- if operations > MAX_UPDATE_OPERATIONS || meta.change_num > 10_000 {
- return Err(SpreadsheetError::TooLarge);
- }
- let current = doc.oplog_vv();
- let preview = doc.fork();
- let imported = preview
- .import(update)
- .map_err(|_| SpreadsheetError::Invalid("Invalid Loro update."))?;
- if imported.pending.is_some() || preview.state_vv() != preview.oplog_vv() {
- return Err(SpreadsheetError::Invalid(
- "The update has missing dependencies.",
- ));
- }
- validate_document(&preview)?;
- let revision = preview.oplog_vv();
- // Recognizing an already applied delta makes a lost HTTP response safely
- // retryable, even if other users have edited after the first application.
- if revision == current {
- return Ok(PreparedUpdate {
- update: update.to_vec(),
- revision: current.encode(),
- applied: false,
- });
- }
- if current != expected {
- return Err(SpreadsheetError::Conflict);
- }
- let update = preview
- .export(ExportMode::Updates {
- from: Cow::Borrowed(&current),
- })
- .map_err(|_| SpreadsheetError::Invalid("The validated update could not be exported."))?;
- if update.len() > MAX_BINARY_BYTES {
- return Err(SpreadsheetError::TooLarge);
- }
- Ok(PreparedUpdate {
- update,
- revision: revision.encode(),
- applied: true,
- })
-}
-
-fn sheet_id(value: &str) -> bool {
- !value.is_empty()
- && value.len() <= 64
- && value
- .bytes()
- .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
-}
-
-fn sheet_name(value: &str) -> bool {
- !value.is_empty()
- && value.trim() == value
- && value.encode_utf16().count() <= 31
- && !value.starts_with('\'')
- && !value.ends_with('\'')
- && !value.chars().any(|c| c < ' ' || "[]:*?/\\".contains(c))
-}
-
-fn sheet_key(key: &str) -> Option<&str> {
- match key.split_once('!') {
- Some((sheet, field)) if sheet_id(sheet) && !field.contains('!') => Some(field),
- Some(_) => None,
- None => Some(key),
- }
-}
-
-fn address(key: &str) -> bool {
- let Some(key) = sheet_key(key) else {
- return false;
- };
- let mut chars = key.chars();
- let Some(column) = chars.next() else {
- return false;
- };
- let row = chars.as_str();
- column.is_ascii_uppercase()
- && !row.starts_with('0')
- && row
- .parse::<u16>()
- .is_ok_and(|row| (1..=1000).contains(&row))
-}
-
-fn integer(value: &Value, minimum: i64, maximum: i64) -> bool {
- value.as_f64().is_some_and(|value| {
- value.is_finite()
- && value.fract() == 0.0
- && value >= minimum as f64
- && value <= maximum as f64
- })
-}
-
-fn choice(value: &Value, choices: &[&str]) -> bool {
- value.as_str().is_some_and(|value| choices.contains(&value))
-}
-
-fn metadata_range(value: &Value) -> bool {
- let Some(range) = value.as_str() else {
- return false;
- };
- let parts: Vec<_> = range.split(':').collect();
- if parts.is_empty()
- || parts.len() > 2
- || !parts.iter().all(|part| {
- address(part)
- && !part.contains('!')
- && part[1..].bytes().all(|byte| byte.is_ascii_digit())
- })
- {
- return false;
- }
- let first = parts[0];
- let last = parts[parts.len() - 1];
- first.as_bytes()[0] <= last.as_bytes()[0]
- && first[1..].parse::<u16>().unwrap_or(0) <= last[1..].parse::<u16>().unwrap_or(0)
-}
-
-fn workbook_metadata(value: &Value) -> bool {
- let Some(encoded) = value
- .as_str()
- .filter(|text| text.encode_utf16().count() <= 100_000)
- else {
- return false;
- };
- let Ok(Value::Object(fields)) = serde_json::from_str::<Value>(encoded) else {
- return false;
- };
- fields.iter().all(|(key, value)| match key.as_str() {
- "merges" => value
- .as_array()
- .is_some_and(|items| items.len() <= 1000 && items.iter().all(metadata_range)),
- "rowHeights" => value.as_object().is_some_and(|items| {
- items.iter().all(|(key, value)| {
- key.bytes().all(|byte| byte.is_ascii_digit())
- && key.parse::<u16>().is_ok_and(|row| row < 1000)
- && value
- .as_f64()
- .is_some_and(|height| (0.0..=409.5).contains(&height))
- })
- }),
- "hiddenRows" | "hiddenColumns" => value.as_array().is_some_and(|items| {
- let limit = if key == "hiddenRows" { 1000 } else { 26 };
- items.len() <= limit && items.iter().all(|item| integer(item, 0, limit as i64 - 1))
- }),
- "hidden" => value.is_boolean(),
- "freeze" => value.as_object().is_some_and(|fields| {
- fields.len() == 2
- && fields
- .get("rows")
- .is_some_and(|value| integer(value, 0, 1000))
- && fields
- .get("columns")
- .is_some_and(|value| integer(value, 0, 26))
- }),
- "autoFilter" => metadata_range(value),
- "definedNames" => value.as_array().is_some_and(|items| {
- items.len() <= 256
- && items.iter().all(|item| {
- item.as_object().is_some_and(|fields| {
- fields
- .keys()
- .all(|key| ["name", "formula", "local"].contains(&key.as_str()))
- && fields
- .get("name")
- .and_then(Value::as_str)
- .is_some_and(|name| {
- !name.is_empty() && name.encode_utf16().count() <= 255
- })
- && fields
- .get("formula")
- .and_then(Value::as_str)
- .is_some_and(|formula| formula.encode_utf16().count() <= 10_000)
- && fields.get("local").is_none_or(Value::is_boolean)
- })
- })
- }),
- _ => false,
- })
-}
-
-fn valid_entry(root: &str, key: &str, value: &Value) -> bool {
- match root {
- "spreadsheetMeta" => key == "formatVersion" && value.as_u64() == Some(1),
- "spreadsheetSheetMetadata" => sheet_id(key) && workbook_metadata(value),
- "spreadsheetSheetNames" => sheet_id(key) && value.as_str().is_some_and(sheet_name),
- "spreadsheetSheetOrder" => sheet_id(key) && value.as_f64().is_some_and(f64::is_finite),
- "spreadsheetDeletedSheets" => sheet_id(key) && value.is_boolean(),
- "spreadsheetSheetRevivals" => key.len() <= 200 && value.as_str().is_some_and(sheet_id),
- "spreadsheetSheetRetentions" => {
- let Some((id, peer)) = key.split_once('!') else {
- return false;
- };
- let Some(encoded) = value.as_str().filter(|value| value.len() <= 512) else {
- return false;
- };
- let Ok(identity) = serde_json::from_str::<Value>(encoded) else {
- return false;
- };
- sheet_id(id)
- && !peer.is_empty()
- && peer.len() <= 64
- && identity
- .get("name")
- .and_then(Value::as_str)
- .is_some_and(sheet_name)
- && identity
- .get("order")
- .and_then(Value::as_f64)
- .is_some_and(f64::is_finite)
- && identity
- .get("revision")
- .and_then(Value::as_f64)
- .is_some_and(|value| value.is_finite() && value >= 0.0)
- }
- "spreadsheetColumnWidths" => {
- sheet_key(key).is_some_and(|key| key.parse::<u8>().is_ok_and(|column| column < 26))
- && integer(value, 64, 640)
- }
- "spreadsheetRowAdditions" => {
- sheet_key(key).is_some_and(|key| !key.is_empty() && key.len() <= 64)
- && integer(value, 1, 1000)
- }
- _ if !address(key) => false,
- "spreadsheetValues" => value
- .as_str()
- .is_some_and(|value| value.encode_utf16().count() <= 10_000),
- "spreadsheetBold"
- | "spreadsheetItalic"
- | "spreadsheetUnderline"
- | "spreadsheetStrikethrough"
- | "spreadsheetWrap"
- | "spreadsheetBorderTop"
- | "spreadsheetBorderRight"
- | "spreadsheetBorderBottom"
- | "spreadsheetBorderLeft" => value.is_boolean(),
- "spreadsheetFontNames" => value.as_str().is_some_and(|value| {
- value.encode_utf16().count() <= 128 && !value.chars().any(|ch| ch <= '\u{001f}')
- }),
- "spreadsheetBorderTopStyles"
- | "spreadsheetBorderRightStyles"
- | "spreadsheetBorderBottomStyles"
- | "spreadsheetBorderLeftStyles" => choice(
- value,
- &[
- "",
- "thin",
- "medium",
- "thick",
- "double",
- "dotted",
- "dashed",
- "dashDot",
- "dashDotDot",
- "slantDashDot",
- "hair",
- "mediumDashed",
- "mediumDashDot",
- "mediumDashDotDot",
- ],
- ),
- "spreadsheetFontFamily" => choice(value, &["sans", "serif", "mono"]),
- "spreadsheetFontSize" => integer(value, 8, 36),
- "spreadsheetBorderTopColors"
- | "spreadsheetBorderRightColors"
- | "spreadsheetBorderBottomColors"
- | "spreadsheetBorderLeftColors"
- | "spreadsheetTextColor"
- | "spreadsheetFillColor" => value.as_str().is_some_and(|value| {
- value.is_empty()
- || (value.len() == 7
- && value.starts_with('#')
- && value[1..].bytes().all(|byte| byte.is_ascii_hexdigit()))
- }),
- "spreadsheetHorizontalAlign" => choice(value, &["auto", "left", "center", "right"]),
- "spreadsheetVerticalAlign" => choice(value, &["top", "middle", "bottom"]),
- "spreadsheetDecimals" => integer(value, -1, 10),
- "spreadsheetNumberFormats" => value.as_str().is_some_and(|value| {
- value.encode_utf16().count() <= 512 && !value.chars().any(|ch| ch <= '\u{001f}')
- }),
- "spreadsheetFormats" => choice(
- value,
- &[
- "general",
- "number",
- "currency",
- "percent",
- "date",
- "time",
- "scientific",
- "text",
- ],
- ),
- _ => false,
- }
-}
-
-fn known_root(root: &str) -> bool {
- matches!(
- root,
- "spreadsheetMeta"
- | "spreadsheetSheetNames"
- | "spreadsheetSheetMetadata"
- | "spreadsheetSheetOrder"
- | "spreadsheetDeletedSheets"
- | "spreadsheetSheetRevivals"
- | "spreadsheetSheetRetentions"
- | "spreadsheetColumnWidths"
- | "spreadsheetRowAdditions"
- | "spreadsheetValues"
- | "spreadsheetBold"
- | "spreadsheetItalic"
- | "spreadsheetUnderline"
- | "spreadsheetStrikethrough"
- | "spreadsheetWrap"
- | "spreadsheetBorderTop"
- | "spreadsheetBorderRight"
- | "spreadsheetBorderBottom"
- | "spreadsheetBorderLeft"
- | "spreadsheetFontNames"
- | "spreadsheetBorderTopStyles"
- | "spreadsheetBorderTopColors"
- | "spreadsheetBorderRightStyles"
- | "spreadsheetBorderRightColors"
- | "spreadsheetBorderBottomStyles"
- | "spreadsheetBorderBottomColors"
- | "spreadsheetBorderLeftStyles"
- | "spreadsheetBorderLeftColors"
- | "spreadsheetFontFamily"
- | "spreadsheetFontSize"
- | "spreadsheetTextColor"
- | "spreadsheetFillColor"
- | "spreadsheetHorizontalAlign"
- | "spreadsheetVerticalAlign"
- | "spreadsheetDecimals"
- | "spreadsheetFormats"
- | "spreadsheetNumberFormats"
- )
-}
-
-fn validate_document(doc: &LoroDoc) -> Result<(), SpreadsheetError> {
- let invalid = || {
- SpreadsheetError::Invalid(
- "Only native spreadsheet maps with supported values may be edited.",
- )
- };
- let LoroValue::Map(roots) = doc.get_value() else {
- return Err(invalid());
- };
- if !roots.contains_key("spreadsheetMeta") {
- return Err(invalid());
- }
- let mut entries = 0_usize;
- for (name, container) in roots.iter() {
- if !known_root(name)
- || !matches!(
- container,
- LoroValue::Container(ContainerID::Root {
- container_type: ContainerType::Map,
- ..
- })
- )
- {
- return Err(invalid());
- }
- let LoroValue::Map(fields) = doc.get_map(name.as_str()).get_value() else {
- return Err(invalid());
- };
- entries += fields.len();
- if entries > 1_000_000 {
- return Err(SpreadsheetError::TooLarge);
- }
- for (key, value) in fields.iter() {
- // Deep JSON turns nested LoroText/Counter containers into apparent
- // scalars. Check their actual CRDT types before any conversion.
- if !matches!(
- value,
- LoroValue::String(_)
- | LoroValue::Bool(_)
- | LoroValue::I64(_)
- | LoroValue::Double(_)
- ) {
- return Err(invalid());
- }
- let value = serde_json::to_value(value).map_err(|_| invalid())?;
- if !valid_entry(name, key, &value) {
- return Err(invalid());
- }
- }
- }
- if doc
- .get_map("spreadsheetMeta")
- .get("formatVersion")
- .and_then(|value| value.into_value().ok())
- != Some(LoroValue::I64(1))
- {
- return Err(invalid());
- }
- Ok(())
-}
-
-#[cfg(test)]
-mod test;
diff --git a/services/sync-service/src/sps.rs b/services/sync-service/src/sps.rs
deleted file mode 100644
index 481150a845..0000000000
--- a/services/sync-service/src/sps.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-use crate::constants::header_names::MACRO_INTERNAL_AUTH_KEY_HEADER_KEY;
-use crate::durable_object::EditAttribution;
-use serde_json::json;
-use tracing::{debug, error};
-use worker::{Fetch, Method, Request, RequestInit};
-
-pub async fn update(
- document_id: &str,
- env: &worker::Env,
- attribution: Option<EditAttribution>,
-) -> worker::Result<()> {
- let internal_auth_key = env
- .secret("SPS_API_SECRET_KEY")
- .inspect_err(|e| error!(error=%e, "Could not find API SPS key binding"))?
- .to_string();
- let url = env
- .var("SPS_URL")
- .inspect_err(|e| error!(error=%e, "Could not find SPS URL binding"))?
- .to_string();
-
- let url = format!("{url}/internal/extract_sync");
- let json_body = json!({
- "documents": [{
- "document_id": document_id,
- "file_type": "md",
- "actor": attribution.as_ref().map(|a| &a.actor),
- "on_behalf_of": attribution.as_ref().and_then(|a| a.on_behalf_of.as_ref()),
- }]
- });
-
- let mut request = Request::new_with_init(
- &url,
- RequestInit::new()
- .with_method(Method::Post)
- .with_body(Some(json_body.to_string().into())),
- )?;
-
- let headers = request.headers_mut()?;
- headers.set("Content-Type", "application/json")?;
- headers.set(MACRO_INTERNAL_AUTH_KEY_HEADER_KEY, &internal_auth_key)?;
-
- let mut response = Fetch::Request(request).send().await?;
- if response.status_code() != 200 {
- error!("non-200 response from sps");
- match response.text().await {
- Ok(body) => error!("non-200 response body:\n{body}"),
- Err(e) => error!("couldn't read non-200 response body to string. Error: {e:?}"),
- }
- }
- debug!("finished sending search index update");
- Ok(())
-}
diff --git a/services/sync-service/src/state.rs b/services/sync-service/src/state.rs
index e35c79d8fb..cef4c599e8 100644
--- a/services/sync-service/src/state.rs
+++ b/services/sync-service/src/state.rs
@@ -13,9 +13,6 @@ const FROM_CLIENT_TAG: &str = "from_client";
const FROM_SERVICE_TAG: &str = "from_service";
const FRONTIERS_ID_SEPERATOR: &str = "|";
-#[cfg(test)]
-mod test;
-
#[derive(Debug)]
pub struct DocumentState {
pub loro_doc: LoroDoc,
@@ -64,10 +61,6 @@ impl DocumentState {
self.loro_doc.get_deep_value().to_json()
}
- /// Only Lexical documents can be consumed by the markdown search extractor.
- pub fn has_markdown_content(&self) -> bool {
- matches!(self.loro_doc.get_value(), LoroValue::Map(roots) if roots.contains_key("root"))
- }
pub fn should_save(&self) -> bool {
let Some(up) = *self
.last_update
diff --git a/services/sync-service/src/state/test.rs b/services/sync-service/src/state/test.rs
deleted file mode 100644
index 84b5306aac..0000000000
--- a/services/sync-service/src/state/test.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use super::*;
-
-#[test]
-fn search_extraction_only_accepts_markdown_snapshots() {
- let markdown = DocumentState::try_from_snapshot(include_bytes!(
- "../../../../static_assets/markdown-golden.1.bin"
- ))
- .unwrap();
- let spreadsheet = DocumentState::try_from_snapshot(include_bytes!(
- "../../../../static_assets/spreadsheet-golden.1.bin"
- ))
- .unwrap();
-
- assert!(markdown.has_markdown_content());
- assert!(!spreadsheet.has_markdown_content());
- assert!(!DocumentState::new().has_markdown_content());
-}
diff --git a/services/sync-service/src/storage.rs b/services/sync-service/src/storage.rs
index da662f1f5d..075708df0e 100644
--- a/services/sync-service/src/storage.rs
+++ b/services/sync-service/src/storage.rs
@@ -16,14 +16,14 @@ pub struct SessionStorage {
oplog: DurableKVStorage,
}
-/// Outbound adapter for atomic spreadsheet updates, sharing the websocket log.
-pub struct SpreadsheetUpdateStorage<'a> {
+/// Outbound adapter for atomic document updates, sharing the websocket log.
+pub struct DocumentUpdateStorage<'a> {
pub document_state: &'a DocumentState,
pub storage: &'a SessionStorage,
- pub attribution: Option<&'a crate::spreadsheet::SpreadsheetAttribution>,
+ pub attribution: Option<&'a crate::domain::document::DocumentAttribution>,
}
-impl crate::spreadsheet::SpreadsheetUpdatePort for SpreadsheetUpdateStorage<'_> {
+impl crate::domain::document::DocumentUpdatePort for DocumentUpdateStorage<'_> {
fn document(&self) -> &loro::LoroDoc {
&self.document_state.loro_doc
}
@@ -31,15 +31,15 @@ impl crate::spreadsheet::SpreadsheetUpdatePort for SpreadsheetUpdateStorage<'_>
async fn apply_and_persist(
&self,
update: &[u8],
- ) -> std::result::Result<(), crate::spreadsheet::SpreadsheetError> {
+ ) -> std::result::Result<(), crate::domain::document::DocumentError> {
self.storage
.oplog
.apply_op_with_attribution(self.document_state, update, self.attribution)
.await
.map(|_| ())
.map_err(|error| {
- tracing::error!(error = ?error, "failed to persist spreadsheet update");
- crate::spreadsheet::SpreadsheetError::Persistence
+ tracing::error!(error = ?error, "failed to persist document update");
+ crate::domain::document::DocumentError::Persistence
})
}
}
diff --git a/services/sync-service/src/storage/backends/durable_kv.rs b/services/sync-service/src/storage/backends/durable_kv.rs
index 48cfd65ffe..025c532c60 100644
--- a/services/sync-service/src/storage/backends/durable_kv.rs
+++ b/services/sync-service/src/storage/backends/durable_kv.rs
@@ -108,7 +108,7 @@ impl DurableKVStorage {
&self,
document_state: &DocumentState,
op_update: &[u8],
- attribution: Option<&crate::spreadsheet::SpreadsheetAttribution>,
+ attribution: Option<&crate::domain::document::DocumentAttribution>,
) -> Result<Vec<String>> {
let op_id = self.ids.id();
let op_key = pending_op_key(&op_id);
@@ -121,7 +121,7 @@ impl DurableKVStorage {
self.inner.put(&all_op_key(&op_id), op_update).await?;
if let Some(attribution) = attribution {
let metadata = serde_json::to_vec(attribution)
- .context("failed to serialize signed spreadsheet attribution")?;
+ .context("failed to serialize signed document attribution")?;
self.inner.put(&format!("actor/{op_id}"), metadata).await?;
}
Ok(touched_nodes)
diff --git a/services/sync-service/tests/spreadsheet_atomic.test.ts b/services/sync-service/tests/document_atomic.test.ts
similarity index 77%
rename from services/sync-service/tests/spreadsheet_atomic.test.ts
rename to services/sync-service/tests/document_atomic.test.ts
index 44e3744b2b..815a9d9b1c 100644
--- a/services/sync-service/tests/spreadsheet_atomic.test.ts
+++ b/services/sync-service/tests/document_atomic.test.ts
@@ -1,4 +1,4 @@
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { LoroDoc, VersionVector } from "loro-crdt";
@@ -14,7 +14,7 @@ import { createTestUser, getTokenForDocument, setupMiniflare } from "./utils";
type User = Awaited<ReturnType<typeof createTestUser>>;
let mf: Miniflare;
const users: User[] = [];
-const persistPath = mkdtempSync(join(tmpdir(), "spreadsheet-atomic-"));
+const persistPath = mkdtempSync(join(tmpdir(), "document-atomic-"));
beforeAll(async () => {
mf = await setupMiniflare({ persistPath });
}, 60_000);
@@ -24,12 +24,19 @@ afterAll(async () => {
await mf?.dispose();
rmSync(persistPath, { recursive: true, force: true });
});
-const auth = (id: string, permission: "owner" | "edit" | "view" = "edit") => ({
+const auth = (
+ id: string,
+ permission: "owner" | "edit" | "comment" | "view" = "edit",
+) => ({
Authorization: `Bearer ${getTokenForDocument(id, "actor", permission)}`,
});
const url = (id: string, action: string) =>
- `http://localhost/document/${id}/spreadsheet-${action}`;
+ `http://localhost/document/${id}/${action === "snapshot" ? "state" : action}`;
async function seed() {
+ const doc = new LoroDoc();
+ doc.getMap("metadata").set("title", "Document");
+ const bytes = doc.export({ mode: "snapshot" });
+ doc.free();
const id = crypto.randomUUID();
const response = await mf.dispatchFetch(
`http://localhost/document/${id}/initialize`,
@@ -37,9 +44,7 @@ async function seed() {
method: "POST",
headers: auth(id),
body: InitializeFromSnapshotRequest.encode({
- snapshot: new Uint8Array(
- readFileSync("../../static_assets/spreadsheet-golden.1.bin"),
- ),
+ snapshot: bytes,
}),
},
);
@@ -68,7 +73,7 @@ function edit(
value = "42",
) {
const from = source.doc.version();
- source.doc.getMap("spreadsheetValues").set(address, value);
+ source.doc.getMap("properties").set(address, value);
source.doc.commit();
return {
expectedRevision: source.revision,
@@ -90,12 +95,13 @@ async function post(
});
}
-describe("atomic native spreadsheet API", () => {
+describe("atomic document API", () => {
it("returns a coherent authorized snapshot and rejects viewers, wrong-document grants, missing and invalid tokens", async () => {
const id = await seed();
const source = await snapshot(id);
const update = edit(source);
expect((await post(id, update, auth(id, "view"))).status).toBe(403);
+ expect((await post(id, update, auth(id, "comment"))).status).toBe(403);
expect((await post(id, update, auth("wrong-document"))).status).toBe(401);
expect(
(await post(id, update, { "x-internal-auth-key": "local" })).status,
@@ -116,7 +122,7 @@ describe("atomic native spreadsheet API", () => {
).status,
).toBe(401);
expect(
- (await snapshot(id)).doc.getMap("spreadsheetValues").get("A1"),
+ (await snapshot(id)).doc.getMap("properties").get("A1"),
).toBeUndefined();
});
@@ -142,7 +148,7 @@ describe("atomic native spreadsheet API", () => {
break;
}
}
- expect(observer.doc.getMap("spreadsheetValues").get("A1")).toBe("42");
+ expect(observer.doc.getMap("properties").get("A1")).toBe("42");
const retry = await post(id, request);
expect(retry.status).toBe(200);
expect(await retry.json()).toEqual({
@@ -172,7 +178,7 @@ describe("atomic native spreadsheet API", () => {
expect(responses.map((response) => response.status).sort()).toEqual([
200, 409,
]);
- const state = (await snapshot(id)).doc.getMap("spreadsheetValues").toJSON();
+ const state = (await snapshot(id)).doc.getMap("properties").toJSON();
expect(state).toEqual(
responses[0].status === 200 ? { A1: "left" } : { B1: "right" },
);
@@ -186,7 +192,7 @@ describe("atomic native spreadsheet API", () => {
permissionLevel: "edit",
});
users.push(writer);
- writer.doc.getMap("spreadsheetValues").set("A1", "human");
+ writer.doc.getMap("properties").set("A1", "human");
writer.doc.commit();
const operationId = crypto.randomUUID();
writer.send(
@@ -201,34 +207,13 @@ describe("atomic native spreadsheet API", () => {
break;
}
expect((await post(id, request)).status).toBe(409);
- expect(
- (await snapshot(id)).doc.getMap("spreadsheetValues").toJSON(),
- ).toEqual({ A1: "human" });
+ expect((await snapshot(id)).doc.getMap("properties").toJSON()).toEqual({
+ A1: "human",
+ });
});
- it("rejects unsupported roots, schemas, snapshot bodies, malformed input, and oversized payloads atomically", async () => {
+ it("rejects snapshot bodies, malformed input, and oversized payloads atomically", async () => {
const id = await seed();
- for (const [root, key, value] of [
- ["root", "content", "no"],
- ["spreadsheetValues", "AA1", "no"],
- ["spreadsheetValues", "A1", "x".repeat(10_001)],
- ["spreadsheetFormats", "A1", "unsupported"],
- ]) {
- const source = await snapshot(id);
- const from = source.doc.version();
- source.doc.getMap(root).set(key, value);
- source.doc.commit();
- expect(
- (
- await post(id, {
- expectedRevision: source.revision,
- update: Buffer.from(
- source.doc.export({ mode: "update", from }),
- ).toString("base64"),
- })
- ).status,
- ).toBe(400);
- }
const original = await snapshot(id);
expect(
(
@@ -250,27 +235,22 @@ describe("atomic native spreadsheet API", () => {
).status,
).toBe(413);
expect((await snapshot(id)).revision).toBe(original.revision);
- const plainId = crypto.randomUUID();
- const initialized = await mf.dispatchFetch(
- `http://localhost/document/${plainId}/initialize`,
- {
- method: "POST",
- headers: auth(plainId),
- body: InitializeFromSnapshotRequest.encode({
- snapshot: new Uint8Array(
- readFileSync("../../static_assets/markdown-golden.1.bin"),
- ),
- }),
- },
- );
- expect(initialized.status).toBe(200);
- expect(
- (
- await mf.dispatchFetch(url(plainId, "snapshot"), {
- headers: auth(plainId),
- })
- ).status,
- ).toBe(400);
+ });
+
+ it("accepts markdown and arbitrary nested CRDT content without interpreting it", async () => {
+ const id = await seed();
+ const source = await snapshot(id);
+ const from = source.doc.version();
+ source.doc.getText("content").insert(0, "markdown text");
+ source.doc.getMap("arbitrary").set("nested", { values: [1, 2, 3] });
+ const response = await post(id, {
+ expectedRevision: source.revision,
+ update: Buffer.from(source.doc.export({ mode: "update", from })).toString(
+ "base64",
+ ),
+ });
+ expect(response.status).toBe(200);
+ expect((await snapshot(id)).doc.toJSON()).toEqual(source.doc.toJSON());
});
it("stores signed actor metadata with the oplog and ignores body attribution", async () => {
@@ -282,7 +262,7 @@ describe("atomic native spreadsheet API", () => {
document_id: id,
user_id: "macro|real-user",
access_level: "edit",
- actor: "bot|spreadsheet",
+ actor: "bot|document-agent",
exp: Math.floor(Date.now() / 1000) + 60,
},
"local",
@@ -298,7 +278,7 @@ describe("atomic native spreadsheet API", () => {
const records = (await response.json()) as [string, number[]][];
expect(records).toHaveLength(1);
expect(JSON.parse(Buffer.from(records[0][1]).toString())).toEqual({
- actor: "bot|spreadsheet",
+ actor: "bot|document-agent",
on_behalf_of: "macro|real-user",
});
});
@@ -309,7 +289,7 @@ describe("atomic native spreadsheet API", () => {
const from = source.doc.version();
for (let row = 1; row <= 100; row++)
source.doc
- .getMap("spreadsheetValues")
+ .getMap("properties")
.set(`A${row}`, `${row}:` + "a".repeat(3000));
source.doc.commit();
const update = source.doc.export({ mode: "update", from });
@@ -319,9 +299,9 @@ describe("atomic native spreadsheet API", () => {
update: Buffer.from(update).toString("base64"),
});
expect(response.status).toBe(200);
- expect(
- (await snapshot(id)).doc.getMap("spreadsheetValues").get("A100"),
- ).toBe("100:" + "a".repeat(3000));
+ expect((await snapshot(id)).doc.getMap("properties").get("A100")).toBe(
+ "100:" + "a".repeat(3000),
+ );
});
it("persists an acknowledged CAS update across worker restart", async () => {
@@ -335,9 +315,7 @@ describe("atomic native spreadsheet API", () => {
await mf.dispose();
mf = await setupMiniflare({ persistPath, migrate: false });
const restored = await snapshot(id);
- expect(restored.doc.getMap("spreadsheetValues").get("Z999")).toBe(
- "=SUM(A1:B2)",
- );
+ expect(restored.doc.getMap("properties").get("Z999")).toBe("=SUM(A1:B2)");
const retry = await post(id, request);
expect(await retry.json()).toEqual({ ...saved, applied: false });
}, 30_000);