Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(initiatives): use collab surface for initiative (#6769)

Will Hutchinson8fce26b1ec83
diff --git a/Cargo.lock b/Cargo.lock
index 630f325f62..e9397d6646 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -16861,6 +16861,7 @@ dependencies = [
"matchit 0.8.4",
"serde",
"serde_json",
+ "sha2 0.10.9",
"subtle",
"sync_service_bebop_schema",
"thiserror 2.0.18",
@@ -16890,9 +16891,11 @@ dependencies = [
"model",
"reqwest 0.13.4",
"serde",
+ "serde_json",
"sync_service_hex",
"tokio",
"tracing",
+ "uuid",
"workspace-hack",
]
diff --git a/crates/macro_sync_service_jwt/src/lib.rs b/crates/macro_sync_service_jwt/src/lib.rs
index 193443f4d5..ff6ab4f6d7 100644
--- a/crates/macro_sync_service_jwt/src/lib.rs
+++ b/crates/macro_sync_service_jwt/src/lib.rs
@@ -1,5 +1,9 @@
-//! Minting and validation for the document-permission JWTs that authorize the
-//! sync service to grant document access.
+//! Minting and validation for sync-service permission JWTs.
+//!
+//! Legacy document consumers use [`encode`] and [`decode`]. Isolated surface
+//! sessions use the distinct identities, claims, and token helpers in [`session`].
+
+pub mod session;
use jsonwebtoken::{
Algorithm, DecodingKey, EncodingKey, Header, Validation, decode as jwt_decode,
diff --git a/crates/macro_sync_service_jwt/src/session.rs b/crates/macro_sync_service_jwt/src/session.rs
new file mode 100644
index 0000000000..a458b5ff9d
--- /dev/null
+++ b/crates/macro_sync_service_jwt/src/session.rs
@@ -0,0 +1,257 @@
+//! Typed sync-session identities and isolated surface grants.
+//!
+//! Surface storage keys are `surface:<lowercase hyphenated UUID>`. Document
+//! keys retain their original spelling and have no prefix. UUID-only identities
+//! keep document keys out of the reserved surface namespace.
+
+use std::{fmt, str::FromStr};
+
+use jsonwebtoken::{
+ Algorithm, DecodingKey, EncodingKey, Header, Validation, decode as jwt_decode,
+ encode as jwt_encode,
+};
+use serde::{Deserialize, Serialize};
+
+use crate::ISSUER;
+
+/// The authorization and storage namespace of a sync session.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SessionKind {
+ /// An existing document session, stored under its unprefixed document ID.
+ Document,
+ /// An isolated collaborative surface session.
+ Surface,
+}
+
+/// An identifier was not a hyphenated UUID (`8-4-4-4-12` ASCII hex digits).
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InvalidSessionId;
+
+impl fmt::Display for InvalidSessionId {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str("session ID must be a hyphenated UUID")
+ }
+}
+
+impl std::error::Error for InvalidSessionId {}
+
+// Validate only the wire grammar; UUID generation/version policy belongs to the
+// caller. This shared worker/backend crate needs no UUID or randomness dependency.
+fn validate_id(id: &str) -> Result<(), InvalidSessionId> {
+ if id.len() != 36 {
+ return Err(InvalidSessionId);
+ }
+ for (index, byte) in id.bytes().enumerate() {
+ let valid = match index {
+ 8 | 13 | 18 | 23 => byte == b'-',
+ _ => byte.is_ascii_hexdigit(),
+ };
+ if !valid {
+ return Err(InvalidSessionId);
+ }
+ }
+ Ok(())
+}
+
+/// A surface UUID in canonical lowercase, hyphenated form.
+///
+/// Construction and deserialization both validate the ID. Alternate UUID forms
+/// (URNs, braces, or unhyphenated hex) and already-prefixed storage keys are rejected.
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(try_from = "String")]
+pub struct SurfaceId(String);
+
+impl SurfaceId {
+ /// Borrow the canonical UUID, without its storage namespace prefix.
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl FromStr for SurfaceId {
+ type Err = InvalidSessionId;
+
+ fn from_str(id: &str) -> Result<Self, Self::Err> {
+ validate_id(id)?;
+ Ok(Self(id.to_ascii_lowercase()))
+ }
+}
+
+impl TryFrom<String> for SurfaceId {
+ type Error = InvalidSessionId;
+
+ fn try_from(id: String) -> Result<Self, Self::Error> {
+ id.parse()
+ }
+}
+
+impl fmt::Display for SurfaceId {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(self.as_str())
+ }
+}
+
+/// A validated authorization target. Matching UUIDs in different kinds are not
+/// equal and never produce the same storage key.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SessionIdentity {
+ kind: SessionKind,
+ id: String,
+}
+
+impl SessionIdentity {
+ /// Validate a route/persisted identity. Document spelling is preserved;
+ /// surface IDs are canonicalized before comparison or storage.
+ pub fn new(kind: SessionKind, id: &str) -> Result<Self, InvalidSessionId> {
+ match kind {
+ SessionKind::Document => {
+ validate_id(id)?;
+ Ok(Self {
+ kind,
+ id: id.to_owned(),
+ })
+ }
+ SessionKind::Surface => Ok(Self::surface(id.parse()?)),
+ }
+ }
+
+ /// Construct an isolated identity from an already validated surface ID.
+ pub fn surface(id: SurfaceId) -> Self {
+ Self {
+ kind: SessionKind::Surface,
+ id: id.0,
+ }
+ }
+
+ /// The namespace that must match the grant as well as the route.
+ pub fn kind(&self) -> SessionKind {
+ self.kind
+ }
+
+ /// The unprefixed ID. Do not use this alone as an authorization target.
+ pub fn id(&self) -> &str {
+ &self.id
+ }
+
+ /// The common base key for Durable Objects and external persistence.
+ /// Existing document names are unchanged.
+ pub fn storage_key(&self) -> String {
+ match self.kind {
+ SessionKind::Document => self.id.clone(),
+ SessionKind::Surface => format!("surface:{}", self.id),
+ }
+ }
+}
+
+/// Permission carried by a surface grant. Internal/admin credentials are not
+/// surface grants; write policy is enforced by the surface access boundary.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum SurfaceAccessLevel {
+ /// Read-only access.
+ View,
+ /// Comment access, not permission for arbitrary CRDT writes.
+ Comment,
+ /// Permission to edit surface content.
+ Edit,
+ /// Owner access.
+ Owner,
+}
+
+/// The mandatory surface-claim discriminator. A document kind cannot be
+/// constructed or deserialized into this type.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SurfaceSessionKind {
+ /// An isolated collaborative surface.
+ Surface,
+}
+
+/// A grant for exactly one isolated surface, never a document grant.
+///
+/// The wire format requires `session_kind: "surface"` and `surface_id`.
+/// There is deliberately no `document_id` fallback. Unknown fields are rejected
+/// so a dual document/surface grant cannot be decoded as a surface grant.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct SurfaceClaims {
+ /// Required explicit discriminator, with no default for legacy tokens.
+ pub session_kind: SurfaceSessionKind,
+ /// The canonical, surface-scoped target ID.
+ pub surface_id: SurfaceId,
+ /// The user, when present.
+ pub user_id: Option<String>,
+ /// The parent-derived surface permission.
+ pub access_level: SurfaceAccessLevel,
+ /// Optional attribution actor, matching the document transport convention.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub actor: Option<String>,
+ /// Expiry as Unix seconds; consumers must retain this for open sockets.
+ pub exp: usize,
+ /// Issuer, validated against [`ISSUER`] by [`decode_surface`].
+ pub iss: String,
+}
+
+impl SurfaceClaims {
+ /// The complete authorization target; compare this with the route identity.
+ pub fn session_identity(&self) -> SessionIdentity {
+ SessionIdentity::surface(self.surface_id.clone())
+ }
+}
+
+/// A signed surface grant, distinct from the legacy document token wrapper.
+#[derive(
+ Debug, Clone, PartialEq, Eq, Serialize, Deserialize, derive_more::Display, derive_more::From,
+)]
+#[serde(transparent)]
+#[display("{_0}")]
+pub struct SurfacePermissionToken(String);
+
+impl SurfacePermissionToken {
+ /// Borrow the encoded JWT.
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+
+ /// Consume the token wrapper.
+ pub fn into_inner(self) -> String {
+ self.0
+ }
+}
+
+/// Sign a surface grant with HS256, without changing legacy document encoding.
+pub fn encode_surface(
+ claims: &SurfaceClaims,
+ secret: &str,
+) -> Result<SurfacePermissionToken, jsonwebtoken::errors::Error> {
+ let token = jwt_encode(
+ &Header::new(Algorithm::HS256),
+ claims,
+ &EncodingKey::from_secret(secret.as_bytes()),
+ )?;
+ Ok(SurfacePermissionToken(token))
+}
+
+/// Verify a surface grant's signature, issuer, expiry, kind, and scoped ID.
+///
+/// Unlike the legacy decoder, this has no expiry leeway. Route target matching,
+/// access-level enforcement, lifecycle checks, and socket expiry enforcement
+/// remain the caller's responsibility.
+pub fn decode_surface(
+ token: &str,
+ secret: &str,
+) -> Result<SurfaceClaims, jsonwebtoken::errors::Error> {
+ let mut validation = Validation::new(Algorithm::HS256);
+ validation.set_issuer(&[ISSUER]);
+ validation.leeway = 0;
+ let data = jwt_decode::<SurfaceClaims>(
+ token,
+ &DecodingKey::from_secret(secret.as_bytes()),
+ &validation,
+ )?;
+ Ok(data.claims)
+}
+
+#[cfg(test)]
+mod test;
diff --git a/crates/macro_sync_service_jwt/src/session/test.rs b/crates/macro_sync_service_jwt/src/session/test.rs
new file mode 100644
index 0000000000..46eaaeda6d
--- /dev/null
+++ b/crates/macro_sync_service_jwt/src/session/test.rs
@@ -0,0 +1,229 @@
+use super::*;
+use crate::{ISSUER, decode, encode};
+use jsonwebtoken::errors::ErrorKind;
+use serde::{Deserialize, Serialize};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+const ID: &str = "0194c5d8-8f00-7000-8000-abcdef012345";
+const SECRET: &str = "session-tests-only";
+
+fn now() -> usize {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap()
+ .as_secs() as usize
+}
+
+fn claims() -> SurfaceClaims {
+ SurfaceClaims {
+ session_kind: SurfaceSessionKind::Surface,
+ surface_id: ID.parse().unwrap(),
+ user_id: Some("macro|[email protected]".into()),
+ access_level: SurfaceAccessLevel::Edit,
+ actor: None,
+ exp: now() + crate::TOKEN_TTL_SECS,
+ iss: ISSUER.into(),
+ }
+}
+
+// Mirrors the existing document issuer and consumer without backend dependencies.
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
+struct LegacyDocumentClaims {
+ user_id: Option<String>,
+ document_id: String,
+ access_level: SurfaceAccessLevel,
+ exp: usize,
+ iss: String,
+}
+
+#[derive(Serialize)]
+struct RawSurfaceClaims<'a> {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ session_kind: Option<&'a str>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ surface_id: Option<&'a str>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ document_id: Option<&'a str>,
+ access_level: &'a str,
+ exp: usize,
+ iss: &'a str,
+}
+
+fn raw_claims() -> RawSurfaceClaims<'static> {
+ RawSurfaceClaims {
+ session_kind: Some("surface"),
+ surface_id: Some(ID),
+ document_id: None,
+ access_level: "edit",
+ exp: now() + crate::TOKEN_TTL_SECS,
+ iss: ISSUER,
+ }
+}
+
+fn rejects(raw: &RawSurfaceClaims<'_>) {
+ let token = encode(raw, SECRET).unwrap();
+ assert!(decode_surface(token.as_str(), SECRET).is_err());
+}
+
+#[test]
+fn identical_uuids_have_distinct_targets_and_storage_keys() {
+ let document = SessionIdentity::new(SessionKind::Document, ID).unwrap();
+ let surface = SessionIdentity::new(SessionKind::Surface, ID).unwrap();
+ assert_ne!(document, surface);
+ assert_eq!(document.kind(), SessionKind::Document);
+ assert_eq!(surface.kind(), SessionKind::Surface);
+ assert_eq!(document.id(), surface.id());
+ assert_eq!(document.storage_key(), ID);
+ assert_eq!(surface.storage_key(), format!("surface:{ID}"));
+}
+
+#[test]
+fn surface_keys_are_canonical_but_document_names_are_unchanged() {
+ let uppercase = ID.to_ascii_uppercase();
+ let surface = SessionIdentity::new(SessionKind::Surface, &uppercase).unwrap();
+ assert_eq!(surface.id(), ID);
+ assert_eq!(surface.storage_key(), format!("surface:{ID}"));
+ let document = SessionIdentity::new(SessionKind::Document, &uppercase).unwrap();
+ assert_eq!(document.id(), uppercase);
+ assert_eq!(document.storage_key(), uppercase);
+ let id: SurfaceId = uppercase.parse().unwrap();
+ assert_eq!(id.as_str(), ID);
+ assert_eq!(id.to_string(), ID);
+}
+
+#[test]
+fn malformed_ids_cannot_enter_identities_or_claims() {
+ for id in [
+ "",
+ "not-a-uuid",
+ "0194c5d88f0070008000abcdef012345",
+ "0194c5d8-8f00-7000-8000-abcdef01234g",
+ "0194c5d8_8f00-7000-8000-abcdef012345",
+ "0194c5d8-8f00-7000-8000-abcdef012345/ops",
+ "surface:0194c5d8-8f00-7000-8000-abcdef012345",
+ "{0194c5d8-8f00-7000-8000-abcdef012345}",
+ " 0194c5d8-8f00-7000-8000-abcdef012345",
+ "0194c5d8-8f00-7000-8000-abcdef01234é",
+ ] {
+ assert!(id.parse::<SurfaceId>().is_err(), "{id}");
+ for kind in [SessionKind::Document, SessionKind::Surface] {
+ assert!(SessionIdentity::new(kind, id).is_err(), "{id}");
+ }
+ let mut raw = raw_claims();
+ raw.surface_id = Some(id);
+ rejects(&raw);
+ }
+}
+
+#[test]
+fn surface_token_round_trip_has_explicit_kind_and_scoped_id() {
+ let expected = claims();
+ let token = encode_surface(&expected, SECRET).unwrap();
+ let actual = decode_surface(token.as_str(), SECRET).unwrap();
+ assert_eq!(actual, expected);
+ assert_eq!(
+ actual.session_identity(),
+ SessionIdentity::new(SessionKind::Surface, ID).unwrap()
+ );
+
+ #[derive(Deserialize)]
+ struct WireClaims {
+ session_kind: SessionKind,
+ surface_id: String,
+ }
+ let wire: WireClaims = decode(token.as_str(), SECRET).unwrap();
+ assert_eq!(wire.session_kind, SessionKind::Surface);
+ assert_eq!(wire.surface_id, ID);
+ assert!(decode::<LegacyDocumentClaims>(token.as_str(), SECRET).is_err());
+}
+
+#[test]
+fn legacy_document_tokens_remain_compatible_but_are_not_surface_grants() {
+ let expected = LegacyDocumentClaims {
+ user_id: None,
+ document_id: ID.into(),
+ access_level: SurfaceAccessLevel::Comment,
+ exp: now() + crate::TOKEN_TTL_SECS,
+ iss: ISSUER.into(),
+ };
+ let token = encode(&expected, SECRET).unwrap();
+ assert_eq!(
+ decode::<LegacyDocumentClaims>(token.as_str(), SECRET).unwrap(),
+ expected
+ );
+ assert!(decode_surface(token.as_str(), SECRET).is_err());
+}
+
+#[test]
+fn surface_claims_require_surface_kind_and_surface_id() {
+ for kind in [None, Some("document"), Some("initiative"), Some("Surface")] {
+ let mut raw = raw_claims();
+ raw.session_kind = kind;
+ rejects(&raw);
+ }
+ let mut raw = raw_claims();
+ raw.surface_id = None;
+ rejects(&raw);
+ raw.document_id = Some(ID);
+ rejects(&raw);
+ raw.surface_id = Some(ID);
+ rejects(&raw); // Reject ambiguous grants containing both target fields.
+}
+
+#[test]
+fn unsupported_access_levels_are_rejected() {
+ for level in ["admin", "write", "Edit", ""] {
+ let mut raw = raw_claims();
+ raw.access_level = level;
+ rejects(&raw);
+ }
+ for level in ["view", "comment", "edit", "owner"] {
+ let mut raw = raw_claims();
+ raw.access_level = level;
+ let token = encode(&raw, SECRET).unwrap();
+ assert!(decode_surface(token.as_str(), SECRET).is_ok());
+ }
+}
+
+#[test]
+fn expired_surface_tokens_are_rejected_without_default_clock_leeway() {
+ let mut expired = claims();
+ expired.exp = now() - 1;
+ let token = encode_surface(&expired, SECRET).unwrap();
+ assert_eq!(
+ decode_surface(token.as_str(), SECRET).unwrap_err().kind(),
+ &ErrorKind::ExpiredSignature
+ );
+}
+
+#[test]
+fn surface_tokens_require_expected_issuer_and_signature() {
+ let valid = claims();
+ let token = encode_surface(&valid, SECRET).unwrap();
+ assert!(decode_surface(token.as_str(), "wrong-secret").is_err());
+ let mut raw = raw_claims();
+ raw.iss = "another-service";
+ rejects(&raw);
+}
+
+#[test]
+fn surface_tokens_require_expiry() {
+ #[derive(Serialize)]
+ struct NoExpiry {
+ session_kind: SessionKind,
+ surface_id: SurfaceId,
+ access_level: SurfaceAccessLevel,
+ iss: &'static str,
+ }
+ let token = encode(
+ &NoExpiry {
+ session_kind: SessionKind::Surface,
+ surface_id: ID.parse().unwrap(),
+ access_level: SurfaceAccessLevel::View,
+ iss: ISSUER,
+ },
+ SECRET,
+ )
+ .unwrap();
+ assert!(decode_surface(token.as_str(), SECRET).is_err());
+}
diff --git a/crates/sync_service_client/Cargo.toml b/crates/sync_service_client/Cargo.toml
index d269976780..431e005b9b 100644
--- a/crates/sync_service_client/Cargo.toml
+++ b/crates/sync_service_client/Cargo.toml
@@ -12,7 +12,9 @@ serde = { workspace = true }
sync_service_hex = { path = "../sync_service", package = "sync_service_hex" }
tokio = { workspace = true }
tracing = { workspace = true }
+uuid = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
+serde_json = { workspace = true }
tokio = { workspace = true }
diff --git a/crates/sync_service_client/src/lib.rs b/crates/sync_service_client/src/lib.rs
index 39690d0c53..a3729a42e4 100644
--- a/crates/sync_service_client/src/lib.rs
+++ b/crates/sync_service_client/src/lib.rs
@@ -4,6 +4,7 @@ pub mod exists;
pub mod get_raw;
pub mod initialize;
pub mod metadata;
+pub mod surface;
pub mod wakeup;
pub(crate) static INTERNAL_ACCESS_HEADER: &str = "x-internal-auth-key";
diff --git a/crates/sync_service_client/src/surface.rs b/crates/sync_service_client/src/surface.rs
new file mode 100644
index 0000000000..1e0c77ee96
--- /dev/null
+++ b/crates/sync_service_client/src/surface.rs
@@ -0,0 +1,208 @@
+//! Internal surface lifecycle transport. Callers own legacy ownership checks and
+//! the durable metadata journal; no method falls back to document initialization.
+
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+use uuid::Uuid;
+
+use crate::SyncServiceClient;
+
+#[cfg(test)]
+mod test;
+
+/// Persist this identity before starting an initialization or migration.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct SurfaceOperationId(pub Uuid);
+
+/// Receipt binding one operation to snapshot bytes, decoded content and both
+/// Loro frontiers. A successful retry returns this same verified receipt.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SnapshotProof {
+ /// Durable operation identity; reuse only for an identical retry.
+ pub operation_id: SurfaceOperationId,
+ /// Legacy source UUID, or None for a newly seeded isolated surface.
+ pub source_id: Option<Uuid>,
+ /// SHA-256 of the full snapshot bytes.
+ pub digest: String,
+ /// SHA-256 of the canonical decoded JSON content.
+ pub content_digest: String,
+ /// Sorted state frontier (peer string, counter) pairs.
+ pub revision: Vec<(String, i32)>,
+ /// Sorted operation-log frontier pairs.
+ pub oplog_revision: Vec<(String, i32)>,
+}
+
+/// Authoritative frozen state, including pending persisted operations.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SurfaceSnapshot {
+ /// Verification receipt for these exact bytes.
+ pub proof: SnapshotProof,
+ /// Full Loro snapshot, not initial Markdown or a shallow client snapshot.
+ pub snapshot: Vec<u8>,
+}
+
+/// Typed transport failures; an existing snapshot is never treated as success.
+#[derive(Debug)]
+pub enum SurfaceSyncError {
+ /// Request or response decoding failed; retry with the same operation.
+ Transport(reqwest::Error),
+ /// HTTP rejection, including conflicts (409) and permanent revocation (403).
+ Rejected(StatusCode),
+}
+
+impl std::fmt::Display for SurfaceSyncError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Transport(error) => write!(f, "surface sync transport: {error}"),
+ Self::Rejected(status) => write!(f, "surface sync rejected: {status}"),
+ }
+ }
+}
+
+impl std::error::Error for SurfaceSyncError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ match self {
+ Self::Transport(error) => Some(error),
+ Self::Rejected(_) => None,
+ }
+ }
+}
+
+impl SyncServiceClient {
+ async fn surface_request(
+ &self,
+ path: &str,
+ body: &impl Serialize,
+ ) -> Result<reqwest::Response, SurfaceSyncError> {
+ let response = self
+ .client
+ .post(format!("{}{path}", self.url))
+ .json(body)
+ .send()
+ .await
+ .map_err(SurfaceSyncError::Transport)?;
+ if response.status() != StatusCode::OK {
+ return Err(SurfaceSyncError::Rejected(response.status()));
+ }
+ Ok(response)
+ }
+
+ async fn surface_json<T: DeserializeOwned>(
+ &self,
+ path: &str,
+ body: &impl Serialize,
+ ) -> Result<T, SurfaceSyncError> {
+ self.surface_request(path, body)
+ .await?
+ .json()
+ .await
+ .map_err(SurfaceSyncError::Transport)
+ }
+
+ /// Seed an isolated surface exactly once. Retries require the same operation
+ /// and snapshot; existing document initialization behavior is unchanged.
+ pub async fn initialize_surface(
+ &self,
+ id: Uuid,
+ operation_id: SurfaceOperationId,
+ snapshot: &[u8],
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ #[derive(Serialize)]
+ struct Initialize<'a> {
+ operation_id: SurfaceOperationId,
+ snapshot: &'a [u8],
+ }
+ self.surface_json(
+ &format!("/surface/{id}/initialize_verified"),
+ &Initialize {
+ operation_id,
+ snapshot,
+ },
+ )
+ .await
+ }
+
+ /// Stop legacy writers and export authoritative state. The caller MUST have
+ /// ruled out live and soft-deleted Document collisions before invoking this.
+ pub async fn freeze_legacy_surface(
+ &self,
+ id: Uuid,
+ operation_id: SurfaceOperationId,
+ ) -> Result<SurfaceSnapshot, SurfaceSyncError> {
+ #[derive(Serialize)]
+ struct Freeze {
+ operation_id: SurfaceOperationId,
+ }
+ self.surface_json(
+ &format!("/document/{id}/migration/freeze"),
+ &Freeze { operation_id },
+ )
+ .await
+ }
+
+ /// Import into a non-writable isolated target. The source and target UUIDs
+ /// must match; conflicting state is never overwritten.
+ pub async fn import_surface(
+ &self,
+ id: Uuid,
+ snapshot: &SurfaceSnapshot,
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ self.surface_json(&format!("/surface/{id}/import"), snapshot)
+ .await
+ }
+
+ /// Read back and verify an import before committing metadata activation.
+ pub async fn verify_surface(
+ &self,
+ id: Uuid,
+ proof: &SnapshotProof,
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ self.surface_json(&format!("/surface/{id}/verify"), proof)
+ .await
+ }
+
+ /// Irreversibly seal the legacy source, then enable the verified target.
+ /// Persist commit intent in the caller's journal BEFORE calling this method.
+ /// Even on a timeout/error, recover forward: the source may already be sealed.
+ /// Activate metadata/token minting only after this operation succeeds.
+ pub async fn activate_surface(
+ &self,
+ id: Uuid,
+ proof: &SnapshotProof,
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ self.surface_json(&format!("/surface/{id}/activate"), proof)
+ .await
+ }
+
+ /// Roll back only the verified original, before activation commit intent.
+ /// A thawed operation cannot subsequently be sealed or reused for a freeze.
+ pub async fn thaw_legacy_surface(
+ &self,
+ id: Uuid,
+ proof: &SnapshotProof,
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ self.surface_json(&format!("/document/{id}/migration/thaw"), proof)
+ .await
+ }
+
+ /// Permanently retire a sealed legacy endpoint, retaining its snapshot/log.
+ pub async fn retire_legacy_surface(
+ &self,
+ id: Uuid,
+ proof: &SnapshotProof,
+ ) -> Result<SnapshotProof, SurfaceSyncError> {
+ self.surface_json(&format!("/document/{id}/migration/retire"), proof)
+ .await
+ }
+
+ /// Permanently deny new and existing surface grants, including pending IDs.
+ pub async fn revoke_surface(&self, id: Uuid) -> Result<(), SurfaceSyncError> {
+ self.surface_request(
+ &format!("/surface/{id}/revoke"),
+ &std::collections::BTreeMap::<String, String>::new(),
+ )
+ .await?;
+ Ok(())
+ }
+}
diff --git a/crates/sync_service_client/src/surface/test.rs b/crates/sync_service_client/src/surface/test.rs
new file mode 100644
index 0000000000..eeb527db29
--- /dev/null
+++ b/crates/sync_service_client/src/surface/test.rs
@@ -0,0 +1,183 @@
+use super::*;
+use std::io::{Read, Write};
+
+fn proof() -> SnapshotProof {
+ SnapshotProof {
+ operation_id: SurfaceOperationId(Uuid::from_u128(1)),
+ source_id: Some(Uuid::from_u128(2)),
+ digest: "snapshot-digest".into(),
+ content_digest: "content-digest".into(),
+ revision: vec![("123".into(), 4)],
+ oplog_revision: vec![("123".into(), 4)],
+ }
+}
+
+#[test]
+fn wire_proof_round_trips_and_rejects_invalid_operation_ids() {
+ let proof = proof();
+ let value = serde_json::to_value(&proof).unwrap();
+ assert_eq!(value["operation_id"], Uuid::from_u128(1).to_string());
+ assert_eq!(
+ serde_json::from_value::<SnapshotProof>(value.clone()).unwrap(),
+ proof
+ );
+ let mut invalid = value;
+ invalid["operation_id"] = "not-a-uuid".into();
+ assert!(serde_json::from_value::<SnapshotProof>(invalid).is_err());
+}
+
+/// Minimal HTTP peer validates transport without another mocking dependency.
+fn server(
+ path: String,
+ expected: serde_json::Value,
+ status: u16,
+ response: String,
+) -> (SyncServiceClient, std::thread::JoinHandle<()>) {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let task = std::thread::spawn(move || {
+ let (mut socket, _) = listener.accept().unwrap();
+ socket
+ .set_read_timeout(Some(std::time::Duration::from_secs(5)))
+ .unwrap();
+ let mut bytes = Vec::new();
+ let (headers, body_start, length) = loop {
+ let mut buf = [0; 4096];
+ let count = socket.read(&mut buf).unwrap();
+ assert_ne!(count, 0);
+ bytes.extend_from_slice(&buf[..count]);
+ if let Some(start) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {
+ let headers = String::from_utf8(bytes[..start].to_vec()).unwrap();
+ let length = headers
+ .lines()
+ .find_map(|line| {
+ line.to_lowercase()
+ .strip_prefix("content-length: ")
+ .map(|n| n.parse::<usize>().unwrap())
+ })
+ .unwrap();
+ break (headers, start + 4, length);
+ }
+ };
+ while bytes.len() < body_start + length {
+ let mut buf = [0; 4096];
+ let count = socket.read(&mut buf).unwrap();
+ assert_ne!(count, 0);
+ bytes.extend_from_slice(&buf[..count]);
+ }
+ assert!(headers.starts_with(&format!("POST {path} HTTP/1.1")));
+ assert!(headers.contains("x-internal-auth-key: secret"));
+ assert_eq!(
+ serde_json::from_slice::<serde_json::Value>(&bytes[body_start..body_start + length])
+ .unwrap(),
+ expected
+ );
+ write!(socket, "HTTP/1.1 {status} Result\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}", response.len()).unwrap();
+ });
+ (
+ SyncServiceClient::new("secret".into(), format!("http://{address}")),
+ task,
+ )
+}
+
+#[tokio::test]
+async fn freeze_uses_legacy_namespace_and_typed_export() {
+ let proof = proof();
+ let id = proof.source_id.unwrap();
+ let export = SurfaceSnapshot {
+ proof: proof.clone(),
+ snapshot: vec![1, 2],
+ };
+ let (client, task) = server(
+ format!("/document/{id}/migration/freeze"),
+ serde_json::json!({ "operation_id": proof.operation_id }),
+ 200,
+ serde_json::to_string(&export).unwrap(),
+ );
+ assert_eq!(
+ client
+ .freeze_legacy_surface(id, proof.operation_id)
+ .await
+ .unwrap(),
+ export
+ );
+ task.join().unwrap();
+}
+
+#[tokio::test]
+async fn surface_lifecycle_methods_use_isolated_routes() {
+ let proof = proof();
+ let id = proof.source_id.unwrap();
+ for operation in ["verify", "activate", "migration/thaw", "migration/retire"] {
+ let kind = if operation.starts_with("migration/") {
+ "document"
+ } else {
+ "surface"
+ };
+ let (client, task) = server(
+ format!("/{kind}/{id}/{operation}"),
+ serde_json::to_value(&proof).unwrap(),
+ 200,
+ serde_json::to_string(&proof).unwrap(),
+ );
+ let result = match operation {
+ "verify" => client.verify_surface(id, &proof).await,
+ "activate" => client.activate_surface(id, &proof).await,
+ "migration/thaw" => client.thaw_legacy_surface(id, &proof).await,
+ _ => client.retire_legacy_surface(id, &proof).await,
+ };
+ assert_eq!(result.unwrap(), proof);
+ task.join().unwrap();
+ }
+ let export = SurfaceSnapshot {
+ proof: proof.clone(),
+ snapshot: vec![1, 2],
+ };
+ let (client, task) = server(
+ format!("/surface/{id}/import"),
+ serde_json::to_value(&export).unwrap(),
+ 200,
+ serde_json::to_string(&proof).unwrap(),
+ );
+ assert_eq!(client.import_surface(id, &export).await.unwrap(), proof);
+ task.join().unwrap();
+ let (client, task) = server(
+ format!("/surface/{id}/initialize_verified"),
+ serde_json::json!({"operation_id": proof.operation_id, "snapshot": [1, 2]}),
+ 200,
+ serde_json::to_string(&proof).unwrap(),
+ );
+ assert_eq!(
+ client
+ .initialize_surface(id, proof.operation_id, &[1, 2])
+ .await
+ .unwrap(),
+ proof
+ );
+ task.join().unwrap();
+ let (client, task) = server(
+ format!("/surface/{id}/revoke"),
+ serde_json::json!({}),
+ 200,
+ String::new(),
+ );
+ client.revoke_surface(id).await.unwrap();
+ task.join().unwrap();
+}
+
+#[tokio::test]
+async fn conflict_is_typed_and_never_string_matched_as_success() {
+ let proof = proof();
+ let id = proof.source_id.unwrap();
+ let (client, task) = server(
+ format!("/surface/{id}/verify"),
+ serde_json::to_value(&proof).unwrap(),
+ 409,
+ "snapshot already exists".into(),
+ );
+ assert!(matches!(
+ client.verify_surface(id, &proof).await,
+ Err(SurfaceSyncError::Rejected(reqwest::StatusCode::CONFLICT))
+ ));
+ task.join().unwrap();
+}
diff --git a/services/sync-service/Cargo.toml b/services/sync-service/Cargo.toml
index a9579d4c5d..6f14c067f4 100644
--- a/services/sync-service/Cargo.toml
+++ b/services/sync-service/Cargo.toml
@@ -9,6 +9,8 @@ publish = false
[features]
default = ["bebop-owned-all", "do-sqlite-snapshot-storage", "search-service"]
no-network-deps = []
+# Test-only persistence fault injection; never enable in deployed workers.
+migration-test-hooks = []
do-sqlite-snapshot-storage = []
kv-snapshot-storage = []
r2-snapshot-storage = []
@@ -39,6 +41,7 @@ time = { workspace = true, features = ['wasm-bindgen'] }
bebop.workspace = true
sync_service_bebop_schema = { path = "bebop-schema" }
serde_json.workspace = true
+sha2.workspace = true
serde.workspace = true
getrandom = { workspace = true, features = ["wasm_js"] }
wasm-bindgen.workspace = true
diff --git a/services/sync-service/src/auth.rs b/services/sync-service/src/auth.rs
index 2aaf059bd7..5939610e76 100644
--- a/services/sync-service/src/auth.rs
+++ b/services/sync-service/src/auth.rs
@@ -1,65 +1,63 @@
+use macro_sync_service_jwt::session::{
+ SessionIdentity, SessionKind, SurfaceAccessLevel, decode_surface,
+};
use serde::{Deserialize, Serialize};
use subtle::ConstantTimeEq;
-use tracing::error;
use worker::Error;
use crate::{constants::header_names, error::ResultExt, secrets::Secrets};
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, PartialOrd, Ord)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Default)]
#[serde(rename_all = "lowercase")]
-#[derive(Default)]
pub enum AccessLevel {
- /// User can view the document
#[default]
View = 0,
- /// User can comment on the document
- /// In this context, this is the same thing as [AccessLevel::View]
+ /// Legacy document sockets permit Comment writes; surfaces do not.
Comment = 1,
- /// User can edit the document
Edit = 2,
- /// User is the owner of the document
Owner = 3,
- /// Internal communication
Admin = 4,
}
impl AccessLevel {
- pub fn can_edit(&self) -> bool {
- self >= &AccessLevel::Comment
+ pub fn can_edit_for(&self, kind: SessionKind) -> bool {
+ let minimum = match kind {
+ SessionKind::Document => Self::Comment,
+ SessionKind::Surface => Self::Edit,
+ };
+ self >= &minimum
}
}
-#[derive(Deserialize, Debug)]
+/// Verified grant normalized at the dedicated session access boundary.
+#[derive(Debug)]
pub struct AuthToken {
pub user_id: Option<String>,
document_id: String,
pub access_level: AccessLevel,
- #[serde(default)]
pub actor: Option<String>,
+ pub session_kind: SessionKind,
+ pub expires_at: Option<usize>,
+}
+
+#[derive(Deserialize)]
+struct DocumentClaims {
+ user_id: Option<String>,
+ document_id: String,
+ access_level: AccessLevel,
+ actor: Option<String>,
+ session_kind: Option<SessionKind>,
+ surface_id: Option<String>,
}
impl AuthToken {
- pub fn has_permission(&self, al: &AccessLevel) -> bool {
- if self.access_level < *al {
- error!(
- "Current permission level [{:?}] is not enough for [{:?}]",
- self.access_level, al
- );
- return false;
- }
- true
+ pub fn has_permission(&self, level: &AccessLevel) -> bool {
+ self.access_level >= *level
}
+
pub fn has_document_id_access(&self, document_id: &str) -> bool {
- if !(self.document_id == document_id || matches!(self.access_level, AccessLevel::Admin)) {
- error!(
- "Don't have permission for document: [{:?}]
-Auth'd document [{:?}]
-access level [{:?}]",
- document_id, self.document_id, self.access_level
- );
- return false;
- }
- true
+ self.session_kind == SessionKind::Document
+ && (self.document_id == document_id || self.access_level == AccessLevel::Admin)
}
}
@@ -73,66 +71,127 @@ pub enum TokenFrom {
QueryParams,
}
-pub fn decode_jwt(
- req: &worker::Request,
- env: &worker::Env,
- token_from: TokenFrom,
-) -> worker::Result<AuthToken> {
- let secrets = Secrets::from(env);
- let token = match token_from {
- TokenFrom::Headers => {
- // NB: rewrite with if/let chain on edition 2024
- let is_admin = match req
- .headers()
- .get(header_names::MACRO_INTERNAL_AUTH_KEY_HEADER_KEY)?
- {
- // sholud we warn when false?
- Some(internal_key) => {
- let res: bool = internal_key
- .as_bytes()
- .ct_eq(secrets.internal_api_secret.as_bytes())
- .into();
-
- if !res {
- error!("provided internal authentication key did not match expected value");
- }
- res
- }
- None => false,
- };
-
- if is_admin {
- return Ok(AuthToken {
- user_id: None,
- document_id: "TODO should be option".to_string(),
- access_level: AccessLevel::Admin,
- actor: None,
- });
- }
-
- match req.headers().get(header_names::AUTHORIZATION)? {
- Some(header) => match header.strip_prefix("Bearer ") {
- Some(token) => token.to_string(),
- None => {
- return Err(Error::from(
- "'Authorization' heard malformed. No 'Bearer ' prefix",
- ));
- }
- },
- None => return Err(Error::from("Missing 'Authorization' header")),
- }
- }
- TokenFrom::QueryParams => req.query::<WebsocketQueryParams>()?.token,
+pub fn is_internal(req: &worker::Request, env: &worker::Env) -> worker::Result<bool> {
+ let Some(key) = req
+ .headers()
+ .get(header_names::MACRO_INTERNAL_AUTH_KEY_HEADER_KEY)?
+ else {
+ return Ok(false);
};
+ Ok(key
+ .as_bytes()
+ .ct_eq(Secrets::from(env).internal_api_secret.as_bytes())
+ .into())
+}
+
+fn request_token(req: &worker::Request, source: TokenFrom) -> worker::Result<String> {
+ match source {
+ TokenFrom::Headers => req
+ .headers()
+ .get(header_names::AUTHORIZATION)?
+ .and_then(|header| header.strip_prefix("Bearer ").map(str::to_owned))
+ .ok_or_else(|| Error::from("Missing or malformed Bearer authorization")),
+ TokenFrom::QueryParams => Ok(req.query::<WebsocketQueryParams>()?.token),
+ }
+}
- let secret = secrets.document_permissions_secret;
+fn decode_document_token(token: &str, secret: &str) -> worker::Result<AuthToken> {
+ let claims = macro_sync_service_jwt::decode::<DocumentClaims>(token, secret)
+ .context("failed to decode document grant")?;
+ if claims
+ .session_kind
+ .is_some_and(|kind| kind != SessionKind::Document)
+ || claims.surface_id.is_some()
+ {
+ return Err(Error::from("surface grant cannot authorize a document"));
+ }
+ Ok(AuthToken {
+ user_id: claims.user_id,
+ document_id: claims.document_id,
+ access_level: claims.access_level,
+ actor: claims.actor,
+ session_kind: SessionKind::Document,
+ expires_at: None,
+ })
+}
- let claims = macro_sync_service_jwt::decode::<AuthToken>(&token, &secret)
- .context("failed to decode `AuthToken`")?;
+fn decode_surface_token(
+ token: &str,
+ secret: &str,
+ identity: &SessionIdentity,
+) -> worker::Result<AuthToken> {
+ let claims = decode_surface(token, secret).context("failed to decode surface grant")?;
+ if claims.session_identity() != *identity {
+ return Err(Error::from("surface grant does not match session"));
+ }
+ let access_level = match claims.access_level {
+ SurfaceAccessLevel::View => AccessLevel::View,
+ SurfaceAccessLevel::Comment => AccessLevel::Comment,
+ SurfaceAccessLevel::Edit => AccessLevel::Edit,
+ SurfaceAccessLevel::Owner => AccessLevel::Owner,
+ };
+ Ok(AuthToken {
+ user_id: claims.user_id,
+ document_id: identity.storage_key(),
+ access_level,
+ actor: claims.actor,
+ session_kind: SessionKind::Surface,
+ expires_at: Some(claims.exp),
+ })
+}
+pub fn surface_access(
+ req: &worker::Request,
+ env: &worker::Env,
+ source: TokenFrom,
+ identity: &SessionIdentity,
+) -> worker::Result<AuthToken> {
+ decode_surface_token(
+ &request_token(req, source)?,
+ &Secrets::from(env).document_permissions_secret,
+ identity,
+ )
+}
+
+/// Authenticate a socket against the persisted, kind-discriminated storage key.
+pub fn socket_access(
+ req: &worker::Request,
+ env: &worker::Env,
+ session_key: &str,
+) -> worker::Result<AuthToken> {
+ if let Some(id) = session_key.strip_prefix("surface:") {
+ let identity = SessionIdentity::new(SessionKind::Surface, id)
+ .map_err(|error| Error::from(error.to_string()))?;
+ return surface_access(req, env, TokenFrom::QueryParams, &identity);
+ }
+ let claims = decode_jwt(req, env, TokenFrom::QueryParams)?;
+ if !claims.has_document_id_access(session_key) {
+ return Err(Error::from("document grant does not match session"));
+ }
Ok(claims)
}
+pub fn decode_jwt(
+ req: &worker::Request,
+ env: &worker::Env,
+ source: TokenFrom,
+) -> worker::Result<AuthToken> {
+ if matches!(source, TokenFrom::Headers) && is_internal(req, env)? {
+ return Ok(AuthToken {
+ user_id: None,
+ document_id: String::new(),
+ access_level: AccessLevel::Admin,
+ actor: None,
+ session_kind: SessionKind::Document,
+ expires_at: None,
+ });
+ }
+ decode_document_token(
+ &request_token(req, source)?,
+ &Secrets::from(env).document_permissions_secret,
+ )
+}
+
/// Dedicated access boundary for document HTTP requests. Internal service
/// credentials do not substitute for a signed, document-scoped user grant.
pub fn document_access(
@@ -144,20 +203,9 @@ pub fn document_access(
crate::domain::document::DocumentError,
> {
use crate::domain::document::DocumentError;
-
- let header = req
- .headers()
- .get(header_names::AUTHORIZATION)
- .map_err(|_| DocumentError::Unauthorized)?
- .ok_or(DocumentError::Unauthorized)?;
- let token = header
- .strip_prefix("Bearer ")
- .ok_or(DocumentError::Unauthorized)?;
- let claims = macro_sync_service_jwt::decode::<AuthToken>(
- token,
- &Secrets::from(env).document_permissions_secret,
- )
- .map_err(|_| DocumentError::Unauthorized)?;
+ let token = request_token(req, TokenFrom::Headers).map_err(|_| DocumentError::Unauthorized)?;
+ let claims = decode_document_token(&token, &Secrets::from(env).document_permissions_secret)
+ .map_err(|_| DocumentError::Unauthorized)?;
let access = crate::domain::document::DocumentAccess::authorize(
document_id,
&claims.document_id,
@@ -167,22 +215,4 @@ pub fn document_access(
}
#[cfg(test)]
-mod test {
- use super::*;
- #[test]
- #[allow(clippy::nonminimal_bool, reason = "demonstrate ordering")]
- fn orderable() {
- let view = AccessLevel::View;
- let comment = AccessLevel::Comment;
- let edit = AccessLevel::Edit;
- let owner = AccessLevel::Owner;
-
- assert!(view < comment);
- assert!(view <= view);
- assert!(!(view < view));
- assert!(edit <= owner);
- assert!(view <= edit);
- assert!(view < owner);
- assert!(!(owner < view));
- }
-}
+mod test;
diff --git a/services/sync-service/src/auth/test.rs b/services/sync-service/src/auth/test.rs
new file mode 100644
index 0000000000..dce4a11e5f
--- /dev/null
+++ b/services/sync-service/src/auth/test.rs
@@ -0,0 +1,74 @@
+use super::*;
+use macro_sync_service_jwt::session::{SurfaceClaims, SurfaceSessionKind};
+
+const ID: &str = "01952cbd-76ad-7a65-9c21-020304050607";
+
+fn surface_token() -> String {
+ macro_sync_service_jwt::session::encode_surface(
+ &SurfaceClaims {
+ session_kind: SurfaceSessionKind::Surface,
+ surface_id: ID.parse().unwrap(),
+ user_id: Some("user".into()),
+ access_level: SurfaceAccessLevel::Edit,
+ actor: None,
+ exp: usize::MAX / 2,
+ iss: macro_sync_service_jwt::ISSUER.into(),
+ },
+ "secret",
+ )
+ .unwrap()
+ .into_inner()
+}
+
+#[test]
+fn same_uuid_does_not_authorize_another_session_kind() {
+ let document = macro_sync_service_jwt::encode(
+ &serde_json::json!({"document_id": ID, "access_level": "edit", "exp": usize::MAX / 2}),
+ "secret",
+ )
+ .unwrap();
+ let surface = SessionIdentity::new(SessionKind::Surface, ID).unwrap();
+ assert!(decode_document_token(&surface_token(), "secret").is_err());
+ assert!(decode_surface_token(document.as_str(), "secret", &surface).is_err());
+ assert!(
+ decode_document_token(document.as_str(), "secret")
+ .unwrap()
+ .has_document_id_access(ID)
+ );
+ let claims = decode_surface_token(&surface_token(), "secret", &surface).unwrap();
+ assert_eq!(claims.session_kind, SessionKind::Surface);
+ assert!(claims.expires_at.is_some());
+ assert!(!claims.has_document_id_access(ID));
+ let other =
+ SessionIdentity::new(SessionKind::Surface, "01952cbd-76ad-7a65-9c21-020304050608").unwrap();
+ assert!(decode_surface_token(&surface_token(), "secret", &other).is_err());
+}
+
+#[test]
+fn dual_kind_document_claims_are_rejected() {
+ for extra in [
+ serde_json::json!({"session_kind": "surface"}),
+ serde_json::json!({"surface_id": ID}),
+ ] {
+ let mut claims =
+ serde_json::json!({"document_id": ID, "access_level": "owner", "exp": usize::MAX / 2});
+ claims
+ .as_object_mut()
+ .unwrap()
+ .extend(extra.as_object().unwrap().clone());
+ let token = macro_sync_service_jwt::encode(&claims, "secret").unwrap();
+ assert!(decode_document_token(token.as_str(), "secret").is_err());
+ }
+}
+
+#[test]
+fn surface_writes_require_edit_without_changing_document_comment_behavior() {
+ assert!(AccessLevel::Comment.can_edit_for(SessionKind::Document));
+ assert!(!AccessLevel::View.can_edit_for(SessionKind::Document));
+ for level in [AccessLevel::View, AccessLevel::Comment] {
+ assert!(!level.can_edit_for(SessionKind::Surface));
+ }
+ for level in [AccessLevel::Edit, AccessLevel::Owner, AccessLevel::Admin] {
+ assert!(level.can_edit_for(SessionKind::Surface));
+ }
+}
diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rs
index bddbe68b04..ba4c0c87fc 100644
--- a/services/sync-service/src/cf_worker.rs
+++ b/services/sync-service/src/cf_worker.rs
@@ -1,9 +1,11 @@
use bebop::{Record, SubRecord};
+use macro_sync_service_jwt::session::{SessionIdentity, SessionKind};
use tracing::{Instrument, error};
use wasm_bindgen::JsValue;
use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Result, Stub};
use crate::{
+ auth::is_internal,
constants::header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},
durable_object::{CopyDocumentRequest, GetSnapshotRequest, response, status_codes},
error::ResultExt,
@@ -20,6 +22,7 @@ mod markers {
pub const SCHEMA: &str = "schema";
pub const COPY: &str = "copy";
pub const REST: &str = "rest";
+ pub const SURFACE: &str = "surface";
}
pub async fn router(env: Env, req: Request) -> Result<Response> {
@@ -31,6 +34,18 @@ pub async fn router(env: Env, req: Request) -> Result<Response> {
markers::ROOT => Response::builder().ok("Hello Sync Service!"),
markers::HEALTH => Response::builder().ok("healthy"),
markers::SCHEMA => Response::builder().ok(include_str!("../bebop/schema.bop")),
+ markers::SURFACE => {
+ let Some(id) = matched.params.get("surface_id") else {
+ return Ok(response(status_codes::NOT_FOUND));
+ };
+ let Ok(identity) = SessionIdentity::new(SessionKind::Surface, id) else {
+ return Ok(response(400));
+ };
+ if matched.params.get("rest") == Some("activate") {
+ return activate_surface(&env, req, &identity).await;
+ }
+ pass_to_session(&env, req, &identity.storage_key()).await
+ }
needs_document_id => {
let document_id = matched.params.get("document_id").with_context(|| {
Error::from(format!(
@@ -47,6 +62,60 @@ pub async fn router(env: Env, req: Request) -> Result<Response> {
}
}
+/// Seal the verified original before enabling the imported target. A failed or
+/// ambiguous response is retried forward; the source can no longer be thawed.
+async fn activate_surface(
+ env: &Env,
+ mut req: Request,
+ identity: &SessionIdentity,
+) -> Result<Response> {
+ if !is_internal(&req, env)? {
+ return Ok(response(401));
+ }
+ if req.method() != Method::Post {
+ return Ok(response(405));
+ }
+ let bytes = req.bytes().await?;
+ let Ok(proof) =
+ serde_json::from_slice::<crate::durable_object::surface_migration::SnapshotProof>(&bytes)
+ else {
+ return Ok(response(400));
+ };
+ let Some(source_id) = proof.source_id else {
+ return Ok(response(409));
+ };
+ if identity.storage_key() != format!("surface:{source_id}") {
+ return Ok(response(409));
+ }
+ for (key, path) in [
+ (
+ identity.storage_key(),
+ format!("/surface/{source_id}/verify"),
+ ),
+ (
+ source_id.to_string(),
+ format!("/document/{source_id}/migration/seal"),
+ ),
+ (
+ identity.storage_key(),
+ format!("/surface/{source_id}/activate"),
+ ),
+ ] {
+ let mut url = req.url()?;
+ url.set_path(&path);
+ let mut init = RequestInit::new();
+ init.with_method(Method::Post)
+ .with_headers(req.headers().clone())
+ .with_body(Some(JsValue::from(bytes.clone())));
+ let request = Request::new_with_init(url.as_ref(), &init)?;
+ let result = pass_to_session(env, request, &key).await?;
+ if result.status_code() != status_codes::OK || path.ends_with("/activate") {
+ return Ok(result);
+ }
+ }
+ Ok(response(409))
+}
+
/// Get the original snapshot then initialize a new document with it.
/// Copying interacts with multiple durable objects so we orchestrate it from the worker.
pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Result<Response> {
@@ -133,6 +202,9 @@ pub static ROUTER: LazyLock<matchit::Router<&str>> = LazyLock::new(|| {
.insert("/document/{document_id}/{*rest}", markers::REST)
.unwrap_context("Router.insert failed");
router
+ .insert("/surface/{surface_id}/{*rest}", markers::SURFACE)
+ .unwrap_context("Router.insert failed");
+ router
});
pub async fn pass_to_durable_object(
@@ -140,6 +212,14 @@ pub async fn pass_to_durable_object(
req: Request,
document_id: &str,
) -> Result<Response> {
+ // Legacy document names remain valid, but cannot address the reserved namespace.
+ if document_id.starts_with("surface:") {
+ return Ok(response(400));
+ }
+ pass_to_session(env, req, document_id).await
+}
+
+async fn pass_to_session(env: &Env, req: Request, document_id: &str) -> Result<Response> {
let stub = get_durable_object(env, document_id)?;
let span = tracing::info_span!("do.fetch", document.id = %document_id);
let req = match worker_rs_otel::traceparent_for_span(&span) {
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index d5eba50472..4ab6d9635a 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -7,6 +7,7 @@ use std::{
use bebop::Record;
use loro::{ExportMode, awareness::EphemeralStore};
+use macro_sync_service_jwt::session::SessionKind;
use matchit::Router;
use serde::{Deserialize, Serialize};
use tracing::{Instrument, debug, error, info, instrument, trace, warn};
@@ -18,11 +19,11 @@ use worker::{
use crate::{
ai_peer::is_ai_peer,
- auth::{AccessLevel, TokenFrom, decode_jwt},
+ auth::{AccessLevel, TokenFrom, decode_jwt, socket_access},
constants::USER_PEER_D1_BINDING,
d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},
domain::document::DocumentAttribution,
- dss_internal::{DssInternal, DssInternalClient, InteractionReason},
+ dss_internal::InteractionReason,
error::ResultExt,
generated::schema::InitializeFromSnapshotRequest,
keepalive::{DEFAULT_TIME_TO_LIVE, keepalive},
@@ -49,6 +50,11 @@ const DOCUMENT_ID_KEY: &str = "DOCUMENT_ID";
mod document_api;
mod document_effects;
+mod surface_api;
+pub(crate) mod surface_migration;
+
+use document_effects::{CloseFlush, report_interaction, report_new_doc_state};
+use surface_api::{SurfaceLifecycle, session_kind_from_storage_key};
mod path {
pub const CONNECT: &str = "connect";
@@ -109,6 +115,10 @@ pub struct WebSocketMetadata {
pub actor: Option<String>,
#[serde(with = "u64_serde_strings")]
pub peer_ids: BTreeSet<u64>,
+ #[serde(default = "surface_api::document_kind")]
+ pub session_kind: SessionKind,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub expires_at: Option<usize>,
}
pub type WsMetaMap = BTreeMap<String, WebSocketMetadata>;
@@ -129,43 +139,6 @@ fn edit_attribution_from_meta<'a>(
})
}
-/// Why a snapshot is published from `websocket_close`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum CloseFlush {
- /// The last peer left; the pre-attribution snapshot-on-idle behaviour.
- LastLeave,
- /// The attributed peer left while humans stay. Its pending edits must
- /// publish now, while its metadata still supplies the actor; the next
- /// alarm would publish them unattributed.
- ActorLeft,
-}
-
-impl CloseFlush {
- /// A human leaving while the actor stays never flushes: the next alarm
- /// still has the actor, and flushing here would `mark_exported` and then
- /// republish the same content on last-leave.
- fn decide(
- is_last_leave: bool,
- leaving_socket_has_actor: bool,
- should_save: bool,
- ) -> Option<Self> {
- if is_last_leave {
- Some(Self::LastLeave)
- } else if leaving_socket_has_actor && should_save {
- Some(Self::ActorLeft)
- } else {
- None
- }
- }
-
- fn interaction_reason(self) -> InteractionReason {
- match self {
- Self::LastLeave => InteractionReason::LastLeave,
- Self::ActorLeft => InteractionReason::Edited,
- }
- }
-}
-
#[durable_object]
pub struct DocumentSyncSession {
state: State,
@@ -182,6 +155,11 @@ pub struct DocumentSyncSession {
msg_buffer: Arc<Mutex<Vec<u8>>>,
/// Buffered blame events. Flushed via D1 batch on each alarm tick.
pending_blame: Arc<Mutex<Vec<crate::d1::BlameEvent>>>,
+ surface_lifecycle: Mutex<Option<SurfaceLifecycle>>,
+ source_migration: Mutex<Option<surface_migration::SourceState>>,
+ // Serialize whole callbacks, including persistence awaits. A freeze must
+ // drain every in-flight mutation before it exports or closes writers.
+ mutation_barrier: futures::lock::Mutex<()>,
}
mod u64_serde_strings {
@@ -228,7 +206,8 @@ mod u64_serde_strings {
#[cfg(test)]
mod actor_attribution_test {
use super::{
- AccessLevel, CloseFlush, DocumentAttribution, WebSocketMetadata, edit_attribution_from_meta,
+ AccessLevel, CloseFlush, DocumentAttribution, SessionKind, WebSocketMetadata,
+ edit_attribution_from_meta,
};
fn meta(actor: Option<&str>, user_id: Option<&str>) -> WebSocketMetadata {
@@ -237,6 +216,8 @@ mod actor_attribution_test {
access_level: AccessLevel::Edit,
actor: actor.map(str::to_string),
peer_ids: Default::default(),
+ session_kind: SessionKind::Document,
+ expires_at: None,
}
}
@@ -356,7 +337,6 @@ impl<'a> Wsm<'a> {
.lock("Wsm::can_edit get")
.get(&ws_id)
.ok_or(Error::from("missing ws metadata"))?
- .access_level
.can_edit())
}
@@ -393,43 +373,6 @@ pub fn get_ws_id(state: &State, ws: &WebSocket) -> Result<String> {
get_ws_id_from_tags(&tags)
}
-/// send a shallow snapshot to cache and search service
-/// we should eagerly call this from tiem to time to keep our backend up to date on
-/// the status of the document:
-/// - every few seconds
-/// - on creation
-/// - on everyone being disconnected
-async fn report_new_doc_state(
- document_id: &str,
- snapshot: &[u8],
- env: &Env,
- attribution: Option<DocumentAttribution>,
-) {
- if let Err(err) = DssInternalClient::new(env)
- .publish_shallow_snapshot(document_id, snapshot)
- .await
- {
- warn!(error=?err, "failed to push snapshot to DSS");
- }
- #[cfg(feature = "search-service")]
- if let Err(err) = DssInternalClient::new(env)
- .publish_sync_content_updated(document_id, attribution)
- .await
- {
- warn!(error=?err, "failed to publish document content change");
- }
-}
-
-/// Report an interaction (join/leave/periodic edit) to DSS.
-async fn report_interaction(document_id: &str, env: &Env, reason: InteractionReason) {
- if let Err(err) = DssInternalClient::new(env)
- .publish_interaction(document_id, reason)
- .await
- {
- warn!(error=?err, "failed to push interaction to DSS");
- }
-}
-
/// Schedule an alarm 5 seconds from now.
async fn bump_alarm(state: &State) -> Result<()> {
let current_alarm = state.storage().get_alarm().await?;
@@ -447,7 +390,7 @@ async fn bump_alarm(state: &State) -> Result<()> {
impl DocumentSyncSession {
pub fn get_websockets(&self) -> Vec<WebSocket> {
- self.state.get_websockets()
+ self.active_websockets()
}
fn edit_attribution(&self) -> Option<DocumentAttribution> {
@@ -515,6 +458,16 @@ impl DocumentSyncSession {
}
async fn inner_fetch(&self, req: Request) -> Result<Response> {
let url = req.url()?;
+ if url.path().starts_with("/surface/") {
+ return self.surface_handler(req).await;
+ }
+ let segments: Vec<_> = url.path().split('/').collect();
+ if let ["", "document", id, "migration", operation] = segments.as_slice() {
+ return self.source_migration_handler(req, id, operation).await;
+ }
+ if self.source_blocked().await? {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
let matched = ROUTER
.at(url.path())
.with_context(|| format!("Failed to route url: [{url}]"))?;
@@ -623,7 +576,11 @@ impl DocumentSyncSession {
.put(DOCUMENT_ID_KEY, document_id.to_string())
.await?;
let dkv_storage = DurableKVStorage::new(self.state.storage());
- let session_storage = Rc::new(SessionStorage::new(storage, dkv_storage));
+ let session_storage = Rc::new(SessionStorage::new(
+ storage,
+ dkv_storage,
+ SessionKind::Document,
+ ));
*self
.session_storage
.lock("DocumentSyncSession::session_storage set within initialize_handler") =
@@ -662,8 +619,11 @@ impl DocumentSyncSession {
/// Active peer ids. With `include_ai = false`, AI editors (peer ids from the
/// reserved AI block) are filtered out so callers see only human collaborators.
async fn active_peer_ids_handler(&self, include_ai: bool) -> Result<Response> {
+ if !self.state.get_websockets().is_empty() {
+ self.validate_surface_sockets(None).await?;
+ }
let mut peer_ids: BTreeSet<u64> = BTreeSet::new();
- for ws in self.state.get_websockets() {
+ for ws in self.get_websockets() {
let new_peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;
peer_ids.extend(
new_peer_ids
@@ -815,8 +775,7 @@ impl DocumentSyncSession {
async fn connect_handler(&self, req: Request, document_id: &str) -> Result<Response> {
let (res, elap) = timeit!({
- let claims = or_unauth!(decode_jwt(&req, &self.env, TokenFrom::QueryParams).ok());
- or_unauth!(claims.has_document_id_access(document_id).then_some(()));
+ let claims = or_unauth!(socket_access(&req, &self.env, document_id).ok());
if self.maybe_set_document_id(document_id).await? {
trace!("init document_id={document_id}");
} else {
@@ -842,6 +801,8 @@ impl DocumentSyncSession {
access_level: claims.access_level,
actor: claims.actor,
peer_ids: Default::default(),
+ session_kind: claims.session_kind,
+ expires_at: claims.expires_at,
};
self.state.storage().put(&ws_id, &ws_meta).await?;
@@ -849,6 +810,14 @@ impl DocumentSyncSession {
.lock("DocumentSyncSession::ws_meta_map insert in connect_handler")
.insert(ws_id, ws_meta);
+ // A surface may have been revoked while socket metadata was being
+ // persisted. Recheck before sending any initial content.
+ if claims.session_kind == SessionKind::Surface
+ && !self.validate_surface_sockets(Some(&pair.server)).await?
+ {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
+
// If the snapshot is already in storage, send the initial sync now.
// Otherwise accept the WS without sending — initialize_handler will
// broadcast initial sync to this socket once /initialize lands.
@@ -887,6 +856,9 @@ impl DocumentSyncSession {
});
}
+ if claims.session_kind == SessionKind::Surface {
+ bump_alarm(&self.state).await?;
+ }
Response::from_websocket(pair.client).context("failed to create websocket response")?
});
@@ -935,7 +907,11 @@ impl DocumentSyncSession {
self.maybe_set_document_id(document_id).await?;
// set up session storage
let dkv_storage = DurableKVStorage::new(self.state.storage());
- let session_storage = Rc::new(SessionStorage::new(snapshot_storage, dkv_storage));
+ let session_storage = Rc::new(SessionStorage::new(
+ snapshot_storage,
+ dkv_storage,
+ session_kind_from_storage_key(document_id),
+ ));
*self
.session_storage
.lock("DocumentSyncSession::session_storage set within exists") =
@@ -993,7 +969,11 @@ impl DocumentSyncSession {
let id = self.document_id().await?.to_string();
let snapshot_storage = get_snapshot_storage(&self.env, &self.state, id.clone())?;
let dkv_storage = DurableKVStorage::new(self.state.storage());
- let ss = Rc::new(SessionStorage::new(snapshot_storage, dkv_storage));
+ let ss = Rc::new(SessionStorage::new(
+ snapshot_storage,
+ dkv_storage,
+ session_kind_from_storage_key(&id),
+ ));
*self
.session_storage
.lock("DocumentSyncSession::session_storage set within main session_storage fn") =
@@ -1095,12 +1075,16 @@ impl DurableObject for DocumentSyncSession {
ws_meta_map: Arc::new(Mutex::new(Default::default())),
msg_buffer: Arc::new(Mutex::new(vec![])),
pending_blame: Arc::new(Mutex::new(Vec::new())),
+ surface_lifecycle: Mutex::new(None),
+ source_migration: Mutex::new(None),
+ mutation_barrier: futures::lock::Mutex::new(()),
}
}
/// Fetch the durable object
/// Upgrades the request to a websocket request connected to the document session
async fn fetch(&self, req: Request) -> Result<Response> {
+ let _barrier = self.mutation_barrier.lock().await;
let set_allow_origin = if let Some(origin) = req
.headers()
.get("Origin")
@@ -1144,6 +1128,14 @@ impl DurableObject for DocumentSyncSession {
}
async fn websocket_message(&self, ws: WebSocket, msg: WebSocketIncomingMessage) -> Result<()> {
+ let _barrier = self.mutation_barrier.lock().await;
+ if self.source_blocked().await? {
+ ws.close(Some(1008), Some("session frozen or retired"))?;
+ return Ok(());
+ }
+ if !self.validate_surface_sockets(Some(&ws)).await? {
+ return Ok(());
+ }
const PONG: &str = "pong";
const PING: &str = "ping";
let binary_message = match msg {
@@ -1221,6 +1213,11 @@ impl DurableObject for DocumentSyncSession {
/// Save document if needed
async fn alarm(&self) -> Result<Response> {
+ let _barrier = self.mutation_barrier.lock().await;
+ if self.source_blocked().await? {
+ return Response::empty();
+ }
+ self.validate_surface_sockets(None).await?;
let span = tracing::info_span!("do.alarm");
worker_rs_otel::scope(&self.env, &self.state, async {
let state = match self
@@ -1301,7 +1298,13 @@ impl DurableObject for DocumentSyncSession {
_reason: String,
_was_clean: bool,
) -> Result<()> {
+ let _barrier = self.mutation_barrier.lock().await;
+ if self.source_blocked().await? {
+ self.forget_websocket_metadata(&ws).await;
+ return Ok(());
+ }
worker_rs_otel::scope(&self.env, &self.state, async {
+ self.validate_surface_sockets(None).await?;
let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;
for peer_id in peer_ids {
self.awareness.delete(&peer_id.to_string());
@@ -1310,7 +1313,7 @@ impl DurableObject for DocumentSyncSession {
// Don't silently discard the error
websocket::broadcast_awareness(
&ws,
- self.state.get_websockets().as_slice(),
+ self.get_websockets().as_slice(),
update.as_slice(),
self.msg_buffer.clone(),
)
diff --git a/services/sync-service/src/durable_object/document_effects.rs b/services/sync-service/src/durable_object/document_effects.rs
index 59685b2b41..84d338ad22 100644
--- a/services/sync-service/src/durable_object/document_effects.rs
+++ b/services/sync-service/src/durable_object/document_effects.rs
@@ -1,15 +1,92 @@
//! Worker notification adapter for the document update use case.
use bebop::{Record, SubRecord};
+use macro_sync_service_jwt::session::SessionKind;
use tracing::{error, warn};
+use worker::Env;
-use super::{DocumentSyncSession, bump_alarm, report_interaction, report_new_doc_state};
+use super::{DocumentSyncSession, bump_alarm, surface_api::session_kind_from_storage_key};
use crate::{
domain::document::{DocumentAttribution, DocumentError, DocumentUpdateEffects},
- dss_internal::InteractionReason,
+ dss_internal::{DssInternal, DssInternalClient, InteractionReason},
state::DocumentState,
};
+/// Why a snapshot is published from `websocket_close`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum CloseFlush {
+ /// The last peer left; the pre-attribution snapshot-on-idle behaviour.
+ LastLeave,
+ /// The attributed peer left while humans stay. Publish pending edits while
+ /// its metadata still supplies the actor, before the next unattributed alarm.
+ ActorLeft,
+}
+
+impl CloseFlush {
+ /// A human leaving while the actor stays never flushes: the next alarm
+ /// still has the actor, and flushing here would `mark_exported` and then
+ /// republish the same content on last-leave.
+ pub(super) fn decide(
+ is_last_leave: bool,
+ leaving_socket_has_actor: bool,
+ should_save: bool,
+ ) -> Option<Self> {
+ if is_last_leave {
+ Some(Self::LastLeave)
+ } else if leaving_socket_has_actor && should_save {
+ Some(Self::ActorLeft)
+ } else {
+ None
+ }
+ }
+
+ pub(super) fn interaction_reason(self) -> InteractionReason {
+ match self {
+ Self::LastLeave => InteractionReason::LastLeave,
+ Self::ActorLeft => InteractionReason::Edited,
+ }
+ }
+}
+
+/// All document-only effects pass through this boundary, including socket joins,
+/// initialization, alarms, HTTP updates and disconnects. The persisted storage key
+/// is a kind-discriminated identity; surface keys never reach document consumers.
+pub(super) async fn report_new_doc_state(
+ session_key: &str,
+ snapshot: &[u8],
+ env: &Env,
+ attribution: Option<DocumentAttribution>,
+) {
+ if session_kind_from_storage_key(session_key) != SessionKind::Document {
+ return;
+ }
+ if let Err(error) = DssInternalClient::new(env)
+ .publish_shallow_snapshot(session_key, snapshot)
+ .await
+ {
+ warn!(error = ?error, "failed to push snapshot to DSS");
+ }
+ #[cfg(feature = "search-service")]
+ if let Err(error) = DssInternalClient::new(env)
+ .publish_sync_content_updated(session_key, attribution)
+ .await
+ {
+ warn!(error = ?error, "failed to publish document content change");
+ }
+}
+
+pub(super) async fn report_interaction(session_key: &str, env: &Env, reason: InteractionReason) {
+ if session_kind_from_storage_key(session_key) != SessionKind::Document {
+ return;
+ }
+ if let Err(error) = DssInternalClient::new(env)
+ .publish_interaction(session_key, reason)
+ .await
+ {
+ warn!(error = ?error, "failed to push interaction to DSS");
+ }
+}
+
pub(super) struct WorkerDocumentEffects<'a> {
pub session: &'a DocumentSyncSession,
pub document_state: &'a DocumentState,
diff --git a/services/sync-service/src/durable_object/surface_api.rs b/services/sync-service/src/durable_object/surface_api.rs
new file mode 100644
index 0000000000..54936ac1b0
--- /dev/null
+++ b/services/sync-service/src/durable_object/surface_api.rs
@@ -0,0 +1,313 @@
+//! Isolated surface transport and persisted lifecycle. No document API fallback.
+
+use bebop::Record;
+use macro_sync_service_jwt::session::{SessionIdentity, SessionKind};
+use serde::{Deserialize, Serialize};
+use worker::{Date, Method, Request, Response, Result, WebSocket};
+
+use super::{DocumentSyncSession, WebSocketMetadata, Wsm, response, status_codes};
+use crate::{
+ auth::{TokenFrom, is_internal, surface_access},
+ error::ResultExt,
+ generated::schema::InitializeFromSnapshotRequest,
+ state::DocumentState,
+ storage::{get_snapshot_storage, snapshot::SnapshotStorage},
+};
+
+#[cfg(test)]
+mod test;
+
+pub(super) const LIFECYCLE_KEY: &str = "SURFACE_LIFECYCLE";
+
+/// The discriminant is part of the persisted DOCUMENT_ID key, not the current
+/// request URL. Legacy document keys are unchanged and cannot use this prefix.
+pub(super) fn session_kind_from_storage_key(key: &str) -> SessionKind {
+ if key.starts_with("surface:") {
+ SessionKind::Surface
+ } else {
+ SessionKind::Document
+ }
+}
+
+pub(super) fn document_kind() -> SessionKind {
+ SessionKind::Document
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub(super) enum SurfaceLifecycle {
+ #[default]
+ Pending,
+ /// In-flight initialization; only kept in memory, never considered ready.
+ Initializing,
+ Ready,
+ Revoked,
+}
+
+impl WebSocketMetadata {
+ pub(super) fn grant_active(&self, kind: SessionKind, now: usize) -> bool {
+ self.session_kind == kind
+ && match kind {
+ SessionKind::Document => true,
+ SessionKind::Surface => self.expires_at.is_some_and(|expiry| expiry > now),
+ }
+ }
+
+ pub(super) fn can_edit(&self) -> bool {
+ self.grant_active(self.session_kind, now_seconds())
+ && self.access_level.can_edit_for(self.session_kind)
+ }
+}
+
+/// A pending session has no writers. Exact seed bytes identify a retry of the
+/// binary initializer, which has no separate operation ID. Unknown seeds must
+/// be rolled back before another initialization can write a replacement.
+async fn prepare_surface_snapshot(storage: &impl SnapshotStorage, snapshot: &[u8]) -> Result<bool> {
+ if storage.has_snapshot().await? {
+ if storage.get_snapshot().await? != snapshot {
+ storage.delete_snapshot().await?;
+ return Ok(false);
+ }
+ } else {
+ storage.store_snapshot(snapshot).await?;
+ }
+ Ok(true)
+}
+
+fn now_seconds() -> usize {
+ (Date::now().as_millis() / 1000) as usize
+}
+
+impl DocumentSyncSession {
+ pub(super) async fn surface_lifecycle(&self) -> Result<SurfaceLifecycle> {
+ if let Some(value) = *self.surface_lifecycle.lock("surface lifecycle cached") {
+ return Ok(value);
+ }
+ let value = self
+ .state
+ .storage()
+ .get(LIFECYCLE_KEY)
+ .await?
+ .unwrap_or_default();
+ let mut cached = self.surface_lifecycle.lock("surface lifecycle loaded");
+ // Do not overwrite a transition made while the storage read was pending.
+ Ok(*cached.get_or_insert(value))
+ }
+
+ pub(super) async fn surface_handler(&self, req: Request) -> Result<Response> {
+ let url = req.url()?;
+ let segments: Vec<_> = url.path().split('/').collect();
+ let ["", "surface", id, operation, rest @ ..] = segments.as_slice() else {
+ return Ok(response(status_codes::NOT_FOUND));
+ };
+ let Ok(identity) = SessionIdentity::new(SessionKind::Surface, id) else {
+ return Ok(response(400));
+ };
+ // Explicit allowlist: no document copy, raw, debug, CAS, or wakeup API.
+ let allowed_method = match (*operation, rest) {
+ ("connect" | "exists" | "active_peers", []) | ("peer", [_]) => Method::Get,
+ (
+ "initialize"
+ | "revoke"
+ | "snapshot"
+ | "initialize_verified"
+ | "import"
+ | "verify"
+ | "activate",
+ [],
+ ) => Method::Post,
+ _ => return Ok(response(status_codes::NOT_FOUND)),
+ };
+ if req.method() != allowed_method {
+ return Ok(response(405));
+ }
+ if matches!(
+ *operation,
+ "initialize" | "revoke" | "initialize_verified" | "import" | "verify" | "activate"
+ ) {
+ if !is_internal(&req, &self.env)? {
+ return Ok(response(status_codes::UNAUTH));
+ }
+ } else {
+ let source = if *operation == "connect" {
+ TokenFrom::QueryParams
+ } else {
+ TokenFrom::Headers
+ };
+ if surface_access(&req, &self.env, source, &identity).is_err() {
+ return Ok(response(status_codes::UNAUTH));
+ }
+ }
+ let key = identity.storage_key();
+ let persisted: Option<String> = self.state.storage().get(super::DOCUMENT_ID_KEY).await?;
+ if persisted.is_some_and(|persisted| persisted != key) {
+ return Ok(response(409));
+ }
+ self.maybe_set_document_id(&key).await?;
+ if *operation == "revoke" {
+ return self.revoke_surface().await;
+ }
+ let lifecycle = self.surface_lifecycle().await?;
+ if lifecycle == SurfaceLifecycle::Revoked {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
+ if matches!(
+ *operation,
+ "initialize_verified" | "import" | "verify" | "activate"
+ ) {
+ return self.target_migration_handler(req, &key, operation).await;
+ }
+ if *operation == "initialize" {
+ if self.has_target_migration().await? {
+ return Ok(response(409));
+ }
+ return self.initialize_surface(req, &key).await;
+ }
+ if lifecycle != SurfaceLifecycle::Ready {
+ return Ok(response(409));
+ }
+ // Missing/corrupt snapshots must not trigger create-default-state, even
+ // when that development feature is enabled for documents.
+ self.document_state().await?;
+ if !self.validate_surface_sockets(None).await? {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
+ match *operation {
+ "connect" => self.connect_handler(req, &key).await,
+ "snapshot" => {
+ let snapshot = self.document_state().await?.export_snapshot(None)?;
+ Ok(Response::builder().body(worker::ResponseBody::Body(snapshot)))
+ }
+ "peer" => self.peer_handler(&key, rest.first().copied()).await,
+ "active_peers" => self.active_peer_ids_handler(true).await,
+ "exists" => Ok(response(status_codes::OK)),
+ _ => Ok(response(status_codes::NOT_FOUND)),
+ }
+ }
+
+ async fn initialize_surface(&self, mut req: Request, key: &str) -> Result<Response> {
+ let bytes = req.bytes().await?;
+ let body = InitializeFromSnapshotRequest::deserialize(&bytes)
+ .context("invalid surface snapshot request")?;
+ DocumentState::try_from_snapshot(&body.snapshot).context("invalid surface snapshot")?;
+ // Claim synchronously before awaiting snapshot IO; a second initializer
+ // must not overwrite either the winner or an existing snapshot.
+ match self.surface_lifecycle().await? {
+ SurfaceLifecycle::Revoked => return Ok(response(status_codes::FORBIDDEN)),
+ SurfaceLifecycle::Ready | SurfaceLifecycle::Initializing => return Ok(response(409)),
+ SurfaceLifecycle::Pending => {}
+ }
+ *self.surface_lifecycle.lock("claim surface initialization") =
+ Some(SurfaceLifecycle::Initializing);
+ let result: Result<Response> = async {
+ let storage = get_snapshot_storage(&self.env, &self.state, key.to_owned())?;
+ if !prepare_surface_snapshot(&storage, &body.snapshot).await? {
+ return Ok(response(409));
+ }
+ #[cfg(feature = "migration-test-hooks")]
+ if req.headers().get("x-sync-test-stop-after")?.as_deref() == Some("surface_snapshot") {
+ return Err(worker::Error::from("injected surface persistence failure"));
+ }
+ if self.surface_lifecycle().await? == SurfaceLifecycle::Revoked {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
+ self.state
+ .storage()
+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Ready)
+ .await?;
+ if self.surface_lifecycle().await? == SurfaceLifecycle::Revoked {
+ return Ok(response(status_codes::FORBIDDEN));
+ }
+ *self.surface_lifecycle.lock("surface ready") = Some(SurfaceLifecycle::Ready);
+ Ok(response(status_codes::OK))
+ }
+ .await;
+ if !result
+ .as_ref()
+ .is_ok_and(|response| response.status_code() == status_codes::OK)
+ {
+ let mut lifecycle = self
+ .surface_lifecycle
+ .lock("release surface initialization");
+ if *lifecycle != Some(SurfaceLifecycle::Revoked) {
+ // A failed Ready write may have committed. Reload durable truth
+ // rather than admitting a new initializer against a ready seed.
+ *lifecycle = None;
+ }
+ }
+ result
+ }
+
+ pub(super) async fn revoke_surface(&self) -> Result<Response> {
+ // Deny in-flight callbacks before yielding, and persist the tombstone
+ // before acknowledging. Retrying revocation is safe, including pending IDs.
+ *self.surface_lifecycle.lock("revoke surface") = Some(SurfaceLifecycle::Revoked);
+ self.state
+ .storage()
+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Revoked)
+ .await?;
+ for socket in self.state.get_websockets() {
+ socket.close(Some(1008), Some("surface revoked"))?;
+ }
+ Ok(response(status_codes::OK))
+ }
+
+ /// Reload socket grants after hibernation before accepting messages or
+ /// selecting broadcast recipients. Cached grants avoid per-message IO.
+ pub(crate) async fn validate_surface_sockets(
+ &self,
+ sender: Option<&WebSocket>,
+ ) -> Result<bool> {
+ let key = self.document_id().await?;
+ if session_kind_from_storage_key(&key) == SessionKind::Document {
+ return Ok(true);
+ }
+ let ready = self.surface_lifecycle().await? == SurfaceLifecycle::Ready;
+ let now = now_seconds();
+ for socket in self.state.get_websockets() {
+ let mut wsm = Wsm::new(self, &socket);
+ wsm.maybe_update_ws_meta_map().await?;
+ let ws_id = wsm.get_ws_id()?;
+ let active = ready
+ && self
+ .ws_meta_map
+ .lock("validate surface grant")
+ .get(ws_id)
+ .is_some_and(|meta| meta.grant_active(SessionKind::Surface, now));
+ if !active {
+ socket.close(Some(1008), Some("surface grant expired or revoked"))?;
+ }
+ }
+ // Metadata restoration can yield. Use current lifecycle and time, not
+ // the values observed before those reads, when authorizing the sender.
+ Ok(self.surface_lifecycle().await? == SurfaceLifecycle::Ready
+ && sender.is_none_or(|sender| self.active_websockets().contains(sender)))
+ }
+
+ pub(super) fn active_websockets(&self) -> Vec<WebSocket> {
+ let key = self.document_id.lock("broadcast session identity");
+ let Some(key) = key.as_ref() else {
+ return Vec::new();
+ };
+ if session_kind_from_storage_key(key) == SessionKind::Document {
+ return self.state.get_websockets();
+ }
+ if *self.surface_lifecycle.lock("broadcast surface lifecycle")
+ != Some(SurfaceLifecycle::Ready)
+ {
+ return Vec::new();
+ }
+ let metas = self.ws_meta_map.lock("broadcast surface grants");
+ let now = now_seconds();
+ self.state
+ .get_websockets()
+ .into_iter()
+ .filter(|socket| {
+ super::get_ws_id(&self.state, socket)
+ .ok()
+ .and_then(|id| metas.get(&id))
+ .is_some_and(|meta| meta.grant_active(SessionKind::Surface, now))
+ })
+ .collect()
+ }
+}
diff --git a/services/sync-service/src/durable_object/surface_api/test.rs b/services/sync-service/src/durable_object/surface_api/test.rs
new file mode 100644
index 0000000000..89dc423f37
--- /dev/null
+++ b/services/sync-service/src/durable_object/surface_api/test.rs
@@ -0,0 +1,125 @@
+use super::*;
+use crate::auth::AccessLevel;
+use std::cell::{Cell, RefCell};
+
+#[derive(Default)]
+struct SnapshotStore {
+ snapshot: RefCell<Option<Vec<u8>>>,
+ writes: Cell<usize>,
+ fail_delete: Cell<bool>,
+}
+
+impl SnapshotStorage for SnapshotStore {
+ async fn store_snapshot(&self, snapshot: &[u8]) -> Result<()> {
+ self.writes.set(self.writes.get() + 1);
+ *self.snapshot.borrow_mut() = Some(snapshot.to_vec());
+ Ok(())
+ }
+
+ async fn get_snapshot(&self) -> Result<Vec<u8>> {
+ Ok(self.snapshot.borrow().clone().unwrap())
+ }
+
+ async fn has_snapshot(&self) -> Result<bool> {
+ Ok(self.snapshot.borrow().is_some())
+ }
+
+ async fn delete_snapshot(&self) -> Result<()> {
+ if self.fail_delete.get() {
+ return Err(worker::Error::from("injected deletion failure"));
+ }
+ *self.snapshot.borrow_mut() = None;
+ Ok(())
+ }
+}
+
+#[test]
+fn pending_initialization_resumes_an_identical_seed_without_rewriting() {
+ futures::executor::block_on(async {
+ let storage = SnapshotStore::default();
+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ // Snapshot committed, but the lifecycle write failed or the DO evicted.
+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ assert_eq!(storage.writes.get(), 1);
+ assert_eq!(storage.get_snapshot().await.unwrap(), b"seed");
+ });
+}
+
+#[test]
+fn unknown_pending_seed_is_removed_before_allowing_a_new_initialization() {
+ futures::executor::block_on(async {
+ let storage = SnapshotStore::default();
+ storage.store_snapshot(b"orphan").await.unwrap();
+ assert!(!prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ assert!(!storage.has_snapshot().await.unwrap());
+ assert_eq!(storage.writes.get(), 1);
+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ assert_eq!(storage.get_snapshot().await.unwrap(), b"seed");
+ });
+}
+
+#[test]
+fn failed_rollback_never_overwrites_the_unidentified_snapshot() {
+ futures::executor::block_on(async {
+ let storage = SnapshotStore::default();
+ storage.store_snapshot(b"orphan").await.unwrap();
+ storage.fail_delete.set(true);
+ for _ in 0..2 {
+ assert!(prepare_surface_snapshot(&storage, b"seed").await.is_err());
+ assert_eq!(storage.get_snapshot().await.unwrap(), b"orphan");
+ assert_eq!(storage.writes.get(), 1);
+ }
+ storage.fail_delete.set(false);
+ assert!(!prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());
+ });
+}
+
+#[test]
+fn hibernated_surface_metadata_retains_kind_and_strict_expiry() {
+ let meta = WebSocketMetadata {
+ user_id: Some("user".into()),
+ access_level: AccessLevel::Edit,
+ actor: Some("actor".into()),
+ peer_ids: [u64::MAX].into(),
+ session_kind: SessionKind::Surface,
+ expires_at: Some(100),
+ };
+ let stored = serde_json::to_vec(&meta).unwrap();
+ let restored: WebSocketMetadata = serde_json::from_slice(&stored).unwrap();
+ assert!(restored.grant_active(SessionKind::Surface, 99));
+ assert!(!restored.grant_active(SessionKind::Surface, 100));
+ assert!(!restored.grant_active(SessionKind::Surface, 101));
+ assert!(!restored.grant_active(SessionKind::Document, 99));
+ assert_eq!(restored.actor, meta.actor);
+ assert_eq!(restored.peer_ids, meta.peer_ids);
+}
+
+#[test]
+fn legacy_metadata_is_only_valid_for_documents() {
+ let legacy: WebSocketMetadata = serde_json::from_value(serde_json::json!({
+ "user_id": "user", "access_level": "comment", "peer_ids": []
+ }))
+ .unwrap();
+ assert!(legacy.grant_active(SessionKind::Document, usize::MAX));
+ assert!(!legacy.grant_active(SessionKind::Surface, 0));
+ let missing_expiry = WebSocketMetadata {
+ session_kind: SessionKind::Surface,
+ ..legacy
+ };
+ assert!(!missing_expiry.grant_active(SessionKind::Surface, 0));
+}
+
+#[test]
+fn persisted_storage_key_discriminates_effect_dispatch_after_eviction() {
+ let id = "01952cbd-76ad-7a65-9c21-020304050607";
+ for kind in [SessionKind::Surface, SessionKind::Document] {
+ let identity = SessionIdentity::new(kind, id).unwrap();
+ assert_eq!(session_kind_from_storage_key(&identity.storage_key()), kind);
+ }
+ let revoked = serde_json::to_vec(&SurfaceLifecycle::Revoked).unwrap();
+ assert_eq!(
+ serde_json::from_slice::<SurfaceLifecycle>(&revoked).unwrap(),
+ SurfaceLifecycle::Revoked
+ );
+}
diff --git a/services/sync-service/src/durable_object/surface_migration.rs b/services/sync-service/src/durable_object/surface_migration.rs
new file mode 100644
index 0000000000..9d5878f742
--- /dev/null
+++ b/services/sync-service/src/durable_object/surface_migration.rs
@@ -0,0 +1,477 @@
+//! Recoverable, internal-only session transfers. The caller must first verify
+//! legacy ownership through the documents domain. This module never infers it.
+//!
+//! Freeze drains the callback barrier and preserves the original snapshot/log.
+//! Activation seals the source in the outer worker BEFORE enabling the target:
+//! a crash can leave both unavailable, but can never leave two writable copies.
+//! Sealing is the irreversible commit intent; recovery thereafter is forward-only.
+
+use loro::{Frontiers, VersionVector};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+use worker::{Method, Request, Response, Result};
+
+use super::{
+ DocumentSyncSession, response,
+ surface_api::{LIFECYCLE_KEY, SurfaceLifecycle},
+};
+use crate::{
+ auth::is_internal,
+ error::ResultExt,
+ state::DocumentState,
+ storage::{
+ backends::durable_sql::DurableSQLStorage, get_snapshot_storage, snapshot::SnapshotStorage,
+ },
+};
+
+const SOURCE_KEY: &str = "SURFACE_MIGRATION_SOURCE";
+const TARGET_KEY: &str = "SURFACE_MIGRATION_TARGET";
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct SnapshotProof {
+ pub operation_id: Uuid,
+ pub source_id: Option<Uuid>,
+ pub digest: String,
+ pub content_digest: String,
+ pub revision: Vec<(String, i32)>,
+ pub oplog_revision: Vec<(String, i32)>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct SnapshotExport {
+ proof: SnapshotProof,
+ snapshot: Vec<u8>,
+}
+
+#[derive(Deserialize)]
+struct FreezeRequest {
+ operation_id: Uuid,
+}
+
+#[derive(Deserialize)]
+struct InitializeRequest {
+ operation_id: Uuid,
+ snapshot: Vec<u8>,
+}
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+#[serde(tag = "phase", content = "proof", rename_all = "snake_case")]
+pub(super) enum SourceState {
+ #[default]
+ Live,
+ Freezing(Uuid),
+ Frozen(SnapshotProof),
+ Sealed(SnapshotProof),
+ Retired(SnapshotProof),
+ Thawed(SnapshotProof),
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum TargetPhase {
+ Claimed,
+ Verified,
+ Active,
+}
+
+#[derive(Serialize, Deserialize)]
+struct TargetState {
+ proof: SnapshotProof,
+ phase: TargetPhase,
+}
+
+fn revision(frontiers: Frontiers) -> Vec<(String, i32)> {
+ let mut ids: Vec<_> = frontiers
+ .iter()
+ .map(|id| (id.peer.to_string(), id.counter))
+ .collect();
+ ids.sort();
+ ids
+}
+
+fn digest(bytes: &[u8]) -> String {
+ format!("{:x}", Sha256::digest(bytes))
+}
+
+impl SnapshotProof {
+ fn from_snapshot(operation_id: Uuid, source_id: Option<Uuid>, snapshot: &[u8]) -> Result<Self> {
+ let state = DocumentState::try_from_snapshot(snapshot)?;
+ // Canonical JSON key order, independent of CRDT map iteration order.
+ let mut content: serde_json::Value = serde_json::from_str(&state.get_json())?;
+ content.sort_all_objects();
+ Ok(Self {
+ operation_id,
+ source_id,
+ digest: digest(snapshot),
+ content_digest: digest(&serde_json::to_vec(&content)?),
+ revision: revision(state.loro_doc.state_frontiers()),
+ oplog_revision: revision(state.loro_doc.oplog_frontiers()),
+ })
+ }
+
+ fn verifies(&self, snapshot: &[u8]) -> Result<bool> {
+ Ok(*self == Self::from_snapshot(self.operation_id, self.source_id, snapshot)?)
+ }
+}
+
+/// Fault injection is absent from production builds, even for internal callers.
+fn persistence_boundary(req: &Request, boundary: &str) -> Result<()> {
+ #[cfg(feature = "migration-test-hooks")]
+ if req.headers().get("x-sync-test-stop-after")?.as_deref() == Some(boundary) {
+ return Err(worker::Error::from(
+ "injected migration persistence failure",
+ ));
+ }
+ let _ = (req, boundary);
+ Ok(())
+}
+
+impl DocumentSyncSession {
+ async fn source_state(&self) -> Result<SourceState> {
+ if let Some(value) = self.source_migration.lock("source state").clone() {
+ return Ok(value);
+ }
+ let value: SourceState = self
+ .state
+ .storage()
+ .get(SOURCE_KEY)
+ .await?
+ .unwrap_or_default();
+ *self.source_migration.lock("load source state") = Some(value.clone());
+ Ok(value)
+ }
+
+ pub(super) async fn source_blocked(&self) -> Result<bool> {
+ Ok(!matches!(
+ self.source_state().await?,
+ SourceState::Live | SourceState::Thawed(_)
+ ))
+ }
+
+ async fn store_source(&self, value: SourceState) -> Result<()> {
+ // A failed write may have committed. Reload durable truth on the next
+ // callback instead of retaining a potentially stale writable cache.
+ *self.source_migration.lock("invalidate source state") = None;
+ self.state.storage().put(SOURCE_KEY, &value).await?;
+ *self.source_migration.lock("persist source state") = Some(value);
+ Ok(())
+ }
+
+ fn close_migration_writers(&self) -> Result<()> {
+ for socket in self.state.get_websockets() {
+ socket.close(Some(1008), Some("session frozen or retired"))?;
+ }
+ Ok(())
+ }
+
+ fn export_storage(&self, operation_id: Uuid) -> Result<DurableSQLStorage> {
+ // Separate, chunked local storage: never replace the original snapshot,
+ // never clear its operation log, and never depend on eventual KV reads.
+ DurableSQLStorage::new(
+ self.state.storage(),
+ format!("migration-export:{operation_id}"),
+ )
+ }
+
+ async fn authoritative_snapshot(&self) -> Result<Vec<u8>> {
+ let storage = self.session_storage().await?;
+ // Strict loading: missing snapshots must not invoke create-default-state;
+ // malformed pending entries must not be silently skipped.
+ let state = DocumentState::try_from_snapshot(&storage.get_snapshot().await?)?;
+ let mut updates: Vec<Vec<u8>> = storage
+ .get_pending_operations()
+ .await?
+ .into_iter()
+ .map(|entry| entry.map(|(_, bytes)| bytes))
+ .collect::<Result<_>>()?;
+ if let Some(current) = self.document_state.lock("freeze current state").as_ref() {
+ updates.push(current.export_snapshot(None)?);
+ }
+ let imported = state
+ .loro_doc
+ .import_batch(&updates)
+ .context("replay frozen operation log")?;
+ if imported.pending.is_some() {
+ return Err(worker::Error::from(
+ "migration has unresolved pending operations",
+ ));
+ }
+ if let Some(bytes) = storage.debug_do_kv_get("LAST_VERSION_VECTOR").await? {
+ let saved = VersionVector::decode(&bytes).context("invalid saved revision")?;
+ if !matches!(
+ state.loro_doc.oplog_vv().partial_cmp(&saved),
+ Some(std::cmp::Ordering::Equal | std::cmp::Ordering::Greater)
+ ) {
+ return Err(worker::Error::from(
+ "migration snapshot is behind saved revision",
+ ));
+ }
+ }
+ state.export_snapshot(None)
+ }
+
+ pub(super) async fn source_migration_handler(
+ &self,
+ mut req: Request,
+ id: &str,
+ operation: &str,
+ ) -> Result<Response> {
+ if !is_internal(&req, &self.env)? {
+ return Ok(response(401));
+ }
+ if req.method() != Method::Post {
+ return Ok(response(405));
+ }
+ let Ok(source_id) = Uuid::parse_str(id) else {
+ return Ok(response(400));
+ };
+ if operation == "freeze" {
+ let Ok(body) = req.json::<FreezeRequest>().await else {
+ return Ok(response(400));
+ };
+ return self
+ .freeze_source(&req, id, source_id, body.operation_id)
+ .await;
+ }
+ if !matches!(operation, "thaw" | "seal" | "retire") {
+ return Ok(response(404));
+ }
+ let Ok(proof) = req.json::<SnapshotProof>().await else {
+ return Ok(response(400));
+ };
+ if proof.source_id != Some(source_id) {
+ return Ok(response(409));
+ }
+ let current = self.source_state().await?;
+ let next = match (operation, &current) {
+ ("thaw", SourceState::Frozen(expected) | SourceState::Thawed(expected))
+ if *expected == proof =>
+ {
+ self.maybe_set_document_id(id).await?;
+ let snapshot = self.authoritative_snapshot().await?;
+ // Snapshot serialization need not be byte-stable on replay. Verify
+ // both frontiers and decoded content against the original proof.
+ let actual =
+ SnapshotProof::from_snapshot(proof.operation_id, proof.source_id, &snapshot)?;
+ if actual.revision != proof.revision
+ || actual.oplog_revision != proof.oplog_revision
+ || actual.content_digest != proof.content_digest
+ {
+ return Ok(response(409));
+ }
+ SourceState::Thawed(proof.clone())
+ }
+ (
+ "seal",
+ SourceState::Frozen(expected)
+ | SourceState::Sealed(expected)
+ | SourceState::Retired(expected),
+ ) if *expected == proof => {
+ if matches!(current, SourceState::Retired(_)) {
+ return Response::from_json(&proof);
+ }
+ SourceState::Sealed(proof.clone())
+ }
+ ("retire", SourceState::Sealed(expected) | SourceState::Retired(expected))
+ if *expected == proof =>
+ {
+ SourceState::Retired(proof.clone())
+ }
+ _ => return Ok(response(409)),
+ };
+ self.store_source(next).await?;
+ persistence_boundary(
+ &req,
+ match operation {
+ "thaw" => "source_thawed",
+ "seal" => "source_sealed",
+ _ => "source_retired",
+ },
+ )?;
+ if operation != "thaw" {
+ self.close_migration_writers()?;
+ }
+ Response::from_json(&proof)
+ }
+
+ async fn freeze_source(
+ &self,
+ req: &Request,
+ id: &str,
+ source_id: Uuid,
+ operation_id: Uuid,
+ ) -> Result<Response> {
+ match self.source_state().await? {
+ SourceState::Frozen(proof)
+ | SourceState::Sealed(proof)
+ | SourceState::Retired(proof)
+ if proof.operation_id == operation_id =>
+ {
+ self.close_migration_writers()?;
+ let snapshot = self.export_storage(operation_id)?.get_snapshot().await?;
+ if !proof.verifies(&snapshot)? {
+ return Ok(response(409));
+ }
+ return Response::from_json(&SnapshotExport { proof, snapshot });
+ }
+ SourceState::Freezing(expected) if expected == operation_id => {}
+ SourceState::Live => {}
+ SourceState::Thawed(proof) if proof.operation_id != operation_id => {}
+ _ => return Ok(response(409)),
+ }
+ let storage = get_snapshot_storage(&self.env, &self.state, id.to_owned())?;
+ if !storage.has_snapshot().await? {
+ return Ok(response(404));
+ }
+ self.maybe_set_document_id(id).await?;
+ self.store_source(SourceState::Freezing(operation_id))
+ .await?;
+ persistence_boundary(req, "source_claim")?;
+ self.close_migration_writers()?;
+ let snapshot = self.authoritative_snapshot().await?;
+ let proof = SnapshotProof::from_snapshot(operation_id, Some(source_id), &snapshot)?;
+ self.export_storage(operation_id)?
+ .store_snapshot(&snapshot)
+ .await?;
+ persistence_boundary(req, "source_export")?;
+ self.store_source(SourceState::Frozen(proof.clone()))
+ .await?;
+ persistence_boundary(req, "source_frozen")?;
+ Response::from_json(&SnapshotExport { proof, snapshot })
+ }
+
+ pub(super) async fn has_target_migration(&self) -> Result<bool> {
+ Ok(self
+ .state
+ .storage()
+ .get::<TargetState>(TARGET_KEY)
+ .await?
+ .is_some())
+ }
+
+ pub(super) async fn target_migration_handler(
+ &self,
+ mut req: Request,
+ key: &str,
+ operation: &str,
+ ) -> Result<Response> {
+ if matches!(operation, "initialize_verified" | "import") {
+ let export = if operation == "initialize_verified" {
+ let Ok(body) = req.json::<InitializeRequest>().await else {
+ return Ok(response(400));
+ };
+ let Ok(proof) =
+ SnapshotProof::from_snapshot(body.operation_id, None, &body.snapshot)
+ else {
+ return Ok(response(400));
+ };
+ SnapshotExport {
+ proof,
+ snapshot: body.snapshot,
+ }
+ } else {
+ let Ok(body) = req.json::<SnapshotExport>().await else {
+ return Ok(response(400));
+ };
+ // Legacy surfaces keep their UUID when moving to the isolated
+ // namespace. This also binds source sealing to exactly one target.
+ if body.proof.source_id.map(|id| format!("surface:{id}")) != Some(key.to_owned()) {
+ return Ok(response(409));
+ }
+ body
+ };
+ if !export.proof.verifies(&export.snapshot).unwrap_or(false) {
+ return Ok(response(400));
+ }
+ return self
+ .import_surface(&req, key, export, operation == "initialize_verified")
+ .await;
+ }
+ let Ok(proof) = req.json::<SnapshotProof>().await else {
+ return Ok(response(400));
+ };
+ let Some(mut target) = self.state.storage().get::<TargetState>(TARGET_KEY).await? else {
+ return Ok(response(409));
+ };
+ if target.proof != proof || target.phase == TargetPhase::Claimed {
+ return Ok(response(409));
+ }
+ if target.phase != TargetPhase::Active {
+ let snapshot = get_snapshot_storage(&self.env, &self.state, key.to_owned())?
+ .get_snapshot()
+ .await?;
+ if !proof.verifies(&snapshot)? {
+ return Ok(response(409));
+ }
+ }
+ if operation == "activate" {
+ target.phase = TargetPhase::Active;
+ self.state.storage().put(TARGET_KEY, &target).await?;
+ persistence_boundary(&req, "target_active")?;
+ self.mark_target_ready(&req).await?;
+ }
+ Response::from_json(&target.proof)
+ }
+
+ async fn import_surface(
+ &self,
+ req: &Request,
+ key: &str,
+ export: SnapshotExport,
+ initialize: bool,
+ ) -> Result<Response> {
+ let storage = get_snapshot_storage(&self.env, &self.state, key.to_owned())?;
+ let target: Option<TargetState> = self.state.storage().get(TARGET_KEY).await?;
+ let mut target = if let Some(target) = target {
+ if target.proof != export.proof {
+ return Ok(response(409));
+ }
+ target
+ } else {
+ if self.surface_lifecycle().await? != SurfaceLifecycle::Pending
+ || storage.has_snapshot().await?
+ {
+ return Ok(response(409));
+ }
+ let target = TargetState {
+ proof: export.proof,
+ phase: TargetPhase::Claimed,
+ };
+ self.state.storage().put(TARGET_KEY, &target).await?;
+ persistence_boundary(req, "target_claim")?;
+ target
+ };
+ // Once active, edits are allowed. A replay of the original operation
+ // returns its durable receipt, never overwriting or re-verifying live data.
+ if target.phase != TargetPhase::Active {
+ if !storage.has_snapshot().await? {
+ storage.store_snapshot(&export.snapshot).await?;
+ persistence_boundary(req, "target_snapshot")?;
+ }
+ // Read back and decode, verifying content, both frontiers and bytes.
+ if !target.proof.verifies(&storage.get_snapshot().await?)? {
+ return Ok(response(409));
+ }
+ target.phase = TargetPhase::Verified;
+ self.state.storage().put(TARGET_KEY, &target).await?;
+ persistence_boundary(req, "target_verified")?;
+ }
+ if initialize {
+ target.phase = TargetPhase::Active;
+ self.state.storage().put(TARGET_KEY, &target).await?;
+ persistence_boundary(req, "target_active")?;
+ self.mark_target_ready(req).await?;
+ }
+ Response::from_json(&target.proof)
+ }
+
+ async fn mark_target_ready(&self, req: &Request) -> Result<()> {
+ self.state
+ .storage()
+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Ready)
+ .await?;
+ *self.surface_lifecycle.lock("verified surface ready") = Some(SurfaceLifecycle::Ready);
+ persistence_boundary(req, "target_ready")
+ }
+}
diff --git a/services/sync-service/src/storage.rs b/services/sync-service/src/storage.rs
index 075708df0e..c910627320 100644
--- a/services/sync-service/src/storage.rs
+++ b/services/sync-service/src/storage.rs
@@ -1,4 +1,5 @@
use backends::Storage;
+use macro_sync_service_jwt::session::SessionKind;
use snapshot::SnapshotStorage;
use tracing::trace;
use worker::{Env, Error, Result, State};
@@ -14,6 +15,7 @@ pub mod snapshot;
pub struct SessionStorage {
snapshot_storage: Storage,
oplog: DurableKVStorage,
+ kind: SessionKind,
}
/// Outbound adapter for atomic document updates, sharing the websocket log.
@@ -45,17 +47,20 @@ impl crate::domain::document::DocumentUpdatePort for DocumentUpdateStorage<'_> {
}
impl SessionStorage {
- pub fn new(snapshot_storage: Storage, oplog: DurableKVStorage) -> Self {
+ pub fn new(snapshot_storage: Storage, oplog: DurableKVStorage, kind: SessionKind) -> Self {
Self {
snapshot_storage,
oplog,
+ kind,
}
}
/// Load document state and apply any pending ops
pub async fn load_document_state(&self) -> Result<DocumentState> {
let snapshot = self.get_snapshot().await;
- let state = match (snapshot, cfg!(feature = "create-default-state")) {
+ let allow_default =
+ self.kind == SessionKind::Document && cfg!(feature = "create-default-state");
+ let state = match (snapshot, allow_default) {
(Ok(snapshot), _) => DocumentState::try_from_snapshot(snapshot.as_slice()),
(Err(_e), true) => {
let state = DocumentState::new();
diff --git a/services/sync-service/src/storage/backends/combined_sql_kv.rs b/services/sync-service/src/storage/backends/combined_sql_kv.rs
index 32a3ea0666..15f98580ad 100644
--- a/services/sync-service/src/storage/backends/combined_sql_kv.rs
+++ b/services/sync-service/src/storage/backends/combined_sql_kv.rs
@@ -31,6 +31,13 @@ impl SnapshotStorage for Storage {
}
}
+ async fn delete_snapshot(&self) -> worker::Result<()> {
+ // Remove the fallback first so a partial rollback cannot expose it after
+ // the authoritative SQL copy is removed.
+ self.kv.delete_snapshot().await?;
+ self.sql.delete_snapshot().await
+ }
+
async fn has_snapshot(&self) -> worker::Result<bool> {
Ok(self.sql.has_snapshot().await? || self.kv.has_snapshot().await?)
}
diff --git a/services/sync-service/src/storage/backends/durable_sql.rs b/services/sync-service/src/storage/backends/durable_sql.rs
index 47285bc187..a6169519d0 100644
--- a/services/sync-service/src/storage/backends/durable_sql.rs
+++ b/services/sync-service/src/storage/backends/durable_sql.rs
@@ -103,7 +103,6 @@ impl DurableSQLStorage {
}
/// Delete a key and all its chunks
- #[expect(unused, reason = "We could use this in place of DO KV")]
pub fn delete(&self, key: &str) -> Result<bool> {
let cursor = self.storage.sql().exec(
"DELETE FROM kv_store WHERE id = ?1",
@@ -206,6 +205,11 @@ impl SnapshotStorage for DurableSQLStorage {
Err(e) => Err(e)?,
}
}
+ async fn delete_snapshot(&self) -> worker::Result<()> {
+ self.delete(&self.document_id)?;
+ Ok(())
+ }
+
/// Checks if a snapshot exists in the storage
async fn has_snapshot(&self) -> worker::Result<bool> {
self.exists(&self.document_id)
diff --git a/services/sync-service/src/storage/backends/kv.rs b/services/sync-service/src/storage/backends/kv.rs
index 1033e45062..06ff0de668 100644
--- a/services/sync-service/src/storage/backends/kv.rs
+++ b/services/sync-service/src/storage/backends/kv.rs
@@ -56,6 +56,12 @@ impl SnapshotStorage for Kv {
Ok(snapshot)
}
+ async fn delete_snapshot(&self) -> worker::Result<()> {
+ let key = format!("{}/{}.snapshot", self.document_id, self.document_id);
+ crate::timeout_ez!(self.inner.delete(&key))
+ .with_context(|| format!("failed to delete snapshot: {key}"))
+ }
+
#[tracing::instrument(skip_all, ret)]
async fn has_snapshot(&self) -> worker::Result<bool> {
Ok(!crate::timeout_ez!(
diff --git a/services/sync-service/src/storage/backends/r2.rs b/services/sync-service/src/storage/backends/r2.rs
index 443ec4c841..14a338397a 100644
--- a/services/sync-service/src/storage/backends/r2.rs
+++ b/services/sync-service/src/storage/backends/r2.rs
@@ -84,6 +84,11 @@ impl SnapshotStorage for R2Storage {
Ok(snapshot)
}
+ async fn delete_snapshot(&self) -> worker::Result<()> {
+ let key = format!("{}/{}.snapshot", self.document_id, self.document_id);
+ self.inner.delete(&key).await
+ }
+
async fn has_snapshot(&self) -> worker::Result<bool> {
let key = format!("{}/{}.snapshot", self.document_id, self.document_id);
let has = self.has(&key).await?;
diff --git a/services/sync-service/src/storage/snapshot.rs b/services/sync-service/src/storage/snapshot.rs
index 395935af6d..6460857c46 100644
--- a/services/sync-service/src/storage/snapshot.rs
+++ b/services/sync-service/src/storage/snapshot.rs
@@ -3,6 +3,8 @@ pub trait SnapshotStorage {
async fn store_snapshot(&self, snapshot: &[u8]) -> worker::Result<()>;
/// Retrieves a snpashot from the storage
async fn get_snapshot(&self) -> worker::Result<Vec<u8>>;
+ /// Deletes a snapshot, including any fallback copy. Safe to retry.
+ async fn delete_snapshot(&self) -> worker::Result<()>;
/// Checks if a snapshot exists in the storage
async fn has_snapshot(&self) -> worker::Result<bool>;
}
diff --git a/services/sync-service/src/websocket.rs b/services/sync-service/src/websocket.rs
index b73c5ad7e6..aec4f53fc4 100644
--- a/services/sync-service/src/websocket.rs
+++ b/services/sync-service/src/websocket.rs
@@ -196,6 +196,11 @@ pub async fn process_message(
dss: &DocumentSyncSession,
telemetry: &mut InboundMessageTelemetry,
) -> Result<()> {
+ // Loading the document after hibernation can yield to a revocation. Check
+ // the surface grant again immediately before handling any protocol message.
+ if !dss.validate_surface_sockets(Some(ws)).await? {
+ return Ok(());
+ }
trace!(
message = tracing::field::display(&message),
"process websocket message"
diff --git a/services/sync-service/tests/surface_isolation.test.ts b/services/sync-service/tests/surface_isolation.test.ts
new file mode 100644
index 0000000000..6f5347231d
--- /dev/null
+++ b/services/sync-service/tests/surface_isolation.test.ts
@@ -0,0 +1,369 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { LoroDoc } from "loro-crdt";
+import { createFetchMock, type Miniflare } from "miniflare";
+import { afterAll, beforeAll, expect, test } from "vitest";
+import {
+ FromPeer,
+ FromRemote,
+ InitializeFromSnapshotRequest,
+} from "../bebop/generated/schema";
+import {
+ createTestWebSocket,
+ getTokenForDocument,
+ getTokenForSurface,
+ setupMiniflare,
+ sleep,
+} from "./utils";
+
+let mf: Miniflare;
+const persistPath = mkdtempSync(join(tmpdir(), "surface-isolation-"));
+type TestClient = {
+ socket: ReturnType<typeof createTestWebSocket>;
+ doc: LoroDoc;
+ closed: Promise<number>;
+};
+const effects: string[] = [];
+const fetchMock = createFetchMock();
+fetchMock.disableNetConnect();
+for (const method of ["POST", "PUT"]) {
+ fetchMock
+ .get("https://dss.test")
+ .intercept({ path: /.*/, method })
+ .reply(200, (options) => {
+ effects.push(options.path);
+ return "";
+ })
+ .persist();
+}
+beforeAll(async () => {
+ mf = await setupMiniflare({ persistPath, fetchMock });
+}, 60_000);
+afterAll(async () => {
+ await mf?.dispose();
+ rmSync(persistPath, { recursive: true, force: true });
+});
+
+function url(
+ kind: "document" | "surface",
+ id: string,
+ operation: string,
+): string {
+ return `http://localhost/${kind}/${id}/${operation}`;
+}
+function token(kind: "document" | "surface", id: string): string {
+ return kind === "surface"
+ ? getTokenForSurface(id)
+ : getTokenForDocument(id, "document-user", "edit");
+}
+async function initialize(
+ kind: "document" | "surface",
+ id: string,
+ content: string,
+): Promise<number> {
+ const doc = new LoroDoc();
+ doc.getText("content").insert(0, content);
+ const body = InitializeFromSnapshotRequest.encode({
+ snapshot: doc.export({ mode: "snapshot" }),
+ });
+ doc.free();
+ const response = await mf.dispatchFetch(url(kind, id, "initialize"), {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ body,
+ });
+ return response.status;
+}
+async function connect(
+ kind: "document" | "surface",
+ id: string,
+ grant = token(kind, id),
+): ReturnType<Miniflare["dispatchFetch"]> {
+ return mf.dispatchFetch(`${url(kind, id, "connect")}?token=${grant}`, {
+ headers: { Upgrade: "websocket" },
+ });
+}
+async function client(
+ kind: "document" | "surface",
+ id: string,
+ grant = token(kind, id),
+): Promise<TestClient> {
+ const response = await connect(kind, id, grant);
+ expect(response.status).toBe(101);
+ const ws = response.webSocket!;
+ const socket = createTestWebSocket(ws);
+ const closed = new Promise<number>((resolve) =>
+ ws.addEventListener("close", (event) => resolve(event.code)),
+ );
+ ws.accept();
+ const message = FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ expect(message.isRemoteInitialSync()).toBe(true);
+ if (!message.isRemoteInitialSync())
+ throw new Error("missing initial snapshot");
+ const doc = new LoroDoc();
+ doc.import(message.value.snapshot);
+ return { socket, doc, closed };
+}
+async function content(
+ kind: "document" | "surface",
+ id: string,
+): Promise<string> {
+ const response = await mf.dispatchFetch(url(kind, id, "snapshot"), {
+ method: "POST",
+ headers: { Authorization: `Bearer ${token(kind, id)}` },
+ });
+ expect(response.status).toBe(200);
+ const doc = new LoroDoc();
+ doc.import(new Uint8Array(await response.arrayBuffer()));
+ const text = doc.getText("content").toString();
+ doc.free();
+ return text;
+}
+function write(entry: TestClient, text: string): void {
+ entry.doc.getText("content").push(text);
+ entry.doc.commit();
+ entry.socket.send(
+ FromPeer.fromPeerUpdate({
+ updates: [entry.doc.export({ mode: "update" })],
+ id: crypto.randomUUID(),
+ }).encode(),
+ );
+}
+
+test("same UUID has distinct grants, snapshots, broadcasts, peers and effects", async () => {
+ const id = crypto.randomUUID();
+ expect(await initialize("document", id, "document")).toBe(200);
+ expect(await initialize("surface", id, "surface")).toBe(200);
+ expect((await connect("document", id, token("surface", id))).status).toBe(
+ 401,
+ );
+ expect((await connect("surface", id, token("document", id))).status).toBe(
+ 401,
+ );
+ for (const [kind, other] of [
+ ["document", "surface"],
+ ["surface", "document"],
+ ] as const) {
+ expect(
+ (
+ await mf.dispatchFetch(url(kind, id, "snapshot"), {
+ method: "POST",
+ headers: { Authorization: `Bearer ${token(other, id)}` },
+ })
+ ).status,
+ ).toBe(401);
+ }
+ expect(
+ (await connect("document", `surface:${id}`, token("surface", id))).status,
+ ).toBe(400);
+ expect(
+ (await connect("surface", "invalid-id", token("surface", id))).status,
+ ).toBe(400);
+ const document = await client("document", id);
+ const surface = await client("surface", id);
+ const observer = await client("surface", id);
+ expect(document.doc.getText("content").toString()).toBe("document");
+ expect(surface.doc.getText("content").toString()).toBe("surface");
+ const peer = 12345n;
+ for (const entry of [document, surface])
+ entry.socket.send(FromPeer.fromPeerRegisterId({ peerid: peer }).encode());
+ write(surface, " edited");
+ const ack = FromRemote.decode(
+ new Uint8Array((await surface.socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ expect(ack.isRemoteUpdateAck()).toBe(true);
+ const broadcast = FromRemote.decode(
+ new Uint8Array((await observer.socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ expect(broadcast.isRemoteUpdate()).toBe(true);
+ expect(await content("document", id)).toBe("document");
+ expect(await content("surface", id)).toBe("surface edited");
+ expect(await content("surface", id.toUpperCase())).toBe("surface edited");
+ const db = await mf.getD1Database("USER_PEER_MAPPING");
+ const peers = await db
+ .prepare("SELECT document_id, user_id FROM peer_user_map WHERE peer_id = ?")
+ .bind(peer.toString())
+ .all();
+ expect(peers.results).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ document_id: id, user_id: "document-user" }),
+ expect.objectContaining({
+ document_id: `surface:${id}`,
+ user_id: "surface-user",
+ }),
+ ]),
+ );
+ for (const entry of [document, surface, observer])
+ entry.socket.getWebSocket().close();
+ await sleep(5500);
+ expect(effects.some((path) => path.includes(id))).toBe(true);
+ expect(effects.every((path) => !path.includes("surface:"))).toBe(true);
+}, 20_000);
+
+test("initialization races cannot overwrite a winner or revive a tombstone", async () => {
+ const id = crypto.randomUUID();
+ const initialized = await Promise.all([
+ initialize("surface", id, "first"),
+ initialize("surface", id, "second"),
+ ]);
+ expect([...initialized].sort()).toEqual([200, 409]);
+ expect(await content("surface", id)).toBe(
+ initialized[0] === 200 ? "first" : "second",
+ );
+ const retiring = crypto.randomUUID();
+ const [initialization, revocation] = await Promise.all([
+ initialize("surface", retiring, "seed"),
+ mf.dispatchFetch(url("surface", retiring, "revoke"), {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ }),
+ ]);
+ expect([200, 403]).toContain(initialization);
+ expect(revocation.status).toBe(200);
+ expect((await connect("surface", retiring)).status).toBe(403);
+ expect(await initialize("surface", retiring, "revive")).toBe(403);
+});
+
+test("Comment cannot write surfaces but retains document websocket behavior", async () => {
+ const id = crypto.randomUUID();
+ for (const kind of ["document", "surface"] as const)
+ expect(await initialize(kind, id, "")).toBe(200);
+ const surface = await client(
+ "surface",
+ id,
+ getTokenForSurface(id, "comment"),
+ );
+ write(surface, "denied");
+ const document = await client(
+ "document",
+ id,
+ getTokenForDocument(id, "commenter", "comment"),
+ );
+ write(document, "allowed");
+ await document.socket.waitForNextMessage();
+ expect(await content("surface", id)).toBe("");
+ expect(await content("document", id)).toBe("allowed");
+});
+
+test("pending, revoked and expired sessions fail closed; unsupported endpoints stay unavailable", async () => {
+ const id = crypto.randomUUID();
+ expect((await connect("surface", id)).status).toBe(409);
+ for (const operation of [
+ "raw",
+ "state",
+ "update",
+ "copy",
+ "debug_dump_operations",
+ "debug_do_kv_list/",
+ ]) {
+ const response = await mf.dispatchFetch(url("surface", id, operation), {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ });
+ expect(response.status).toBe(404);
+ }
+ expect(
+ (
+ await mf.dispatchFetch(url("surface", id, "initialize"), {
+ method: "POST",
+ headers: { Authorization: `Bearer ${getTokenForSurface(id, "owner")}` },
+ })
+ ).status,
+ ).toBe(401);
+ expect(await initialize("surface", id, "ready")).toBe(200);
+ expect(
+ (
+ await connect(
+ "surface",
+ id,
+ getTokenForSurface(id, "edit", Math.floor(Date.now() / 1000) - 1),
+ )
+ ).status,
+ ).toBe(401);
+ const active = await client("surface", id);
+ expect(
+ (
+ await mf.dispatchFetch(url("surface", id, "revoke"), {
+ method: "POST",
+ headers: { Authorization: `Bearer ${token("surface", id)}` },
+ })
+ ).status,
+ ).toBe(401);
+ expect(
+ (
+ await mf.dispatchFetch(url("surface", id, "revoke"), {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ })
+ ).status,
+ ).toBe(200);
+ write(active, "denied");
+ active.socket.send(FromPeer.fromPeerRequestSnapshot({}).encode());
+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(
+ "Timed out",
+ );
+ expect((await connect("surface", id)).status).toBe(403);
+ expect(await initialize("surface", id, "revive")).toBe(403);
+ // This workerd version buffers close delivery until teardown, even for a
+ // minimal JS Durable Object. Check the actual server close code then restart.
+ await mf.dispose();
+ expect(await active.closed).toBe(1008);
+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });
+ expect((await connect("surface", id)).status).toBe(403);
+});
+
+test("surface expiry stops idle sockets and survives persisted state reload", async () => {
+ const id = crypto.randomUUID();
+ expect(await initialize("surface", id, "persisted")).toBe(200);
+ const active = await client(
+ "surface",
+ id,
+ getTokenForSurface(id, "edit", Math.floor(Date.now() / 1000) + 2),
+ );
+ await sleep(5500);
+ write(active, "denied");
+ active.socket.send(FromPeer.fromPeerRequestSnapshot({}).encode());
+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(
+ "Timed out",
+ );
+ expect(await content("surface", id)).toBe("persisted");
+ const writer = await client("surface", id);
+ write(writer, " live");
+ await writer.socket.waitForNextMessage();
+ expect(await content("surface", id)).toBe("persisted live");
+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(
+ "Timed out",
+ );
+ expect(effects.some((path) => path.includes(id))).toBe(false);
+ const revoked = crypto.randomUUID();
+ await mf.dispatchFetch(url("surface", revoked, "revoke"), {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ });
+ await mf.dispose();
+ expect(await active.closed).toBe(1008);
+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });
+ expect((await connect("surface", revoked)).status).toBe(403);
+ expect(await content("surface", id)).toBe("persisted live");
+ expect(effects.some((path) => path.includes(id))).toBe(false);
+}, 30_000);
+
+test("same-UUID operation logs replay independently after eviction before the snapshot alarm", async () => {
+ const id = crypto.randomUUID();
+ for (const kind of ["surface", "document"] as const) {
+ expect(await initialize(kind, id, kind)).toBe(200);
+ const writer = await client(kind, id);
+ write(writer, " persisted op");
+ const ack = FromRemote.decode(
+ new Uint8Array((await writer.socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ expect(ack.isRemoteUpdateAck()).toBe(true);
+ }
+ await mf.dispose();
+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });
+ expect(await content("surface", id)).toBe("surface persisted op");
+ expect(await content("document", id)).toBe("document persisted op");
+});
diff --git a/services/sync-service/tests/surface_migration.test.ts b/services/sync-service/tests/surface_migration.test.ts
new file mode 100644
index 0000000000..8791aaa240
--- /dev/null
+++ b/services/sync-service/tests/surface_migration.test.ts
@@ -0,0 +1,560 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { LoroDoc } from "loro-crdt";
+import type { Miniflare } from "miniflare";
+import { afterAll, beforeAll, expect, test } from "vitest";
+import {
+ FromPeer,
+ FromRemote,
+ InitializeFromSnapshotRequest,
+} from "../bebop/generated/schema";
+import {
+ createTestWebSocket,
+ getTokenForDocument,
+ getTokenForSurface,
+ setupMiniflare,
+} from "./utils";
+
+type Proof = {
+ operation_id: string;
+ source_id: string | null;
+ digest: string;
+ content_digest: string;
+ revision: [string, number][];
+ oplog_revision: [string, number][];
+};
+type Export = { proof: Proof; snapshot: number[] };
+let mf: Miniflare;
+const persistPath = mkdtempSync(join(tmpdir(), "surface-migration-"));
+beforeAll(async () => {
+ mf = await setupMiniflare({ persistPath });
+}, 60_000);
+afterAll(async () => {
+ await mf?.dispose();
+ rmSync(persistPath, { recursive: true, force: true });
+});
+async function restart(): Promise<void> {
+ await mf.dispose();
+ mf = await setupMiniflare({ persistPath, migrate: false });
+}
+async function post(
+ kind: string,
+ id: string,
+ operation: string,
+ body: unknown,
+ stop?: string,
+): ReturnType<Miniflare["dispatchFetch"]> {
+ return mf.dispatchFetch(`http://localhost/${kind}/${id}/${operation}`, {
+ method: "POST",
+ headers: {
+ "x-internal-auth-key": "local",
+ "Content-Type": "application/json",
+ ...(stop ? { "x-sync-test-stop-after": stop } : {}),
+ },
+ body: JSON.stringify(body),
+ });
+}
+async function seed(id: string): Promise<void> {
+ const doc = new LoroDoc();
+ doc.getText("content").push("original");
+ const response = await mf.dispatchFetch(
+ `http://localhost/document/${id}/initialize`,
+ {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ body: InitializeFromSnapshotRequest.encode({
+ snapshot: doc.export({ mode: "snapshot" }),
+ }),
+ },
+ );
+ doc.free();
+ expect(response.status).toBe(200);
+}
+async function connect(
+ kind: "surface" | "document",
+ id: string,
+): ReturnType<Miniflare["dispatchFetch"]> {
+ const token =
+ kind === "surface"
+ ? getTokenForSurface(id)
+ : getTokenForDocument(id, "writer", "edit");
+ return mf.dispatchFetch(
+ `http://localhost/${kind}/${id}/connect?token=${token}`,
+ { headers: { Upgrade: "websocket" } },
+ );
+}
+async function freeze(
+ id: string,
+ operation_id = crypto.randomUUID(),
+): Promise<Export> {
+ const response = await post("document", id, "migration/freeze", {
+ operation_id,
+ });
+ expect(response.status).toBe(200);
+ return response.json<Export>();
+}
+function text(snapshot: number[]): string {
+ const doc = new LoroDoc();
+ doc.import(new Uint8Array(snapshot));
+ const value = doc.getText("content").toString();
+ doc.free();
+ return value;
+}
+
+test("freeze drains acknowledged and racing edits, closes writers, and replays pending logs after restart", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const connected = await connect("document", id);
+ const ws = connected.webSocket!;
+ const socket = createTestWebSocket(ws);
+ const closed = new Promise<number>((resolve) =>
+ ws.addEventListener("close", (event) => resolve(event.code)),
+ );
+ ws.accept();
+ const initial = FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");
+ const doc = new LoroDoc();
+ doc.import(initial.value.snapshot);
+ function edit(value: string): void {
+ doc.getText("content").push(value);
+ socket.send(
+ FromPeer.fromPeerUpdate({
+ updates: [doc.export({ mode: "update" })],
+ id: crypto.randomUUID(),
+ }).encode(),
+ );
+ }
+ edit(" acknowledged");
+ expect(
+ FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ ).isRemoteUpdateAck(),
+ ).toBe(true);
+ edit(" racing");
+ const exported = await freeze(id);
+ expect(text(exported.snapshot)).toMatch(/^original acknowledged( racing)?$/);
+ // If the racing mutation is acknowledged, it must be present in the export.
+ const racing = await socket.waitForNextMessage(100).catch(() => undefined);
+ if (racing)
+ expect(text(exported.snapshot)).toBe("original acknowledged racing");
+ expect((await connect("document", id)).status).toBe(403);
+ await restart();
+ expect(await closed).toBe(1008);
+ expect(await freeze(id, exported.proof.operation_id)).toEqual(exported);
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ expect((await connect("surface", id)).status).toBe(409);
+ expect((await post("surface", id, "verify", exported.proof)).status).toBe(
+ 200,
+ );
+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(
+ 200,
+ );
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(409);
+ expect(
+ (await post("document", id, "migration/retire", exported.proof)).status,
+ ).toBe(200);
+ await restart();
+ expect((await connect("document", id)).status).toBe(403);
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(409);
+ const migrated = await connect("surface", id);
+ expect(migrated.status).toBe(101);
+ migrated.webSocket!.accept();
+ migrated.webSocket!.close();
+ expect((await post("surface", id, "revoke", {})).status).toBe(200);
+ await restart();
+ expect((await connect("surface", id)).status).toBe(403);
+ expect((await post("surface", id, "import", exported)).status).toBe(403);
+ doc.free();
+}, 30_000);
+
+test("verified rollback is allowed only before sealing; stale operations and conflicting targets fail closed", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const exported = await freeze(id);
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ expect(
+ (
+ await post("surface", id, "import", {
+ ...exported,
+ proof: { ...exported.proof, operation_id: crypto.randomUUID() },
+ })
+ ).status,
+ ).toBe(409);
+ expect(
+ (await post("surface", id, "import", { ...exported, snapshot: [1, 2, 3] }))
+ .status,
+ ).toBe(400);
+ expect(
+ (await post("document", id, "migration/retire", exported.proof)).status,
+ ).toBe(409);
+ expect(
+ (
+ await post("document", id, "migration/thaw", {
+ ...exported.proof,
+ digest: "wrong",
+ })
+ ).status,
+ ).toBe(409);
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(200);
+ await restart();
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(200);
+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(
+ 409,
+ );
+ expect(
+ (
+ await post("document", id, "migration/freeze", {
+ operation_id: exported.proof.operation_id,
+ })
+ ).status,
+ ).toBe(409);
+ const next = await freeze(id);
+ expect(text(next.snapshot)).toBe("original");
+ expect((await post("surface", id, "import", next)).status).toBe(409);
+});
+
+test("internal-only operations and missing sources never create blank replacement content", async () => {
+ const id = crypto.randomUUID();
+ expect(
+ (
+ await post("document", id, "migration/freeze", {
+ operation_id: crypto.randomUUID(),
+ })
+ ).status,
+ ).toBe(404);
+ for (const [kind, operation] of [
+ ["document", "migration/freeze"],
+ ["document", "migration/thaw"],
+ ["document", "migration/seal"],
+ ["document", "migration/retire"],
+ ["surface", "import"],
+ ["surface", "verify"],
+ ["surface", "activate"],
+ ["surface", "initialize_verified"],
+ ]) {
+ const response = await mf.dispatchFetch(
+ `http://localhost/${kind}/${id}/${operation}`,
+ { method: "POST" },
+ );
+ expect(response.status).toBe(401);
+ }
+});
+
+test("isolated initialization acknowledges only the identical verified operation and seed", async () => {
+ const id = crypto.randomUUID();
+ const doc = new LoroDoc();
+ doc.getText("content").push("seed");
+ const body = {
+ operation_id: crypto.randomUUID(),
+ snapshot: Array.from(doc.export({ mode: "snapshot" })),
+ };
+ const response = await post("surface", id, "initialize_verified", body);
+ expect(response.status).toBe(200);
+ const receipt = await response.json();
+ await restart();
+ expect(
+ await (await post("surface", id, "initialize_verified", body)).json(),
+ ).toEqual(receipt);
+ expect(
+ (
+ await post("surface", id, "initialize_verified", {
+ ...body,
+ operation_id: crypto.randomUUID(),
+ })
+ ).status,
+ ).toBe(409);
+ doc.getText("content").push("different");
+ expect(
+ (
+ await post("surface", id, "initialize_verified", {
+ ...body,
+ snapshot: Array.from(doc.export({ mode: "snapshot" })),
+ })
+ ).status,
+ ).toBe(409);
+ doc.free();
+});
+
+test("cold freeze replays acknowledged operation logs and rollback preserves the original", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const response = await connect("document", id);
+ const ws = response.webSocket!;
+ const socket = createTestWebSocket(ws);
+ ws.accept();
+ const initial = FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");
+ const doc = new LoroDoc();
+ doc.import(initial.value.snapshot);
+ doc.getText("content").push(" pending");
+ socket.send(
+ FromPeer.fromPeerUpdate({
+ updates: [doc.export({ mode: "update" })],
+ id: crypto.randomUUID(),
+ }).encode(),
+ );
+ expect(
+ FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ ).isRemoteUpdateAck(),
+ ).toBe(true);
+ await restart();
+ const exported = await freeze(id);
+ expect(text(exported.snapshot)).toBe("original pending");
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(200);
+ const original = await mf.dispatchFetch(
+ `http://localhost/document/${id}/snapshot`,
+ {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ },
+ );
+ expect(text(Array.from(new Uint8Array(await original.arrayBuffer())))).toBe(
+ "original pending",
+ );
+ doc.free();
+});
+
+test("competing freezes and activation versus rollback never yield two writable copies", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const results = await Promise.all([
+ post("document", id, "migration/freeze", {
+ operation_id: crypto.randomUUID(),
+ }),
+ post("document", id, "migration/freeze", {
+ operation_id: crypto.randomUUID(),
+ }),
+ ]);
+ expect(results.map((response) => response.status).sort()).toEqual([200, 409]);
+ const exported = await results
+ .find((response) => response.status === 200)!
+ .json<Export>();
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ const [activation, rollback] = await Promise.all([
+ post("surface", id, "activate", exported.proof),
+ post("document", id, "migration/thaw", exported.proof),
+ ]);
+ expect([activation.status, rollback.status].sort()).toEqual([200, 409]);
+ await restart();
+ const source = await connect("document", id);
+ const target = await connect("surface", id);
+ expect(
+ [source.status, target.status].filter((status) => status === 101),
+ ).toHaveLength(1);
+ for (const response of [source, target]) {
+ response.webSocket?.accept();
+ response.webSocket?.close();
+ }
+});
+
+test("an occupied target and forged revision/digest are not overwritten", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const exported = await freeze(id);
+ for (const changed of [
+ { digest: "bad" },
+ { content_digest: "bad" },
+ { revision: [] },
+ { oplog_revision: [] },
+ ]) {
+ expect(
+ (
+ await post("surface", id, "import", {
+ ...exported,
+ proof: { ...exported.proof, ...changed },
+ })
+ ).status,
+ ).toBe(400);
+ }
+ const doc = new LoroDoc();
+ doc.getText("content").push("unrelated target");
+ const occupied = await mf.dispatchFetch(
+ `http://localhost/surface/${id}/initialize`,
+ {
+ method: "POST",
+ headers: { "x-internal-auth-key": "local" },
+ body: InitializeFromSnapshotRequest.encode({
+ snapshot: doc.export({ mode: "snapshot" }),
+ }),
+ },
+ );
+ expect(occupied.status).toBe(200);
+ expect((await post("surface", id, "import", exported)).status).toBe(409);
+ const snapshot = await mf.dispatchFetch(
+ `http://localhost/surface/${id}/snapshot`,
+ {
+ method: "POST",
+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },
+ },
+ );
+ expect(text(Array.from(new Uint8Array(await snapshot.arrayBuffer())))).toBe(
+ "unrelated target",
+ );
+ doc.free();
+});
+
+test("retries after activation preserve subsequent acknowledged surface edits", async () => {
+ const id = crypto.randomUUID();
+ await seed(id);
+ const exported = await freeze(id);
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(
+ 200,
+ );
+ const response = await connect("surface", id);
+ const ws = response.webSocket!;
+ const socket = createTestWebSocket(ws);
+ ws.accept();
+ const initial = FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ );
+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");
+ const doc = new LoroDoc();
+ doc.import(initial.value.snapshot);
+ doc.getText("content").push(" after activation");
+ socket.send(
+ FromPeer.fromPeerUpdate({
+ updates: [doc.export({ mode: "update" })],
+ id: crypto.randomUUID(),
+ }).encode(),
+ );
+ expect(
+ FromRemote.decode(
+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),
+ ).isRemoteUpdateAck(),
+ ).toBe(true);
+ await restart();
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(
+ 200,
+ );
+ const snapshot = await mf.dispatchFetch(
+ `http://localhost/surface/${id}/snapshot`,
+ {
+ method: "POST",
+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },
+ },
+ );
+ expect(text(Array.from(new Uint8Array(await snapshot.arrayBuffer())))).toBe(
+ "original after activation",
+ );
+ doc.free();
+});
+
+// Run with a migration-test-hooks worker build to inject a crash immediately
+// after each durable write. Production builds do not recognize this header.
+const faultTests =
+ process.env.SYNC_MIGRATION_FAULT_TESTS === "1" ? test : test.skip;
+faultTests.each([
+ "source_claim",
+ "source_export",
+ "source_frozen",
+ "target_claim",
+ "target_snapshot",
+ "target_verified",
+ "source_sealed",
+ "target_active",
+ "target_ready",
+ "source_retired",
+ "source_thawed",
+])(
+ "recovers after persistence boundary %s",
+ async (boundary) => {
+ const id = crypto.randomUUID();
+ const operation_id = crypto.randomUUID();
+ await seed(id);
+ if (
+ boundary.startsWith("source_") &&
+ ["source_claim", "source_export", "source_frozen"].includes(boundary)
+ ) {
+ expect(
+ (
+ await post(
+ "document",
+ id,
+ "migration/freeze",
+ { operation_id },
+ boundary,
+ )
+ ).status,
+ ).toBe(500);
+ await restart();
+ }
+ const exported = await freeze(id, operation_id);
+ if (boundary === "source_thawed") {
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof, boundary))
+ .status,
+ ).toBe(500);
+ await restart();
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(200);
+ expect(
+ (await post("document", id, "migration/seal", exported.proof)).status,
+ ).toBe(409);
+ return;
+ }
+ if (
+ ["target_claim", "target_snapshot", "target_verified"].includes(boundary)
+ ) {
+ expect(
+ (await post("surface", id, "import", exported, boundary)).status,
+ ).toBe(500);
+ await restart();
+ // The old initializer cannot steal a claimed-but-not-yet-written target.
+ expect((await post("surface", id, "initialize", {})).status).toBe(409);
+ }
+ expect((await post("surface", id, "import", exported)).status).toBe(200);
+ if (["source_sealed", "target_active", "target_ready"].includes(boundary)) {
+ expect(
+ (await post("surface", id, "activate", exported.proof, boundary))
+ .status,
+ ).toBe(500);
+ await restart();
+ expect(
+ (await post("document", id, "migration/thaw", exported.proof)).status,
+ ).toBe(409);
+ }
+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(
+ 200,
+ );
+ if (boundary === "source_retired") {
+ expect(
+ (
+ await post(
+ "document",
+ id,
+ "migration/retire",
+ exported.proof,
+ boundary,
+ )
+ ).status,
+ ).toBe(500);
+ await restart();
+ }
+ expect(
+ (await post("document", id, "migration/retire", exported.proof)).status,
+ ).toBe(200);
+ expect(await freeze(id, operation_id)).toEqual(exported);
+ expect((await connect("document", id)).status).toBe(403);
+ },
+ 30_000,
+);
diff --git a/services/sync-service/tests/surface_recovery.test.ts b/services/sync-service/tests/surface_recovery.test.ts
new file mode 100644
index 0000000000..24a3306f67
--- /dev/null
+++ b/services/sync-service/tests/surface_recovery.test.ts
@@ -0,0 +1,122 @@
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { LoroDoc } from "loro-crdt";
+import type { Miniflare } from "miniflare";
+import { afterAll, beforeAll, expect, test } from "vitest";
+import { InitializeFromSnapshotRequest } from "../bebop/generated/schema";
+import { getTokenForSurface, setupMiniflare } from "./utils";
+
+let mf: Miniflare;
+const persistPath = mkdtempSync(join(tmpdir(), "surface-recovery-"));
+beforeAll(async () => {
+ mf = await setupMiniflare({ persistPath });
+}, 60_000);
+afterAll(async () => {
+ await mf?.dispose();
+ rmSync(persistPath, { recursive: true, force: true });
+});
+
+function seed(content: string): Uint8Array {
+ const doc = new LoroDoc();
+ doc.getText("content").insert(0, content);
+ const snapshot = doc.export({ mode: "snapshot" });
+ doc.free();
+ return snapshot;
+}
+
+async function initialize(id: string, snapshot: Uint8Array, stop = false) {
+ return mf.dispatchFetch(`http://localhost/surface/${id}/initialize`, {
+ method: "POST",
+ headers: {
+ "x-internal-auth-key": "local",
+ ...(stop ? { "x-sync-test-stop-after": "surface_snapshot" } : {}),
+ },
+ body: InitializeFromSnapshotRequest.encode({ snapshot }),
+ });
+}
+
+async function exists(id: string): Promise<number> {
+ return (
+ await mf.dispatchFetch(`http://localhost/surface/${id}/exists`, {
+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },
+ })
+ ).status;
+}
+
+async function content(id: string): Promise<string> {
+ const response = await mf.dispatchFetch(`http://localhost/surface/${id}/snapshot`, {
+ method: "POST",
+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },
+ });
+ expect(response.status).toBe(200);
+ const doc = new LoroDoc();
+ doc.import(new Uint8Array(await response.arrayBuffer()));
+ const text = doc.getText("content").toString();
+ doc.free();
+ return text;
+}
+
+async function restart(): Promise<void> {
+ await mf.dispose();
+ mf = await setupMiniflare({ persistPath, migrate: false });
+}
+
+test("pending legacy snapshot resumes only for the identical seed", async () => {
+ const id = crypto.randomUUID();
+ const snapshot = seed("original");
+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");
+ await kv.put(`surface:${id}/surface:${id}.snapshot`, snapshot);
+ expect(await exists(id)).toBe(409);
+ expect((await initialize(id, snapshot)).status).toBe(200);
+ expect(await content(id)).toBe("original");
+ await restart();
+ expect(await exists(id)).toBe(200);
+ expect((await initialize(id, seed("replacement"))).status).toBe(409);
+ expect(await content(id)).toBe("original");
+});
+
+test("unidentified pending snapshot is rolled back including its KV fallback", async () => {
+ const id = crypto.randomUUID();
+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");
+ const key = `surface:${id}/surface:${id}.snapshot`;
+ await kv.put(key, seed("orphan"));
+ const snapshot = seed("replacement");
+ expect((await initialize(id, snapshot)).status).toBe(409);
+ expect(await kv.get(key)).toBeNull();
+ expect(await exists(id)).toBe(409);
+ await restart();
+ expect((await initialize(id, snapshot)).status).toBe(200);
+ expect(await content(id)).toBe("replacement");
+});
+
+const faultTests = process.env.SYNC_MIGRATION_FAULT_TESTS === "1" ? test : test.skip;
+faultTests.each([false, true])(
+ "snapshot committed before Ready failure recovers (restart: %s)",
+ async (cold) => {
+ const id = crypto.randomUUID();
+ const snapshot = seed("durable seed");
+ expect((await initialize(id, snapshot, true)).status).toBe(500);
+ expect(await exists(id)).toBe(409);
+ if (cold) await restart();
+ expect((await initialize(id, snapshot)).status).toBe(200);
+ expect(await content(id)).toBe("durable seed");
+ await restart();
+ expect(await exists(id)).toBe(200);
+ },
+);
+
+faultTests("conflicting retry rolls back SQL and KV snapshots before a new seed", async () => {
+ const id = crypto.randomUUID();
+ const original = seed("original");
+ expect((await initialize(id, original, true)).status).toBe(500);
+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");
+ const key = `surface:${id}/surface:${id}.snapshot`;
+ await kv.put(key, original);
+ const replacement = seed("replacement");
+ expect((await initialize(id, replacement)).status).toBe(409);
+ expect(await kv.get(key)).toBeNull();
+ await restart();
+ expect((await initialize(id, replacement)).status).toBe(200);
+ expect(await content(id)).toBe("replacement");
+});
diff --git a/services/sync-service/tests/utils.ts b/services/sync-service/tests/utils.ts
index 0b73054d4a..9733da437b 100644
--- a/services/sync-service/tests/utils.ts
+++ b/services/sync-service/tests/utils.ts
@@ -1,5 +1,5 @@
import { EphemeralStore, LoroDoc } from 'loro-crdt';
-import { Miniflare, type WebSocket } from 'miniflare';
+import { Miniflare, type MiniflareOptions, type WebSocket } from 'miniflare';
import { assert } from 'vitest';
import jwt from 'jsonwebtoken';
import {
@@ -27,7 +27,7 @@ async function migrateDatabase(mf: Miniflare) {
}
}
-export async function setupMiniflare(options: { persistPath?: string; migrate?: boolean } = {}) {
+export async function setupMiniflare(options: { persistPath?: string; migrate?: boolean; fetchMock?: MiniflareOptions['fetchMock'] } = {}) {
const persist = (name: string) => options.persistPath ? `${options.persistPath}/${name}` : false;
const mf = new Miniflare({
d1Databases: {
@@ -55,7 +55,9 @@ export async function setupMiniflare(options: { persistPath?: string; migrate?:
d1Persist: persist('d1'),
kvPersist: persist('kv'),
r2Persist: persist('r2'),
+ fetchMock: options.fetchMock,
bindings: {
+ ...(options.fetchMock ? { DSS_URL: "https://dss.test", DSS_INTERNAL_AUTH_KEY: "local" } : {}),
DOCUMENT_PERMISSIONS_SECRET: "local",
INTERNAL_API_SECRET_KEY: "INTERNAL_API_SECRET",
INTERNAL_API_SECRET,
@@ -222,7 +224,7 @@ export function createTestWebSocket(ws: WebSocket) {
return new Promise<ArrayBuffer>((resolve, reject) => {
const timeoutId = setTimeout(() => {
- const index = waiters.indexOf(resolve);
+ const index = waiters.indexOf(wrappedResolve);
if (index !== -1) {
waiters.splice(index, 1);
}
@@ -242,8 +244,8 @@ export function createTestWebSocket(ws: WebSocket) {
});
},
- send(message: string | ArrayBuffer) {
- ws.send(message);
+ send(message: string | ArrayBuffer | Uint8Array): void {
+ ws.send(message instanceof Uint8Array ? new Uint8Array(message) : message);
},
getWebSocket() {
@@ -255,7 +257,7 @@ export function createTestWebSocket(ws: WebSocket) {
export function getTokenForDocument(
documentId: string,
userId: string,
- permissionLevel: 'view' | 'edit' | 'owner'
+ permissionLevel: 'view' | 'edit' | 'owner' | 'comment'
): string {
const token = jwt.sign({
user_id: userId,
@@ -268,6 +270,21 @@ export function getTokenForDocument(
}
+export function getTokenForSurface(
+ surfaceId: string,
+ permissionLevel: 'view' | 'comment' | 'edit' | 'owner' = 'edit',
+ expiresAt = Math.floor(Date.now() / 1000) + 60,
+): string {
+ return jwt.sign({
+ session_kind: 'surface',
+ surface_id: surfaceId,
+ user_id: 'surface-user',
+ access_level: permissionLevel,
+ exp: expiresAt,
+ iss: 'document_storage_service',
+ }, 'local', { noTimestamp: true });
+}
+
export async function connectToDocumentForTesting(
mf: Miniflare,
documentId: string,