Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(initiatives): use collab surface for initiative (#6769)
Will Hutchinson8fce26b1ec83
diff --git a/Cargo.lock b/Cargo.lockindex 630f325f62..e9397d6646 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -16861,6 +16861,7 @@ dependencies = ["matchit 0.8.4","serde","serde_json",+ "sha2 0.10.9","subtle","sync_service_bebop_schema","thiserror 2.0.18",@@ -16890,9 +16891,11 @@ dependencies = ["model","reqwest 0.13.4","serde",+ "serde_json","sync_service_hex","tokio","tracing",+ "uuid","workspace-hack",]diff --git a/crates/macro_sync_service_jwt/src/lib.rs b/crates/macro_sync_service_jwt/src/lib.rsindex 193443f4d5..ff6ab4f6d7 100644--- a/crates/macro_sync_service_jwt/src/lib.rs+++ b/crates/macro_sync_service_jwt/src/lib.rs@@ -1,5 +1,9 @@-//! Minting and validation for the document-permission JWTs that authorize the-//! sync service to grant document access.+//! Minting and validation for sync-service permission JWTs.+//!+//! Legacy document consumers use [`encode`] and [`decode`]. Isolated surface+//! sessions use the distinct identities, claims, and token helpers in [`session`].++pub mod session;use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode as jwt_decode,diff --git a/crates/macro_sync_service_jwt/src/session.rs b/crates/macro_sync_service_jwt/src/session.rsnew file mode 100644index 0000000000..a458b5ff9d--- /dev/null+++ b/crates/macro_sync_service_jwt/src/session.rs@@ -0,0 +1,257 @@+//! Typed sync-session identities and isolated surface grants.+//!+//! Surface storage keys are `surface:<lowercase hyphenated UUID>`. Document+//! keys retain their original spelling and have no prefix. UUID-only identities+//! keep document keys out of the reserved surface namespace.++use std::{fmt, str::FromStr};++use jsonwebtoken::{+ Algorithm, DecodingKey, EncodingKey, Header, Validation, decode as jwt_decode,+ encode as jwt_encode,+};+use serde::{Deserialize, Serialize};++use crate::ISSUER;++/// The authorization and storage namespace of a sync session.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum SessionKind {+ /// An existing document session, stored under its unprefixed document ID.+ Document,+ /// An isolated collaborative surface session.+ Surface,+}++/// An identifier was not a hyphenated UUID (`8-4-4-4-12` ASCII hex digits).+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct InvalidSessionId;++impl fmt::Display for InvalidSessionId {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str("session ID must be a hyphenated UUID")+ }+}++impl std::error::Error for InvalidSessionId {}++// Validate only the wire grammar; UUID generation/version policy belongs to the+// caller. This shared worker/backend crate needs no UUID or randomness dependency.+fn validate_id(id: &str) -> Result<(), InvalidSessionId> {+ if id.len() != 36 {+ return Err(InvalidSessionId);+ }+ for (index, byte) in id.bytes().enumerate() {+ let valid = match index {+ 8 | 13 | 18 | 23 => byte == b'-',+ _ => byte.is_ascii_hexdigit(),+ };+ if !valid {+ return Err(InvalidSessionId);+ }+ }+ Ok(())+}++/// A surface UUID in canonical lowercase, hyphenated form.+///+/// Construction and deserialization both validate the ID. Alternate UUID forms+/// (URNs, braces, or unhyphenated hex) and already-prefixed storage keys are rejected.+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(try_from = "String")]+pub struct SurfaceId(String);++impl SurfaceId {+ /// Borrow the canonical UUID, without its storage namespace prefix.+ pub fn as_str(&self) -> &str {+ &self.0+ }+}++impl FromStr for SurfaceId {+ type Err = InvalidSessionId;++ fn from_str(id: &str) -> Result<Self, Self::Err> {+ validate_id(id)?;+ Ok(Self(id.to_ascii_lowercase()))+ }+}++impl TryFrom<String> for SurfaceId {+ type Error = InvalidSessionId;++ fn try_from(id: String) -> Result<Self, Self::Error> {+ id.parse()+ }+}++impl fmt::Display for SurfaceId {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(self.as_str())+ }+}++/// A validated authorization target. Matching UUIDs in different kinds are not+/// equal and never produce the same storage key.+#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct SessionIdentity {+ kind: SessionKind,+ id: String,+}++impl SessionIdentity {+ /// Validate a route/persisted identity. Document spelling is preserved;+ /// surface IDs are canonicalized before comparison or storage.+ pub fn new(kind: SessionKind, id: &str) -> Result<Self, InvalidSessionId> {+ match kind {+ SessionKind::Document => {+ validate_id(id)?;+ Ok(Self {+ kind,+ id: id.to_owned(),+ })+ }+ SessionKind::Surface => Ok(Self::surface(id.parse()?)),+ }+ }++ /// Construct an isolated identity from an already validated surface ID.+ pub fn surface(id: SurfaceId) -> Self {+ Self {+ kind: SessionKind::Surface,+ id: id.0,+ }+ }++ /// The namespace that must match the grant as well as the route.+ pub fn kind(&self) -> SessionKind {+ self.kind+ }++ /// The unprefixed ID. Do not use this alone as an authorization target.+ pub fn id(&self) -> &str {+ &self.id+ }++ /// The common base key for Durable Objects and external persistence.+ /// Existing document names are unchanged.+ pub fn storage_key(&self) -> String {+ match self.kind {+ SessionKind::Document => self.id.clone(),+ SessionKind::Surface => format!("surface:{}", self.id),+ }+ }+}++/// Permission carried by a surface grant. Internal/admin credentials are not+/// surface grants; write policy is enforced by the surface access boundary.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "lowercase")]+pub enum SurfaceAccessLevel {+ /// Read-only access.+ View,+ /// Comment access, not permission for arbitrary CRDT writes.+ Comment,+ /// Permission to edit surface content.+ Edit,+ /// Owner access.+ Owner,+}++/// The mandatory surface-claim discriminator. A document kind cannot be+/// constructed or deserialized into this type.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum SurfaceSessionKind {+ /// An isolated collaborative surface.+ Surface,+}++/// A grant for exactly one isolated surface, never a document grant.+///+/// The wire format requires `session_kind: "surface"` and `surface_id`.+/// There is deliberately no `document_id` fallback. Unknown fields are rejected+/// so a dual document/surface grant cannot be decoded as a surface grant.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct SurfaceClaims {+ /// Required explicit discriminator, with no default for legacy tokens.+ pub session_kind: SurfaceSessionKind,+ /// The canonical, surface-scoped target ID.+ pub surface_id: SurfaceId,+ /// The user, when present.+ pub user_id: Option<String>,+ /// The parent-derived surface permission.+ pub access_level: SurfaceAccessLevel,+ /// Optional attribution actor, matching the document transport convention.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub actor: Option<String>,+ /// Expiry as Unix seconds; consumers must retain this for open sockets.+ pub exp: usize,+ /// Issuer, validated against [`ISSUER`] by [`decode_surface`].+ pub iss: String,+}++impl SurfaceClaims {+ /// The complete authorization target; compare this with the route identity.+ pub fn session_identity(&self) -> SessionIdentity {+ SessionIdentity::surface(self.surface_id.clone())+ }+}++/// A signed surface grant, distinct from the legacy document token wrapper.+#[derive(+ Debug, Clone, PartialEq, Eq, Serialize, Deserialize, derive_more::Display, derive_more::From,+)]+#[serde(transparent)]+#[display("{_0}")]+pub struct SurfacePermissionToken(String);++impl SurfacePermissionToken {+ /// Borrow the encoded JWT.+ pub fn as_str(&self) -> &str {+ &self.0+ }++ /// Consume the token wrapper.+ pub fn into_inner(self) -> String {+ self.0+ }+}++/// Sign a surface grant with HS256, without changing legacy document encoding.+pub fn encode_surface(+ claims: &SurfaceClaims,+ secret: &str,+) -> Result<SurfacePermissionToken, jsonwebtoken::errors::Error> {+ let token = jwt_encode(+ &Header::new(Algorithm::HS256),+ claims,+ &EncodingKey::from_secret(secret.as_bytes()),+ )?;+ Ok(SurfacePermissionToken(token))+}++/// Verify a surface grant's signature, issuer, expiry, kind, and scoped ID.+///+/// Unlike the legacy decoder, this has no expiry leeway. Route target matching,+/// access-level enforcement, lifecycle checks, and socket expiry enforcement+/// remain the caller's responsibility.+pub fn decode_surface(+ token: &str,+ secret: &str,+) -> Result<SurfaceClaims, jsonwebtoken::errors::Error> {+ let mut validation = Validation::new(Algorithm::HS256);+ validation.set_issuer(&[ISSUER]);+ validation.leeway = 0;+ let data = jwt_decode::<SurfaceClaims>(+ token,+ &DecodingKey::from_secret(secret.as_bytes()),+ &validation,+ )?;+ Ok(data.claims)+}++#[cfg(test)]+mod test;diff --git a/crates/macro_sync_service_jwt/src/session/test.rs b/crates/macro_sync_service_jwt/src/session/test.rsnew file mode 100644index 0000000000..46eaaeda6d--- /dev/null+++ b/crates/macro_sync_service_jwt/src/session/test.rs@@ -0,0 +1,229 @@+use super::*;+use crate::{ISSUER, decode, encode};+use jsonwebtoken::errors::ErrorKind;+use serde::{Deserialize, Serialize};+use std::time::{SystemTime, UNIX_EPOCH};++const ID: &str = "0194c5d8-8f00-7000-8000-abcdef012345";+const SECRET: &str = "session-tests-only";++fn now() -> usize {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap()+ .as_secs() as usize+}++fn claims() -> SurfaceClaims {+ SurfaceClaims {+ session_kind: SurfaceSessionKind::Surface,+ surface_id: ID.parse().unwrap(),+ user_id: Some("macro|[email protected]".into()),+ access_level: SurfaceAccessLevel::Edit,+ actor: None,+ exp: now() + crate::TOKEN_TTL_SECS,+ iss: ISSUER.into(),+ }+}++// Mirrors the existing document issuer and consumer without backend dependencies.+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]+struct LegacyDocumentClaims {+ user_id: Option<String>,+ document_id: String,+ access_level: SurfaceAccessLevel,+ exp: usize,+ iss: String,+}++#[derive(Serialize)]+struct RawSurfaceClaims<'a> {+ #[serde(skip_serializing_if = "Option::is_none")]+ session_kind: Option<&'a str>,+ #[serde(skip_serializing_if = "Option::is_none")]+ surface_id: Option<&'a str>,+ #[serde(skip_serializing_if = "Option::is_none")]+ document_id: Option<&'a str>,+ access_level: &'a str,+ exp: usize,+ iss: &'a str,+}++fn raw_claims() -> RawSurfaceClaims<'static> {+ RawSurfaceClaims {+ session_kind: Some("surface"),+ surface_id: Some(ID),+ document_id: None,+ access_level: "edit",+ exp: now() + crate::TOKEN_TTL_SECS,+ iss: ISSUER,+ }+}++fn rejects(raw: &RawSurfaceClaims<'_>) {+ let token = encode(raw, SECRET).unwrap();+ assert!(decode_surface(token.as_str(), SECRET).is_err());+}++#[test]+fn identical_uuids_have_distinct_targets_and_storage_keys() {+ let document = SessionIdentity::new(SessionKind::Document, ID).unwrap();+ let surface = SessionIdentity::new(SessionKind::Surface, ID).unwrap();+ assert_ne!(document, surface);+ assert_eq!(document.kind(), SessionKind::Document);+ assert_eq!(surface.kind(), SessionKind::Surface);+ assert_eq!(document.id(), surface.id());+ assert_eq!(document.storage_key(), ID);+ assert_eq!(surface.storage_key(), format!("surface:{ID}"));+}++#[test]+fn surface_keys_are_canonical_but_document_names_are_unchanged() {+ let uppercase = ID.to_ascii_uppercase();+ let surface = SessionIdentity::new(SessionKind::Surface, &uppercase).unwrap();+ assert_eq!(surface.id(), ID);+ assert_eq!(surface.storage_key(), format!("surface:{ID}"));+ let document = SessionIdentity::new(SessionKind::Document, &uppercase).unwrap();+ assert_eq!(document.id(), uppercase);+ assert_eq!(document.storage_key(), uppercase);+ let id: SurfaceId = uppercase.parse().unwrap();+ assert_eq!(id.as_str(), ID);+ assert_eq!(id.to_string(), ID);+}++#[test]+fn malformed_ids_cannot_enter_identities_or_claims() {+ for id in [+ "",+ "not-a-uuid",+ "0194c5d88f0070008000abcdef012345",+ "0194c5d8-8f00-7000-8000-abcdef01234g",+ "0194c5d8_8f00-7000-8000-abcdef012345",+ "0194c5d8-8f00-7000-8000-abcdef012345/ops",+ "surface:0194c5d8-8f00-7000-8000-abcdef012345",+ "{0194c5d8-8f00-7000-8000-abcdef012345}",+ " 0194c5d8-8f00-7000-8000-abcdef012345",+ "0194c5d8-8f00-7000-8000-abcdef01234é",+ ] {+ assert!(id.parse::<SurfaceId>().is_err(), "{id}");+ for kind in [SessionKind::Document, SessionKind::Surface] {+ assert!(SessionIdentity::new(kind, id).is_err(), "{id}");+ }+ let mut raw = raw_claims();+ raw.surface_id = Some(id);+ rejects(&raw);+ }+}++#[test]+fn surface_token_round_trip_has_explicit_kind_and_scoped_id() {+ let expected = claims();+ let token = encode_surface(&expected, SECRET).unwrap();+ let actual = decode_surface(token.as_str(), SECRET).unwrap();+ assert_eq!(actual, expected);+ assert_eq!(+ actual.session_identity(),+ SessionIdentity::new(SessionKind::Surface, ID).unwrap()+ );++ #[derive(Deserialize)]+ struct WireClaims {+ session_kind: SessionKind,+ surface_id: String,+ }+ let wire: WireClaims = decode(token.as_str(), SECRET).unwrap();+ assert_eq!(wire.session_kind, SessionKind::Surface);+ assert_eq!(wire.surface_id, ID);+ assert!(decode::<LegacyDocumentClaims>(token.as_str(), SECRET).is_err());+}++#[test]+fn legacy_document_tokens_remain_compatible_but_are_not_surface_grants() {+ let expected = LegacyDocumentClaims {+ user_id: None,+ document_id: ID.into(),+ access_level: SurfaceAccessLevel::Comment,+ exp: now() + crate::TOKEN_TTL_SECS,+ iss: ISSUER.into(),+ };+ let token = encode(&expected, SECRET).unwrap();+ assert_eq!(+ decode::<LegacyDocumentClaims>(token.as_str(), SECRET).unwrap(),+ expected+ );+ assert!(decode_surface(token.as_str(), SECRET).is_err());+}++#[test]+fn surface_claims_require_surface_kind_and_surface_id() {+ for kind in [None, Some("document"), Some("initiative"), Some("Surface")] {+ let mut raw = raw_claims();+ raw.session_kind = kind;+ rejects(&raw);+ }+ let mut raw = raw_claims();+ raw.surface_id = None;+ rejects(&raw);+ raw.document_id = Some(ID);+ rejects(&raw);+ raw.surface_id = Some(ID);+ rejects(&raw); // Reject ambiguous grants containing both target fields.+}++#[test]+fn unsupported_access_levels_are_rejected() {+ for level in ["admin", "write", "Edit", ""] {+ let mut raw = raw_claims();+ raw.access_level = level;+ rejects(&raw);+ }+ for level in ["view", "comment", "edit", "owner"] {+ let mut raw = raw_claims();+ raw.access_level = level;+ let token = encode(&raw, SECRET).unwrap();+ assert!(decode_surface(token.as_str(), SECRET).is_ok());+ }+}++#[test]+fn expired_surface_tokens_are_rejected_without_default_clock_leeway() {+ let mut expired = claims();+ expired.exp = now() - 1;+ let token = encode_surface(&expired, SECRET).unwrap();+ assert_eq!(+ decode_surface(token.as_str(), SECRET).unwrap_err().kind(),+ &ErrorKind::ExpiredSignature+ );+}++#[test]+fn surface_tokens_require_expected_issuer_and_signature() {+ let valid = claims();+ let token = encode_surface(&valid, SECRET).unwrap();+ assert!(decode_surface(token.as_str(), "wrong-secret").is_err());+ let mut raw = raw_claims();+ raw.iss = "another-service";+ rejects(&raw);+}++#[test]+fn surface_tokens_require_expiry() {+ #[derive(Serialize)]+ struct NoExpiry {+ session_kind: SessionKind,+ surface_id: SurfaceId,+ access_level: SurfaceAccessLevel,+ iss: &'static str,+ }+ let token = encode(+ &NoExpiry {+ session_kind: SessionKind::Surface,+ surface_id: ID.parse().unwrap(),+ access_level: SurfaceAccessLevel::View,+ iss: ISSUER,+ },+ SECRET,+ )+ .unwrap();+ assert!(decode_surface(token.as_str(), SECRET).is_err());+}diff --git a/crates/sync_service_client/Cargo.toml b/crates/sync_service_client/Cargo.tomlindex d269976780..431e005b9b 100644--- a/crates/sync_service_client/Cargo.toml+++ b/crates/sync_service_client/Cargo.toml@@ -12,7 +12,9 @@ serde = { workspace = true }sync_service_hex = { path = "../sync_service", package = "sync_service_hex" }tokio = { workspace = true }tracing = { workspace = true }+uuid = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]+serde_json = { workspace = true }tokio = { workspace = true }diff --git a/crates/sync_service_client/src/lib.rs b/crates/sync_service_client/src/lib.rsindex 39690d0c53..a3729a42e4 100644--- a/crates/sync_service_client/src/lib.rs+++ b/crates/sync_service_client/src/lib.rs@@ -4,6 +4,7 @@ pub mod exists;pub mod get_raw;pub mod initialize;pub mod metadata;+pub mod surface;pub mod wakeup;pub(crate) static INTERNAL_ACCESS_HEADER: &str = "x-internal-auth-key";diff --git a/crates/sync_service_client/src/surface.rs b/crates/sync_service_client/src/surface.rsnew file mode 100644index 0000000000..1e0c77ee96--- /dev/null+++ b/crates/sync_service_client/src/surface.rs@@ -0,0 +1,208 @@+//! Internal surface lifecycle transport. Callers own legacy ownership checks and+//! the durable metadata journal; no method falls back to document initialization.++use reqwest::StatusCode;+use serde::{Deserialize, Serialize, de::DeserializeOwned};+use uuid::Uuid;++use crate::SyncServiceClient;++#[cfg(test)]+mod test;++/// Persist this identity before starting an initialization or migration.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]+#[serde(transparent)]+pub struct SurfaceOperationId(pub Uuid);++/// Receipt binding one operation to snapshot bytes, decoded content and both+/// Loro frontiers. A successful retry returns this same verified receipt.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct SnapshotProof {+ /// Durable operation identity; reuse only for an identical retry.+ pub operation_id: SurfaceOperationId,+ /// Legacy source UUID, or None for a newly seeded isolated surface.+ pub source_id: Option<Uuid>,+ /// SHA-256 of the full snapshot bytes.+ pub digest: String,+ /// SHA-256 of the canonical decoded JSON content.+ pub content_digest: String,+ /// Sorted state frontier (peer string, counter) pairs.+ pub revision: Vec<(String, i32)>,+ /// Sorted operation-log frontier pairs.+ pub oplog_revision: Vec<(String, i32)>,+}++/// Authoritative frozen state, including pending persisted operations.+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub struct SurfaceSnapshot {+ /// Verification receipt for these exact bytes.+ pub proof: SnapshotProof,+ /// Full Loro snapshot, not initial Markdown or a shallow client snapshot.+ pub snapshot: Vec<u8>,+}++/// Typed transport failures; an existing snapshot is never treated as success.+#[derive(Debug)]+pub enum SurfaceSyncError {+ /// Request or response decoding failed; retry with the same operation.+ Transport(reqwest::Error),+ /// HTTP rejection, including conflicts (409) and permanent revocation (403).+ Rejected(StatusCode),+}++impl std::fmt::Display for SurfaceSyncError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Transport(error) => write!(f, "surface sync transport: {error}"),+ Self::Rejected(status) => write!(f, "surface sync rejected: {status}"),+ }+ }+}++impl std::error::Error for SurfaceSyncError {+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {+ match self {+ Self::Transport(error) => Some(error),+ Self::Rejected(_) => None,+ }+ }+}++impl SyncServiceClient {+ async fn surface_request(+ &self,+ path: &str,+ body: &impl Serialize,+ ) -> Result<reqwest::Response, SurfaceSyncError> {+ let response = self+ .client+ .post(format!("{}{path}", self.url))+ .json(body)+ .send()+ .await+ .map_err(SurfaceSyncError::Transport)?;+ if response.status() != StatusCode::OK {+ return Err(SurfaceSyncError::Rejected(response.status()));+ }+ Ok(response)+ }++ async fn surface_json<T: DeserializeOwned>(+ &self,+ path: &str,+ body: &impl Serialize,+ ) -> Result<T, SurfaceSyncError> {+ self.surface_request(path, body)+ .await?+ .json()+ .await+ .map_err(SurfaceSyncError::Transport)+ }++ /// Seed an isolated surface exactly once. Retries require the same operation+ /// and snapshot; existing document initialization behavior is unchanged.+ pub async fn initialize_surface(+ &self,+ id: Uuid,+ operation_id: SurfaceOperationId,+ snapshot: &[u8],+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ #[derive(Serialize)]+ struct Initialize<'a> {+ operation_id: SurfaceOperationId,+ snapshot: &'a [u8],+ }+ self.surface_json(+ &format!("/surface/{id}/initialize_verified"),+ &Initialize {+ operation_id,+ snapshot,+ },+ )+ .await+ }++ /// Stop legacy writers and export authoritative state. The caller MUST have+ /// ruled out live and soft-deleted Document collisions before invoking this.+ pub async fn freeze_legacy_surface(+ &self,+ id: Uuid,+ operation_id: SurfaceOperationId,+ ) -> Result<SurfaceSnapshot, SurfaceSyncError> {+ #[derive(Serialize)]+ struct Freeze {+ operation_id: SurfaceOperationId,+ }+ self.surface_json(+ &format!("/document/{id}/migration/freeze"),+ &Freeze { operation_id },+ )+ .await+ }++ /// Import into a non-writable isolated target. The source and target UUIDs+ /// must match; conflicting state is never overwritten.+ pub async fn import_surface(+ &self,+ id: Uuid,+ snapshot: &SurfaceSnapshot,+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ self.surface_json(&format!("/surface/{id}/import"), snapshot)+ .await+ }++ /// Read back and verify an import before committing metadata activation.+ pub async fn verify_surface(+ &self,+ id: Uuid,+ proof: &SnapshotProof,+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ self.surface_json(&format!("/surface/{id}/verify"), proof)+ .await+ }++ /// Irreversibly seal the legacy source, then enable the verified target.+ /// Persist commit intent in the caller's journal BEFORE calling this method.+ /// Even on a timeout/error, recover forward: the source may already be sealed.+ /// Activate metadata/token minting only after this operation succeeds.+ pub async fn activate_surface(+ &self,+ id: Uuid,+ proof: &SnapshotProof,+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ self.surface_json(&format!("/surface/{id}/activate"), proof)+ .await+ }++ /// Roll back only the verified original, before activation commit intent.+ /// A thawed operation cannot subsequently be sealed or reused for a freeze.+ pub async fn thaw_legacy_surface(+ &self,+ id: Uuid,+ proof: &SnapshotProof,+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ self.surface_json(&format!("/document/{id}/migration/thaw"), proof)+ .await+ }++ /// Permanently retire a sealed legacy endpoint, retaining its snapshot/log.+ pub async fn retire_legacy_surface(+ &self,+ id: Uuid,+ proof: &SnapshotProof,+ ) -> Result<SnapshotProof, SurfaceSyncError> {+ self.surface_json(&format!("/document/{id}/migration/retire"), proof)+ .await+ }++ /// Permanently deny new and existing surface grants, including pending IDs.+ pub async fn revoke_surface(&self, id: Uuid) -> Result<(), SurfaceSyncError> {+ self.surface_request(+ &format!("/surface/{id}/revoke"),+ &std::collections::BTreeMap::<String, String>::new(),+ )+ .await?;+ Ok(())+ }+}diff --git a/crates/sync_service_client/src/surface/test.rs b/crates/sync_service_client/src/surface/test.rsnew file mode 100644index 0000000000..eeb527db29--- /dev/null+++ b/crates/sync_service_client/src/surface/test.rs@@ -0,0 +1,183 @@+use super::*;+use std::io::{Read, Write};++fn proof() -> SnapshotProof {+ SnapshotProof {+ operation_id: SurfaceOperationId(Uuid::from_u128(1)),+ source_id: Some(Uuid::from_u128(2)),+ digest: "snapshot-digest".into(),+ content_digest: "content-digest".into(),+ revision: vec![("123".into(), 4)],+ oplog_revision: vec![("123".into(), 4)],+ }+}++#[test]+fn wire_proof_round_trips_and_rejects_invalid_operation_ids() {+ let proof = proof();+ let value = serde_json::to_value(&proof).unwrap();+ assert_eq!(value["operation_id"], Uuid::from_u128(1).to_string());+ assert_eq!(+ serde_json::from_value::<SnapshotProof>(value.clone()).unwrap(),+ proof+ );+ let mut invalid = value;+ invalid["operation_id"] = "not-a-uuid".into();+ assert!(serde_json::from_value::<SnapshotProof>(invalid).is_err());+}++/// Minimal HTTP peer validates transport without another mocking dependency.+fn server(+ path: String,+ expected: serde_json::Value,+ status: u16,+ response: String,+) -> (SyncServiceClient, std::thread::JoinHandle<()>) {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let task = std::thread::spawn(move || {+ let (mut socket, _) = listener.accept().unwrap();+ socket+ .set_read_timeout(Some(std::time::Duration::from_secs(5)))+ .unwrap();+ let mut bytes = Vec::new();+ let (headers, body_start, length) = loop {+ let mut buf = [0; 4096];+ let count = socket.read(&mut buf).unwrap();+ assert_ne!(count, 0);+ bytes.extend_from_slice(&buf[..count]);+ if let Some(start) = bytes.windows(4).position(|window| window == b"\r\n\r\n") {+ let headers = String::from_utf8(bytes[..start].to_vec()).unwrap();+ let length = headers+ .lines()+ .find_map(|line| {+ line.to_lowercase()+ .strip_prefix("content-length: ")+ .map(|n| n.parse::<usize>().unwrap())+ })+ .unwrap();+ break (headers, start + 4, length);+ }+ };+ while bytes.len() < body_start + length {+ let mut buf = [0; 4096];+ let count = socket.read(&mut buf).unwrap();+ assert_ne!(count, 0);+ bytes.extend_from_slice(&buf[..count]);+ }+ assert!(headers.starts_with(&format!("POST {path} HTTP/1.1")));+ assert!(headers.contains("x-internal-auth-key: secret"));+ assert_eq!(+ serde_json::from_slice::<serde_json::Value>(&bytes[body_start..body_start + length])+ .unwrap(),+ expected+ );+ write!(socket, "HTTP/1.1 {status} Result\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response}", response.len()).unwrap();+ });+ (+ SyncServiceClient::new("secret".into(), format!("http://{address}")),+ task,+ )+}++#[tokio::test]+async fn freeze_uses_legacy_namespace_and_typed_export() {+ let proof = proof();+ let id = proof.source_id.unwrap();+ let export = SurfaceSnapshot {+ proof: proof.clone(),+ snapshot: vec![1, 2],+ };+ let (client, task) = server(+ format!("/document/{id}/migration/freeze"),+ serde_json::json!({ "operation_id": proof.operation_id }),+ 200,+ serde_json::to_string(&export).unwrap(),+ );+ assert_eq!(+ client+ .freeze_legacy_surface(id, proof.operation_id)+ .await+ .unwrap(),+ export+ );+ task.join().unwrap();+}++#[tokio::test]+async fn surface_lifecycle_methods_use_isolated_routes() {+ let proof = proof();+ let id = proof.source_id.unwrap();+ for operation in ["verify", "activate", "migration/thaw", "migration/retire"] {+ let kind = if operation.starts_with("migration/") {+ "document"+ } else {+ "surface"+ };+ let (client, task) = server(+ format!("/{kind}/{id}/{operation}"),+ serde_json::to_value(&proof).unwrap(),+ 200,+ serde_json::to_string(&proof).unwrap(),+ );+ let result = match operation {+ "verify" => client.verify_surface(id, &proof).await,+ "activate" => client.activate_surface(id, &proof).await,+ "migration/thaw" => client.thaw_legacy_surface(id, &proof).await,+ _ => client.retire_legacy_surface(id, &proof).await,+ };+ assert_eq!(result.unwrap(), proof);+ task.join().unwrap();+ }+ let export = SurfaceSnapshot {+ proof: proof.clone(),+ snapshot: vec![1, 2],+ };+ let (client, task) = server(+ format!("/surface/{id}/import"),+ serde_json::to_value(&export).unwrap(),+ 200,+ serde_json::to_string(&proof).unwrap(),+ );+ assert_eq!(client.import_surface(id, &export).await.unwrap(), proof);+ task.join().unwrap();+ let (client, task) = server(+ format!("/surface/{id}/initialize_verified"),+ serde_json::json!({"operation_id": proof.operation_id, "snapshot": [1, 2]}),+ 200,+ serde_json::to_string(&proof).unwrap(),+ );+ assert_eq!(+ client+ .initialize_surface(id, proof.operation_id, &[1, 2])+ .await+ .unwrap(),+ proof+ );+ task.join().unwrap();+ let (client, task) = server(+ format!("/surface/{id}/revoke"),+ serde_json::json!({}),+ 200,+ String::new(),+ );+ client.revoke_surface(id).await.unwrap();+ task.join().unwrap();+}++#[tokio::test]+async fn conflict_is_typed_and_never_string_matched_as_success() {+ let proof = proof();+ let id = proof.source_id.unwrap();+ let (client, task) = server(+ format!("/surface/{id}/verify"),+ serde_json::to_value(&proof).unwrap(),+ 409,+ "snapshot already exists".into(),+ );+ assert!(matches!(+ client.verify_surface(id, &proof).await,+ Err(SurfaceSyncError::Rejected(reqwest::StatusCode::CONFLICT))+ ));+ task.join().unwrap();+}diff --git a/services/sync-service/Cargo.toml b/services/sync-service/Cargo.tomlindex a9579d4c5d..6f14c067f4 100644--- a/services/sync-service/Cargo.toml+++ b/services/sync-service/Cargo.toml@@ -9,6 +9,8 @@ publish = false[features]default = ["bebop-owned-all", "do-sqlite-snapshot-storage", "search-service"]no-network-deps = []+# Test-only persistence fault injection; never enable in deployed workers.+migration-test-hooks = []do-sqlite-snapshot-storage = []kv-snapshot-storage = []r2-snapshot-storage = []@@ -39,6 +41,7 @@ time = { workspace = true, features = ['wasm-bindgen'] }bebop.workspace = truesync_service_bebop_schema = { path = "bebop-schema" }serde_json.workspace = true+sha2.workspace = trueserde.workspace = truegetrandom = { workspace = true, features = ["wasm_js"] }wasm-bindgen.workspace = truediff --git a/services/sync-service/src/auth.rs b/services/sync-service/src/auth.rsindex 2aaf059bd7..5939610e76 100644--- a/services/sync-service/src/auth.rs+++ b/services/sync-service/src/auth.rs@@ -1,65 +1,63 @@+use macro_sync_service_jwt::session::{+ SessionIdentity, SessionKind, SurfaceAccessLevel, decode_surface,+};use serde::{Deserialize, Serialize};use subtle::ConstantTimeEq;-use tracing::error;use worker::Error;use crate::{constants::header_names, error::ResultExt, secrets::Secrets};-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, PartialOrd, Ord)]+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Default)]#[serde(rename_all = "lowercase")]-#[derive(Default)]pub enum AccessLevel {- /// User can view the document#[default]View = 0,- /// User can comment on the document- /// In this context, this is the same thing as [AccessLevel::View]+ /// Legacy document sockets permit Comment writes; surfaces do not.Comment = 1,- /// User can edit the documentEdit = 2,- /// User is the owner of the documentOwner = 3,- /// Internal communicationAdmin = 4,}impl AccessLevel {- pub fn can_edit(&self) -> bool {- self >= &AccessLevel::Comment+ pub fn can_edit_for(&self, kind: SessionKind) -> bool {+ let minimum = match kind {+ SessionKind::Document => Self::Comment,+ SessionKind::Surface => Self::Edit,+ };+ self >= &minimum}}-#[derive(Deserialize, Debug)]+/// Verified grant normalized at the dedicated session access boundary.+#[derive(Debug)]pub struct AuthToken {pub user_id: Option<String>,document_id: String,pub access_level: AccessLevel,- #[serde(default)]pub actor: Option<String>,+ pub session_kind: SessionKind,+ pub expires_at: Option<usize>,+}++#[derive(Deserialize)]+struct DocumentClaims {+ user_id: Option<String>,+ document_id: String,+ access_level: AccessLevel,+ actor: Option<String>,+ session_kind: Option<SessionKind>,+ surface_id: Option<String>,}impl AuthToken {- pub fn has_permission(&self, al: &AccessLevel) -> bool {- if self.access_level < *al {- error!(- "Current permission level [{:?}] is not enough for [{:?}]",- self.access_level, al- );- return false;- }- true+ pub fn has_permission(&self, level: &AccessLevel) -> bool {+ self.access_level >= *level}+pub fn has_document_id_access(&self, document_id: &str) -> bool {- if !(self.document_id == document_id || matches!(self.access_level, AccessLevel::Admin)) {- error!(- "Don't have permission for document: [{:?}]-Auth'd document [{:?}]-access level [{:?}]",- document_id, self.document_id, self.access_level- );- return false;- }- true+ self.session_kind == SessionKind::Document+ && (self.document_id == document_id || self.access_level == AccessLevel::Admin)}}@@ -73,66 +71,127 @@ pub enum TokenFrom {QueryParams,}-pub fn decode_jwt(- req: &worker::Request,- env: &worker::Env,- token_from: TokenFrom,-) -> worker::Result<AuthToken> {- let secrets = Secrets::from(env);- let token = match token_from {- TokenFrom::Headers => {- // NB: rewrite with if/let chain on edition 2024- let is_admin = match req- .headers()- .get(header_names::MACRO_INTERNAL_AUTH_KEY_HEADER_KEY)?- {- // sholud we warn when false?- Some(internal_key) => {- let res: bool = internal_key- .as_bytes()- .ct_eq(secrets.internal_api_secret.as_bytes())- .into();-- if !res {- error!("provided internal authentication key did not match expected value");- }- res- }- None => false,- };-- if is_admin {- return Ok(AuthToken {- user_id: None,- document_id: "TODO should be option".to_string(),- access_level: AccessLevel::Admin,- actor: None,- });- }-- match req.headers().get(header_names::AUTHORIZATION)? {- Some(header) => match header.strip_prefix("Bearer ") {- Some(token) => token.to_string(),- None => {- return Err(Error::from(- "'Authorization' heard malformed. No 'Bearer ' prefix",- ));- }- },- None => return Err(Error::from("Missing 'Authorization' header")),- }- }- TokenFrom::QueryParams => req.query::<WebsocketQueryParams>()?.token,+pub fn is_internal(req: &worker::Request, env: &worker::Env) -> worker::Result<bool> {+ let Some(key) = req+ .headers()+ .get(header_names::MACRO_INTERNAL_AUTH_KEY_HEADER_KEY)?+ else {+ return Ok(false);};+ Ok(key+ .as_bytes()+ .ct_eq(Secrets::from(env).internal_api_secret.as_bytes())+ .into())+}++fn request_token(req: &worker::Request, source: TokenFrom) -> worker::Result<String> {+ match source {+ TokenFrom::Headers => req+ .headers()+ .get(header_names::AUTHORIZATION)?+ .and_then(|header| header.strip_prefix("Bearer ").map(str::to_owned))+ .ok_or_else(|| Error::from("Missing or malformed Bearer authorization")),+ TokenFrom::QueryParams => Ok(req.query::<WebsocketQueryParams>()?.token),+ }+}- let secret = secrets.document_permissions_secret;+fn decode_document_token(token: &str, secret: &str) -> worker::Result<AuthToken> {+ let claims = macro_sync_service_jwt::decode::<DocumentClaims>(token, secret)+ .context("failed to decode document grant")?;+ if claims+ .session_kind+ .is_some_and(|kind| kind != SessionKind::Document)+ || claims.surface_id.is_some()+ {+ return Err(Error::from("surface grant cannot authorize a document"));+ }+ Ok(AuthToken {+ user_id: claims.user_id,+ document_id: claims.document_id,+ access_level: claims.access_level,+ actor: claims.actor,+ session_kind: SessionKind::Document,+ expires_at: None,+ })+}- let claims = macro_sync_service_jwt::decode::<AuthToken>(&token, &secret)- .context("failed to decode `AuthToken`")?;+fn decode_surface_token(+ token: &str,+ secret: &str,+ identity: &SessionIdentity,+) -> worker::Result<AuthToken> {+ let claims = decode_surface(token, secret).context("failed to decode surface grant")?;+ if claims.session_identity() != *identity {+ return Err(Error::from("surface grant does not match session"));+ }+ let access_level = match claims.access_level {+ SurfaceAccessLevel::View => AccessLevel::View,+ SurfaceAccessLevel::Comment => AccessLevel::Comment,+ SurfaceAccessLevel::Edit => AccessLevel::Edit,+ SurfaceAccessLevel::Owner => AccessLevel::Owner,+ };+ Ok(AuthToken {+ user_id: claims.user_id,+ document_id: identity.storage_key(),+ access_level,+ actor: claims.actor,+ session_kind: SessionKind::Surface,+ expires_at: Some(claims.exp),+ })+}+pub fn surface_access(+ req: &worker::Request,+ env: &worker::Env,+ source: TokenFrom,+ identity: &SessionIdentity,+) -> worker::Result<AuthToken> {+ decode_surface_token(+ &request_token(req, source)?,+ &Secrets::from(env).document_permissions_secret,+ identity,+ )+}++/// Authenticate a socket against the persisted, kind-discriminated storage key.+pub fn socket_access(+ req: &worker::Request,+ env: &worker::Env,+ session_key: &str,+) -> worker::Result<AuthToken> {+ if let Some(id) = session_key.strip_prefix("surface:") {+ let identity = SessionIdentity::new(SessionKind::Surface, id)+ .map_err(|error| Error::from(error.to_string()))?;+ return surface_access(req, env, TokenFrom::QueryParams, &identity);+ }+ let claims = decode_jwt(req, env, TokenFrom::QueryParams)?;+ if !claims.has_document_id_access(session_key) {+ return Err(Error::from("document grant does not match session"));+ }Ok(claims)}+pub fn decode_jwt(+ req: &worker::Request,+ env: &worker::Env,+ source: TokenFrom,+) -> worker::Result<AuthToken> {+ if matches!(source, TokenFrom::Headers) && is_internal(req, env)? {+ return Ok(AuthToken {+ user_id: None,+ document_id: String::new(),+ access_level: AccessLevel::Admin,+ actor: None,+ session_kind: SessionKind::Document,+ expires_at: None,+ });+ }+ decode_document_token(+ &request_token(req, source)?,+ &Secrets::from(env).document_permissions_secret,+ )+}+/// Dedicated access boundary for document HTTP requests. Internal service/// credentials do not substitute for a signed, document-scoped user grant.pub fn document_access(@@ -144,20 +203,9 @@ pub fn document_access(crate::domain::document::DocumentError,> {use crate::domain::document::DocumentError;-- let header = req- .headers()- .get(header_names::AUTHORIZATION)- .map_err(|_| DocumentError::Unauthorized)?- .ok_or(DocumentError::Unauthorized)?;- let token = header- .strip_prefix("Bearer ")- .ok_or(DocumentError::Unauthorized)?;- let claims = macro_sync_service_jwt::decode::<AuthToken>(- token,- &Secrets::from(env).document_permissions_secret,- )- .map_err(|_| DocumentError::Unauthorized)?;+ let token = request_token(req, TokenFrom::Headers).map_err(|_| DocumentError::Unauthorized)?;+ let claims = decode_document_token(&token, &Secrets::from(env).document_permissions_secret)+ .map_err(|_| DocumentError::Unauthorized)?;let access = crate::domain::document::DocumentAccess::authorize(document_id,&claims.document_id,@@ -167,22 +215,4 @@ pub fn document_access(}#[cfg(test)]-mod test {- use super::*;- #[test]- #[allow(clippy::nonminimal_bool, reason = "demonstrate ordering")]- fn orderable() {- let view = AccessLevel::View;- let comment = AccessLevel::Comment;- let edit = AccessLevel::Edit;- let owner = AccessLevel::Owner;-- assert!(view < comment);- assert!(view <= view);- assert!(!(view < view));- assert!(edit <= owner);- assert!(view <= edit);- assert!(view < owner);- assert!(!(owner < view));- }-}+mod test;diff --git a/services/sync-service/src/auth/test.rs b/services/sync-service/src/auth/test.rsnew file mode 100644index 0000000000..dce4a11e5f--- /dev/null+++ b/services/sync-service/src/auth/test.rs@@ -0,0 +1,74 @@+use super::*;+use macro_sync_service_jwt::session::{SurfaceClaims, SurfaceSessionKind};++const ID: &str = "01952cbd-76ad-7a65-9c21-020304050607";++fn surface_token() -> String {+ macro_sync_service_jwt::session::encode_surface(+ &SurfaceClaims {+ session_kind: SurfaceSessionKind::Surface,+ surface_id: ID.parse().unwrap(),+ user_id: Some("user".into()),+ access_level: SurfaceAccessLevel::Edit,+ actor: None,+ exp: usize::MAX / 2,+ iss: macro_sync_service_jwt::ISSUER.into(),+ },+ "secret",+ )+ .unwrap()+ .into_inner()+}++#[test]+fn same_uuid_does_not_authorize_another_session_kind() {+ let document = macro_sync_service_jwt::encode(+ &serde_json::json!({"document_id": ID, "access_level": "edit", "exp": usize::MAX / 2}),+ "secret",+ )+ .unwrap();+ let surface = SessionIdentity::new(SessionKind::Surface, ID).unwrap();+ assert!(decode_document_token(&surface_token(), "secret").is_err());+ assert!(decode_surface_token(document.as_str(), "secret", &surface).is_err());+ assert!(+ decode_document_token(document.as_str(), "secret")+ .unwrap()+ .has_document_id_access(ID)+ );+ let claims = decode_surface_token(&surface_token(), "secret", &surface).unwrap();+ assert_eq!(claims.session_kind, SessionKind::Surface);+ assert!(claims.expires_at.is_some());+ assert!(!claims.has_document_id_access(ID));+ let other =+ SessionIdentity::new(SessionKind::Surface, "01952cbd-76ad-7a65-9c21-020304050608").unwrap();+ assert!(decode_surface_token(&surface_token(), "secret", &other).is_err());+}++#[test]+fn dual_kind_document_claims_are_rejected() {+ for extra in [+ serde_json::json!({"session_kind": "surface"}),+ serde_json::json!({"surface_id": ID}),+ ] {+ let mut claims =+ serde_json::json!({"document_id": ID, "access_level": "owner", "exp": usize::MAX / 2});+ claims+ .as_object_mut()+ .unwrap()+ .extend(extra.as_object().unwrap().clone());+ let token = macro_sync_service_jwt::encode(&claims, "secret").unwrap();+ assert!(decode_document_token(token.as_str(), "secret").is_err());+ }+}++#[test]+fn surface_writes_require_edit_without_changing_document_comment_behavior() {+ assert!(AccessLevel::Comment.can_edit_for(SessionKind::Document));+ assert!(!AccessLevel::View.can_edit_for(SessionKind::Document));+ for level in [AccessLevel::View, AccessLevel::Comment] {+ assert!(!level.can_edit_for(SessionKind::Surface));+ }+ for level in [AccessLevel::Edit, AccessLevel::Owner, AccessLevel::Admin] {+ assert!(level.can_edit_for(SessionKind::Surface));+ }+}diff --git a/services/sync-service/src/cf_worker.rs b/services/sync-service/src/cf_worker.rsindex bddbe68b04..ba4c0c87fc 100644--- a/services/sync-service/src/cf_worker.rs+++ b/services/sync-service/src/cf_worker.rs@@ -1,9 +1,11 @@use bebop::{Record, SubRecord};+use macro_sync_service_jwt::session::{SessionIdentity, SessionKind};use tracing::{Instrument, error};use wasm_bindgen::JsValue;use worker::{Env, Error, Headers, Method, Request, RequestInit, Response, Result, Stub};use crate::{+ auth::is_internal,constants::header_names::{AUTHORIZATION, MACRO_INTERNAL_AUTH_KEY_HEADER_KEY},durable_object::{CopyDocumentRequest, GetSnapshotRequest, response, status_codes},error::ResultExt,@@ -20,6 +22,7 @@ mod markers {pub const SCHEMA: &str = "schema";pub const COPY: &str = "copy";pub const REST: &str = "rest";+ pub const SURFACE: &str = "surface";}pub async fn router(env: Env, req: Request) -> Result<Response> {@@ -31,6 +34,18 @@ pub async fn router(env: Env, req: Request) -> Result<Response> {markers::ROOT => Response::builder().ok("Hello Sync Service!"),markers::HEALTH => Response::builder().ok("healthy"),markers::SCHEMA => Response::builder().ok(include_str!("../bebop/schema.bop")),+ markers::SURFACE => {+ let Some(id) = matched.params.get("surface_id") else {+ return Ok(response(status_codes::NOT_FOUND));+ };+ let Ok(identity) = SessionIdentity::new(SessionKind::Surface, id) else {+ return Ok(response(400));+ };+ if matched.params.get("rest") == Some("activate") {+ return activate_surface(&env, req, &identity).await;+ }+ pass_to_session(&env, req, &identity.storage_key()).await+ }needs_document_id => {let document_id = matched.params.get("document_id").with_context(|| {Error::from(format!(@@ -47,6 +62,60 @@ pub async fn router(env: Env, req: Request) -> Result<Response> {}}+/// Seal the verified original before enabling the imported target. A failed or+/// ambiguous response is retried forward; the source can no longer be thawed.+async fn activate_surface(+ env: &Env,+ mut req: Request,+ identity: &SessionIdentity,+) -> Result<Response> {+ if !is_internal(&req, env)? {+ return Ok(response(401));+ }+ if req.method() != Method::Post {+ return Ok(response(405));+ }+ let bytes = req.bytes().await?;+ let Ok(proof) =+ serde_json::from_slice::<crate::durable_object::surface_migration::SnapshotProof>(&bytes)+ else {+ return Ok(response(400));+ };+ let Some(source_id) = proof.source_id else {+ return Ok(response(409));+ };+ if identity.storage_key() != format!("surface:{source_id}") {+ return Ok(response(409));+ }+ for (key, path) in [+ (+ identity.storage_key(),+ format!("/surface/{source_id}/verify"),+ ),+ (+ source_id.to_string(),+ format!("/document/{source_id}/migration/seal"),+ ),+ (+ identity.storage_key(),+ format!("/surface/{source_id}/activate"),+ ),+ ] {+ let mut url = req.url()?;+ url.set_path(&path);+ let mut init = RequestInit::new();+ init.with_method(Method::Post)+ .with_headers(req.headers().clone())+ .with_body(Some(JsValue::from(bytes.clone())));+ let request = Request::new_with_init(url.as_ref(), &init)?;+ let result = pass_to_session(env, request, &key).await?;+ if result.status_code() != status_codes::OK || path.ends_with("/activate") {+ return Ok(result);+ }+ }+ Ok(response(409))+}+/// Get the original snapshot then initialize a new document with it./// Copying interacts with multiple durable objects so we orchestrate it from the worker.pub async fn copy_handler(env: Env, mut req: Request, document_id: &str) -> Result<Response> {@@ -133,6 +202,9 @@ pub static ROUTER: LazyLock<matchit::Router<&str>> = LazyLock::new(|| {.insert("/document/{document_id}/{*rest}", markers::REST).unwrap_context("Router.insert failed");router+ .insert("/surface/{surface_id}/{*rest}", markers::SURFACE)+ .unwrap_context("Router.insert failed");+ router});pub async fn pass_to_durable_object(@@ -140,6 +212,14 @@ pub async fn pass_to_durable_object(req: Request,document_id: &str,) -> Result<Response> {+ // Legacy document names remain valid, but cannot address the reserved namespace.+ if document_id.starts_with("surface:") {+ return Ok(response(400));+ }+ pass_to_session(env, req, document_id).await+}++async fn pass_to_session(env: &Env, req: Request, document_id: &str) -> Result<Response> {let stub = get_durable_object(env, document_id)?;let span = tracing::info_span!("do.fetch", document.id = %document_id);let req = match worker_rs_otel::traceparent_for_span(&span) {diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rsindex d5eba50472..4ab6d9635a 100644--- a/services/sync-service/src/durable_object.rs+++ b/services/sync-service/src/durable_object.rs@@ -7,6 +7,7 @@ use std::{use bebop::Record;use loro::{ExportMode, awareness::EphemeralStore};+use macro_sync_service_jwt::session::SessionKind;use matchit::Router;use serde::{Deserialize, Serialize};use tracing::{Instrument, debug, error, info, instrument, trace, warn};@@ -18,11 +19,11 @@ use worker::{use crate::{ai_peer::is_ai_peer,- auth::{AccessLevel, TokenFrom, decode_jwt},+ auth::{AccessLevel, TokenFrom, decode_jwt, socket_access},constants::USER_PEER_D1_BINDING,d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},domain::document::DocumentAttribution,- dss_internal::{DssInternal, DssInternalClient, InteractionReason},+ dss_internal::InteractionReason,error::ResultExt,generated::schema::InitializeFromSnapshotRequest,keepalive::{DEFAULT_TIME_TO_LIVE, keepalive},@@ -49,6 +50,11 @@ const DOCUMENT_ID_KEY: &str = "DOCUMENT_ID";mod document_api;mod document_effects;+mod surface_api;+pub(crate) mod surface_migration;++use document_effects::{CloseFlush, report_interaction, report_new_doc_state};+use surface_api::{SurfaceLifecycle, session_kind_from_storage_key};mod path {pub const CONNECT: &str = "connect";@@ -109,6 +115,10 @@ pub struct WebSocketMetadata {pub actor: Option<String>,#[serde(with = "u64_serde_strings")]pub peer_ids: BTreeSet<u64>,+ #[serde(default = "surface_api::document_kind")]+ pub session_kind: SessionKind,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub expires_at: Option<usize>,}pub type WsMetaMap = BTreeMap<String, WebSocketMetadata>;@@ -129,43 +139,6 @@ fn edit_attribution_from_meta<'a>(})}-/// Why a snapshot is published from `websocket_close`.-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum CloseFlush {- /// The last peer left; the pre-attribution snapshot-on-idle behaviour.- LastLeave,- /// The attributed peer left while humans stay. Its pending edits must- /// publish now, while its metadata still supplies the actor; the next- /// alarm would publish them unattributed.- ActorLeft,-}--impl CloseFlush {- /// A human leaving while the actor stays never flushes: the next alarm- /// still has the actor, and flushing here would `mark_exported` and then- /// republish the same content on last-leave.- fn decide(- is_last_leave: bool,- leaving_socket_has_actor: bool,- should_save: bool,- ) -> Option<Self> {- if is_last_leave {- Some(Self::LastLeave)- } else if leaving_socket_has_actor && should_save {- Some(Self::ActorLeft)- } else {- None- }- }-- fn interaction_reason(self) -> InteractionReason {- match self {- Self::LastLeave => InteractionReason::LastLeave,- Self::ActorLeft => InteractionReason::Edited,- }- }-}-#[durable_object]pub struct DocumentSyncSession {state: State,@@ -182,6 +155,11 @@ pub struct DocumentSyncSession {msg_buffer: Arc<Mutex<Vec<u8>>>,/// Buffered blame events. Flushed via D1 batch on each alarm tick.pending_blame: Arc<Mutex<Vec<crate::d1::BlameEvent>>>,+ surface_lifecycle: Mutex<Option<SurfaceLifecycle>>,+ source_migration: Mutex<Option<surface_migration::SourceState>>,+ // Serialize whole callbacks, including persistence awaits. A freeze must+ // drain every in-flight mutation before it exports or closes writers.+ mutation_barrier: futures::lock::Mutex<()>,}mod u64_serde_strings {@@ -228,7 +206,8 @@ mod u64_serde_strings {#[cfg(test)]mod actor_attribution_test {use super::{- AccessLevel, CloseFlush, DocumentAttribution, WebSocketMetadata, edit_attribution_from_meta,+ AccessLevel, CloseFlush, DocumentAttribution, SessionKind, WebSocketMetadata,+ edit_attribution_from_meta,};fn meta(actor: Option<&str>, user_id: Option<&str>) -> WebSocketMetadata {@@ -237,6 +216,8 @@ mod actor_attribution_test {access_level: AccessLevel::Edit,actor: actor.map(str::to_string),peer_ids: Default::default(),+ session_kind: SessionKind::Document,+ expires_at: None,}}@@ -356,7 +337,6 @@ impl<'a> Wsm<'a> {.lock("Wsm::can_edit get").get(&ws_id).ok_or(Error::from("missing ws metadata"))?- .access_level.can_edit())}@@ -393,43 +373,6 @@ pub fn get_ws_id(state: &State, ws: &WebSocket) -> Result<String> {get_ws_id_from_tags(&tags)}-/// send a shallow snapshot to cache and search service-/// we should eagerly call this from tiem to time to keep our backend up to date on-/// the status of the document:-/// - every few seconds-/// - on creation-/// - on everyone being disconnected-async fn report_new_doc_state(- document_id: &str,- snapshot: &[u8],- env: &Env,- attribution: Option<DocumentAttribution>,-) {- if let Err(err) = DssInternalClient::new(env)- .publish_shallow_snapshot(document_id, snapshot)- .await- {- warn!(error=?err, "failed to push snapshot to DSS");- }- #[cfg(feature = "search-service")]- if let Err(err) = DssInternalClient::new(env)- .publish_sync_content_updated(document_id, attribution)- .await- {- warn!(error=?err, "failed to publish document content change");- }-}--/// Report an interaction (join/leave/periodic edit) to DSS.-async fn report_interaction(document_id: &str, env: &Env, reason: InteractionReason) {- if let Err(err) = DssInternalClient::new(env)- .publish_interaction(document_id, reason)- .await- {- warn!(error=?err, "failed to push interaction to DSS");- }-}-/// Schedule an alarm 5 seconds from now.async fn bump_alarm(state: &State) -> Result<()> {let current_alarm = state.storage().get_alarm().await?;@@ -447,7 +390,7 @@ async fn bump_alarm(state: &State) -> Result<()> {impl DocumentSyncSession {pub fn get_websockets(&self) -> Vec<WebSocket> {- self.state.get_websockets()+ self.active_websockets()}fn edit_attribution(&self) -> Option<DocumentAttribution> {@@ -515,6 +458,16 @@ impl DocumentSyncSession {}async fn inner_fetch(&self, req: Request) -> Result<Response> {let url = req.url()?;+ if url.path().starts_with("/surface/") {+ return self.surface_handler(req).await;+ }+ let segments: Vec<_> = url.path().split('/').collect();+ if let ["", "document", id, "migration", operation] = segments.as_slice() {+ return self.source_migration_handler(req, id, operation).await;+ }+ if self.source_blocked().await? {+ return Ok(response(status_codes::FORBIDDEN));+ }let matched = ROUTER.at(url.path()).with_context(|| format!("Failed to route url: [{url}]"))?;@@ -623,7 +576,11 @@ impl DocumentSyncSession {.put(DOCUMENT_ID_KEY, document_id.to_string()).await?;let dkv_storage = DurableKVStorage::new(self.state.storage());- let session_storage = Rc::new(SessionStorage::new(storage, dkv_storage));+ let session_storage = Rc::new(SessionStorage::new(+ storage,+ dkv_storage,+ SessionKind::Document,+ ));*self.session_storage.lock("DocumentSyncSession::session_storage set within initialize_handler") =@@ -662,8 +619,11 @@ impl DocumentSyncSession {/// Active peer ids. With `include_ai = false`, AI editors (peer ids from the/// reserved AI block) are filtered out so callers see only human collaborators.async fn active_peer_ids_handler(&self, include_ai: bool) -> Result<Response> {+ if !self.state.get_websockets().is_empty() {+ self.validate_surface_sockets(None).await?;+ }let mut peer_ids: BTreeSet<u64> = BTreeSet::new();- for ws in self.state.get_websockets() {+ for ws in self.get_websockets() {let new_peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;peer_ids.extend(new_peer_ids@@ -815,8 +775,7 @@ impl DocumentSyncSession {async fn connect_handler(&self, req: Request, document_id: &str) -> Result<Response> {let (res, elap) = timeit!({- let claims = or_unauth!(decode_jwt(&req, &self.env, TokenFrom::QueryParams).ok());- or_unauth!(claims.has_document_id_access(document_id).then_some(()));+ let claims = or_unauth!(socket_access(&req, &self.env, document_id).ok());if self.maybe_set_document_id(document_id).await? {trace!("init document_id={document_id}");} else {@@ -842,6 +801,8 @@ impl DocumentSyncSession {access_level: claims.access_level,actor: claims.actor,peer_ids: Default::default(),+ session_kind: claims.session_kind,+ expires_at: claims.expires_at,};self.state.storage().put(&ws_id, &ws_meta).await?;@@ -849,6 +810,14 @@ impl DocumentSyncSession {.lock("DocumentSyncSession::ws_meta_map insert in connect_handler").insert(ws_id, ws_meta);+ // A surface may have been revoked while socket metadata was being+ // persisted. Recheck before sending any initial content.+ if claims.session_kind == SessionKind::Surface+ && !self.validate_surface_sockets(Some(&pair.server)).await?+ {+ return Ok(response(status_codes::FORBIDDEN));+ }+// If the snapshot is already in storage, send the initial sync now.// Otherwise accept the WS without sending — initialize_handler will// broadcast initial sync to this socket once /initialize lands.@@ -887,6 +856,9 @@ impl DocumentSyncSession {});}+ if claims.session_kind == SessionKind::Surface {+ bump_alarm(&self.state).await?;+ }Response::from_websocket(pair.client).context("failed to create websocket response")?});@@ -935,7 +907,11 @@ impl DocumentSyncSession {self.maybe_set_document_id(document_id).await?;// set up session storagelet dkv_storage = DurableKVStorage::new(self.state.storage());- let session_storage = Rc::new(SessionStorage::new(snapshot_storage, dkv_storage));+ let session_storage = Rc::new(SessionStorage::new(+ snapshot_storage,+ dkv_storage,+ session_kind_from_storage_key(document_id),+ ));*self.session_storage.lock("DocumentSyncSession::session_storage set within exists") =@@ -993,7 +969,11 @@ impl DocumentSyncSession {let id = self.document_id().await?.to_string();let snapshot_storage = get_snapshot_storage(&self.env, &self.state, id.clone())?;let dkv_storage = DurableKVStorage::new(self.state.storage());- let ss = Rc::new(SessionStorage::new(snapshot_storage, dkv_storage));+ let ss = Rc::new(SessionStorage::new(+ snapshot_storage,+ dkv_storage,+ session_kind_from_storage_key(&id),+ ));*self.session_storage.lock("DocumentSyncSession::session_storage set within main session_storage fn") =@@ -1095,12 +1075,16 @@ impl DurableObject for DocumentSyncSession {ws_meta_map: Arc::new(Mutex::new(Default::default())),msg_buffer: Arc::new(Mutex::new(vec![])),pending_blame: Arc::new(Mutex::new(Vec::new())),+ surface_lifecycle: Mutex::new(None),+ source_migration: Mutex::new(None),+ mutation_barrier: futures::lock::Mutex::new(()),}}/// Fetch the durable object/// Upgrades the request to a websocket request connected to the document sessionasync fn fetch(&self, req: Request) -> Result<Response> {+ let _barrier = self.mutation_barrier.lock().await;let set_allow_origin = if let Some(origin) = req.headers().get("Origin")@@ -1144,6 +1128,14 @@ impl DurableObject for DocumentSyncSession {}async fn websocket_message(&self, ws: WebSocket, msg: WebSocketIncomingMessage) -> Result<()> {+ let _barrier = self.mutation_barrier.lock().await;+ if self.source_blocked().await? {+ ws.close(Some(1008), Some("session frozen or retired"))?;+ return Ok(());+ }+ if !self.validate_surface_sockets(Some(&ws)).await? {+ return Ok(());+ }const PONG: &str = "pong";const PING: &str = "ping";let binary_message = match msg {@@ -1221,6 +1213,11 @@ impl DurableObject for DocumentSyncSession {/// Save document if neededasync fn alarm(&self) -> Result<Response> {+ let _barrier = self.mutation_barrier.lock().await;+ if self.source_blocked().await? {+ return Response::empty();+ }+ self.validate_surface_sockets(None).await?;let span = tracing::info_span!("do.alarm");worker_rs_otel::scope(&self.env, &self.state, async {let state = match self@@ -1301,7 +1298,13 @@ impl DurableObject for DocumentSyncSession {_reason: String,_was_clean: bool,) -> Result<()> {+ let _barrier = self.mutation_barrier.lock().await;+ if self.source_blocked().await? {+ self.forget_websocket_metadata(&ws).await;+ return Ok(());+ }worker_rs_otel::scope(&self.env, &self.state, async {+ self.validate_surface_sockets(None).await?;let peer_ids = Wsm::new(self, &ws).get_peer_ids().await?;for peer_id in peer_ids {self.awareness.delete(&peer_id.to_string());@@ -1310,7 +1313,7 @@ impl DurableObject for DocumentSyncSession {// Don't silently discard the errorwebsocket::broadcast_awareness(&ws,- self.state.get_websockets().as_slice(),+ self.get_websockets().as_slice(),update.as_slice(),self.msg_buffer.clone(),)diff --git a/services/sync-service/src/durable_object/document_effects.rs b/services/sync-service/src/durable_object/document_effects.rsindex 59685b2b41..84d338ad22 100644--- a/services/sync-service/src/durable_object/document_effects.rs+++ b/services/sync-service/src/durable_object/document_effects.rs@@ -1,15 +1,92 @@//! Worker notification adapter for the document update use case.use bebop::{Record, SubRecord};+use macro_sync_service_jwt::session::SessionKind;use tracing::{error, warn};+use worker::Env;-use super::{DocumentSyncSession, bump_alarm, report_interaction, report_new_doc_state};+use super::{DocumentSyncSession, bump_alarm, surface_api::session_kind_from_storage_key};use crate::{domain::document::{DocumentAttribution, DocumentError, DocumentUpdateEffects},- dss_internal::InteractionReason,+ dss_internal::{DssInternal, DssInternalClient, InteractionReason},state::DocumentState,};+/// Why a snapshot is published from `websocket_close`.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum CloseFlush {+ /// The last peer left; the pre-attribution snapshot-on-idle behaviour.+ LastLeave,+ /// The attributed peer left while humans stay. Publish pending edits while+ /// its metadata still supplies the actor, before the next unattributed alarm.+ ActorLeft,+}++impl CloseFlush {+ /// A human leaving while the actor stays never flushes: the next alarm+ /// still has the actor, and flushing here would `mark_exported` and then+ /// republish the same content on last-leave.+ pub(super) fn decide(+ is_last_leave: bool,+ leaving_socket_has_actor: bool,+ should_save: bool,+ ) -> Option<Self> {+ if is_last_leave {+ Some(Self::LastLeave)+ } else if leaving_socket_has_actor && should_save {+ Some(Self::ActorLeft)+ } else {+ None+ }+ }++ pub(super) fn interaction_reason(self) -> InteractionReason {+ match self {+ Self::LastLeave => InteractionReason::LastLeave,+ Self::ActorLeft => InteractionReason::Edited,+ }+ }+}++/// All document-only effects pass through this boundary, including socket joins,+/// initialization, alarms, HTTP updates and disconnects. The persisted storage key+/// is a kind-discriminated identity; surface keys never reach document consumers.+pub(super) async fn report_new_doc_state(+ session_key: &str,+ snapshot: &[u8],+ env: &Env,+ attribution: Option<DocumentAttribution>,+) {+ if session_kind_from_storage_key(session_key) != SessionKind::Document {+ return;+ }+ if let Err(error) = DssInternalClient::new(env)+ .publish_shallow_snapshot(session_key, snapshot)+ .await+ {+ warn!(error = ?error, "failed to push snapshot to DSS");+ }+ #[cfg(feature = "search-service")]+ if let Err(error) = DssInternalClient::new(env)+ .publish_sync_content_updated(session_key, attribution)+ .await+ {+ warn!(error = ?error, "failed to publish document content change");+ }+}++pub(super) async fn report_interaction(session_key: &str, env: &Env, reason: InteractionReason) {+ if session_kind_from_storage_key(session_key) != SessionKind::Document {+ return;+ }+ if let Err(error) = DssInternalClient::new(env)+ .publish_interaction(session_key, reason)+ .await+ {+ warn!(error = ?error, "failed to push interaction to DSS");+ }+}+pub(super) struct WorkerDocumentEffects<'a> {pub session: &'a DocumentSyncSession,pub document_state: &'a DocumentState,diff --git a/services/sync-service/src/durable_object/surface_api.rs b/services/sync-service/src/durable_object/surface_api.rsnew file mode 100644index 0000000000..54936ac1b0--- /dev/null+++ b/services/sync-service/src/durable_object/surface_api.rs@@ -0,0 +1,313 @@+//! Isolated surface transport and persisted lifecycle. No document API fallback.++use bebop::Record;+use macro_sync_service_jwt::session::{SessionIdentity, SessionKind};+use serde::{Deserialize, Serialize};+use worker::{Date, Method, Request, Response, Result, WebSocket};++use super::{DocumentSyncSession, WebSocketMetadata, Wsm, response, status_codes};+use crate::{+ auth::{TokenFrom, is_internal, surface_access},+ error::ResultExt,+ generated::schema::InitializeFromSnapshotRequest,+ state::DocumentState,+ storage::{get_snapshot_storage, snapshot::SnapshotStorage},+};++#[cfg(test)]+mod test;++pub(super) const LIFECYCLE_KEY: &str = "SURFACE_LIFECYCLE";++/// The discriminant is part of the persisted DOCUMENT_ID key, not the current+/// request URL. Legacy document keys are unchanged and cannot use this prefix.+pub(super) fn session_kind_from_storage_key(key: &str) -> SessionKind {+ if key.starts_with("surface:") {+ SessionKind::Surface+ } else {+ SessionKind::Document+ }+}++pub(super) fn document_kind() -> SessionKind {+ SessionKind::Document+}++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub(super) enum SurfaceLifecycle {+ #[default]+ Pending,+ /// In-flight initialization; only kept in memory, never considered ready.+ Initializing,+ Ready,+ Revoked,+}++impl WebSocketMetadata {+ pub(super) fn grant_active(&self, kind: SessionKind, now: usize) -> bool {+ self.session_kind == kind+ && match kind {+ SessionKind::Document => true,+ SessionKind::Surface => self.expires_at.is_some_and(|expiry| expiry > now),+ }+ }++ pub(super) fn can_edit(&self) -> bool {+ self.grant_active(self.session_kind, now_seconds())+ && self.access_level.can_edit_for(self.session_kind)+ }+}++/// A pending session has no writers. Exact seed bytes identify a retry of the+/// binary initializer, which has no separate operation ID. Unknown seeds must+/// be rolled back before another initialization can write a replacement.+async fn prepare_surface_snapshot(storage: &impl SnapshotStorage, snapshot: &[u8]) -> Result<bool> {+ if storage.has_snapshot().await? {+ if storage.get_snapshot().await? != snapshot {+ storage.delete_snapshot().await?;+ return Ok(false);+ }+ } else {+ storage.store_snapshot(snapshot).await?;+ }+ Ok(true)+}++fn now_seconds() -> usize {+ (Date::now().as_millis() / 1000) as usize+}++impl DocumentSyncSession {+ pub(super) async fn surface_lifecycle(&self) -> Result<SurfaceLifecycle> {+ if let Some(value) = *self.surface_lifecycle.lock("surface lifecycle cached") {+ return Ok(value);+ }+ let value = self+ .state+ .storage()+ .get(LIFECYCLE_KEY)+ .await?+ .unwrap_or_default();+ let mut cached = self.surface_lifecycle.lock("surface lifecycle loaded");+ // Do not overwrite a transition made while the storage read was pending.+ Ok(*cached.get_or_insert(value))+ }++ pub(super) async fn surface_handler(&self, req: Request) -> Result<Response> {+ let url = req.url()?;+ let segments: Vec<_> = url.path().split('/').collect();+ let ["", "surface", id, operation, rest @ ..] = segments.as_slice() else {+ return Ok(response(status_codes::NOT_FOUND));+ };+ let Ok(identity) = SessionIdentity::new(SessionKind::Surface, id) else {+ return Ok(response(400));+ };+ // Explicit allowlist: no document copy, raw, debug, CAS, or wakeup API.+ let allowed_method = match (*operation, rest) {+ ("connect" | "exists" | "active_peers", []) | ("peer", [_]) => Method::Get,+ (+ "initialize"+ | "revoke"+ | "snapshot"+ | "initialize_verified"+ | "import"+ | "verify"+ | "activate",+ [],+ ) => Method::Post,+ _ => return Ok(response(status_codes::NOT_FOUND)),+ };+ if req.method() != allowed_method {+ return Ok(response(405));+ }+ if matches!(+ *operation,+ "initialize" | "revoke" | "initialize_verified" | "import" | "verify" | "activate"+ ) {+ if !is_internal(&req, &self.env)? {+ return Ok(response(status_codes::UNAUTH));+ }+ } else {+ let source = if *operation == "connect" {+ TokenFrom::QueryParams+ } else {+ TokenFrom::Headers+ };+ if surface_access(&req, &self.env, source, &identity).is_err() {+ return Ok(response(status_codes::UNAUTH));+ }+ }+ let key = identity.storage_key();+ let persisted: Option<String> = self.state.storage().get(super::DOCUMENT_ID_KEY).await?;+ if persisted.is_some_and(|persisted| persisted != key) {+ return Ok(response(409));+ }+ self.maybe_set_document_id(&key).await?;+ if *operation == "revoke" {+ return self.revoke_surface().await;+ }+ let lifecycle = self.surface_lifecycle().await?;+ if lifecycle == SurfaceLifecycle::Revoked {+ return Ok(response(status_codes::FORBIDDEN));+ }+ if matches!(+ *operation,+ "initialize_verified" | "import" | "verify" | "activate"+ ) {+ return self.target_migration_handler(req, &key, operation).await;+ }+ if *operation == "initialize" {+ if self.has_target_migration().await? {+ return Ok(response(409));+ }+ return self.initialize_surface(req, &key).await;+ }+ if lifecycle != SurfaceLifecycle::Ready {+ return Ok(response(409));+ }+ // Missing/corrupt snapshots must not trigger create-default-state, even+ // when that development feature is enabled for documents.+ self.document_state().await?;+ if !self.validate_surface_sockets(None).await? {+ return Ok(response(status_codes::FORBIDDEN));+ }+ match *operation {+ "connect" => self.connect_handler(req, &key).await,+ "snapshot" => {+ let snapshot = self.document_state().await?.export_snapshot(None)?;+ Ok(Response::builder().body(worker::ResponseBody::Body(snapshot)))+ }+ "peer" => self.peer_handler(&key, rest.first().copied()).await,+ "active_peers" => self.active_peer_ids_handler(true).await,+ "exists" => Ok(response(status_codes::OK)),+ _ => Ok(response(status_codes::NOT_FOUND)),+ }+ }++ async fn initialize_surface(&self, mut req: Request, key: &str) -> Result<Response> {+ let bytes = req.bytes().await?;+ let body = InitializeFromSnapshotRequest::deserialize(&bytes)+ .context("invalid surface snapshot request")?;+ DocumentState::try_from_snapshot(&body.snapshot).context("invalid surface snapshot")?;+ // Claim synchronously before awaiting snapshot IO; a second initializer+ // must not overwrite either the winner or an existing snapshot.+ match self.surface_lifecycle().await? {+ SurfaceLifecycle::Revoked => return Ok(response(status_codes::FORBIDDEN)),+ SurfaceLifecycle::Ready | SurfaceLifecycle::Initializing => return Ok(response(409)),+ SurfaceLifecycle::Pending => {}+ }+ *self.surface_lifecycle.lock("claim surface initialization") =+ Some(SurfaceLifecycle::Initializing);+ let result: Result<Response> = async {+ let storage = get_snapshot_storage(&self.env, &self.state, key.to_owned())?;+ if !prepare_surface_snapshot(&storage, &body.snapshot).await? {+ return Ok(response(409));+ }+ #[cfg(feature = "migration-test-hooks")]+ if req.headers().get("x-sync-test-stop-after")?.as_deref() == Some("surface_snapshot") {+ return Err(worker::Error::from("injected surface persistence failure"));+ }+ if self.surface_lifecycle().await? == SurfaceLifecycle::Revoked {+ return Ok(response(status_codes::FORBIDDEN));+ }+ self.state+ .storage()+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Ready)+ .await?;+ if self.surface_lifecycle().await? == SurfaceLifecycle::Revoked {+ return Ok(response(status_codes::FORBIDDEN));+ }+ *self.surface_lifecycle.lock("surface ready") = Some(SurfaceLifecycle::Ready);+ Ok(response(status_codes::OK))+ }+ .await;+ if !result+ .as_ref()+ .is_ok_and(|response| response.status_code() == status_codes::OK)+ {+ let mut lifecycle = self+ .surface_lifecycle+ .lock("release surface initialization");+ if *lifecycle != Some(SurfaceLifecycle::Revoked) {+ // A failed Ready write may have committed. Reload durable truth+ // rather than admitting a new initializer against a ready seed.+ *lifecycle = None;+ }+ }+ result+ }++ pub(super) async fn revoke_surface(&self) -> Result<Response> {+ // Deny in-flight callbacks before yielding, and persist the tombstone+ // before acknowledging. Retrying revocation is safe, including pending IDs.+ *self.surface_lifecycle.lock("revoke surface") = Some(SurfaceLifecycle::Revoked);+ self.state+ .storage()+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Revoked)+ .await?;+ for socket in self.state.get_websockets() {+ socket.close(Some(1008), Some("surface revoked"))?;+ }+ Ok(response(status_codes::OK))+ }++ /// Reload socket grants after hibernation before accepting messages or+ /// selecting broadcast recipients. Cached grants avoid per-message IO.+ pub(crate) async fn validate_surface_sockets(+ &self,+ sender: Option<&WebSocket>,+ ) -> Result<bool> {+ let key = self.document_id().await?;+ if session_kind_from_storage_key(&key) == SessionKind::Document {+ return Ok(true);+ }+ let ready = self.surface_lifecycle().await? == SurfaceLifecycle::Ready;+ let now = now_seconds();+ for socket in self.state.get_websockets() {+ let mut wsm = Wsm::new(self, &socket);+ wsm.maybe_update_ws_meta_map().await?;+ let ws_id = wsm.get_ws_id()?;+ let active = ready+ && self+ .ws_meta_map+ .lock("validate surface grant")+ .get(ws_id)+ .is_some_and(|meta| meta.grant_active(SessionKind::Surface, now));+ if !active {+ socket.close(Some(1008), Some("surface grant expired or revoked"))?;+ }+ }+ // Metadata restoration can yield. Use current lifecycle and time, not+ // the values observed before those reads, when authorizing the sender.+ Ok(self.surface_lifecycle().await? == SurfaceLifecycle::Ready+ && sender.is_none_or(|sender| self.active_websockets().contains(sender)))+ }++ pub(super) fn active_websockets(&self) -> Vec<WebSocket> {+ let key = self.document_id.lock("broadcast session identity");+ let Some(key) = key.as_ref() else {+ return Vec::new();+ };+ if session_kind_from_storage_key(key) == SessionKind::Document {+ return self.state.get_websockets();+ }+ if *self.surface_lifecycle.lock("broadcast surface lifecycle")+ != Some(SurfaceLifecycle::Ready)+ {+ return Vec::new();+ }+ let metas = self.ws_meta_map.lock("broadcast surface grants");+ let now = now_seconds();+ self.state+ .get_websockets()+ .into_iter()+ .filter(|socket| {+ super::get_ws_id(&self.state, socket)+ .ok()+ .and_then(|id| metas.get(&id))+ .is_some_and(|meta| meta.grant_active(SessionKind::Surface, now))+ })+ .collect()+ }+}diff --git a/services/sync-service/src/durable_object/surface_api/test.rs b/services/sync-service/src/durable_object/surface_api/test.rsnew file mode 100644index 0000000000..89dc423f37--- /dev/null+++ b/services/sync-service/src/durable_object/surface_api/test.rs@@ -0,0 +1,125 @@+use super::*;+use crate::auth::AccessLevel;+use std::cell::{Cell, RefCell};++#[derive(Default)]+struct SnapshotStore {+ snapshot: RefCell<Option<Vec<u8>>>,+ writes: Cell<usize>,+ fail_delete: Cell<bool>,+}++impl SnapshotStorage for SnapshotStore {+ async fn store_snapshot(&self, snapshot: &[u8]) -> Result<()> {+ self.writes.set(self.writes.get() + 1);+ *self.snapshot.borrow_mut() = Some(snapshot.to_vec());+ Ok(())+ }++ async fn get_snapshot(&self) -> Result<Vec<u8>> {+ Ok(self.snapshot.borrow().clone().unwrap())+ }++ async fn has_snapshot(&self) -> Result<bool> {+ Ok(self.snapshot.borrow().is_some())+ }++ async fn delete_snapshot(&self) -> Result<()> {+ if self.fail_delete.get() {+ return Err(worker::Error::from("injected deletion failure"));+ }+ *self.snapshot.borrow_mut() = None;+ Ok(())+ }+}++#[test]+fn pending_initialization_resumes_an_identical_seed_without_rewriting() {+ futures::executor::block_on(async {+ let storage = SnapshotStore::default();+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ // Snapshot committed, but the lifecycle write failed or the DO evicted.+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ assert_eq!(storage.writes.get(), 1);+ assert_eq!(storage.get_snapshot().await.unwrap(), b"seed");+ });+}++#[test]+fn unknown_pending_seed_is_removed_before_allowing_a_new_initialization() {+ futures::executor::block_on(async {+ let storage = SnapshotStore::default();+ storage.store_snapshot(b"orphan").await.unwrap();+ assert!(!prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ assert!(!storage.has_snapshot().await.unwrap());+ assert_eq!(storage.writes.get(), 1);+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ assert_eq!(storage.get_snapshot().await.unwrap(), b"seed");+ });+}++#[test]+fn failed_rollback_never_overwrites_the_unidentified_snapshot() {+ futures::executor::block_on(async {+ let storage = SnapshotStore::default();+ storage.store_snapshot(b"orphan").await.unwrap();+ storage.fail_delete.set(true);+ for _ in 0..2 {+ assert!(prepare_surface_snapshot(&storage, b"seed").await.is_err());+ assert_eq!(storage.get_snapshot().await.unwrap(), b"orphan");+ assert_eq!(storage.writes.get(), 1);+ }+ storage.fail_delete.set(false);+ assert!(!prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ assert!(prepare_surface_snapshot(&storage, b"seed").await.unwrap());+ });+}++#[test]+fn hibernated_surface_metadata_retains_kind_and_strict_expiry() {+ let meta = WebSocketMetadata {+ user_id: Some("user".into()),+ access_level: AccessLevel::Edit,+ actor: Some("actor".into()),+ peer_ids: [u64::MAX].into(),+ session_kind: SessionKind::Surface,+ expires_at: Some(100),+ };+ let stored = serde_json::to_vec(&meta).unwrap();+ let restored: WebSocketMetadata = serde_json::from_slice(&stored).unwrap();+ assert!(restored.grant_active(SessionKind::Surface, 99));+ assert!(!restored.grant_active(SessionKind::Surface, 100));+ assert!(!restored.grant_active(SessionKind::Surface, 101));+ assert!(!restored.grant_active(SessionKind::Document, 99));+ assert_eq!(restored.actor, meta.actor);+ assert_eq!(restored.peer_ids, meta.peer_ids);+}++#[test]+fn legacy_metadata_is_only_valid_for_documents() {+ let legacy: WebSocketMetadata = serde_json::from_value(serde_json::json!({+ "user_id": "user", "access_level": "comment", "peer_ids": []+ }))+ .unwrap();+ assert!(legacy.grant_active(SessionKind::Document, usize::MAX));+ assert!(!legacy.grant_active(SessionKind::Surface, 0));+ let missing_expiry = WebSocketMetadata {+ session_kind: SessionKind::Surface,+ ..legacy+ };+ assert!(!missing_expiry.grant_active(SessionKind::Surface, 0));+}++#[test]+fn persisted_storage_key_discriminates_effect_dispatch_after_eviction() {+ let id = "01952cbd-76ad-7a65-9c21-020304050607";+ for kind in [SessionKind::Surface, SessionKind::Document] {+ let identity = SessionIdentity::new(kind, id).unwrap();+ assert_eq!(session_kind_from_storage_key(&identity.storage_key()), kind);+ }+ let revoked = serde_json::to_vec(&SurfaceLifecycle::Revoked).unwrap();+ assert_eq!(+ serde_json::from_slice::<SurfaceLifecycle>(&revoked).unwrap(),+ SurfaceLifecycle::Revoked+ );+}diff --git a/services/sync-service/src/durable_object/surface_migration.rs b/services/sync-service/src/durable_object/surface_migration.rsnew file mode 100644index 0000000000..9d5878f742--- /dev/null+++ b/services/sync-service/src/durable_object/surface_migration.rs@@ -0,0 +1,477 @@+//! Recoverable, internal-only session transfers. The caller must first verify+//! legacy ownership through the documents domain. This module never infers it.+//!+//! Freeze drains the callback barrier and preserves the original snapshot/log.+//! Activation seals the source in the outer worker BEFORE enabling the target:+//! a crash can leave both unavailable, but can never leave two writable copies.+//! Sealing is the irreversible commit intent; recovery thereafter is forward-only.++use loro::{Frontiers, VersionVector};+use serde::{Deserialize, Serialize};+use sha2::{Digest, Sha256};+use uuid::Uuid;+use worker::{Method, Request, Response, Result};++use super::{+ DocumentSyncSession, response,+ surface_api::{LIFECYCLE_KEY, SurfaceLifecycle},+};+use crate::{+ auth::is_internal,+ error::ResultExt,+ state::DocumentState,+ storage::{+ backends::durable_sql::DurableSQLStorage, get_snapshot_storage, snapshot::SnapshotStorage,+ },+};++const SOURCE_KEY: &str = "SURFACE_MIGRATION_SOURCE";+const TARGET_KEY: &str = "SURFACE_MIGRATION_TARGET";++#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]+pub(crate) struct SnapshotProof {+ pub operation_id: Uuid,+ pub source_id: Option<Uuid>,+ pub digest: String,+ pub content_digest: String,+ pub revision: Vec<(String, i32)>,+ pub oplog_revision: Vec<(String, i32)>,+}++#[derive(Serialize, Deserialize)]+struct SnapshotExport {+ proof: SnapshotProof,+ snapshot: Vec<u8>,+}++#[derive(Deserialize)]+struct FreezeRequest {+ operation_id: Uuid,+}++#[derive(Deserialize)]+struct InitializeRequest {+ operation_id: Uuid,+ snapshot: Vec<u8>,+}++#[derive(Clone, Default, Serialize, Deserialize)]+#[serde(tag = "phase", content = "proof", rename_all = "snake_case")]+pub(super) enum SourceState {+ #[default]+ Live,+ Freezing(Uuid),+ Frozen(SnapshotProof),+ Sealed(SnapshotProof),+ Retired(SnapshotProof),+ Thawed(SnapshotProof),+}++#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+enum TargetPhase {+ Claimed,+ Verified,+ Active,+}++#[derive(Serialize, Deserialize)]+struct TargetState {+ proof: SnapshotProof,+ phase: TargetPhase,+}++fn revision(frontiers: Frontiers) -> Vec<(String, i32)> {+ let mut ids: Vec<_> = frontiers+ .iter()+ .map(|id| (id.peer.to_string(), id.counter))+ .collect();+ ids.sort();+ ids+}++fn digest(bytes: &[u8]) -> String {+ format!("{:x}", Sha256::digest(bytes))+}++impl SnapshotProof {+ fn from_snapshot(operation_id: Uuid, source_id: Option<Uuid>, snapshot: &[u8]) -> Result<Self> {+ let state = DocumentState::try_from_snapshot(snapshot)?;+ // Canonical JSON key order, independent of CRDT map iteration order.+ let mut content: serde_json::Value = serde_json::from_str(&state.get_json())?;+ content.sort_all_objects();+ Ok(Self {+ operation_id,+ source_id,+ digest: digest(snapshot),+ content_digest: digest(&serde_json::to_vec(&content)?),+ revision: revision(state.loro_doc.state_frontiers()),+ oplog_revision: revision(state.loro_doc.oplog_frontiers()),+ })+ }++ fn verifies(&self, snapshot: &[u8]) -> Result<bool> {+ Ok(*self == Self::from_snapshot(self.operation_id, self.source_id, snapshot)?)+ }+}++/// Fault injection is absent from production builds, even for internal callers.+fn persistence_boundary(req: &Request, boundary: &str) -> Result<()> {+ #[cfg(feature = "migration-test-hooks")]+ if req.headers().get("x-sync-test-stop-after")?.as_deref() == Some(boundary) {+ return Err(worker::Error::from(+ "injected migration persistence failure",+ ));+ }+ let _ = (req, boundary);+ Ok(())+}++impl DocumentSyncSession {+ async fn source_state(&self) -> Result<SourceState> {+ if let Some(value) = self.source_migration.lock("source state").clone() {+ return Ok(value);+ }+ let value: SourceState = self+ .state+ .storage()+ .get(SOURCE_KEY)+ .await?+ .unwrap_or_default();+ *self.source_migration.lock("load source state") = Some(value.clone());+ Ok(value)+ }++ pub(super) async fn source_blocked(&self) -> Result<bool> {+ Ok(!matches!(+ self.source_state().await?,+ SourceState::Live | SourceState::Thawed(_)+ ))+ }++ async fn store_source(&self, value: SourceState) -> Result<()> {+ // A failed write may have committed. Reload durable truth on the next+ // callback instead of retaining a potentially stale writable cache.+ *self.source_migration.lock("invalidate source state") = None;+ self.state.storage().put(SOURCE_KEY, &value).await?;+ *self.source_migration.lock("persist source state") = Some(value);+ Ok(())+ }++ fn close_migration_writers(&self) -> Result<()> {+ for socket in self.state.get_websockets() {+ socket.close(Some(1008), Some("session frozen or retired"))?;+ }+ Ok(())+ }++ fn export_storage(&self, operation_id: Uuid) -> Result<DurableSQLStorage> {+ // Separate, chunked local storage: never replace the original snapshot,+ // never clear its operation log, and never depend on eventual KV reads.+ DurableSQLStorage::new(+ self.state.storage(),+ format!("migration-export:{operation_id}"),+ )+ }++ async fn authoritative_snapshot(&self) -> Result<Vec<u8>> {+ let storage = self.session_storage().await?;+ // Strict loading: missing snapshots must not invoke create-default-state;+ // malformed pending entries must not be silently skipped.+ let state = DocumentState::try_from_snapshot(&storage.get_snapshot().await?)?;+ let mut updates: Vec<Vec<u8>> = storage+ .get_pending_operations()+ .await?+ .into_iter()+ .map(|entry| entry.map(|(_, bytes)| bytes))+ .collect::<Result<_>>()?;+ if let Some(current) = self.document_state.lock("freeze current state").as_ref() {+ updates.push(current.export_snapshot(None)?);+ }+ let imported = state+ .loro_doc+ .import_batch(&updates)+ .context("replay frozen operation log")?;+ if imported.pending.is_some() {+ return Err(worker::Error::from(+ "migration has unresolved pending operations",+ ));+ }+ if let Some(bytes) = storage.debug_do_kv_get("LAST_VERSION_VECTOR").await? {+ let saved = VersionVector::decode(&bytes).context("invalid saved revision")?;+ if !matches!(+ state.loro_doc.oplog_vv().partial_cmp(&saved),+ Some(std::cmp::Ordering::Equal | std::cmp::Ordering::Greater)+ ) {+ return Err(worker::Error::from(+ "migration snapshot is behind saved revision",+ ));+ }+ }+ state.export_snapshot(None)+ }++ pub(super) async fn source_migration_handler(+ &self,+ mut req: Request,+ id: &str,+ operation: &str,+ ) -> Result<Response> {+ if !is_internal(&req, &self.env)? {+ return Ok(response(401));+ }+ if req.method() != Method::Post {+ return Ok(response(405));+ }+ let Ok(source_id) = Uuid::parse_str(id) else {+ return Ok(response(400));+ };+ if operation == "freeze" {+ let Ok(body) = req.json::<FreezeRequest>().await else {+ return Ok(response(400));+ };+ return self+ .freeze_source(&req, id, source_id, body.operation_id)+ .await;+ }+ if !matches!(operation, "thaw" | "seal" | "retire") {+ return Ok(response(404));+ }+ let Ok(proof) = req.json::<SnapshotProof>().await else {+ return Ok(response(400));+ };+ if proof.source_id != Some(source_id) {+ return Ok(response(409));+ }+ let current = self.source_state().await?;+ let next = match (operation, ¤t) {+ ("thaw", SourceState::Frozen(expected) | SourceState::Thawed(expected))+ if *expected == proof =>+ {+ self.maybe_set_document_id(id).await?;+ let snapshot = self.authoritative_snapshot().await?;+ // Snapshot serialization need not be byte-stable on replay. Verify+ // both frontiers and decoded content against the original proof.+ let actual =+ SnapshotProof::from_snapshot(proof.operation_id, proof.source_id, &snapshot)?;+ if actual.revision != proof.revision+ || actual.oplog_revision != proof.oplog_revision+ || actual.content_digest != proof.content_digest+ {+ return Ok(response(409));+ }+ SourceState::Thawed(proof.clone())+ }+ (+ "seal",+ SourceState::Frozen(expected)+ | SourceState::Sealed(expected)+ | SourceState::Retired(expected),+ ) if *expected == proof => {+ if matches!(current, SourceState::Retired(_)) {+ return Response::from_json(&proof);+ }+ SourceState::Sealed(proof.clone())+ }+ ("retire", SourceState::Sealed(expected) | SourceState::Retired(expected))+ if *expected == proof =>+ {+ SourceState::Retired(proof.clone())+ }+ _ => return Ok(response(409)),+ };+ self.store_source(next).await?;+ persistence_boundary(+ &req,+ match operation {+ "thaw" => "source_thawed",+ "seal" => "source_sealed",+ _ => "source_retired",+ },+ )?;+ if operation != "thaw" {+ self.close_migration_writers()?;+ }+ Response::from_json(&proof)+ }++ async fn freeze_source(+ &self,+ req: &Request,+ id: &str,+ source_id: Uuid,+ operation_id: Uuid,+ ) -> Result<Response> {+ match self.source_state().await? {+ SourceState::Frozen(proof)+ | SourceState::Sealed(proof)+ | SourceState::Retired(proof)+ if proof.operation_id == operation_id =>+ {+ self.close_migration_writers()?;+ let snapshot = self.export_storage(operation_id)?.get_snapshot().await?;+ if !proof.verifies(&snapshot)? {+ return Ok(response(409));+ }+ return Response::from_json(&SnapshotExport { proof, snapshot });+ }+ SourceState::Freezing(expected) if expected == operation_id => {}+ SourceState::Live => {}+ SourceState::Thawed(proof) if proof.operation_id != operation_id => {}+ _ => return Ok(response(409)),+ }+ let storage = get_snapshot_storage(&self.env, &self.state, id.to_owned())?;+ if !storage.has_snapshot().await? {+ return Ok(response(404));+ }+ self.maybe_set_document_id(id).await?;+ self.store_source(SourceState::Freezing(operation_id))+ .await?;+ persistence_boundary(req, "source_claim")?;+ self.close_migration_writers()?;+ let snapshot = self.authoritative_snapshot().await?;+ let proof = SnapshotProof::from_snapshot(operation_id, Some(source_id), &snapshot)?;+ self.export_storage(operation_id)?+ .store_snapshot(&snapshot)+ .await?;+ persistence_boundary(req, "source_export")?;+ self.store_source(SourceState::Frozen(proof.clone()))+ .await?;+ persistence_boundary(req, "source_frozen")?;+ Response::from_json(&SnapshotExport { proof, snapshot })+ }++ pub(super) async fn has_target_migration(&self) -> Result<bool> {+ Ok(self+ .state+ .storage()+ .get::<TargetState>(TARGET_KEY)+ .await?+ .is_some())+ }++ pub(super) async fn target_migration_handler(+ &self,+ mut req: Request,+ key: &str,+ operation: &str,+ ) -> Result<Response> {+ if matches!(operation, "initialize_verified" | "import") {+ let export = if operation == "initialize_verified" {+ let Ok(body) = req.json::<InitializeRequest>().await else {+ return Ok(response(400));+ };+ let Ok(proof) =+ SnapshotProof::from_snapshot(body.operation_id, None, &body.snapshot)+ else {+ return Ok(response(400));+ };+ SnapshotExport {+ proof,+ snapshot: body.snapshot,+ }+ } else {+ let Ok(body) = req.json::<SnapshotExport>().await else {+ return Ok(response(400));+ };+ // Legacy surfaces keep their UUID when moving to the isolated+ // namespace. This also binds source sealing to exactly one target.+ if body.proof.source_id.map(|id| format!("surface:{id}")) != Some(key.to_owned()) {+ return Ok(response(409));+ }+ body+ };+ if !export.proof.verifies(&export.snapshot).unwrap_or(false) {+ return Ok(response(400));+ }+ return self+ .import_surface(&req, key, export, operation == "initialize_verified")+ .await;+ }+ let Ok(proof) = req.json::<SnapshotProof>().await else {+ return Ok(response(400));+ };+ let Some(mut target) = self.state.storage().get::<TargetState>(TARGET_KEY).await? else {+ return Ok(response(409));+ };+ if target.proof != proof || target.phase == TargetPhase::Claimed {+ return Ok(response(409));+ }+ if target.phase != TargetPhase::Active {+ let snapshot = get_snapshot_storage(&self.env, &self.state, key.to_owned())?+ .get_snapshot()+ .await?;+ if !proof.verifies(&snapshot)? {+ return Ok(response(409));+ }+ }+ if operation == "activate" {+ target.phase = TargetPhase::Active;+ self.state.storage().put(TARGET_KEY, &target).await?;+ persistence_boundary(&req, "target_active")?;+ self.mark_target_ready(&req).await?;+ }+ Response::from_json(&target.proof)+ }++ async fn import_surface(+ &self,+ req: &Request,+ key: &str,+ export: SnapshotExport,+ initialize: bool,+ ) -> Result<Response> {+ let storage = get_snapshot_storage(&self.env, &self.state, key.to_owned())?;+ let target: Option<TargetState> = self.state.storage().get(TARGET_KEY).await?;+ let mut target = if let Some(target) = target {+ if target.proof != export.proof {+ return Ok(response(409));+ }+ target+ } else {+ if self.surface_lifecycle().await? != SurfaceLifecycle::Pending+ || storage.has_snapshot().await?+ {+ return Ok(response(409));+ }+ let target = TargetState {+ proof: export.proof,+ phase: TargetPhase::Claimed,+ };+ self.state.storage().put(TARGET_KEY, &target).await?;+ persistence_boundary(req, "target_claim")?;+ target+ };+ // Once active, edits are allowed. A replay of the original operation+ // returns its durable receipt, never overwriting or re-verifying live data.+ if target.phase != TargetPhase::Active {+ if !storage.has_snapshot().await? {+ storage.store_snapshot(&export.snapshot).await?;+ persistence_boundary(req, "target_snapshot")?;+ }+ // Read back and decode, verifying content, both frontiers and bytes.+ if !target.proof.verifies(&storage.get_snapshot().await?)? {+ return Ok(response(409));+ }+ target.phase = TargetPhase::Verified;+ self.state.storage().put(TARGET_KEY, &target).await?;+ persistence_boundary(req, "target_verified")?;+ }+ if initialize {+ target.phase = TargetPhase::Active;+ self.state.storage().put(TARGET_KEY, &target).await?;+ persistence_boundary(req, "target_active")?;+ self.mark_target_ready(req).await?;+ }+ Response::from_json(&target.proof)+ }++ async fn mark_target_ready(&self, req: &Request) -> Result<()> {+ self.state+ .storage()+ .put(LIFECYCLE_KEY, SurfaceLifecycle::Ready)+ .await?;+ *self.surface_lifecycle.lock("verified surface ready") = Some(SurfaceLifecycle::Ready);+ persistence_boundary(req, "target_ready")+ }+}diff --git a/services/sync-service/src/storage.rs b/services/sync-service/src/storage.rsindex 075708df0e..c910627320 100644--- a/services/sync-service/src/storage.rs+++ b/services/sync-service/src/storage.rs@@ -1,4 +1,5 @@use backends::Storage;+use macro_sync_service_jwt::session::SessionKind;use snapshot::SnapshotStorage;use tracing::trace;use worker::{Env, Error, Result, State};@@ -14,6 +15,7 @@ pub mod snapshot;pub struct SessionStorage {snapshot_storage: Storage,oplog: DurableKVStorage,+ kind: SessionKind,}/// Outbound adapter for atomic document updates, sharing the websocket log.@@ -45,17 +47,20 @@ impl crate::domain::document::DocumentUpdatePort for DocumentUpdateStorage<'_> {}impl SessionStorage {- pub fn new(snapshot_storage: Storage, oplog: DurableKVStorage) -> Self {+ pub fn new(snapshot_storage: Storage, oplog: DurableKVStorage, kind: SessionKind) -> Self {Self {snapshot_storage,oplog,+ kind,}}/// Load document state and apply any pending opspub async fn load_document_state(&self) -> Result<DocumentState> {let snapshot = self.get_snapshot().await;- let state = match (snapshot, cfg!(feature = "create-default-state")) {+ let allow_default =+ self.kind == SessionKind::Document && cfg!(feature = "create-default-state");+ let state = match (snapshot, allow_default) {(Ok(snapshot), _) => DocumentState::try_from_snapshot(snapshot.as_slice()),(Err(_e), true) => {let state = DocumentState::new();diff --git a/services/sync-service/src/storage/backends/combined_sql_kv.rs b/services/sync-service/src/storage/backends/combined_sql_kv.rsindex 32a3ea0666..15f98580ad 100644--- a/services/sync-service/src/storage/backends/combined_sql_kv.rs+++ b/services/sync-service/src/storage/backends/combined_sql_kv.rs@@ -31,6 +31,13 @@ impl SnapshotStorage for Storage {}}+ async fn delete_snapshot(&self) -> worker::Result<()> {+ // Remove the fallback first so a partial rollback cannot expose it after+ // the authoritative SQL copy is removed.+ self.kv.delete_snapshot().await?;+ self.sql.delete_snapshot().await+ }+async fn has_snapshot(&self) -> worker::Result<bool> {Ok(self.sql.has_snapshot().await? || self.kv.has_snapshot().await?)}diff --git a/services/sync-service/src/storage/backends/durable_sql.rs b/services/sync-service/src/storage/backends/durable_sql.rsindex 47285bc187..a6169519d0 100644--- a/services/sync-service/src/storage/backends/durable_sql.rs+++ b/services/sync-service/src/storage/backends/durable_sql.rs@@ -103,7 +103,6 @@ impl DurableSQLStorage {}/// Delete a key and all its chunks- #[expect(unused, reason = "We could use this in place of DO KV")]pub fn delete(&self, key: &str) -> Result<bool> {let cursor = self.storage.sql().exec("DELETE FROM kv_store WHERE id = ?1",@@ -206,6 +205,11 @@ impl SnapshotStorage for DurableSQLStorage {Err(e) => Err(e)?,}}+ async fn delete_snapshot(&self) -> worker::Result<()> {+ self.delete(&self.document_id)?;+ Ok(())+ }+/// Checks if a snapshot exists in the storageasync fn has_snapshot(&self) -> worker::Result<bool> {self.exists(&self.document_id)diff --git a/services/sync-service/src/storage/backends/kv.rs b/services/sync-service/src/storage/backends/kv.rsindex 1033e45062..06ff0de668 100644--- a/services/sync-service/src/storage/backends/kv.rs+++ b/services/sync-service/src/storage/backends/kv.rs@@ -56,6 +56,12 @@ impl SnapshotStorage for Kv {Ok(snapshot)}+ async fn delete_snapshot(&self) -> worker::Result<()> {+ let key = format!("{}/{}.snapshot", self.document_id, self.document_id);+ crate::timeout_ez!(self.inner.delete(&key))+ .with_context(|| format!("failed to delete snapshot: {key}"))+ }+#[tracing::instrument(skip_all, ret)]async fn has_snapshot(&self) -> worker::Result<bool> {Ok(!crate::timeout_ez!(diff --git a/services/sync-service/src/storage/backends/r2.rs b/services/sync-service/src/storage/backends/r2.rsindex 443ec4c841..14a338397a 100644--- a/services/sync-service/src/storage/backends/r2.rs+++ b/services/sync-service/src/storage/backends/r2.rs@@ -84,6 +84,11 @@ impl SnapshotStorage for R2Storage {Ok(snapshot)}+ async fn delete_snapshot(&self) -> worker::Result<()> {+ let key = format!("{}/{}.snapshot", self.document_id, self.document_id);+ self.inner.delete(&key).await+ }+async fn has_snapshot(&self) -> worker::Result<bool> {let key = format!("{}/{}.snapshot", self.document_id, self.document_id);let has = self.has(&key).await?;diff --git a/services/sync-service/src/storage/snapshot.rs b/services/sync-service/src/storage/snapshot.rsindex 395935af6d..6460857c46 100644--- a/services/sync-service/src/storage/snapshot.rs+++ b/services/sync-service/src/storage/snapshot.rs@@ -3,6 +3,8 @@ pub trait SnapshotStorage {async fn store_snapshot(&self, snapshot: &[u8]) -> worker::Result<()>;/// Retrieves a snpashot from the storageasync fn get_snapshot(&self) -> worker::Result<Vec<u8>>;+ /// Deletes a snapshot, including any fallback copy. Safe to retry.+ async fn delete_snapshot(&self) -> worker::Result<()>;/// Checks if a snapshot exists in the storageasync fn has_snapshot(&self) -> worker::Result<bool>;}diff --git a/services/sync-service/src/websocket.rs b/services/sync-service/src/websocket.rsindex b73c5ad7e6..aec4f53fc4 100644--- a/services/sync-service/src/websocket.rs+++ b/services/sync-service/src/websocket.rs@@ -196,6 +196,11 @@ pub async fn process_message(dss: &DocumentSyncSession,telemetry: &mut InboundMessageTelemetry,) -> Result<()> {+ // Loading the document after hibernation can yield to a revocation. Check+ // the surface grant again immediately before handling any protocol message.+ if !dss.validate_surface_sockets(Some(ws)).await? {+ return Ok(());+ }trace!(message = tracing::field::display(&message),"process websocket message"diff --git a/services/sync-service/tests/surface_isolation.test.ts b/services/sync-service/tests/surface_isolation.test.tsnew file mode 100644index 0000000000..6f5347231d--- /dev/null+++ b/services/sync-service/tests/surface_isolation.test.ts@@ -0,0 +1,369 @@+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { LoroDoc } from "loro-crdt";+import { createFetchMock, type Miniflare } from "miniflare";+import { afterAll, beforeAll, expect, test } from "vitest";+import {+ FromPeer,+ FromRemote,+ InitializeFromSnapshotRequest,+} from "../bebop/generated/schema";+import {+ createTestWebSocket,+ getTokenForDocument,+ getTokenForSurface,+ setupMiniflare,+ sleep,+} from "./utils";++let mf: Miniflare;+const persistPath = mkdtempSync(join(tmpdir(), "surface-isolation-"));+type TestClient = {+ socket: ReturnType<typeof createTestWebSocket>;+ doc: LoroDoc;+ closed: Promise<number>;+};+const effects: string[] = [];+const fetchMock = createFetchMock();+fetchMock.disableNetConnect();+for (const method of ["POST", "PUT"]) {+ fetchMock+ .get("https://dss.test")+ .intercept({ path: /.*/, method })+ .reply(200, (options) => {+ effects.push(options.path);+ return "";+ })+ .persist();+}+beforeAll(async () => {+ mf = await setupMiniflare({ persistPath, fetchMock });+}, 60_000);+afterAll(async () => {+ await mf?.dispose();+ rmSync(persistPath, { recursive: true, force: true });+});++function url(+ kind: "document" | "surface",+ id: string,+ operation: string,+): string {+ return `http://localhost/${kind}/${id}/${operation}`;+}+function token(kind: "document" | "surface", id: string): string {+ return kind === "surface"+ ? getTokenForSurface(id)+ : getTokenForDocument(id, "document-user", "edit");+}+async function initialize(+ kind: "document" | "surface",+ id: string,+ content: string,+): Promise<number> {+ const doc = new LoroDoc();+ doc.getText("content").insert(0, content);+ const body = InitializeFromSnapshotRequest.encode({+ snapshot: doc.export({ mode: "snapshot" }),+ });+ doc.free();+ const response = await mf.dispatchFetch(url(kind, id, "initialize"), {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ body,+ });+ return response.status;+}+async function connect(+ kind: "document" | "surface",+ id: string,+ grant = token(kind, id),+): ReturnType<Miniflare["dispatchFetch"]> {+ return mf.dispatchFetch(`${url(kind, id, "connect")}?token=${grant}`, {+ headers: { Upgrade: "websocket" },+ });+}+async function client(+ kind: "document" | "surface",+ id: string,+ grant = token(kind, id),+): Promise<TestClient> {+ const response = await connect(kind, id, grant);+ expect(response.status).toBe(101);+ const ws = response.webSocket!;+ const socket = createTestWebSocket(ws);+ const closed = new Promise<number>((resolve) =>+ ws.addEventListener("close", (event) => resolve(event.code)),+ );+ ws.accept();+ const message = FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ );+ expect(message.isRemoteInitialSync()).toBe(true);+ if (!message.isRemoteInitialSync())+ throw new Error("missing initial snapshot");+ const doc = new LoroDoc();+ doc.import(message.value.snapshot);+ return { socket, doc, closed };+}+async function content(+ kind: "document" | "surface",+ id: string,+): Promise<string> {+ const response = await mf.dispatchFetch(url(kind, id, "snapshot"), {+ method: "POST",+ headers: { Authorization: `Bearer ${token(kind, id)}` },+ });+ expect(response.status).toBe(200);+ const doc = new LoroDoc();+ doc.import(new Uint8Array(await response.arrayBuffer()));+ const text = doc.getText("content").toString();+ doc.free();+ return text;+}+function write(entry: TestClient, text: string): void {+ entry.doc.getText("content").push(text);+ entry.doc.commit();+ entry.socket.send(+ FromPeer.fromPeerUpdate({+ updates: [entry.doc.export({ mode: "update" })],+ id: crypto.randomUUID(),+ }).encode(),+ );+}++test("same UUID has distinct grants, snapshots, broadcasts, peers and effects", async () => {+ const id = crypto.randomUUID();+ expect(await initialize("document", id, "document")).toBe(200);+ expect(await initialize("surface", id, "surface")).toBe(200);+ expect((await connect("document", id, token("surface", id))).status).toBe(+ 401,+ );+ expect((await connect("surface", id, token("document", id))).status).toBe(+ 401,+ );+ for (const [kind, other] of [+ ["document", "surface"],+ ["surface", "document"],+ ] as const) {+ expect(+ (+ await mf.dispatchFetch(url(kind, id, "snapshot"), {+ method: "POST",+ headers: { Authorization: `Bearer ${token(other, id)}` },+ })+ ).status,+ ).toBe(401);+ }+ expect(+ (await connect("document", `surface:${id}`, token("surface", id))).status,+ ).toBe(400);+ expect(+ (await connect("surface", "invalid-id", token("surface", id))).status,+ ).toBe(400);+ const document = await client("document", id);+ const surface = await client("surface", id);+ const observer = await client("surface", id);+ expect(document.doc.getText("content").toString()).toBe("document");+ expect(surface.doc.getText("content").toString()).toBe("surface");+ const peer = 12345n;+ for (const entry of [document, surface])+ entry.socket.send(FromPeer.fromPeerRegisterId({ peerid: peer }).encode());+ write(surface, " edited");+ const ack = FromRemote.decode(+ new Uint8Array((await surface.socket.waitForNextMessage()) as ArrayBuffer),+ );+ expect(ack.isRemoteUpdateAck()).toBe(true);+ const broadcast = FromRemote.decode(+ new Uint8Array((await observer.socket.waitForNextMessage()) as ArrayBuffer),+ );+ expect(broadcast.isRemoteUpdate()).toBe(true);+ expect(await content("document", id)).toBe("document");+ expect(await content("surface", id)).toBe("surface edited");+ expect(await content("surface", id.toUpperCase())).toBe("surface edited");+ const db = await mf.getD1Database("USER_PEER_MAPPING");+ const peers = await db+ .prepare("SELECT document_id, user_id FROM peer_user_map WHERE peer_id = ?")+ .bind(peer.toString())+ .all();+ expect(peers.results).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ document_id: id, user_id: "document-user" }),+ expect.objectContaining({+ document_id: `surface:${id}`,+ user_id: "surface-user",+ }),+ ]),+ );+ for (const entry of [document, surface, observer])+ entry.socket.getWebSocket().close();+ await sleep(5500);+ expect(effects.some((path) => path.includes(id))).toBe(true);+ expect(effects.every((path) => !path.includes("surface:"))).toBe(true);+}, 20_000);++test("initialization races cannot overwrite a winner or revive a tombstone", async () => {+ const id = crypto.randomUUID();+ const initialized = await Promise.all([+ initialize("surface", id, "first"),+ initialize("surface", id, "second"),+ ]);+ expect([...initialized].sort()).toEqual([200, 409]);+ expect(await content("surface", id)).toBe(+ initialized[0] === 200 ? "first" : "second",+ );+ const retiring = crypto.randomUUID();+ const [initialization, revocation] = await Promise.all([+ initialize("surface", retiring, "seed"),+ mf.dispatchFetch(url("surface", retiring, "revoke"), {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ }),+ ]);+ expect([200, 403]).toContain(initialization);+ expect(revocation.status).toBe(200);+ expect((await connect("surface", retiring)).status).toBe(403);+ expect(await initialize("surface", retiring, "revive")).toBe(403);+});++test("Comment cannot write surfaces but retains document websocket behavior", async () => {+ const id = crypto.randomUUID();+ for (const kind of ["document", "surface"] as const)+ expect(await initialize(kind, id, "")).toBe(200);+ const surface = await client(+ "surface",+ id,+ getTokenForSurface(id, "comment"),+ );+ write(surface, "denied");+ const document = await client(+ "document",+ id,+ getTokenForDocument(id, "commenter", "comment"),+ );+ write(document, "allowed");+ await document.socket.waitForNextMessage();+ expect(await content("surface", id)).toBe("");+ expect(await content("document", id)).toBe("allowed");+});++test("pending, revoked and expired sessions fail closed; unsupported endpoints stay unavailable", async () => {+ const id = crypto.randomUUID();+ expect((await connect("surface", id)).status).toBe(409);+ for (const operation of [+ "raw",+ "state",+ "update",+ "copy",+ "debug_dump_operations",+ "debug_do_kv_list/",+ ]) {+ const response = await mf.dispatchFetch(url("surface", id, operation), {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ });+ expect(response.status).toBe(404);+ }+ expect(+ (+ await mf.dispatchFetch(url("surface", id, "initialize"), {+ method: "POST",+ headers: { Authorization: `Bearer ${getTokenForSurface(id, "owner")}` },+ })+ ).status,+ ).toBe(401);+ expect(await initialize("surface", id, "ready")).toBe(200);+ expect(+ (+ await connect(+ "surface",+ id,+ getTokenForSurface(id, "edit", Math.floor(Date.now() / 1000) - 1),+ )+ ).status,+ ).toBe(401);+ const active = await client("surface", id);+ expect(+ (+ await mf.dispatchFetch(url("surface", id, "revoke"), {+ method: "POST",+ headers: { Authorization: `Bearer ${token("surface", id)}` },+ })+ ).status,+ ).toBe(401);+ expect(+ (+ await mf.dispatchFetch(url("surface", id, "revoke"), {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ })+ ).status,+ ).toBe(200);+ write(active, "denied");+ active.socket.send(FromPeer.fromPeerRequestSnapshot({}).encode());+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(+ "Timed out",+ );+ expect((await connect("surface", id)).status).toBe(403);+ expect(await initialize("surface", id, "revive")).toBe(403);+ // This workerd version buffers close delivery until teardown, even for a+ // minimal JS Durable Object. Check the actual server close code then restart.+ await mf.dispose();+ expect(await active.closed).toBe(1008);+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });+ expect((await connect("surface", id)).status).toBe(403);+});++test("surface expiry stops idle sockets and survives persisted state reload", async () => {+ const id = crypto.randomUUID();+ expect(await initialize("surface", id, "persisted")).toBe(200);+ const active = await client(+ "surface",+ id,+ getTokenForSurface(id, "edit", Math.floor(Date.now() / 1000) + 2),+ );+ await sleep(5500);+ write(active, "denied");+ active.socket.send(FromPeer.fromPeerRequestSnapshot({}).encode());+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(+ "Timed out",+ );+ expect(await content("surface", id)).toBe("persisted");+ const writer = await client("surface", id);+ write(writer, " live");+ await writer.socket.waitForNextMessage();+ expect(await content("surface", id)).toBe("persisted live");+ await expect(active.socket.waitForNextMessage(100)).rejects.toThrow(+ "Timed out",+ );+ expect(effects.some((path) => path.includes(id))).toBe(false);+ const revoked = crypto.randomUUID();+ await mf.dispatchFetch(url("surface", revoked, "revoke"), {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ });+ await mf.dispose();+ expect(await active.closed).toBe(1008);+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });+ expect((await connect("surface", revoked)).status).toBe(403);+ expect(await content("surface", id)).toBe("persisted live");+ expect(effects.some((path) => path.includes(id))).toBe(false);+}, 30_000);++test("same-UUID operation logs replay independently after eviction before the snapshot alarm", async () => {+ const id = crypto.randomUUID();+ for (const kind of ["surface", "document"] as const) {+ expect(await initialize(kind, id, kind)).toBe(200);+ const writer = await client(kind, id);+ write(writer, " persisted op");+ const ack = FromRemote.decode(+ new Uint8Array((await writer.socket.waitForNextMessage()) as ArrayBuffer),+ );+ expect(ack.isRemoteUpdateAck()).toBe(true);+ }+ await mf.dispose();+ mf = await setupMiniflare({ persistPath, migrate: false, fetchMock });+ expect(await content("surface", id)).toBe("surface persisted op");+ expect(await content("document", id)).toBe("document persisted op");+});diff --git a/services/sync-service/tests/surface_migration.test.ts b/services/sync-service/tests/surface_migration.test.tsnew file mode 100644index 0000000000..8791aaa240--- /dev/null+++ b/services/sync-service/tests/surface_migration.test.ts@@ -0,0 +1,560 @@+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { LoroDoc } from "loro-crdt";+import type { Miniflare } from "miniflare";+import { afterAll, beforeAll, expect, test } from "vitest";+import {+ FromPeer,+ FromRemote,+ InitializeFromSnapshotRequest,+} from "../bebop/generated/schema";+import {+ createTestWebSocket,+ getTokenForDocument,+ getTokenForSurface,+ setupMiniflare,+} from "./utils";++type Proof = {+ operation_id: string;+ source_id: string | null;+ digest: string;+ content_digest: string;+ revision: [string, number][];+ oplog_revision: [string, number][];+};+type Export = { proof: Proof; snapshot: number[] };+let mf: Miniflare;+const persistPath = mkdtempSync(join(tmpdir(), "surface-migration-"));+beforeAll(async () => {+ mf = await setupMiniflare({ persistPath });+}, 60_000);+afterAll(async () => {+ await mf?.dispose();+ rmSync(persistPath, { recursive: true, force: true });+});+async function restart(): Promise<void> {+ await mf.dispose();+ mf = await setupMiniflare({ persistPath, migrate: false });+}+async function post(+ kind: string,+ id: string,+ operation: string,+ body: unknown,+ stop?: string,+): ReturnType<Miniflare["dispatchFetch"]> {+ return mf.dispatchFetch(`http://localhost/${kind}/${id}/${operation}`, {+ method: "POST",+ headers: {+ "x-internal-auth-key": "local",+ "Content-Type": "application/json",+ ...(stop ? { "x-sync-test-stop-after": stop } : {}),+ },+ body: JSON.stringify(body),+ });+}+async function seed(id: string): Promise<void> {+ const doc = new LoroDoc();+ doc.getText("content").push("original");+ const response = await mf.dispatchFetch(+ `http://localhost/document/${id}/initialize`,+ {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ body: InitializeFromSnapshotRequest.encode({+ snapshot: doc.export({ mode: "snapshot" }),+ }),+ },+ );+ doc.free();+ expect(response.status).toBe(200);+}+async function connect(+ kind: "surface" | "document",+ id: string,+): ReturnType<Miniflare["dispatchFetch"]> {+ const token =+ kind === "surface"+ ? getTokenForSurface(id)+ : getTokenForDocument(id, "writer", "edit");+ return mf.dispatchFetch(+ `http://localhost/${kind}/${id}/connect?token=${token}`,+ { headers: { Upgrade: "websocket" } },+ );+}+async function freeze(+ id: string,+ operation_id = crypto.randomUUID(),+): Promise<Export> {+ const response = await post("document", id, "migration/freeze", {+ operation_id,+ });+ expect(response.status).toBe(200);+ return response.json<Export>();+}+function text(snapshot: number[]): string {+ const doc = new LoroDoc();+ doc.import(new Uint8Array(snapshot));+ const value = doc.getText("content").toString();+ doc.free();+ return value;+}++test("freeze drains acknowledged and racing edits, closes writers, and replays pending logs after restart", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const connected = await connect("document", id);+ const ws = connected.webSocket!;+ const socket = createTestWebSocket(ws);+ const closed = new Promise<number>((resolve) =>+ ws.addEventListener("close", (event) => resolve(event.code)),+ );+ ws.accept();+ const initial = FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ );+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");+ const doc = new LoroDoc();+ doc.import(initial.value.snapshot);+ function edit(value: string): void {+ doc.getText("content").push(value);+ socket.send(+ FromPeer.fromPeerUpdate({+ updates: [doc.export({ mode: "update" })],+ id: crypto.randomUUID(),+ }).encode(),+ );+ }+ edit(" acknowledged");+ expect(+ FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ ).isRemoteUpdateAck(),+ ).toBe(true);+ edit(" racing");+ const exported = await freeze(id);+ expect(text(exported.snapshot)).toMatch(/^original acknowledged( racing)?$/);+ // If the racing mutation is acknowledged, it must be present in the export.+ const racing = await socket.waitForNextMessage(100).catch(() => undefined);+ if (racing)+ expect(text(exported.snapshot)).toBe("original acknowledged racing");+ expect((await connect("document", id)).status).toBe(403);+ await restart();+ expect(await closed).toBe(1008);+ expect(await freeze(id, exported.proof.operation_id)).toEqual(exported);+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ expect((await connect("surface", id)).status).toBe(409);+ expect((await post("surface", id, "verify", exported.proof)).status).toBe(+ 200,+ );+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(+ 200,+ );+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(409);+ expect(+ (await post("document", id, "migration/retire", exported.proof)).status,+ ).toBe(200);+ await restart();+ expect((await connect("document", id)).status).toBe(403);+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(409);+ const migrated = await connect("surface", id);+ expect(migrated.status).toBe(101);+ migrated.webSocket!.accept();+ migrated.webSocket!.close();+ expect((await post("surface", id, "revoke", {})).status).toBe(200);+ await restart();+ expect((await connect("surface", id)).status).toBe(403);+ expect((await post("surface", id, "import", exported)).status).toBe(403);+ doc.free();+}, 30_000);++test("verified rollback is allowed only before sealing; stale operations and conflicting targets fail closed", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const exported = await freeze(id);+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ expect(+ (+ await post("surface", id, "import", {+ ...exported,+ proof: { ...exported.proof, operation_id: crypto.randomUUID() },+ })+ ).status,+ ).toBe(409);+ expect(+ (await post("surface", id, "import", { ...exported, snapshot: [1, 2, 3] }))+ .status,+ ).toBe(400);+ expect(+ (await post("document", id, "migration/retire", exported.proof)).status,+ ).toBe(409);+ expect(+ (+ await post("document", id, "migration/thaw", {+ ...exported.proof,+ digest: "wrong",+ })+ ).status,+ ).toBe(409);+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(200);+ await restart();+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(200);+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(+ 409,+ );+ expect(+ (+ await post("document", id, "migration/freeze", {+ operation_id: exported.proof.operation_id,+ })+ ).status,+ ).toBe(409);+ const next = await freeze(id);+ expect(text(next.snapshot)).toBe("original");+ expect((await post("surface", id, "import", next)).status).toBe(409);+});++test("internal-only operations and missing sources never create blank replacement content", async () => {+ const id = crypto.randomUUID();+ expect(+ (+ await post("document", id, "migration/freeze", {+ operation_id: crypto.randomUUID(),+ })+ ).status,+ ).toBe(404);+ for (const [kind, operation] of [+ ["document", "migration/freeze"],+ ["document", "migration/thaw"],+ ["document", "migration/seal"],+ ["document", "migration/retire"],+ ["surface", "import"],+ ["surface", "verify"],+ ["surface", "activate"],+ ["surface", "initialize_verified"],+ ]) {+ const response = await mf.dispatchFetch(+ `http://localhost/${kind}/${id}/${operation}`,+ { method: "POST" },+ );+ expect(response.status).toBe(401);+ }+});++test("isolated initialization acknowledges only the identical verified operation and seed", async () => {+ const id = crypto.randomUUID();+ const doc = new LoroDoc();+ doc.getText("content").push("seed");+ const body = {+ operation_id: crypto.randomUUID(),+ snapshot: Array.from(doc.export({ mode: "snapshot" })),+ };+ const response = await post("surface", id, "initialize_verified", body);+ expect(response.status).toBe(200);+ const receipt = await response.json();+ await restart();+ expect(+ await (await post("surface", id, "initialize_verified", body)).json(),+ ).toEqual(receipt);+ expect(+ (+ await post("surface", id, "initialize_verified", {+ ...body,+ operation_id: crypto.randomUUID(),+ })+ ).status,+ ).toBe(409);+ doc.getText("content").push("different");+ expect(+ (+ await post("surface", id, "initialize_verified", {+ ...body,+ snapshot: Array.from(doc.export({ mode: "snapshot" })),+ })+ ).status,+ ).toBe(409);+ doc.free();+});++test("cold freeze replays acknowledged operation logs and rollback preserves the original", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const response = await connect("document", id);+ const ws = response.webSocket!;+ const socket = createTestWebSocket(ws);+ ws.accept();+ const initial = FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ );+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");+ const doc = new LoroDoc();+ doc.import(initial.value.snapshot);+ doc.getText("content").push(" pending");+ socket.send(+ FromPeer.fromPeerUpdate({+ updates: [doc.export({ mode: "update" })],+ id: crypto.randomUUID(),+ }).encode(),+ );+ expect(+ FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ ).isRemoteUpdateAck(),+ ).toBe(true);+ await restart();+ const exported = await freeze(id);+ expect(text(exported.snapshot)).toBe("original pending");+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(200);+ const original = await mf.dispatchFetch(+ `http://localhost/document/${id}/snapshot`,+ {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ },+ );+ expect(text(Array.from(new Uint8Array(await original.arrayBuffer())))).toBe(+ "original pending",+ );+ doc.free();+});++test("competing freezes and activation versus rollback never yield two writable copies", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const results = await Promise.all([+ post("document", id, "migration/freeze", {+ operation_id: crypto.randomUUID(),+ }),+ post("document", id, "migration/freeze", {+ operation_id: crypto.randomUUID(),+ }),+ ]);+ expect(results.map((response) => response.status).sort()).toEqual([200, 409]);+ const exported = await results+ .find((response) => response.status === 200)!+ .json<Export>();+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ const [activation, rollback] = await Promise.all([+ post("surface", id, "activate", exported.proof),+ post("document", id, "migration/thaw", exported.proof),+ ]);+ expect([activation.status, rollback.status].sort()).toEqual([200, 409]);+ await restart();+ const source = await connect("document", id);+ const target = await connect("surface", id);+ expect(+ [source.status, target.status].filter((status) => status === 101),+ ).toHaveLength(1);+ for (const response of [source, target]) {+ response.webSocket?.accept();+ response.webSocket?.close();+ }+});++test("an occupied target and forged revision/digest are not overwritten", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const exported = await freeze(id);+ for (const changed of [+ { digest: "bad" },+ { content_digest: "bad" },+ { revision: [] },+ { oplog_revision: [] },+ ]) {+ expect(+ (+ await post("surface", id, "import", {+ ...exported,+ proof: { ...exported.proof, ...changed },+ })+ ).status,+ ).toBe(400);+ }+ const doc = new LoroDoc();+ doc.getText("content").push("unrelated target");+ const occupied = await mf.dispatchFetch(+ `http://localhost/surface/${id}/initialize`,+ {+ method: "POST",+ headers: { "x-internal-auth-key": "local" },+ body: InitializeFromSnapshotRequest.encode({+ snapshot: doc.export({ mode: "snapshot" }),+ }),+ },+ );+ expect(occupied.status).toBe(200);+ expect((await post("surface", id, "import", exported)).status).toBe(409);+ const snapshot = await mf.dispatchFetch(+ `http://localhost/surface/${id}/snapshot`,+ {+ method: "POST",+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },+ },+ );+ expect(text(Array.from(new Uint8Array(await snapshot.arrayBuffer())))).toBe(+ "unrelated target",+ );+ doc.free();+});++test("retries after activation preserve subsequent acknowledged surface edits", async () => {+ const id = crypto.randomUUID();+ await seed(id);+ const exported = await freeze(id);+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(+ 200,+ );+ const response = await connect("surface", id);+ const ws = response.webSocket!;+ const socket = createTestWebSocket(ws);+ ws.accept();+ const initial = FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ );+ if (!initial.isRemoteInitialSync()) throw new Error("missing initial sync");+ const doc = new LoroDoc();+ doc.import(initial.value.snapshot);+ doc.getText("content").push(" after activation");+ socket.send(+ FromPeer.fromPeerUpdate({+ updates: [doc.export({ mode: "update" })],+ id: crypto.randomUUID(),+ }).encode(),+ );+ expect(+ FromRemote.decode(+ new Uint8Array((await socket.waitForNextMessage()) as ArrayBuffer),+ ).isRemoteUpdateAck(),+ ).toBe(true);+ await restart();+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(+ 200,+ );+ const snapshot = await mf.dispatchFetch(+ `http://localhost/surface/${id}/snapshot`,+ {+ method: "POST",+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },+ },+ );+ expect(text(Array.from(new Uint8Array(await snapshot.arrayBuffer())))).toBe(+ "original after activation",+ );+ doc.free();+});++// Run with a migration-test-hooks worker build to inject a crash immediately+// after each durable write. Production builds do not recognize this header.+const faultTests =+ process.env.SYNC_MIGRATION_FAULT_TESTS === "1" ? test : test.skip;+faultTests.each([+ "source_claim",+ "source_export",+ "source_frozen",+ "target_claim",+ "target_snapshot",+ "target_verified",+ "source_sealed",+ "target_active",+ "target_ready",+ "source_retired",+ "source_thawed",+])(+ "recovers after persistence boundary %s",+ async (boundary) => {+ const id = crypto.randomUUID();+ const operation_id = crypto.randomUUID();+ await seed(id);+ if (+ boundary.startsWith("source_") &&+ ["source_claim", "source_export", "source_frozen"].includes(boundary)+ ) {+ expect(+ (+ await post(+ "document",+ id,+ "migration/freeze",+ { operation_id },+ boundary,+ )+ ).status,+ ).toBe(500);+ await restart();+ }+ const exported = await freeze(id, operation_id);+ if (boundary === "source_thawed") {+ expect(+ (await post("document", id, "migration/thaw", exported.proof, boundary))+ .status,+ ).toBe(500);+ await restart();+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(200);+ expect(+ (await post("document", id, "migration/seal", exported.proof)).status,+ ).toBe(409);+ return;+ }+ if (+ ["target_claim", "target_snapshot", "target_verified"].includes(boundary)+ ) {+ expect(+ (await post("surface", id, "import", exported, boundary)).status,+ ).toBe(500);+ await restart();+ // The old initializer cannot steal a claimed-but-not-yet-written target.+ expect((await post("surface", id, "initialize", {})).status).toBe(409);+ }+ expect((await post("surface", id, "import", exported)).status).toBe(200);+ if (["source_sealed", "target_active", "target_ready"].includes(boundary)) {+ expect(+ (await post("surface", id, "activate", exported.proof, boundary))+ .status,+ ).toBe(500);+ await restart();+ expect(+ (await post("document", id, "migration/thaw", exported.proof)).status,+ ).toBe(409);+ }+ expect((await post("surface", id, "activate", exported.proof)).status).toBe(+ 200,+ );+ if (boundary === "source_retired") {+ expect(+ (+ await post(+ "document",+ id,+ "migration/retire",+ exported.proof,+ boundary,+ )+ ).status,+ ).toBe(500);+ await restart();+ }+ expect(+ (await post("document", id, "migration/retire", exported.proof)).status,+ ).toBe(200);+ expect(await freeze(id, operation_id)).toEqual(exported);+ expect((await connect("document", id)).status).toBe(403);+ },+ 30_000,+);diff --git a/services/sync-service/tests/surface_recovery.test.ts b/services/sync-service/tests/surface_recovery.test.tsnew file mode 100644index 0000000000..24a3306f67--- /dev/null+++ b/services/sync-service/tests/surface_recovery.test.ts@@ -0,0 +1,122 @@+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { LoroDoc } from "loro-crdt";+import type { Miniflare } from "miniflare";+import { afterAll, beforeAll, expect, test } from "vitest";+import { InitializeFromSnapshotRequest } from "../bebop/generated/schema";+import { getTokenForSurface, setupMiniflare } from "./utils";++let mf: Miniflare;+const persistPath = mkdtempSync(join(tmpdir(), "surface-recovery-"));+beforeAll(async () => {+ mf = await setupMiniflare({ persistPath });+}, 60_000);+afterAll(async () => {+ await mf?.dispose();+ rmSync(persistPath, { recursive: true, force: true });+});++function seed(content: string): Uint8Array {+ const doc = new LoroDoc();+ doc.getText("content").insert(0, content);+ const snapshot = doc.export({ mode: "snapshot" });+ doc.free();+ return snapshot;+}++async function initialize(id: string, snapshot: Uint8Array, stop = false) {+ return mf.dispatchFetch(`http://localhost/surface/${id}/initialize`, {+ method: "POST",+ headers: {+ "x-internal-auth-key": "local",+ ...(stop ? { "x-sync-test-stop-after": "surface_snapshot" } : {}),+ },+ body: InitializeFromSnapshotRequest.encode({ snapshot }),+ });+}++async function exists(id: string): Promise<number> {+ return (+ await mf.dispatchFetch(`http://localhost/surface/${id}/exists`, {+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },+ })+ ).status;+}++async function content(id: string): Promise<string> {+ const response = await mf.dispatchFetch(`http://localhost/surface/${id}/snapshot`, {+ method: "POST",+ headers: { Authorization: `Bearer ${getTokenForSurface(id)}` },+ });+ expect(response.status).toBe(200);+ const doc = new LoroDoc();+ doc.import(new Uint8Array(await response.arrayBuffer()));+ const text = doc.getText("content").toString();+ doc.free();+ return text;+}++async function restart(): Promise<void> {+ await mf.dispose();+ mf = await setupMiniflare({ persistPath, migrate: false });+}++test("pending legacy snapshot resumes only for the identical seed", async () => {+ const id = crypto.randomUUID();+ const snapshot = seed("original");+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");+ await kv.put(`surface:${id}/surface:${id}.snapshot`, snapshot);+ expect(await exists(id)).toBe(409);+ expect((await initialize(id, snapshot)).status).toBe(200);+ expect(await content(id)).toBe("original");+ await restart();+ expect(await exists(id)).toBe(200);+ expect((await initialize(id, seed("replacement"))).status).toBe(409);+ expect(await content(id)).toBe("original");+});++test("unidentified pending snapshot is rolled back including its KV fallback", async () => {+ const id = crypto.randomUUID();+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");+ const key = `surface:${id}/surface:${id}.snapshot`;+ await kv.put(key, seed("orphan"));+ const snapshot = seed("replacement");+ expect((await initialize(id, snapshot)).status).toBe(409);+ expect(await kv.get(key)).toBeNull();+ expect(await exists(id)).toBe(409);+ await restart();+ expect((await initialize(id, snapshot)).status).toBe(200);+ expect(await content(id)).toBe("replacement");+});++const faultTests = process.env.SYNC_MIGRATION_FAULT_TESTS === "1" ? test : test.skip;+faultTests.each([false, true])(+ "snapshot committed before Ready failure recovers (restart: %s)",+ async (cold) => {+ const id = crypto.randomUUID();+ const snapshot = seed("durable seed");+ expect((await initialize(id, snapshot, true)).status).toBe(500);+ expect(await exists(id)).toBe(409);+ if (cold) await restart();+ expect((await initialize(id, snapshot)).status).toBe(200);+ expect(await content(id)).toBe("durable seed");+ await restart();+ expect(await exists(id)).toBe(200);+ },+);++faultTests("conflicting retry rolls back SQL and KV snapshots before a new seed", async () => {+ const id = crypto.randomUUID();+ const original = seed("original");+ expect((await initialize(id, original, true)).status).toBe(500);+ const kv = await mf.getKVNamespace("SNAPSHOT_STORE_KV");+ const key = `surface:${id}/surface:${id}.snapshot`;+ await kv.put(key, original);+ const replacement = seed("replacement");+ expect((await initialize(id, replacement)).status).toBe(409);+ expect(await kv.get(key)).toBeNull();+ await restart();+ expect((await initialize(id, replacement)).status).toBe(200);+ expect(await content(id)).toBe("replacement");+});diff --git a/services/sync-service/tests/utils.ts b/services/sync-service/tests/utils.tsindex 0b73054d4a..9733da437b 100644--- a/services/sync-service/tests/utils.ts+++ b/services/sync-service/tests/utils.ts@@ -1,5 +1,5 @@import { EphemeralStore, LoroDoc } from 'loro-crdt';-import { Miniflare, type WebSocket } from 'miniflare';+import { Miniflare, type MiniflareOptions, type WebSocket } from 'miniflare';import { assert } from 'vitest';import jwt from 'jsonwebtoken';import {@@ -27,7 +27,7 @@ async function migrateDatabase(mf: Miniflare) {}}-export async function setupMiniflare(options: { persistPath?: string; migrate?: boolean } = {}) {+export async function setupMiniflare(options: { persistPath?: string; migrate?: boolean; fetchMock?: MiniflareOptions['fetchMock'] } = {}) {const persist = (name: string) => options.persistPath ? `${options.persistPath}/${name}` : false;const mf = new Miniflare({d1Databases: {@@ -55,7 +55,9 @@ export async function setupMiniflare(options: { persistPath?: string; migrate?:d1Persist: persist('d1'),kvPersist: persist('kv'),r2Persist: persist('r2'),+ fetchMock: options.fetchMock,bindings: {+ ...(options.fetchMock ? { DSS_URL: "https://dss.test", DSS_INTERNAL_AUTH_KEY: "local" } : {}),DOCUMENT_PERMISSIONS_SECRET: "local",INTERNAL_API_SECRET_KEY: "INTERNAL_API_SECRET",INTERNAL_API_SECRET,@@ -222,7 +224,7 @@ export function createTestWebSocket(ws: WebSocket) {return new Promise<ArrayBuffer>((resolve, reject) => {const timeoutId = setTimeout(() => {- const index = waiters.indexOf(resolve);+ const index = waiters.indexOf(wrappedResolve);if (index !== -1) {waiters.splice(index, 1);}@@ -242,8 +244,8 @@ export function createTestWebSocket(ws: WebSocket) {});},- send(message: string | ArrayBuffer) {- ws.send(message);+ send(message: string | ArrayBuffer | Uint8Array): void {+ ws.send(message instanceof Uint8Array ? new Uint8Array(message) : message);},getWebSocket() {@@ -255,7 +257,7 @@ export function createTestWebSocket(ws: WebSocket) {export function getTokenForDocument(documentId: string,userId: string,- permissionLevel: 'view' | 'edit' | 'owner'+ permissionLevel: 'view' | 'edit' | 'owner' | 'comment'): string {const token = jwt.sign({user_id: userId,@@ -268,6 +270,21 @@ export function getTokenForDocument(}+export function getTokenForSurface(+ surfaceId: string,+ permissionLevel: 'view' | 'comment' | 'edit' | 'owner' = 'edit',+ expiresAt = Math.floor(Date.now() / 1000) + 60,+): string {+ return jwt.sign({+ session_kind: 'surface',+ surface_id: surfaceId,+ user_id: 'surface-user',+ access_level: permissionLevel,+ exp: expiresAt,+ iss: 'document_storage_service',+ }, 'local', { noTimestamp: true });+}+export async function connectToDocumentForTesting(mf: Miniflare,documentId: string,