Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore: start 'chunking' sync service websockets (#4507)

Wolf Mermelstein00b79b1e8cc4
diff --git a/packages/collaboration/src/sync-service/socket.ts b/packages/collaboration/src/sync-service/socket.ts
index c3eb6ea753..1fd09a91a5 100644
--- a/packages/collaboration/src/sync-service/socket.ts
+++ b/packages/collaboration/src/sync-service/socket.ts
@@ -9,6 +9,8 @@ import {
type WebsocketEvent,
type WebsocketEventListener,
} from '../websocket';
+import { platformWebSocketFactory } from '../websocket/platform/factory';
+import { framedWebSocketFactory } from '../websocket/platform/framing';
import { FromPeer, FromRemote, type IFromPeer } from './generated/schema';
/** The concrete sync websocket: a {@link Websocket} speaking the Bebop protocol. */
@@ -44,6 +46,7 @@ export function createSyncSocket(getUrl: UrlResolver): SyncWebsocket {
return (
new WebsocketBuilder(getUrl)
.withSerializer(new BebopSerializer(FromPeer, FromRemote))
+ .withFactory(framedWebSocketFactory(platformWebSocketFactory))
// Capped exponential backoff. The scheduler calls next() before the first
// retry, so the delays are 250*2^1 = 500ms doubling to a 250*2^5 = 8s
// cap; 20 retries ≈ 2 minutes of automatic attempts, after which
diff --git a/packages/collaboration/src/websocket/core/websocket-builder.ts b/packages/collaboration/src/websocket/core/websocket-builder.ts
index a22c70bbb0..18c15d71f2 100644
--- a/packages/collaboration/src/websocket/core/websocket-builder.ts
+++ b/packages/collaboration/src/websocket/core/websocket-builder.ts
@@ -192,6 +192,7 @@ export class WebsocketBuilder<Send = WebsocketData, Receive = WebsocketData> {
listeners: this._options?.listeners,
serializer,
buffer: this._options?.buffer,
+ factory: this._options?.factory,
} as unknown as WebsocketOptions<NewSend, NewReceive>;
if (serializer.binaryType !== undefined) {
diff --git a/packages/collaboration/src/websocket/platform/framing/framed-websocket.test.ts b/packages/collaboration/src/websocket/platform/framing/framed-websocket.test.ts
new file mode 100644
index 0000000000..63de8de9ad
--- /dev/null
+++ b/packages/collaboration/src/websocket/platform/framing/framed-websocket.test.ts
@@ -0,0 +1,111 @@
+import { describe, expect, test } from 'vitest';
+import type { MinimalWebSocket } from '../minimal-websocket';
+import { framedWebSocketFactory } from './framed-websocket';
+
+type MockWebSocket = MinimalWebSocket & {
+ /** Payloads passed to `send`, in order. */
+ readonly sent: (string | ArrayBufferLike | Blob | ArrayBufferView)[];
+ /** Fire a 'message' event to registered listeners, as the real socket would. */
+ emit(data: string | ArrayBuffer): void;
+};
+
+function createMockWebSocket(): MockWebSocket {
+ const listeners = new Set<EventListener>();
+ const sent: (string | ArrayBufferLike | Blob | ArrayBufferView)[] = [];
+
+ return {
+ binaryType: 'arraybuffer',
+ bufferedAmount: 0,
+ extensions: '',
+ protocol: '',
+ readyState: 1,
+ url: 'ws://test',
+ onopen: null,
+ onclose: null,
+ onerror: null,
+ onmessage: null,
+ CONNECTING: 0,
+ OPEN: 1,
+ CLOSING: 2,
+ CLOSED: 3,
+ addEventListener: (type, listener) => {
+ if (type === 'message') listeners.add(listener as EventListener);
+ },
+ removeEventListener: (type, listener) => {
+ if (type === 'message') listeners.delete(listener as EventListener);
+ },
+ dispatchEvent: () => true,
+ close: () => {},
+ send: (data) => {
+ sent.push(data);
+ },
+ sent,
+ emit: (data) => {
+ const event = new MessageEvent('message', { data });
+ listeners.forEach((listener) => listener(event));
+ },
+ };
+}
+
+/** Wraps a mock socket in a FramedWebSocket via the public factory. */
+function framedOver(mock: MockWebSocket) {
+ return framedWebSocketFactory(() => mock)('ws://test');
+}
+
+describe('FramedWebSocket', () => {
+ test('frames outbound binary messages', () => {
+ const mock = createMockWebSocket();
+ framedOver(mock).send(new Uint8Array([9, 8, 7]));
+
+ expect(mock.sent.length).toBe(1);
+ const frame = mock.sent[0] as Uint8Array;
+ expect(frame[0]).toBe(1); // isFinal
+ expect(Array.from(frame.subarray(1))).toEqual([9, 8, 7]);
+ });
+
+ test('passes outbound strings through unframed', () => {
+ const mock = createMockWebSocket();
+ framedOver(mock).send('ping');
+ expect(mock.sent).toEqual(['ping']);
+ });
+
+ test('reassembles a single inbound binary frame', () => {
+ const mock = createMockWebSocket();
+ const ws = framedOver(mock);
+ const received: number[][] = [];
+ ws.addEventListener('message', (e) =>
+ received.push(Array.from(new Uint8Array((e as MessageEvent).data)))
+ );
+
+ mock.emit(new Uint8Array([1, 5, 6]).buffer); // isFinal, payload [5,6]
+
+ expect(received).toEqual([[5, 6]]);
+ });
+
+ test('dispatches once after the final frame of a multi-frame message', () => {
+ const mock = createMockWebSocket();
+ const ws = framedOver(mock);
+ const received: number[][] = [];
+ ws.addEventListener('message', (e) =>
+ received.push(Array.from(new Uint8Array((e as MessageEvent).data)))
+ );
+
+ mock.emit(new Uint8Array([0, 1, 2]).buffer);
+ mock.emit(new Uint8Array([1, 3]).buffer);
+
+ expect(received).toEqual([[1, 2, 3]]);
+ });
+
+ test('passes inbound strings (pong) straight through', () => {
+ const mock = createMockWebSocket();
+ const ws = framedOver(mock);
+ const received: unknown[] = [];
+ ws.addEventListener('message', (e) =>
+ received.push((e as MessageEvent).data)
+ );
+
+ mock.emit('pong');
+
+ expect(received).toEqual(['pong']);
+ });
+});
diff --git a/packages/collaboration/src/websocket/platform/framing/framed-websocket.ts b/packages/collaboration/src/websocket/platform/framing/framed-websocket.ts
new file mode 100644
index 0000000000..aadb77702a
--- /dev/null
+++ b/packages/collaboration/src/websocket/platform/framing/framed-websocket.ts
@@ -0,0 +1,155 @@
+import type { MinimalWebSocket, WebSocketFactory } from '../minimal-websocket';
+import { intoFrames, Reassembler } from './frames';
+
+class FramedWebSocket implements MinimalWebSocket {
+ /** 'message' listeners, routed through us so reassembly can interpose. */
+ private readonly messageListeners = new Set<EventListener>();
+
+ /** Per-connection inbound reassembly (frames arrive in order over one socket). */
+ private readonly reassembler = new Reassembler();
+
+ constructor(private readonly inner: MinimalWebSocket) {
+ this.inner.addEventListener('message', this.handleInnerMessage);
+ }
+
+ send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
+ // Strings (heartbeat ping/pong) ride the text channel — never framed.
+ if (typeof data === 'string') {
+ this.inner.send(data);
+ return;
+ }
+ for (const frame of intoFrames(toBytes(data))) {
+ this.inner.send(frame);
+ }
+ }
+
+ private handleInnerMessage = (event: MessageEvent): void => {
+ // Strings (heartbeat pong) pass straight through.
+ if (typeof event.data === 'string') {
+ this.dispatchMessage(event);
+ return;
+ }
+ const message = this.reassembler.push(new Uint8Array(event.data));
+ if (message === null) return; // partial — waiting for more frames
+ this.dispatchMessage(new MessageEvent('message', { data: message.buffer }));
+ };
+
+ private dispatchMessage(event: MessageEvent): void {
+ this.messageListeners.forEach((listener) => listener(event));
+ }
+
+ addEventListener<K extends keyof WebSocketEventMap>(
+ type: K,
+ listener: (this: WebSocket, ev: WebSocketEventMap[K]) => any,
+ options?: boolean | AddEventListenerOptions
+ ): void {
+ if (type === 'message') {
+ this.messageListeners.add(listener as EventListener);
+ } else {
+ this.inner.addEventListener(type, listener, options);
+ }
+ }
+
+ removeEventListener<K extends keyof WebSocketEventMap>(
+ type: K,
+ listener: (this: WebSocket, ev: WebSocketEventMap[K]) => any,
+ options?: boolean | EventListenerOptions
+ ): void {
+ if (type === 'message') {
+ this.messageListeners.delete(listener as EventListener);
+ } else {
+ this.inner.removeEventListener(type, listener, options);
+ }
+ }
+
+ dispatchEvent(event: Event): boolean {
+ return this.inner.dispatchEvent(event);
+ }
+
+ get binaryType(): BinaryType {
+ return this.inner.binaryType;
+ }
+
+ set binaryType(value: BinaryType) {
+ this.inner.binaryType = value;
+ }
+
+ get bufferedAmount(): number {
+ return this.inner.bufferedAmount;
+ }
+
+ get extensions(): string {
+ return this.inner.extensions;
+ }
+
+ get protocol(): string {
+ return this.inner.protocol;
+ }
+
+ get readyState(): number {
+ return this.inner.readyState;
+ }
+
+ get url(): string {
+ return this.inner.url;
+ }
+
+ close(code?: number, reason?: string): void {
+ this.inner.close(code, reason);
+ }
+
+ get onopen() {
+ return this.inner.onopen;
+ }
+
+ set onopen(value) {
+ this.inner.onopen = value;
+ }
+
+ get onclose() {
+ return this.inner.onclose;
+ }
+
+ set onclose(value) {
+ this.inner.onclose = value;
+ }
+
+ get onerror() {
+ return this.inner.onerror;
+ }
+
+ set onerror(value) {
+ this.inner.onerror = value;
+ }
+
+ get onmessage() {
+ return this.inner.onmessage;
+ }
+
+ set onmessage(value) {
+ this.inner.onmessage = value;
+ }
+
+ readonly CONNECTING = 0 as const;
+ readonly OPEN = 1 as const;
+ readonly CLOSING = 2 as const;
+ readonly CLOSED = 3 as const;
+}
+
+/**
+ * Wraps a base factory so the sockets it produces run through.
+ */
+export const framedWebSocketFactory =
+ (base: WebSocketFactory): WebSocketFactory =>
+ (url, protocols) =>
+ new FramedWebSocket(base(url, protocols));
+
+/** Normalizes a binary payload to a `Uint8Array` view for framing. */
+function toBytes(data: ArrayBufferLike | Blob | ArrayBufferView): Uint8Array {
+ if (data instanceof Uint8Array) return data;
+ if (ArrayBuffer.isView(data)) {
+ return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+ }
+ if (data instanceof ArrayBuffer) return new Uint8Array(data);
+ throw new Error('FramedWebSocket: unsupported binary payload type');
+}
diff --git a/packages/collaboration/src/websocket/platform/framing/frames.test.ts b/packages/collaboration/src/websocket/platform/framing/frames.test.ts
new file mode 100644
index 0000000000..cc6cad9918
--- /dev/null
+++ b/packages/collaboration/src/websocket/platform/framing/frames.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, test } from 'vitest';
+import { intoFrames, MAX_CHUNK_SIZE, Reassembler } from './frames';
+
+describe('intoFrames', () => {
+ test('small payload → one final frame', () => {
+ const frames = intoFrames(new Uint8Array([1, 2, 3]));
+ expect(frames.length).toBe(1);
+ expect(frames[0][0]).toBe(1);
+ expect(Array.from(frames[0].subarray(1))).toEqual([1, 2, 3]);
+ });
+
+ test('exact boundary → one frame', () => {
+ const frames = intoFrames(new Uint8Array([1, 2, 3, 4]), 4);
+ expect(frames.length).toBe(1);
+ expect(frames[0][0]).toBe(1);
+ });
+
+ test('multi-frame splits and flags all but last as non-final', () => {
+ const payload = new Uint8Array(Array.from({ length: 10 }, (_, i) => i));
+ const frames = intoFrames(payload, 4); // 4 + 4 + 2 => 3 frames
+ expect(frames.length).toBe(3);
+ expect(frames.map((f) => f[0])).toEqual([0, 0, 1]);
+
+ const body = frames.flatMap((f) => Array.from(f.subarray(1)));
+ expect(body).toEqual(Array.from(payload));
+ });
+
+ test('empty payload → single final empty frame', () => {
+ const frames = intoFrames(new Uint8Array([]), 4);
+ expect(frames.length).toBe(1);
+ expect(Array.from(frames[0])).toEqual([1]);
+ });
+});
+
+describe('Reassembler', () => {
+ test('single final frame returns whole message', () => {
+ const out = new Reassembler().push(new Uint8Array([1, 9, 8, 7]));
+ expect(out && Array.from(out)).toEqual([9, 8, 7]);
+ });
+
+ test('returns null until the final frame', () => {
+ const r = new Reassembler();
+ expect(r.push(new Uint8Array([0, 1, 2]))).toBeNull();
+ expect(r.push(new Uint8Array([0, 3]))).toBeNull();
+ const out = r.push(new Uint8Array([1, 4, 5]));
+ expect(out && Array.from(out)).toEqual([1, 2, 3, 4, 5]);
+ });
+
+ test('round-trips with intoFrames', () => {
+ const payload = new Uint8Array(
+ Array.from({ length: 1000 }, (_, i) => i % 256)
+ );
+ const r = new Reassembler();
+ let out: Uint8Array | null = null;
+ for (const frame of intoFrames(payload, 64)) {
+ out = r.push(frame);
+ }
+ expect(out && Array.from(out)).toEqual(Array.from(payload));
+ });
+});
+
+test('MAX_CHUNK_SIZE stays under the 1MB durable object limit', () => {
+ expect(MAX_CHUNK_SIZE).toBeLessThan(1_000_000);
+});
diff --git a/packages/collaboration/src/websocket/platform/framing/frames.ts b/packages/collaboration/src/websocket/platform/framing/frames.ts
new file mode 100644
index 0000000000..6a946e0fb2
--- /dev/null
+++ b/packages/collaboration/src/websocket/platform/framing/frames.ts
@@ -0,0 +1,59 @@
+/** Max bytes of message payload per frame. Mirrors the Rust `MAX_CHUNK_SIZE`. */
+export const MAX_CHUNK_SIZE = 900_000;
+
+/**
+ * Splits a serialized message into wire frames of `[isFinal: u8, ...chunk]`.
+ */
+export function intoFrames(
+ payload: Uint8Array,
+ maxChunk = MAX_CHUNK_SIZE
+): Uint8Array[] {
+ const frames: Uint8Array[] = [];
+ // Always emit at least one frame, even for an empty payload.
+ for (
+ let offset = 0;
+ offset < payload.length || frames.length === 0;
+ offset += maxChunk
+ ) {
+ const chunk = payload.subarray(offset, offset + maxChunk);
+ const isFinal = offset + maxChunk >= payload.length;
+ const frame = new Uint8Array(1 + chunk.length);
+ frame[0] = isFinal ? 1 : 0;
+ frame.set(chunk, 1);
+ frames.push(frame);
+ }
+ return frames;
+}
+
+/**
+ * Reassembles frames produced by {@link intoFrames} back into whole messages.
+ * One instance per connection; frames arrive in order over a single socket.
+ */
+export class Reassembler {
+ private chunks: Uint8Array[] = [];
+
+ /**
+ * Feeds one frame. Returns the complete message once the final frame arrives,
+ * otherwise `null` (more frames pending).
+ */
+ push(frame: Uint8Array): Uint8Array | null {
+ const isFinal = frame[0] === 1;
+ this.chunks.push(frame.subarray(1));
+ if (!isFinal) return null;
+
+ const message = concat(this.chunks);
+ this.chunks = [];
+ return message;
+ }
+}
+
+function concat(chunks: Uint8Array[]): Uint8Array {
+ const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+ const out = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) {
+ out.set(chunk, offset);
+ offset += chunk.length;
+ }
+ return out;
+}
diff --git a/packages/collaboration/src/websocket/platform/framing/index.ts b/packages/collaboration/src/websocket/platform/framing/index.ts
new file mode 100644
index 0000000000..ad132b7a92
--- /dev/null
+++ b/packages/collaboration/src/websocket/platform/framing/index.ts
@@ -0,0 +1,2 @@
+export { framedWebSocketFactory } from './framed-websocket';
+export { intoFrames, MAX_CHUNK_SIZE, Reassembler } from './frames';
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index ba0e2db0fd..5f6dee7a24 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -1,6 +1,6 @@
use std::{
borrow::Cow,
- collections::{BTreeMap, BTreeSet},
+ collections::{BTreeMap, BTreeSet, HashMap},
rc::Rc,
sync::{Arc, LazyLock},
};
@@ -28,13 +28,14 @@ use crate::{
generated::schema::InitializeFromSnapshotRequest,
keepalive::{DEFAULT_TIME_TO_LIVE, keepalive},
mutex::Mutex,
+ socket::{InboundBuffers, Socket, protocol},
state::DocumentState,
storage::{
SessionStorage, backends::durable_kv::DurableKVStorage, get_snapshot_storage,
snapshot::SnapshotStorage,
},
tags::{get_ws_id_from_tags, new_ws_id},
- timeit, websocket,
+ timeit,
};
pub const NO_SUCH_VALUE_ERR_STR: &str = "No such value in storage.";
@@ -136,7 +137,8 @@ pub struct DocumentSyncSession {
awareness: EphemeralStore,
/// a map from websocket's ID's to websocket metadata
ws_meta_map: Arc<Mutex<WsMetaMap>>,
- msg_buffer: Arc<Mutex<Vec<u8>>>,
+ /// Partial inbound messages, isolated by connection.
+ inbound: InboundBuffers,
/// Buffered blame events. Flushed via D1 batch on each alarm tick.
pending_blame: Arc<Mutex<Vec<crate::d1::BlameEvent>>>,
/// Distinct editors since the last snapshot notification.
@@ -323,6 +325,17 @@ fn take_editors(editors: &Mutex<BTreeSet<DocumentAttribution>>) -> Vec<DocumentA
}
impl DocumentSyncSession {
+ pub(crate) fn socket_for(&self, ws: &WebSocket) -> Socket {
+ Socket::new(ws.clone(), self.inbound.clone())
+ }
+
+ pub(crate) fn get_sockets(&self) -> Vec<Socket> {
+ self.active_websockets()
+ .into_iter()
+ .map(|ws| self.socket_for(&ws))
+ .collect()
+ }
+
pub fn get_websockets(&self) -> Vec<WebSocket> {
self.active_websockets()
}
@@ -339,6 +352,7 @@ impl DocumentSyncSession {
let Ok(ws_id) = get_ws_id(&self.state, ws) else {
return;
};
+ self.socket_for(ws).forget(&ws_id);
self.ws_meta_map
.lock("DocumentSyncSession::forget_websocket_metadata")
.remove(&ws_id);
@@ -512,11 +526,10 @@ impl DocumentSyncSession {
{
let awareness = self.awareness.encode_all();
for ws in &self.state.get_websockets() {
- if let Err(e) = websocket::send_initial_sync(
- ws,
+ if let Err(e) = protocol::send_initial_sync(
+ &self.socket_for(ws),
snapshot.as_slice(),
awareness.as_slice(),
- self.msg_buffer.clone(),
) {
warn!(
error =? e,
@@ -748,11 +761,10 @@ impl DocumentSyncSession {
// Size of the initial sync this connect sent — the server end
// of the client's `doc.sync.initial-sync` span.
tracing::Span::current().record("snapshot.bytes", snapshot.len());
- websocket::send_initial_sync(
- &pair.server,
+ protocol::send_initial_sync(
+ &self.socket_for(&pair.server),
snapshot.as_slice(),
self.awareness.encode_all().as_slice(),
- self.msg_buffer.clone(),
)
.context("failed to send initial sync message")?;
} else {
@@ -991,7 +1003,7 @@ impl DurableObject for DocumentSyncSession {
session_storage: Mutex::new(None),
awareness: EphemeralStore::new(5_000),
ws_meta_map: Arc::new(Mutex::new(Default::default())),
- msg_buffer: Arc::new(Mutex::new(vec![])),
+ inbound: Arc::new(Mutex::new(HashMap::new())),
pending_blame: Arc::new(Mutex::new(Vec::new())),
pending_editors: Arc::new(Mutex::new(Default::default())),
surface_lifecycle: Mutex::new(None),
@@ -1073,8 +1085,13 @@ impl DurableObject for DocumentSyncSession {
}
WebSocketIncomingMessage::Binary(bm) => bm,
};
+ let socket = self.socket_for(&ws);
+ let ws_id = get_ws_id(&self.state, &ws)?;
+ let Some(binary_message) = socket.receive(&ws_id, &binary_message)? else {
+ return Ok(());
+ };
worker_rs_otel::scope(&self.env, &self.state, async {
- let mut telemetry = websocket::InboundMessageTelemetry::new(binary_message.len());
+ let mut telemetry = protocol::InboundMessageTelemetry::new(binary_message.len());
let res: Result<()> = async {
let document_id = self.document_id().await.inspect_err(|_| {
@@ -1083,21 +1100,19 @@ impl DurableObject for DocumentSyncSession {
let ws_id = get_ws_id_from_tags(&self.state.get_tags(&ws)).ok();
telemetry.record_context(&document_id, ws_id);
- let message =
- websocket::deserialize_message(&binary_message).inspect_err(|_| {
- telemetry.record_message_type("invalid");
- telemetry.record_error_stage("deserialize");
- })?;
- telemetry.record_message_type(websocket::message_type(&message));
+ let message = protocol::deserialize_message(&binary_message).inspect_err(|_| {
+ telemetry.record_message_type("invalid");
+ telemetry.record_error_stage("deserialize");
+ })?;
+ telemetry.record_message_type(protocol::message_type(&message));
- websocket::process_message(
- &ws,
+ protocol::process_message(
+ &self.socket_for(&ws),
&document_id,
&*self.document_state().await?,
&*self.session_storage().await?,
&self.awareness,
message,
- self.msg_buffer.clone(),
self,
&mut telemetry,
)
@@ -1231,13 +1246,8 @@ impl DurableObject for DocumentSyncSession {
let update = self.awareness.encode(&peer_id.to_string());
// Don't silently discard the error
- websocket::broadcast_awareness(
- &ws,
- self.get_websockets().as_slice(),
- update.as_slice(),
- self.msg_buffer.clone(),
- )
- .context("failed to broadcast awareness")?;
+ protocol::broadcast_awareness(self, &self.socket_for(&ws), update.as_slice())
+ .context("failed to broadcast awareness")?;
}
if self.state.get_websockets().len() == 1
diff --git a/services/sync-service/src/durable_object/document_effects.rs b/services/sync-service/src/durable_object/document_effects.rs
index 6e5fa2b54c..45e0e2782f 100644
--- a/services/sync-service/src/durable_object/document_effects.rs
+++ b/services/sync-service/src/durable_object/document_effects.rs
@@ -1,6 +1,5 @@
//! Worker notification adapter for the document update use case.
-use bebop::{Record, SubRecord};
use macro_sync_service_jwt::session::SessionKind;
use tracing::{error, warn};
use worker::Env;
@@ -67,15 +66,10 @@ fn notification_error(error: impl std::fmt::Debug) -> DocumentError {
impl DocumentUpdateEffects for WorkerDocumentEffects<'_> {
fn broadcast(&self, update: &[u8]) -> Result<(), DocumentError> {
- let message = crate::generated::schema::FromRemote::RemoteUpdate {
- update: bebop::SliceWrapper::Raw(update),
- };
- let mut message_bytes = Vec::with_capacity(message.serialized_size());
- message
- .serialize(&mut message_bytes)
- .map_err(notification_error)?;
- for socket in self.session.get_websockets() {
- if let Err(error) = socket.send_with_bytes(&message_bytes) {
+ for socket in self.session.get_sockets() {
+ if let Err(error) = socket.send(crate::generated::schema::FromRemote::RemoteUpdate {
+ update: bebop::SliceWrapper::Raw(update),
+ }) {
warn!(error = ?error, "failed to broadcast document update; continuing");
}
}
diff --git a/services/sync-service/src/lib.rs b/services/sync-service/src/lib.rs
index 33fe9e9a52..2bc13987d9 100644
--- a/services/sync-service/src/lib.rs
+++ b/services/sync-service/src/lib.rs
@@ -12,11 +12,11 @@ pub mod keepalive;
mod metrics;
mod mutex;
mod secrets;
+mod socket;
mod state;
mod storage;
mod tags;
mod timeout;
-mod websocket;
use tracing_subscriber::{
EnvFilter, fmt::time::UtcTime, layer::SubscriberExt, util::SubscriberInitExt,
diff --git a/services/sync-service/src/socket/framing.rs b/services/sync-service/src/socket/framing.rs
new file mode 100644
index 0000000000..0a24447105
--- /dev/null
+++ b/services/sync-service/src/socket/framing.rs
@@ -0,0 +1,150 @@
+use worker::{Error, Result};
+
+/// Max bytes of message payload per frame. Kept under the durable object's ~1 MB
+/// outbound websocket limit (and the server's inbound `MAX_MESSAGE_SIZE`),
+/// leaving headroom for the 1-byte frame header and websocket overhead.
+pub(crate) const MAX_CHUNK_SIZE: usize = 900_000;
+
+/// Splits a serialized message into wire frames of `[is_final: u8][chunk bytes]`.
+///
+/// `is_final` is 1 on the last (or only) frame, 0 otherwise. Always yields at
+/// least one frame, so an empty payload produces a single final empty frame.
+/// The receiver concatenates chunk bodies until it sees `is_final == 1`.
+pub(crate) fn into_frames(payload: &[u8], max_chunk: usize) -> Vec<Vec<u8>> {
+ let mut chunks = payload.chunks(max_chunk).peekable();
+
+ // `chunks` is empty only when `payload` is empty; emit one empty final frame.
+ if chunks.peek().is_none() {
+ return vec![vec![1]];
+ }
+
+ let mut frames = Vec::new();
+ while let Some(chunk) = chunks.next() {
+ let is_final = chunks.peek().is_none();
+ let mut frame = Vec::with_capacity(1 + chunk.len());
+ frame.push(u8::from(is_final));
+ frame.extend_from_slice(chunk);
+ frames.push(frame);
+ }
+ frames
+}
+
+/// Splits a received frame into its `is_final` flag and payload slice.
+pub(crate) fn strip_frame(frame: &[u8]) -> Result<(bool, &[u8])> {
+ let (&header, payload) = frame
+ .split_first()
+ .ok_or_else(|| Error::from("received empty websocket frame"))?;
+ Ok((header != 0, payload))
+}
+
+/// Reassembles inbound frames (produced by [`into_frames`]) back into whole
+/// messages. One per connection; frames arrive in order over a single socket.
+///
+/// Held in memory, so a partial mid-message is lost if the durable object
+/// hibernates between frames — acceptable because clients only ever send small,
+/// single-frame messages (a dropped partial just fails to deserialize and the
+/// client resends).
+#[derive(Default)]
+pub(crate) struct Reassembler {
+ buffer: Vec<u8>,
+}
+
+impl Reassembler {
+ /// Feeds one frame. Returns the complete message once the final frame
+ /// arrives, otherwise `None` (more frames pending).
+ pub(crate) fn push(&mut self, frame: &[u8]) -> Result<Option<Vec<u8>>> {
+ let (is_final, payload) = strip_frame(frame)?;
+ self.buffer.extend_from_slice(payload);
+ if is_final {
+ Ok(Some(std::mem::take(&mut self.buffer)))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn single_frame_small_payload() {
+ let payload = b"hello";
+ let frames = into_frames(payload, MAX_CHUNK_SIZE);
+ assert_eq!(frames.len(), 1);
+ assert_eq!(frames[0][0], 1);
+ assert_eq!(&frames[0][1..], payload);
+ }
+
+ #[test]
+ fn exact_boundary_is_one_frame() {
+ let payload = vec![7u8; 4];
+ let frames = into_frames(&payload, 4);
+ assert_eq!(frames.len(), 1);
+ assert_eq!(frames[0][0], 1);
+ assert_eq!(&frames[0][1..], payload.as_slice());
+ }
+
+ #[test]
+ fn multi_frame_splits_and_flags() {
+ let payload: Vec<u8> = (0..10).collect();
+ let frames = into_frames(&payload, 4); // 4 + 4 + 2 => 3 frames
+ assert_eq!(frames.len(), 3);
+ assert_eq!(frames[0][0], 0);
+ assert_eq!(frames[1][0], 0);
+ assert_eq!(frames[2][0], 1);
+
+ let reassembled: Vec<u8> = frames.iter().flat_map(|f| f[1..].iter().copied()).collect();
+ assert_eq!(reassembled, payload);
+ }
+
+ #[test]
+ fn empty_payload_yields_one_final_frame() {
+ assert_eq!(into_frames(&[], 4), vec![vec![1u8]]);
+ }
+
+ #[test]
+ fn strip_frame_reads_header() {
+ assert_eq!(strip_frame(&[1, 2, 3]).unwrap(), (true, &[2, 3][..]));
+ assert_eq!(strip_frame(&[0, 9]).unwrap(), (false, &[9][..]));
+ assert!(strip_frame(&[]).is_err());
+ }
+
+ #[test]
+ fn round_trip() {
+ let payload: Vec<u8> = (0..1000u32).map(|i| i as u8).collect();
+ let frames = into_frames(&payload, 64);
+
+ let mut out = Vec::new();
+ for frame in &frames {
+ let (_is_final, body) = strip_frame(frame).unwrap();
+ out.extend_from_slice(body);
+ }
+ assert_eq!(out, payload);
+ }
+
+ #[test]
+ fn reassembler_single_final_frame() {
+ let mut r = Reassembler::default();
+ assert_eq!(r.push(&[1, 9, 8, 7]).unwrap(), Some(vec![9, 8, 7]));
+ }
+
+ #[test]
+ fn reassembler_returns_none_until_final() {
+ let mut r = Reassembler::default();
+ assert_eq!(r.push(&[0, 1, 2]).unwrap(), None);
+ assert_eq!(r.push(&[0, 3]).unwrap(), None);
+ assert_eq!(r.push(&[1, 4, 5]).unwrap(), Some(vec![1, 2, 3, 4, 5]));
+ }
+
+ #[test]
+ fn reassembler_round_trips_with_into_frames() {
+ let payload: Vec<u8> = (0..1000u32).map(|i| i as u8).collect();
+ let mut r = Reassembler::default();
+ let mut out = None;
+ for frame in into_frames(&payload, 64) {
+ out = r.push(&frame).unwrap();
+ }
+ assert_eq!(out, Some(payload));
+ }
+}
diff --git a/services/sync-service/src/socket/mod.rs b/services/sync-service/src/socket/mod.rs
new file mode 100644
index 0000000000..618344a517
--- /dev/null
+++ b/services/sync-service/src/socket/mod.rs
@@ -0,0 +1,9 @@
+pub(crate) mod framing;
+pub(crate) mod protocol;
+#[allow(
+ clippy::module_inception,
+ reason = "Socket type lives in socket.rs within the socket module"
+)]
+mod socket;
+
+pub(crate) use socket::{InboundBuffers, Socket};
diff --git a/services/sync-service/src/websocket.rs b/services/sync-service/src/socket/protocol.rs
similarity index 68%
rename from services/sync-service/src/websocket.rs
rename to services/sync-service/src/socket/protocol.rs
index 09fba2757b..cd204b77fb 100644
--- a/services/sync-service/src/websocket.rs
+++ b/services/sync-service/src/socket/protocol.rs
@@ -1,28 +1,17 @@
-use std::sync::Arc;
-
use bebop::{Record, SliceWrapper};
use loro::{VersionVector, awareness::EphemeralStore};
use tracing::trace;
-use worker::{Result, WebSocket};
+use worker::Result;
use crate::{
durable_object::{DocumentSyncSession, Wsm},
error::ResultExt,
generated::schema::{FromPeer, FromRemote},
- mutex::Mutex,
+ socket::Socket,
state::DocumentState,
storage::SessionStorage,
};
-fn serialize<'a, T: bebop::Record<'a>>(
- obj: T,
- msg_buf: &mut Vec<u8>,
-) -> std::result::Result<&[u8], bebop::SerializeError> {
- msg_buf.clear();
- obj.serialize(msg_buf)?;
- Ok(msg_buf)
-}
-
/// Decodes an inbound WebSocket payload once, before its trace span is created.
pub fn deserialize_message(message: &[u8]) -> Result<FromPeer<'_>> {
if message.len() > MAX_MESSAGE_SIZE {
@@ -135,46 +124,32 @@ impl InboundMessageTelemetry {
}
}
-/// Sends the initial sync message to the client over the websocket
+/// Sends the initial sync message to a single peer.
/// The initial sync message contains the snapshot of the current state of the document
-pub fn send_initial_sync(
- ws: &WebSocket,
- snapshot: &[u8],
- awareness: &[u8],
- buf: Arc<Mutex<Vec<u8>>>,
-) -> Result<()> {
- let message = FromRemote::RemoteInitialSync {
+pub fn send_initial_sync(socket: &Socket, snapshot: &[u8], awareness: &[u8]) -> Result<()> {
+ socket.send(FromRemote::RemoteInitialSync {
snapshot: SliceWrapper::Raw(snapshot),
awareness: SliceWrapper::Raw(awareness),
- };
-
- let mut buf = buf.lock("serialize RemoteInitialSync in send_initial_sync");
- let serialized = serialize(message, &mut buf).context("Failed serializing snapshot")?;
- ws.send_with_bytes(serialized)
- .context("failed to send initial sync message")?;
- Ok(())
+ })
}
+/// Broadcasts an awareness update to every connected peer except `from`.
pub fn broadcast_awareness(
- from_socket: &WebSocket,
- sockets: &[WebSocket],
+ dss: &DocumentSyncSession,
+ from: &Socket,
awareness: &[u8],
- buf: Arc<Mutex<Vec<u8>>>,
) -> Result<()> {
- let message = FromRemote::RemoteAwareness {
- awareness: SliceWrapper::Raw(awareness),
- };
-
- let mut buf = buf.lock("serialize RemoteAwareness in broadcast_awareness");
- let serialized = serialize(message, &mut buf).context("failed serializing RemoteAwareness")?;
-
- for w in sockets.iter().filter(|w| w != &from_socket) {
+ for socket in dss.get_sockets() {
+ if socket == *from {
+ continue;
+ }
// A dead peer socket must not abort delivery to the remaining peers.
- if let Err(e) = w.send_with_bytes(serialized) {
+ if let Err(e) = socket.send(FromRemote::RemoteAwareness {
+ awareness: SliceWrapper::Raw(awareness),
+ }) {
tracing::warn!(error = ?e, "failed to send awareness to a peer; continuing");
}
}
-
Ok(())
}
@@ -186,19 +161,21 @@ const MAX_MESSAGE_SIZE: usize = 1000 * 1000;
reason = "lots of args lets us avoid having multiple mutable refs to same object"
)]
pub async fn process_message(
- ws: &WebSocket,
+ sender: &Socket,
document_id: &str,
document_state: &DocumentState,
session_storage: &SessionStorage,
awareness: &EphemeralStore,
message: FromPeer<'_>,
- buf: Arc<Mutex<Vec<u8>>>,
dss: &DocumentSyncSession,
telemetry: &mut InboundMessageTelemetry,
) -> Result<()> {
// Loading the document after hibernation can yield to a revocation. Check
// the surface grant again immediately before handling any protocol message.
- if !dss.validate_surface_sockets(Some(ws)).await? {
+ if !dss
+ .validate_surface_sockets(Some(sender.websocket()))
+ .await?
+ {
return Ok(());
}
trace!(
@@ -210,7 +187,7 @@ pub async fn process_message(
// This registers a peer id to the owner of the current websocket
FromPeer::PeerRegisterId { peerid } => {
telemetry.peer_id = Some(peerid);
- Wsm::new(dss, ws)
+ Wsm::new(dss, sender.websocket())
.add_new_peerid(peerid, document_id)
.await?;
}
@@ -220,19 +197,22 @@ pub async fn process_message(
FromPeer::PeerUpdate { updates, id } => {
telemetry.op_id = Some(id.to_string());
telemetry.update_bytes = Some(updates.iter().map(|u| u.len()).sum());
- if !Wsm::new(dss, ws).can_edit().await? {
+ if !Wsm::new(dss, sender.websocket()).can_edit().await? {
tracing::warn!("received update from peer without edit permission");
return Ok(());
}
- let attribution = match Wsm::new(dss, ws).edit_attribution().await {
+ let attribution = match Wsm::new(dss, sender.websocket()).edit_attribution().await {
Ok(attribution) => attribution,
Err(error) => {
tracing::warn!(error = ?error, "failed to resolve optional edit attribution");
None
}
};
- let peer_ids = Wsm::new(dss, ws).get_peer_ids().await.unwrap_or_default();
+ let peer_ids = Wsm::new(dss, sender.websocket())
+ .get_peer_ids()
+ .await
+ .unwrap_or_default();
let peer_id = peer_ids.first().copied();
telemetry.peer_id = peer_id;
let now_ms = web_time::SystemTime::now()
@@ -264,40 +244,23 @@ pub async fn process_message(
}
}
- {
- // ACK the sender before broadcasting: the batch is durably
- // stored at this point, and a failed broadcast to some other
- // peer must not block the ack.
- let message = FromRemote::RemoteUpdateAck { id };
- let mut buf = buf.lock("serialize RemoteUpdateAck in PeerUpdate handler");
- let serialized =
- serialize(message, &mut buf).context("Failed serializing update")?;
- ws.send_with_bytes(serialized)
- .inspect_err(|error| {
- tracing::error!(error = ?error, op.id = %id, "failed to send update ack");
- })
- .context("failed to send update ack")?;
- }
+ // ACK only after the update is durably stored.
+ sender
+ .send(FromRemote::RemoteUpdateAck { id })
+ .inspect_err(|error| {
+ tracing::error!(error = ?error, op.id = %id, "failed to send update ack");
+ })
+ .context("failed to send update ack")?;
- // Peers receiving the rebroadcast (everyone but the sender). The
- // count is the key Teo/Hutch diagnostic: if it excludes a peer the
- // DO thinks isn't connected, that's the dropped-update bug.
- let sockets = dss.get_websockets();
- let targets: Vec<&WebSocket> = sockets.iter().filter(|w| w != &ws).collect();
+ let sockets = dss.get_sockets();
+ let targets: Vec<&Socket> = sockets.iter().filter(|socket| *socket != sender).collect();
telemetry.broadcast_targets = Some(targets.len());
for update in &updates {
- // broadcast each update to other peers
- let message = FromRemote::RemoteUpdate {
- update: SliceWrapper::Raw(update),
- };
- let mut buf = buf.lock("serialize RemoteUpdate in PeerUpdate handler");
- let serialized =
- serialize(message, &mut buf).context("Failed serializing update")?;
- for w in &targets {
- // A dead peer socket must not abort delivery to the
- // remaining peers.
- if let Err(e) = w.send_with_bytes(serialized) {
- tracing::warn!(error = ?e, "failed to send update to a peer; continuing");
+ for socket in &targets {
+ if let Err(error) = socket.send(FromRemote::RemoteUpdate {
+ update: SliceWrapper::Raw(update),
+ }) {
+ tracing::warn!(error = ?error, "failed to send update to a peer; continuing");
}
}
}
@@ -312,11 +275,10 @@ pub async fn process_message(
return Ok(());
}
let encodede = awareness.encode_all();
- let sockets = dss.get_websockets();
+ let sockets = dss.get_sockets();
telemetry.broadcast_targets =
- Some(sockets.iter().filter(|socket| socket != &ws).count());
- broadcast_awareness(ws, &sockets, &encodede, buf)
- .context("failed to broadcast awareness")?;
+ Some(sockets.iter().filter(|socket| *socket != sender).count());
+ broadcast_awareness(dss, sender, &encodede).context("failed to broadcast awareness")?;
}
// Handle a peer requesting a specific set of updates from the document.
// The client sends a version vector (not frontiers) so unknown peers
@@ -336,15 +298,11 @@ pub async fn process_message(
// sent; `decode(vv).encode()` is not guaranteed to reproduce the same
// bytes for a multi-peer version vector, which would make the client
// discard a perfectly good response and time out.
- let message = FromRemote::RemoteUpdateSince {
- update: SliceWrapper::Raw(&update),
- vv,
- };
-
- let mut buf = buf.lock("serialize RemoteUpdate in PeerRequestSince handler");
- let serialized =
- serialize(message, &mut buf).context("failed serializing PeerRequestSince")?;
- ws.send_with_bytes(serialized)
+ sender
+ .send(FromRemote::RemoteUpdateSince {
+ update: SliceWrapper::Raw(&update),
+ vv,
+ })
.context("failed to send update")?;
}
// Peer is requesting a snapshot from the remote
@@ -352,14 +310,11 @@ pub async fn process_message(
let snapshot = document_state.export_shallow_snapshot()?;
telemetry.response_bytes = Some(snapshot.len());
- let message = FromRemote::RemoteSnapshot {
- snapshot: SliceWrapper::Raw(&snapshot),
- };
- let mut buf = buf.lock("serialize RemoteSnapshot in PeerRequestSnapshot handler");
- let serialized =
- serialize(message, &mut buf).context("failed serializing PeerRequestSnapshot")?;
- ws.send_with_bytes(serialized)
- .context("failed to send update")?;
+ sender
+ .send(FromRemote::RemoteSnapshot {
+ snapshot: SliceWrapper::Raw(&snapshot),
+ })
+ .context("failed to send snapshot")?;
}
FromPeer::Unknown => {
return Err(worker::Error::from("unknown message type"));
diff --git a/services/sync-service/src/socket/socket.rs b/services/sync-service/src/socket/socket.rs
new file mode 100644
index 0000000000..e254bcdfb3
--- /dev/null
+++ b/services/sync-service/src/socket/socket.rs
@@ -0,0 +1,70 @@
+use std::{collections::HashMap, sync::Arc};
+
+use bebop::Record;
+use worker::{Result, WebSocket};
+
+use super::framing::{self, Reassembler};
+use crate::{error::ResultExt, mutex::Mutex};
+
+/// Per-connection inbound reassembly buffers, keyed by websocket id. Lives on the
+/// durable object (shared via `Arc`) so partial messages persist across the
+/// `websocket_message` events that deliver each frame.
+pub(crate) type InboundBuffers = Arc<Mutex<HashMap<String, Reassembler>>>;
+
+/// A single peer websocket, but abstracted so that we can wrap our own transport layer.
+pub(crate) struct Socket {
+ ws: WebSocket,
+ inbound: InboundBuffers,
+}
+
+/// Sockets are equal when they wrap the same peer websocket; the shared inbound
+/// buffer handle is not part of identity (used to skip the sender in broadcasts).
+impl PartialEq for Socket {
+ fn eq(&self, other: &Self) -> bool {
+ self.ws == other.ws
+ }
+}
+impl Eq for Socket {}
+
+impl Socket {
+ pub(crate) fn new(ws: WebSocket, inbound: InboundBuffers) -> Self {
+ Self { ws, inbound }
+ }
+
+ /// The underlying peer websocket.
+ pub(crate) fn websocket(&self) -> &WebSocket {
+ &self.ws
+ }
+
+ /// Serialize and send one message to this peer, split into wire frames so it
+ /// stays under the durable object's max outbound websocket message size.
+ pub(crate) fn send<'m, T: Record<'m>>(&self, msg: T) -> Result<()> {
+ let mut buf = Vec::new();
+ msg.serialize(&mut buf)
+ .context("failed to serialize message")?;
+ for frame in framing::into_frames(&buf, framing::MAX_CHUNK_SIZE) {
+ self.ws
+ .send_with_bytes(&frame)
+ .context("failed to send frame")?;
+ }
+ Ok(())
+ }
+
+ /// Feeds one inbound frame for the connection identified by `ws_id`. Returns
+ /// the complete message once its final frame arrives, otherwise `None` while
+ /// earlier frames are still buffering.
+ pub(crate) fn receive(&self, ws_id: &str, frame: &[u8]) -> Result<Option<Vec<u8>>> {
+ let mut inbound = self.inbound.lock("Socket::receive");
+ let message = inbound.entry(ws_id.to_string()).or_default().push(frame)?;
+ if message.is_some() {
+ inbound.remove(ws_id);
+ }
+ Ok(message)
+ }
+
+ /// Discards any partially-buffered inbound message for a connection. Call
+ /// when the connection closes so a dropped mid-message doesn't leak.
+ pub(crate) fn forget(&self, ws_id: &str) {
+ self.inbound.lock("Socket::forget").remove(ws_id);
+ }
+}
diff --git a/services/sync-service/tests/chunking.test.ts b/services/sync-service/tests/chunking.test.ts
new file mode 100644
index 0000000000..126b1a343b
--- /dev/null
+++ b/services/sync-service/tests/chunking.test.ts
@@ -0,0 +1,102 @@
+import { randomBytes } from "node:crypto";
+import { LoroDoc } from "loro-crdt";
+import type { Miniflare } from "miniflare";
+import { afterAll, beforeAll, expect, test } from "vitest";
+import { MAX_CHUNK_SIZE } from "../../../packages/collaboration/src/websocket/platform/framing/frames";
+import {
+ FromPeer,
+ InitializeFromSnapshotRequest,
+} from "../bebop/generated/schema";
+import { createTestUser, getTokenForDocument, setupMiniflare } from "./utils";
+
+let mf: Miniflare;
+const users: Awaited<ReturnType<typeof createTestUser>>[] = [];
+beforeAll(async () => {
+ mf = await setupMiniflare();
+}, 60_000);
+afterAll(async () => {
+ for (const user of users)
+ if (user.getWebSocket().readyState === 1) user.getWebSocket().close();
+ await mf?.dispose();
+});
+
+test("large update batches, snapshots and reconnects use bounded wire frames", async () => {
+ const id = crypto.randomUUID();
+ const seed = new LoroDoc();
+ seed.getText("content").push("Chunked sync\n");
+ seed.commit();
+ const initialized = await mf.dispatchFetch(
+ `http://localhost/document/${id}/initialize`,
+ {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${getTokenForDocument(id, "owner", "owner")}`,
+ },
+ body: InitializeFromSnapshotRequest.encode({
+ snapshot: seed.export({ mode: "snapshot" }),
+ }),
+ },
+ );
+ expect(initialized.status).toBe(200);
+ const alice = await createTestUser(mf, id);
+ const bob = await createTestUser(mf, id);
+ users.push(alice, bob);
+ const incoming: number[] = [];
+ bob.getWebSocket().addEventListener("message", (event) => {
+ if (typeof event.data !== "string") incoming.push(event.data.byteLength);
+ });
+ const text = randomBytes(1_800_000).toString("base64");
+ const updates: Uint8Array[] = [];
+ for (let offset = 0; offset < text.length; offset += 40_000) {
+ const from = alice.doc.version();
+ alice.doc.getText("content").push(text.slice(offset, offset + 40_000));
+ alice.doc.commit();
+ updates.push(alice.doc.export({ mode: "update", from }));
+ }
+ const message = FromPeer.fromPeerUpdate({
+ updates,
+ id: "large-update",
+ }).encode();
+ expect(message.byteLength).toBeGreaterThan(MAX_CHUNK_SIZE);
+ alice.send(message);
+ const ack = await alice.readNextMessage();
+ expect(ack.isRemoteUpdateAck() && ack.value.id).toBe("large-update");
+ for (const _ of updates) await bob.importNextUpdate();
+ expect(bob.getState()).toBe(alice.getState());
+ expect(incoming.length).toBe(updates.length);
+ expect(Math.max(...incoming)).toBeLessThanOrEqual(MAX_CHUNK_SIZE + 1);
+
+ const bobFrom = bob.doc.version();
+ bob.doc.getText("content").push("\nSmall reverse edit");
+ bob.doc.commit();
+ bob.send(
+ FromPeer.fromPeerUpdate({
+ updates: [bob.doc.export({ mode: "update", from: bobFrom })],
+ id: "small-update",
+ }).encode(),
+ );
+ await alice.importNextUpdate();
+ expect(alice.getState()).toBe(bob.getState());
+
+ const snapshotFrames: number[] = [];
+ alice.getWebSocket().addEventListener("message", (event) => {
+ if (typeof event.data !== "string")
+ snapshotFrames.push(event.data.byteLength);
+ });
+ alice.send(FromPeer.fromPeerRequestSnapshot({}).encode());
+ const response = await alice.readNextMessage();
+ expect(response.isRemoteSnapshot()).toBe(true);
+ expect(snapshotFrames.length).toBeGreaterThan(1);
+ expect(Math.max(...snapshotFrames)).toBeLessThanOrEqual(MAX_CHUNK_SIZE + 1);
+ if (!response.isRemoteSnapshot()) throw new Error("Expected snapshot");
+ const snapshot = new LoroDoc();
+ snapshot.import(response.value.snapshot);
+ expect(snapshot.getText("content").toString()).toBe(alice.getState());
+
+ bob.getWebSocket().close();
+ const fresh = await createTestUser(mf, id);
+ users.push(fresh);
+ expect(fresh.getState()).toBe(alice.getState());
+ fresh.send("ping");
+ expect(await fresh.waitForNextMessage()).toBe("pong");
+}, 60_000);
diff --git a/services/sync-service/tests/utils.ts b/services/sync-service/tests/utils.ts
index bc139269f8..805908b87d 100644
--- a/services/sync-service/tests/utils.ts
+++ b/services/sync-service/tests/utils.ts
@@ -1,3 +1,4 @@
+import { intoFrames, Reassembler } from '../../../packages/collaboration/src/websocket/platform/framing/frames';
import { EphemeralStore, LoroDoc } from 'loro-crdt';
import { Miniflare, type MiniflareOptions, type WebSocket } from 'miniflare';
import { assert } from 'vitest';
@@ -161,7 +162,7 @@ export async function createTestUser(mf: Miniflare, documentId = 'test-doc', opt
const registrationMessage = FromPeer.fromPeerRegisterId({ peerid: loroDoc.peerId }).encode();
- connection.getWebSocket().send(registrationMessage);
+ connection.send(registrationMessage);
return {
doc: loroDoc,
@@ -210,11 +211,15 @@ export async function createTestUser(mf: Miniflare, documentId = 'test-doc', opt
}
export function createTestWebSocket(ws: WebSocket) {
- const messages: ArrayBuffer[] = [];
- const waiters: ((message: ArrayBuffer) => void)[] = [];
+ const reassembler = new Reassembler();
+ const messages: (ArrayBuffer | string)[] = [];
+ const waiters: ((message: ArrayBuffer | string) => void)[] = [];
ws.addEventListener('message', (event) => {
- const message = event.data as ArrayBuffer;
+ const message = typeof event.data === 'string'
+ ? event.data
+ : reassembler.push(new Uint8Array(event.data as ArrayBuffer))?.buffer;
+ if (message === undefined) return;
if (waiters.length > 0) {
const resolve = waiters.shift()!;
@@ -230,7 +235,7 @@ export function createTestWebSocket(ws: WebSocket) {
return messages.shift()!;
}
- return new Promise<ArrayBuffer>((resolve, reject) => {
+ return new Promise<ArrayBuffer | string>((resolve, reject) => {
const timeoutId = setTimeout(() => {
const index = waiters.indexOf(wrappedResolve);
if (index !== -1) {
@@ -243,7 +248,7 @@ export function createTestWebSocket(ws: WebSocket) {
);
}, timeout);
- const wrappedResolve = (message: ArrayBuffer) => {
+ const wrappedResolve = (message: ArrayBuffer | string) => {
clearTimeout(timeoutId);
resolve(message);
};
@@ -253,7 +258,11 @@ export function createTestWebSocket(ws: WebSocket) {
},
send(message: string | ArrayBuffer | Uint8Array): void {
- ws.send(message instanceof Uint8Array ? new Uint8Array(message) : message);
+ if (typeof message === 'string') {
+ ws.send(message);
+ return;
+ }
+ for (const frame of intoFrames(new Uint8Array(message))) ws.send(frame);
},
getWebSocket() {