Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
import { type InferType, SyncDirection } from '@loro-mirror/core';
import type { Attributes, Span } from '@macro-inc/observability';
import { Mutex } from 'async-mutex';
import type { VersionVector } from 'loro-crdt';
import type { ResultAsync } from 'neverthrow';
import {
type Accessor,
createEffect,
createSignal,
on,
onCleanup,
} from 'solid-js';
import { match } from 'ts-pattern';
import type { Awareness } from './awareness';
import { BroadcastChannelChatter, type Chatter, noopChatter } from './chatter';
import { logSyncService } from './logger';
import {
LoroManagerError,
LoroStateTag,
type StateUpdate,
type SyncEngineManager,
} from './manager';
import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';
import type { SnapshotStore } from './snapshot-store';
import { peerCounterAttr, telemetrySpan } from './telemetry';
// SnapshotStore in the engine is always Loro updates — RawUpdate.
type LoroSnapshotStore = SnapshotStore<RawUpdate>;
import type { LiveSyncSource, SyncError, SyncSourceEvent } from './source';
import type { WALSyncer } from './wal';
const SNAPSHOT_INTERVAL_MS = 5_000;
const REQUEST_UPDATES_MAX_ATTEMPTS = 3;
const REQUEST_UPDATES_RETRY_DELAY_MS = 2_000;
/** A version vector as a compact `peer:counter` span attribute. */
function vvAttr(vv: VersionVector): string {
try {
return peerCounterAttr(vv.toJSON().entries());
} catch {
return 'unavailable';
}
}
export type EngineBindings<S extends GenericRootSchema> = {
onRemoteState: (state: InferType<S>) => void;
};
export type SyncSources = {
wal: WALSyncer<RawUpdate>;
live: LiveSyncSource;
};
export type SyncEngineParams<S extends GenericRootSchema, D> = {
loroManager: SyncEngineManager<S>;
parentSpan?: Span;
awareness: Awareness<D>;
syncs: SyncSources;
bindings: EngineBindings<S>;
readonly?: () => boolean;
onRunningChange?: (v: boolean) => void;
snapshotStore?: LoroSnapshotStore;
makeChatter?: (documentId: string) => Chatter;
};
type SnapshotThunk = () => ResultAsync<Uint8Array, SyncError>;
export class SyncEngine<S extends GenericRootSchema, D> {
private _isRunning = false;
get isRunning() {
return this._isRunning;
}
private readonly parentSpan?: Span;
private readonly loroManager: SyncEngineManager<S>;
private readonly awareness: Awareness<D>;
private readonly syncs: SyncSources;
private readonly bindings: EngineBindings<S>;
private readonly readonly: () => boolean;
private readonly syncLock = new Mutex();
private unsubscribe?: () => void;
private liveUnsubscribe?: () => void;
private snapshotInterval?: ReturnType<typeof setInterval>;
private readonly snapshotStore?: LoroSnapshotStore;
private readonly defaultSnapshotThunk: SnapshotThunk;
private readonly onRunningChange: (v: boolean) => void;
private readonly makeChatter: (documentId: string) => Chatter;
private chatter?: Chatter;
private chatterUnsub?: () => void;
private lifecycleGeneration = 0;
private convergence?: {
generation: number;
promise: Promise<void>;
rerunRequested: boolean;
};
constructor({
loroManager,
parentSpan,
awareness,
syncs,
bindings,
readonly = () => false,
onRunningChange = () => {},
snapshotStore,
makeChatter = () => noopChatter(),
}: SyncEngineParams<S, D>) {
this.loroManager = loroManager;
this.parentSpan = parentSpan;
this.awareness = awareness;
this.syncs = syncs;
this.bindings = bindings;
this.readonly = readonly;
this.defaultSnapshotThunk = syncs.live.requestSnapshot;
this.onRunningChange = onRunningChange;
this.snapshotStore = snapshotStore;
this.makeChatter = makeChatter;
}
private log(
level: Parameters<typeof logSyncService>[0]['level'],
message: string,
extra?: Attributes
) {
logSyncService({
documentId: this.syncs.live.documentId,
level,
context: extra ? { misc: extra } : {},
message,
});
}
public start(): boolean {
if (this._isRunning) return true; // already running — idempotent
if (!this.loroManager.initialized) {
this.log('warn', 'engine.start: manager not initialized, aborting');
return false;
}
this.unsubscribe?.();
this.unsubscribe = this.loroManager.doc.subscribeLocalUpdates((update) => {
this.handleLocalUpdates(update);
});
this.chatter = this.makeChatter(this.syncs.live.documentId);
this.chatterUnsub = this.chatter.subscribe((msg) =>
match(msg)
.with({ type: 'update' }, (m) => void this.handleRemoteUpdate(m.data))
.with({ type: 'awareness' }, (m) =>
this.awareness.importRemoteAwareness(m.data)
)
.exhaustive(() =>
this.log('warn', 'engine.chatter: unknown message ignored')
)
);
this._isRunning = true;
this.lifecycleGeneration++;
this.liveUnsubscribe?.();
this.liveUnsubscribe = this.syncs.live.listen((event) =>
this.handleSourceEvent(event)
);
this.syncs.live.registerPeerId(this.loroManager.peerId);
if (this.snapshotStore && this.snapshotInterval === undefined) {
this.snapshotInterval = setInterval(
() => void this.persistSnapshot(),
SNAPSHOT_INTERVAL_MS
);
}
this.onRunningChange(true);
this.log('info', 'engine.start: ok');
void this.convergeFromServer();
return true;
}
public stop() {
this._isRunning = false;
this.lifecycleGeneration++;
this.unsubscribe?.();
this.unsubscribe = undefined;
this.liveUnsubscribe?.();
this.liveUnsubscribe = undefined;
this.chatterUnsub?.();
this.chatter?.close();
this.chatterUnsub = undefined;
this.chatter = undefined;
if (this.snapshotInterval !== undefined) {
clearInterval(this.snapshotInterval);
this.snapshotInterval = undefined;
}
this.awareness.updateLocalAwareness(undefined);
this.syncs.live.pushAwareness(this.awareness.getEncodedLocalAwareness());
this.onRunningChange(false);
this.log('info', 'engine.stop: ok');
}
public async syncStateToLoro(state: InferType<S>) {
if (!this._isRunning) return;
await this.syncLock.runExclusive(async () => {
const syncResult = await this.loroManager.syncToLoro(state);
if (syncResult.isErr()) {
this.log('error', 'syncStateToLoro: failed, resetting engine', {
err: JSON.stringify(syncResult.error),
});
this.reset();
}
});
}
public syncAwarenessToLoro(awarenessUpdate: D) {
if (!this._isRunning) return;
this.awareness.updateLocalAwareness(awarenessUpdate);
this.syncs.live.pushAwareness(this.awareness.getEncodedLocalAwareness());
}
public async reset(snapshotThunk?: SnapshotThunk) {
const wasRunning = this._isRunning;
if (wasRunning) {
this.stop();
}
const generation = this.lifecycleGeneration;
const replaced = await this.syncLock.runExclusive(async () => {
if (generation !== this.lifecycleGeneration) return false;
this.log('info', 'engine.reset: starting');
// The snapshot replaces this document, and this peer's unsent edits
// can't be merged into it locally afterwards; deliver them first so the
// server's snapshot holds them. Any the server doesn't acknowledge stay
// in the WAL for later delivery (the canvas replays them onto the
// replacement).
await this.flushUnsentEdits();
if (generation !== this.lifecycleGeneration) return false;
const snapshot = await (snapshotThunk ?? this.defaultSnapshotThunk)();
if (generation !== this.lifecycleGeneration) return false;
if (snapshot.isErr()) {
this.log('error', 'engine.reset: failed to get snapshot', {
err: String(snapshot.error),
});
return false;
}
const resetResult = await this.loroManager.reset(snapshot.value);
if (resetResult.isErr()) {
this.log('error', 'engine.reset: loro manager reset failed', {
err: JSON.stringify(resetResult.error),
});
return false;
}
if (generation !== this.lifecycleGeneration) return false;
if (!wasRunning) return true;
// The binding still shows the replaced document, and local edits sync
// its whole state. Restart and hand it the new state under this lock,
// so nothing the restart triggers can write the stale one back first.
this.start();
const state = this.loroManager.state?.state;
if (state) this.bindings.onRemoteState(state);
return true;
});
if (!replaced && wasRunning && generation === this.lifecycleGeneration) {
this.start();
}
}
private async flushUnsentEdits(): Promise<void> {
try {
await this.syncs.wal.flushPending();
} catch {
// The WAL reports its own failures.
}
}
public onStateUpdate(stateUpdate: StateUpdate<S> | undefined) {
if (!this._isRunning || !stateUpdate) return;
if (stateUpdate.metadata.direction === SyncDirection.TO_LORO) return;
if (stateUpdate.metadata.tags?.includes(LoroStateTag.Initialize)) return;
this.syncLock.runExclusive(() =>
this.bindings.onRemoteState(stateUpdate.state)
);
}
public onLocalAwarenessChange() {
if (!this._isRunning) return;
const awarenessUpdate = this.awareness.getEncodedLocalAwareness();
if (!awarenessUpdate) return;
this.syncs.live.pushAwareness(awarenessUpdate);
this.chatter?.post({ type: 'awareness', data: awarenessUpdate });
}
private async handleLocalUpdates(update: LoroRawUpdate) {
if (this.readonly()) return;
this.log('debug', 'engine: local update, appending to WAL');
void this.syncs.wal.append(update);
this.chatter?.post({ type: 'update', data: update });
}
private async persistSnapshot() {
if (!this.snapshotStore) return;
void this.syncs.wal.flush(); // unawaited
try {
const doc = this.loroManager.doc;
this.log('debug', 'engine: persisting snapshot', {
operations: doc.opCount(),
});
const snapshot = doc.export({
mode: 'shallow-snapshot',
frontiers: doc.oplogFrontiers(),
});
await this.snapshotStore.save(snapshot);
// now safe to drop WAL entries it captures. we prune only
// after the save succeeds so that we can always recover fully.
await this.syncs.wal.pruneDelivered();
this.log('debug', 'engine: snapshot persisted ok');
} catch (err) {
this.log('error', 'engine: failed to persist snapshot', {
errName: err instanceof Error ? err.name : undefined,
errMessage: err instanceof Error ? err.message : String(err),
});
}
}
private async handleRemoteUpdate(
update: RawUpdate
): Promise<'applied' | 'pending' | 'reset'> {
const generation = this.lifecycleGeneration;
return this.syncLock.runExclusive(() => {
// A queued packet can outlive stop/reset and its document's WASM owner.
if (!this.isActiveGeneration(generation)) return 'applied';
return telemetrySpan(
this.syncs.live.documentId,
'edit.apply',
async (span) => {
span.setAttr('update.bytes', update.length);
const importResult = this.loroManager.importUpdate(update);
await Promise.resolve();
if (!this.isActiveGeneration(generation)) return 'applied';
if (importResult.isErr()) {
const pendingOnly = importResult.error.every(
(e) => e.code === LoroManagerError.ImportPending
);
if (pendingOnly) {
// Loro retains causally-ahead updates. Pull the missing operations
// rather than resetting the document.
this.log('debug', 'engine: remote update pending on missing ops');
span.setAttr('outcome', 'pending');
void this.convergeFromServer();
return 'pending';
}
this.log(
'error',
'engine: failed to import remote update, resetting',
{
err: JSON.stringify(importResult.error),
}
);
span.error(importResult.error);
span.setAttr('outcome', 'reset');
this.reset();
return 'reset';
}
span.setAttr('outcome', 'applied');
span.setAttr('did_change', importResult.value);
return 'applied';
}
);
});
}
private handleSourceEvent(event: SyncSourceEvent) {
switch (event.type) {
case 'update':
this.handleRemoteUpdate(event.update);
break;
case 'awareness':
this.awareness.importRemoteAwareness(event.awareness);
break;
case 'incremental_snapshot':
this.log('debug', 'engine: source event: incremental_snapshot');
this.handleRemoteUpdate(event.snapshot);
break;
case 'reconnect':
this.log(
'info',
'engine: reconnect, requesting updates since current version'
);
void this.convergeFromServer({ rerunIfInFlight: true });
break;
}
}
/**
* Pull every server operation missing from the current local seed.
*
* Snapshot sources are intentionally transport-agnostic and first-wins, so
* this anti-entropy step is what makes optimistic, IDB, and S3 seeds converge
* to server truth. Calls within one engine lifecycle are coalesced;
* `rerunIfInFlight` additionally queues one fresh pass after the in-flight
* one settles, so a reconnect is never absorbed into an attempt that goes
* on to exhaust its retries.
*/
private convergeFromServer({
rerunIfInFlight = false,
}: {
rerunIfInFlight?: boolean;
} = {}): Promise<void> {
const generation = this.lifecycleGeneration;
if (!this.isActiveGeneration(generation)) return Promise.resolve();
const inFlight = this.convergence;
if (inFlight?.generation === generation) {
if (rerunIfInFlight) inFlight.rerunRequested = true;
return inFlight.promise;
}
const since = this.loroManager.doc.version();
const promise = this.requestAndHandleUpdatesSince(since, 1, generation);
const record = { generation, promise, rerunRequested: false };
this.convergence = record;
const settle = () => {
if (this.convergence !== record) return;
this.convergence = undefined;
if (record.rerunRequested) void this.convergeFromServer();
};
void promise.then(settle, settle);
return promise;
}
private isActiveGeneration(generation: number): boolean {
return this._isRunning && this.lifecycleGeneration === generation;
}
private async requestAndHandleUpdatesSince(
since: VersionVector,
attempt: number,
generation: number
) {
if (!this.isActiveGeneration(generation)) return;
// The catch-up flow: "give me everything after the version I hold".
// The span shows the anchor version, what came back, and whether it
// applied — the trail for "client was stale and (never) reconciled".
const span = telemetrySpan(
this.syncs.live.documentId,
'doc.sync.catchup',
undefined,
this.parentSpan
);
span.setAttr('since.version', vvAttr(since));
span.setAttr('attempt', attempt);
this.log('debug', `engine: requestUpdatesSince (attempt ${attempt})`);
const updates = await this.syncs.live.requestUpdatesSince(since);
if (!this.isActiveGeneration(generation)) {
span.setAttr('outcome', 'cancelled');
span.end();
return;
}
if (updates.isErr() || !updates.value) {
this.log('error', 'engine: requestUpdatesSince failed', {
err: updates.isErr() ? String(updates.error) : 'update is undefined',
});
const retrying =
updates.isErr() && attempt < REQUEST_UPDATES_MAX_ATTEMPTS;
span.error(updates.isErr() ? updates.error : 'update is undefined');
span.setAttr('outcome', retrying ? 'retrying' : 'failed');
span.end();
if (retrying) {
await new Promise((resolve) =>
setTimeout(resolve, REQUEST_UPDATES_RETRY_DELAY_MS)
);
await this.requestAndHandleUpdatesSince(since, attempt + 1, generation);
}
return;
}
if (updates.value.length === 0) {
// Nothing to converge. Zero bytes are not a valid Loro payload (real
// sources encode "no new ops" as a non-empty update), so importing them
// would throw and trigger a reset — the noop live source used by
// non-propagating AI edit sessions answers with exactly this.
this.log('debug', 'engine: requestUpdatesSince ok, empty update');
span.setAttr('outcome', 'noop');
span.setAttr('update.bytes', 0);
span.end();
return;
}
this.log('debug', 'engine: requestUpdatesSince ok, applying update');
const outcome = await this.handleRemoteUpdate(updates.value);
span.setAttr('outcome', outcome);
span.setAttr('update.bytes', updates.value.length);
span.end();
}
}
export type ReactiveSyncEngine<S extends GenericRootSchema, D> = {
isRunning: Accessor<boolean>;
start: () => void;
stop: () => void;
reset: (
snapshotThunk?: () => ResultAsync<Uint8Array, SyncError>
) => Promise<void>;
syncStateToLoro: (state: InferType<S>) => Promise<void>;
syncAwarenessToLoro: (awareness: D) => void;
};
export function createSyncEngine<
D,
S extends GenericRootSchema = GenericRootSchema,
>(
params: Omit<SyncEngineParams<S, D>, 'onRunningChange'> & {
readonly?: Accessor<boolean>;
}
): ReactiveSyncEngine<S, D> {
const [isRunning, setIsRunning] = createSignal(false);
const engine = new SyncEngine({
...params,
onRunningChange: setIsRunning,
// In the browser, gossip local edits to other tabs of the same doc.
makeChatter:
params.makeChatter ?? ((id) => new BroadcastChannelChatter(id)),
});
const { loroManager, awareness } = params;
onCleanup(
loroManager.onStateChange((update) => engine.onStateUpdate(update))
);
createEffect(on(awareness.local, () => engine.onLocalAwarenessChange()));
return {
isRunning,
start: () => engine.start(),
stop: () => engine.stop(),
reset: (t) => engine.reset(t),
syncStateToLoro: (state) => engine.syncStateToLoro(state),
syncAwarenessToLoro: (a) => engine.syncAwarenessToLoro(a),
};
}