Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
import {
type InferType,
Mirror,
SyncDirection,
type UpdateMetadata,
} from '@loro-mirror/core';
import type { Span } from '@macro-inc/observability';
import {
type Container,
type ContainerID,
type Cursor,
type ImportStatus,
LoroDoc,
type PeerID,
type Side,
type UndoConfig,
UndoManager,
type VersionVector,
} from 'loro-crdt';
import { err, ok, type Result } from 'neverthrow';
import { onCleanup } from 'solid-js';
import type { ResultError } from '../internal/result';
import { logSyncService } from './logger';
import { LoroManagerError } from './manager-error';
import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';
import {
disposeTelemetryFor,
frontiersAttr,
setTelemetryAttr,
telemetrySpan,
} from './telemetry';
export { LoroManagerError };
export enum LoroStateTag {
Initialize = 'INITIALIZE',
FromManager = 'FROM_MANAGER',
}
export type StateUpdate<S extends GenericRootSchema> = {
state: InferType<S>;
metadata: UpdateMetadata;
};
export type SnapshotIngest =
| { kind: 'optimistic'; snapshot: RawUpdate }
| {
kind: 'local';
snapshot: RawUpdate;
/** Optional WAL update entries to replay on top of the snapshot. */
walUpdates?: RawUpdate[];
}
| { kind: 's3'; snapshot: RawUpdate }
| { kind: 'dss'; snapshot: RawUpdate };
export type LoroManagerOptions = {
documentId: string;
/** Temporary peers share the live document's telemetry without owning it. */
ownsTelemetry?: boolean;
/** Caller-owned initial document flow, never ended by the manager. */
parentSpan?: Span;
/**
* Keep a JSON mirror of the schema's state and report it through
* `onStateChange` (the default). Editors that read the LoroDoc directly
* turn it off: the mirror re-reads the whole document on every change.
* State updates then carry no state, only the initialization signal.
*/
mirror?: boolean;
};
/** Map Loro's {@link ImportStatus} onto our Result: ok(didChange), or
* {@link LoroManagerError.ImportPending} when ops were held back waiting on
* missing causal dependencies. */
function importStatusToResult(
importStatus: ImportStatus
): Result<boolean, ResultError<LoroManagerError>[]> {
if ((importStatus.pending?.size ?? 0) > 0) {
return err([
{
code: LoroManagerError.ImportPending,
message: 'Import held back pending missing causal dependencies',
},
]);
}
return ok(importStatus.success.size > 0);
}
export interface SyncEngineManager<
S extends GenericRootSchema = GenericRootSchema,
> {
readonly doc: LoroDoc;
readonly initialized: boolean;
readonly peerId: bigint;
/** The latest state the manager published, if any. */
readonly state: StateUpdate<S> | undefined;
importUpdate(
update: LoroRawUpdate
): Result<boolean, ResultError<LoroManagerError>[]>;
syncToLoro(
state: InferType<S>
): Promise<Result<void, ResultError<LoroManagerError>[]>>;
reset(
snapshot: LoroRawUpdate
): Promise<Result<void, ResultError<LoroManagerError>[]>>;
/** Subscribe to state changes; `createSyncEngine` uses this to wire the
* manager's updates into the engine. Returns an unsubscribe fn. */
onStateChange(listener: (update: StateUpdate<S>) => void): () => void;
}
/**
* The LoroManager manages the state of a LoroDoc by syncing arbitrary JSON
* state to and from it via the {@link Mirror}, which incrementally diffs the
* incoming JSON state and applies it to the LoroDoc.
*
* ┌────────────┬──────────────────────────────────────┐
* │ Manager │ │
* ├────────────┘ │
* │ .───────. │
* │ ╱ ╲ │
* │ ( State )◀─┐ │
* │ `. ,' │ │
* │ │─────' │ │
* │ ┌───┬────────┴───────────┘ │
* │ │ ▼ │
* │ ┌──────────────┐ ┌──────────────┐ │
* │ │ │◀─────────┤ │ │
* │ │ Mirror │ │ LoroDoc │ │
* │ │ ├─────────▶│ │ │
* │ └──────────────┘ └──────────────┘ │
* └───────────────────────────────────────────────────┘
*/
export class LoroManager<S extends GenericRootSchema = GenericRootSchema>
implements SyncEngineManager<S>
{
/** The current schema of the manager. */
readonly schema: S;
private _initialized = false;
private _doc: LoroDoc = createLoroDoc();
private _mirror?: Mirror<S>;
private _state?: StateUpdate<S>;
private mirrorUnsub?: () => void;
private readonly options: LoroManagerOptions;
private readonly stateListeners = new Set<(u: StateUpdate<S>) => void>();
private readonly initListeners = new Set<(v: boolean) => void>();
constructor(schema: S, options: LoroManagerOptions) {
this.schema = schema;
this.options = options;
// Stamp this doc's telemetry with the peer identity as soon as it
// exists, so every span and log record carries who we are.
if (options.ownsTelemetry !== false) {
setTelemetryAttr(options.documentId, 'loro.peer_id', this._doc.peerIdStr);
}
}
/** The inner LoroDoc. Only touch this if you know what you're doing. */
get doc(): LoroDoc {
return this._doc;
}
/** An undo manager over this document, so callers need not import loro eagerly. */
createUndoManager(config: UndoConfig): UndoManager {
return new UndoManager(this._doc, config);
}
/** The inner Mirror, once initialized. */
get mirror(): Mirror<S> | undefined {
return this._mirror;
}
/** The current mirrored state of the loro doc
*
* ┌─────────────┐
* │ Local State │ ┌─────────────┐
* │ Update │ │Remote Import│
* └─────────────┘ └─────────────┘
* │ │
* │ │
* ▼ ▼
* ┌──────────┐ ┌ ─ ─ ─ ─ ┐ ┌──────────┐ ┌ ─ ─ ─ ─ ─ ┐
* │ Mirror │─────▶ Diff │ LoroDoc │─────▶ exportJSON
* └──────────┘ └ ─ ─ ─ ─ ┘ └──────────┘ └ ─ ─ ─ ─ ─ ┘
* │ │
* │ │
* ▼ ▼
* ┌───────────┐ ┌───────────┐
* │ LoroDoc │ │ State │
* └───────────┘ └───────────┘
* */
get state(): StateUpdate<S> | undefined {
return this._state;
}
/** Whether the manager has been initialized from a snapshot. */
get initialized(): boolean {
return this._initialized;
}
get documentId(): string {
return this.options.documentId;
}
get peerId(): bigint {
return this._doc.peerId;
}
get peerIdStr(): PeerID {
return this._doc.peerIdStr;
}
get version(): VersionVector {
return this._doc.version();
}
/** Subscribe to state changes (both directions, with metadata). Returns an
* unsubscribe fn. */
onStateChange(listener: (update: StateUpdate<S>) => void): () => void {
this.stateListeners.add(listener);
return () => this.stateListeners.delete(listener);
}
/** Subscribe to `initialized` flips. Returns an unsubscribe fn. */
onInitializedChange(listener: (initialized: boolean) => void): () => void {
this.initListeners.add(listener);
return () => this.initListeners.delete(listener);
}
private setInitialized(value: boolean) {
if (this._initialized === value) return;
this._initialized = value;
for (const listener of this.initListeners) listener(value);
}
private emitState(update: StateUpdate<S>) {
this._state = update;
for (const listener of this.stateListeners) listener(update);
}
/** Subscribe to the given mirror, dropping any prior subscription. The mirror
* fires for every state change (both directions, with metadata). */
private subscribeMirror(mirror: Mirror<S>) {
this.mirrorUnsub?.();
this._mirror = mirror;
this.mirrorUnsub = mirror.subscribe((update, metadata) => {
this.emitState({ state: update, metadata });
});
}
/** Util for awaiting the sync of the mirror to finish. */
private async awaitMirrorSync() {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
importUpdate(
update: LoroRawUpdate
): Result<boolean, ResultError<LoroManagerError>[]> {
let importStatus: ImportStatus;
try {
importStatus = this._doc.import(update);
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `importUpdate failed: ${e}`,
});
return err([
{
code: LoroManagerError.ImportFailed,
message: `Failed to import update: ${e}`,
},
]);
}
return importStatusToResult(importStatus);
}
importBatchUpdates(
updates: LoroRawUpdate[]
): Result<boolean, ResultError<LoroManagerError>[]> {
let importStatus: ImportStatus;
try {
importStatus = this._doc.importBatch(updates);
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `importBatchUpdates failed: ${e}`,
});
return err([
{
code: LoroManagerError.ImportFailed,
message: `Failed to import update: ${e}`,
},
]);
}
return importStatusToResult(importStatus);
}
async initializeFromSnapshot(
snapshot: LoroRawUpdate
): Promise<Result<void, ResultError<LoroManagerError>[]>> {
const importResult = this.importUpdate(snapshot);
if (importResult.isErr()) {
const [error] = importResult.error;
return err([{ code: error.code, message: error.message }]);
}
if (this.options.mirror === false) {
this.emitState({
state: {} as InferType<S>,
metadata: {
direction: SyncDirection.TO_LORO,
tags: ['INITIALIZE'],
},
});
this.setInitialized(true);
return ok(undefined);
}
const mirror = createMirror(this._doc, this.schema);
try {
await this.awaitMirrorSync();
const mirrorState = mirror.getState();
this.emitState({
state: mirrorState,
metadata: {
direction: SyncDirection.TO_LORO,
tags: ['INITIALIZE'],
},
});
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `initializeFromSnapshot: mirror sync failed: ${e}`,
});
return err([
{
code: LoroManagerError.InitializeFailed,
message: `Failed to sync mirror: ${e}`,
},
]);
}
this.setInitialized(true);
this.subscribeMirror(mirror);
return ok(undefined);
}
getUpdateSince(
lastVersionVector: VersionVector
): Result<Uint8Array | undefined, ResultError<LoroManagerError>[]> {
if (!this._initialized) {
return err([
{ code: LoroManagerError.NotInitialized, message: 'Not initialized' },
]);
}
const currentVersionVector = this._doc.version();
/** Comparison between the current state, and the last synced state */
const vvDiff = lastVersionVector.compare(currentVersionVector);
if (vvDiff === 0) {
return ok(undefined);
}
const spans = this._doc.findIdSpansBetween(
this._doc.vvToFrontiers(lastVersionVector),
this._doc.vvToFrontiers(currentVersionVector)
);
const localSpans = spans.forward.filter(
(span) => span.peer === this._doc.peerIdStr
);
let update: RawUpdate;
try {
update = this._doc.export({
mode: 'updates-in-range',
spans: localSpans.map((span) => ({
id: { peer: span.peer, counter: span.counter },
len: span.length,
})),
});
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `getUpdateSince: export failed: ${e}`,
});
return err([
{
code: LoroManagerError.ExportFailed,
message: `Failed to export update: ${e}`,
},
]);
}
return ok(update);
}
getAllContainerIds(): Result<ContainerID[], ResultError<LoroManagerError>[]> {
if (!this._initialized || !this._mirror) {
return err([
{ code: LoroManagerError.NotInitialized, message: 'Not initialized' },
]);
}
return ok(this._mirror.getContainerIds());
}
async syncToLoro(
state: InferType<S>
): Promise<Result<void, ResultError<LoroManagerError>[]>> {
if (!this._initialized || !this._mirror) {
return err([
{ code: LoroManagerError.NotInitialized, message: 'Not initialized' },
]);
}
try {
this._mirror.setState(state, {
tags: LoroStateTag.FromManager,
});
await this.awaitMirrorSync();
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `syncToLoro failed: ${e}`,
});
return err([
{
code: LoroManagerError.SyncFailed,
message: `Failed to sync to loro: ${e}`,
},
]);
}
return ok(undefined);
}
async reset(
snapshot: LoroRawUpdate
): Promise<Result<void, ResultError<LoroManagerError>[]>> {
this.mirrorUnsub?.();
this.mirrorUnsub = undefined;
this._mirror?.dispose();
this._doc.free();
const newDoc = createLoroDoc();
this._doc = newDoc;
let importStatus: ImportStatus;
try {
importStatus = newDoc.import(snapshot);
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `reset: snapshot import failed: ${e}`,
});
return err([
{
code: LoroManagerError.ImportFailed,
message: `Failed to import snapshot: ${e}`,
},
]);
}
const statusResult = importStatusToResult(importStatus);
if (statusResult.isErr()) {
// A reset snapshot must be self-contained; pending deps mean a bad seed.
return err(statusResult.error);
}
if (this.options.mirror === false) {
this.emitState({
state: {} as InferType<S>,
metadata: {
direction: SyncDirection.TO_LORO,
tags: [LoroStateTag.Initialize],
},
});
this.setInitialized(true);
return ok(undefined);
}
const newMirror = createMirror(newDoc, this.schema);
this.subscribeMirror(newMirror);
await this.awaitMirrorSync();
const state = newMirror.getState();
this.emitState({
state,
metadata: {
direction: SyncDirection.TO_LORO,
tags: [LoroStateTag.Initialize],
},
});
this.setInitialized(true);
return ok(undefined);
}
getContainerById(
id: ContainerID
): Result<Container | undefined, ResultError<LoroManagerError>[]> {
let container: Container | undefined;
try {
container = this._doc.getContainerById(id);
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `getContainerById failed: ${e}`,
});
return err([
{ code: LoroManagerError.GetContainerByIdFailed, message: String(e) },
]);
}
return ok(container);
}
getCursorPos(cursor: Cursor): Result<
{
update?: Cursor;
offset: number;
side: Side;
},
ResultError<LoroManagerError>[]
> {
let pos: { update?: Cursor; offset: number; side: Side } | undefined;
try {
pos = this._doc.getCursorPos(cursor);
if (!pos) {
return err([
{
code: LoroManagerError.GetCursorPosFailed,
message: "loro didn't give us a cursor position",
},
]);
}
} catch (e) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `getCursorPos failed: ${e}`,
});
return err([
{ code: LoroManagerError.GetCursorPosFailed, message: String(e) },
]);
}
return ok(pos);
}
/**
* First snapshot received is the one ingested, all others are automatically
* ignored. Returns whether this snapshot became the seed.
*/
async ingest(input: SnapshotIngest): Promise<boolean> {
if (this._initialized) return false;
// The winner of the snapshot race seeds the doc; its kind, size, and
// resulting version are the anchor every later sync question ("since
// what?") is asked against.
return telemetrySpan(
this.options.documentId,
'doc.seed',
async (span) => {
span.setAttr('seed.kind', input.kind);
span.setAttr('snapshot.bytes', input.snapshot.length);
const initResult = await this.initializeFromSnapshot(input.snapshot);
if (initResult.isErr()) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `ingest(${input.kind}): seed failed`,
});
span.error(`seed from ${input.kind} snapshot failed`);
return false;
}
// WAL entries are causally anchored to the local snapshot, so they're only
// safe to replay on top of the local seed we just applied.
if (input.kind === 'local' && input.walUpdates?.length) {
span.setAttr('wal.replayed', input.walUpdates.length);
const replayResult = this.importBatchUpdates(input.walUpdates);
if (replayResult.isErr()) {
logSyncService({
documentId: this.options.documentId,
level: 'error',
context: {},
message: `ingest(${input.kind}): WAL replay failed`,
});
span.error('WAL replay onto local seed failed');
}
}
span.setAttr('version.after', frontiersAttr(this._doc));
return true;
},
this.options.parentSpan
);
}
/** Tear down subscriptions and free the underlying doc. */
dispose() {
this.mirrorUnsub?.();
this.mirrorUnsub = undefined;
this.stateListeners.clear();
this.initListeners.clear();
this._mirror?.dispose();
this._doc.free();
if (this.options.ownsTelemetry !== false) {
disposeTelemetryFor(this.options.documentId);
}
}
}
export function createLoroManager<S extends GenericRootSchema>(
schema: S,
options: LoroManagerOptions
): LoroManager<S> {
const manager = new LoroManager(schema, options);
onCleanup(() => manager.dispose());
return manager;
}
export function createLoroDoc(): LoroDoc {
const doc = new LoroDoc();
doc.setRecordTimestamp(true);
return doc;
}
function createMirror<S extends GenericRootSchema>(
doc: LoroDoc,
schema: S
): Mirror<S> {
return new Mirror({
doc,
schema,
});
}