Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

20.1 KB
Raw Blame
import {
  type InferType,
  Mirror,
  SyncDirection,
  type UpdateMetadata,
} from '@loro-mirror/core';
import type { Span } from '@macro-inc/observability';
import {
  type Container,
  type ContainerID,
  type Cursor,
  type ImportStatus,
  LoroDoc,
  type PeerID,
  type Side,
  type UndoConfig,
  UndoManager,
  type VersionVector,
} from 'loro-crdt';
import { err, ok, type Result } from 'neverthrow';
import { onCleanup } from 'solid-js';
import type { ResultError } from '../internal/result';
import { logSyncService } from './logger';
import { LoroManagerError } from './manager-error';
import type { GenericRootSchema, LoroRawUpdate, RawUpdate } from './shared';
import {
  disposeTelemetryFor,
  frontiersAttr,
  setTelemetryAttr,
  telemetrySpan,
} from './telemetry';

export { LoroManagerError };

export enum LoroStateTag {
  Initialize = 'INITIALIZE',
  FromManager = 'FROM_MANAGER',
}

export type StateUpdate<S extends GenericRootSchema> = {
  state: InferType<S>;
  metadata: UpdateMetadata;
};

export type SnapshotIngest =
  | { kind: 'optimistic'; snapshot: RawUpdate }
  | {
      kind: 'local';
      snapshot: RawUpdate;
      /** Optional WAL update entries to replay on top of the snapshot. */
      walUpdates?: RawUpdate[];
    }
  | { kind: 's3'; snapshot: RawUpdate }
  | { kind: 'dss'; snapshot: RawUpdate };

export type LoroManagerOptions = {
  documentId: string;
  /** Temporary peers share the live document's telemetry without owning it. */
  ownsTelemetry?: boolean;
  /** Caller-owned initial document flow, never ended by the manager. */
  parentSpan?: Span;
  /**
   * Keep a JSON mirror of the schema's state and report it through
   * `onStateChange` (the default). Editors that read the LoroDoc directly
   * turn it off: the mirror re-reads the whole document on every change.
   * State updates then carry no state, only the initialization signal.
   */
  mirror?: boolean;
};

/** Map Loro's {@link ImportStatus} onto our Result: ok(didChange), or
 *  {@link LoroManagerError.ImportPending} when ops were held back waiting on
 *  missing causal dependencies. */
function importStatusToResult(
  importStatus: ImportStatus
): Result<boolean, ResultError<LoroManagerError>[]> {
  if ((importStatus.pending?.size ?? 0) > 0) {
    return err([
      {
        code: LoroManagerError.ImportPending,
        message: 'Import held back pending missing causal dependencies',
      },
    ]);
  }

  return ok(importStatus.success.size > 0);
}

export interface SyncEngineManager<
  S extends GenericRootSchema = GenericRootSchema,
> {
  readonly doc: LoroDoc;
  readonly initialized: boolean;
  readonly peerId: bigint;
  /** The latest state the manager published, if any. */
  readonly state: StateUpdate<S> | undefined;
  importUpdate(
    update: LoroRawUpdate
  ): Result<boolean, ResultError<LoroManagerError>[]>;
  syncToLoro(
    state: InferType<S>
  ): Promise<Result<void, ResultError<LoroManagerError>[]>>;
  reset(
    snapshot: LoroRawUpdate
  ): Promise<Result<void, ResultError<LoroManagerError>[]>>;
  /** Subscribe to state changes; `createSyncEngine` uses this to wire the
   *  manager's updates into the engine. Returns an unsubscribe fn. */
  onStateChange(listener: (update: StateUpdate<S>) => void): () => void;
}

/**
 * The LoroManager manages the state of a LoroDoc by syncing arbitrary JSON
 * state to and from it via the {@link Mirror}, which incrementally diffs the
 * incoming JSON state and applies it to the LoroDoc.
 *
 * ┌────────────┬──────────────────────────────────────┐
 * │  Manager   │                                      │
 * ├────────────┘                                      │
 * │                    .───────.                      │
 * │                   ╱         ╲                     │
 * │                  (   State   )◀─┐                 │
 * │                   `.       ,'   │                 │
 * │                     │─────'     │                 │
 * │        ┌───┬────────┴───────────┘                 │
 * │        │   ▼                                      │
 * │    ┌──────────────┐          ┌──────────────┐     │
 * │    │              │◀─────────┤              │     │
 * │    │    Mirror    │          │   LoroDoc    │     │
 * │    │              ├─────────▶│              │     │
 * │    └──────────────┘          └──────────────┘     │
 * └───────────────────────────────────────────────────┘
 */
export class LoroManager<S extends GenericRootSchema = GenericRootSchema>
  implements SyncEngineManager<S>
{
  /** The current schema of the manager. */
  readonly schema: S;

  private _initialized = false;
  private _doc: LoroDoc = createLoroDoc();
  private _mirror?: Mirror<S>;
  private _state?: StateUpdate<S>;

  private mirrorUnsub?: () => void;
  private readonly options: LoroManagerOptions;

  private readonly stateListeners = new Set<(u: StateUpdate<S>) => void>();
  private readonly initListeners = new Set<(v: boolean) => void>();

  constructor(schema: S, options: LoroManagerOptions) {
    this.schema = schema;
    this.options = options;
    // Stamp this doc's telemetry with the peer identity as soon as it
    // exists, so every span and log record carries who we are.
    if (options.ownsTelemetry !== false) {
      setTelemetryAttr(options.documentId, 'loro.peer_id', this._doc.peerIdStr);
    }
  }

  /** The inner LoroDoc. Only touch this if you know what you're doing. */
  get doc(): LoroDoc {
    return this._doc;
  }

  /** An undo manager over this document, so callers need not import loro eagerly. */
  createUndoManager(config: UndoConfig): UndoManager {
    return new UndoManager(this._doc, config);
  }

  /** The inner Mirror, once initialized. */
  get mirror(): Mirror<S> | undefined {
    return this._mirror;
  }
  /** The current mirrored state of the loro doc
   *
   * ┌─────────────┐
   * │ Local State │                    ┌─────────────┐
   * │   Update    │                    │Remote Import│
   * └─────────────┘                    └─────────────┘
   *       │                                  │
   *       │                                  │
   *       ▼                                  ▼
   * ┌──────────┐      ┌ ─ ─ ─ ─ ┐      ┌──────────┐      ┌ ─ ─ ─ ─ ─ ┐
   * │  Mirror  │─────▶   Diff          │ LoroDoc  │─────▶ exportJSON
   * └──────────┘      └ ─ ─ ─ ─ ┘      └──────────┘      └ ─ ─ ─ ─ ─ ┘
   *                         │                                   │
   *                         │                                   │
   *                         ▼                                   ▼
   *                   ┌───────────┐                       ┌───────────┐
   *                   │  LoroDoc  │                       │   State   │
   *                   └───────────┘                       └───────────┘
   * */
  get state(): StateUpdate<S> | undefined {
    return this._state;
  }
  /** Whether the manager has been initialized from a snapshot. */
  get initialized(): boolean {
    return this._initialized;
  }
  get documentId(): string {
    return this.options.documentId;
  }
  get peerId(): bigint {
    return this._doc.peerId;
  }
  get peerIdStr(): PeerID {
    return this._doc.peerIdStr;
  }
  get version(): VersionVector {
    return this._doc.version();
  }

  /** Subscribe to state changes (both directions, with metadata). Returns an
   *  unsubscribe fn. */
  onStateChange(listener: (update: StateUpdate<S>) => void): () => void {
    this.stateListeners.add(listener);
    return () => this.stateListeners.delete(listener);
  }

  /** Subscribe to `initialized` flips. Returns an unsubscribe fn. */
  onInitializedChange(listener: (initialized: boolean) => void): () => void {
    this.initListeners.add(listener);
    return () => this.initListeners.delete(listener);
  }

  private setInitialized(value: boolean) {
    if (this._initialized === value) return;
    this._initialized = value;
    for (const listener of this.initListeners) listener(value);
  }

  private emitState(update: StateUpdate<S>) {
    this._state = update;
    for (const listener of this.stateListeners) listener(update);
  }

  /** Subscribe to the given mirror, dropping any prior subscription. The mirror
   *  fires for every state change (both directions, with metadata). */
  private subscribeMirror(mirror: Mirror<S>) {
    this.mirrorUnsub?.();
    this._mirror = mirror;
    this.mirrorUnsub = mirror.subscribe((update, metadata) => {
      this.emitState({ state: update, metadata });
    });
  }

  /** Util for awaiting the sync of the mirror to finish. */
  private async awaitMirrorSync() {
    await Promise.resolve();
    await Promise.resolve();
    await Promise.resolve();
  }

  importUpdate(
    update: LoroRawUpdate
  ): Result<boolean, ResultError<LoroManagerError>[]> {
    let importStatus: ImportStatus;

    try {
      importStatus = this._doc.import(update);
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `importUpdate failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.ImportFailed,
          message: `Failed to import update: ${e}`,
        },
      ]);
    }

    return importStatusToResult(importStatus);
  }

  importBatchUpdates(
    updates: LoroRawUpdate[]
  ): Result<boolean, ResultError<LoroManagerError>[]> {
    let importStatus: ImportStatus;

    try {
      importStatus = this._doc.importBatch(updates);
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `importBatchUpdates failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.ImportFailed,
          message: `Failed to import update: ${e}`,
        },
      ]);
    }

    return importStatusToResult(importStatus);
  }

  async initializeFromSnapshot(
    snapshot: LoroRawUpdate
  ): Promise<Result<void, ResultError<LoroManagerError>[]>> {
    const importResult = this.importUpdate(snapshot);

    if (importResult.isErr()) {
      const [error] = importResult.error;
      return err([{ code: error.code, message: error.message }]);
    }

    if (this.options.mirror === false) {
      this.emitState({
        state: {} as InferType<S>,
        metadata: {
          direction: SyncDirection.TO_LORO,
          tags: ['INITIALIZE'],
        },
      });
      this.setInitialized(true);
      return ok(undefined);
    }

    const mirror = createMirror(this._doc, this.schema);

    try {
      await this.awaitMirrorSync();
      const mirrorState = mirror.getState();
      this.emitState({
        state: mirrorState,
        metadata: {
          direction: SyncDirection.TO_LORO,
          tags: ['INITIALIZE'],
        },
      });
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `initializeFromSnapshot: mirror sync failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.InitializeFailed,
          message: `Failed to sync mirror: ${e}`,
        },
      ]);
    }

    this.setInitialized(true);
    this.subscribeMirror(mirror);

    return ok(undefined);
  }

  getUpdateSince(
    lastVersionVector: VersionVector
  ): Result<Uint8Array | undefined, ResultError<LoroManagerError>[]> {
    if (!this._initialized) {
      return err([
        { code: LoroManagerError.NotInitialized, message: 'Not initialized' },
      ]);
    }
    const currentVersionVector = this._doc.version();
    /** Comparison between the current state, and the last synced state */
    const vvDiff = lastVersionVector.compare(currentVersionVector);

    if (vvDiff === 0) {
      return ok(undefined);
    }

    const spans = this._doc.findIdSpansBetween(
      this._doc.vvToFrontiers(lastVersionVector),
      this._doc.vvToFrontiers(currentVersionVector)
    );

    const localSpans = spans.forward.filter(
      (span) => span.peer === this._doc.peerIdStr
    );

    let update: RawUpdate;

    try {
      update = this._doc.export({
        mode: 'updates-in-range',
        spans: localSpans.map((span) => ({
          id: { peer: span.peer, counter: span.counter },
          len: span.length,
        })),
      });
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `getUpdateSince: export failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.ExportFailed,
          message: `Failed to export update: ${e}`,
        },
      ]);
    }

    return ok(update);
  }

  getAllContainerIds(): Result<ContainerID[], ResultError<LoroManagerError>[]> {
    if (!this._initialized || !this._mirror) {
      return err([
        { code: LoroManagerError.NotInitialized, message: 'Not initialized' },
      ]);
    }

    return ok(this._mirror.getContainerIds());
  }

  async syncToLoro(
    state: InferType<S>
  ): Promise<Result<void, ResultError<LoroManagerError>[]>> {
    if (!this._initialized || !this._mirror) {
      return err([
        { code: LoroManagerError.NotInitialized, message: 'Not initialized' },
      ]);
    }

    try {
      this._mirror.setState(state, {
        tags: LoroStateTag.FromManager,
      });

      await this.awaitMirrorSync();
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `syncToLoro failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.SyncFailed,
          message: `Failed to sync to loro: ${e}`,
        },
      ]);
    }

    return ok(undefined);
  }

  async reset(
    snapshot: LoroRawUpdate
  ): Promise<Result<void, ResultError<LoroManagerError>[]>> {
    this.mirrorUnsub?.();
    this.mirrorUnsub = undefined;
    this._mirror?.dispose();
    this._doc.free();

    const newDoc = createLoroDoc();
    this._doc = newDoc;

    let importStatus: ImportStatus;
    try {
      importStatus = newDoc.import(snapshot);
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `reset: snapshot import failed: ${e}`,
      });
      return err([
        {
          code: LoroManagerError.ImportFailed,
          message: `Failed to import snapshot: ${e}`,
        },
      ]);
    }

    const statusResult = importStatusToResult(importStatus);
    if (statusResult.isErr()) {
      // A reset snapshot must be self-contained; pending deps mean a bad seed.
      return err(statusResult.error);
    }

    if (this.options.mirror === false) {
      this.emitState({
        state: {} as InferType<S>,
        metadata: {
          direction: SyncDirection.TO_LORO,
          tags: [LoroStateTag.Initialize],
        },
      });
      this.setInitialized(true);
      return ok(undefined);
    }

    const newMirror = createMirror(newDoc, this.schema);
    this.subscribeMirror(newMirror);

    await this.awaitMirrorSync();

    const state = newMirror.getState();
    this.emitState({
      state,
      metadata: {
        direction: SyncDirection.TO_LORO,
        tags: [LoroStateTag.Initialize],
      },
    });

    this.setInitialized(true);

    return ok(undefined);
  }

  getContainerById(
    id: ContainerID
  ): Result<Container | undefined, ResultError<LoroManagerError>[]> {
    let container: Container | undefined;

    try {
      container = this._doc.getContainerById(id);
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `getContainerById failed: ${e}`,
      });
      return err([
        { code: LoroManagerError.GetContainerByIdFailed, message: String(e) },
      ]);
    }

    return ok(container);
  }

  getCursorPos(cursor: Cursor): Result<
    {
      update?: Cursor;
      offset: number;
      side: Side;
    },
    ResultError<LoroManagerError>[]
  > {
    let pos: { update?: Cursor; offset: number; side: Side } | undefined;
    try {
      pos = this._doc.getCursorPos(cursor);
      if (!pos) {
        return err([
          {
            code: LoroManagerError.GetCursorPosFailed,
            message: "loro didn't give us a cursor position",
          },
        ]);
      }
    } catch (e) {
      logSyncService({
        documentId: this.options.documentId,
        level: 'error',
        context: {},
        message: `getCursorPos failed: ${e}`,
      });
      return err([
        { code: LoroManagerError.GetCursorPosFailed, message: String(e) },
      ]);
    }

    return ok(pos);
  }

  /**
   * First snapshot received is the one ingested, all others are automatically
   * ignored. Returns whether this snapshot became the seed.
   */
  async ingest(input: SnapshotIngest): Promise<boolean> {
    if (this._initialized) return false;

    // The winner of the snapshot race seeds the doc; its kind, size, and
    // resulting version are the anchor every later sync question ("since
    // what?") is asked against.
    return telemetrySpan(
      this.options.documentId,
      'doc.seed',
      async (span) => {
        span.setAttr('seed.kind', input.kind);
        span.setAttr('snapshot.bytes', input.snapshot.length);
        const initResult = await this.initializeFromSnapshot(input.snapshot);
        if (initResult.isErr()) {
          logSyncService({
            documentId: this.options.documentId,
            level: 'error',
            context: {},
            message: `ingest(${input.kind}): seed failed`,
          });
          span.error(`seed from ${input.kind} snapshot failed`);
          return false;
        }

        // WAL entries are causally anchored to the local snapshot, so they're only
        // safe to replay on top of the local seed we just applied.
        if (input.kind === 'local' && input.walUpdates?.length) {
          span.setAttr('wal.replayed', input.walUpdates.length);
          const replayResult = this.importBatchUpdates(input.walUpdates);
          if (replayResult.isErr()) {
            logSyncService({
              documentId: this.options.documentId,
              level: 'error',
              context: {},
              message: `ingest(${input.kind}): WAL replay failed`,
            });
            span.error('WAL replay onto local seed failed');
          }
        }

        span.setAttr('version.after', frontiersAttr(this._doc));
        return true;
      },
      this.options.parentSpan
    );
  }

  /** Tear down subscriptions and free the underlying doc. */
  dispose() {
    this.mirrorUnsub?.();
    this.mirrorUnsub = undefined;
    this.stateListeners.clear();
    this.initListeners.clear();
    this._mirror?.dispose();
    this._doc.free();
    if (this.options.ownsTelemetry !== false) {
      disposeTelemetryFor(this.options.documentId);
    }
  }
}

export function createLoroManager<S extends GenericRootSchema>(
  schema: S,
  options: LoroManagerOptions
): LoroManager<S> {
  const manager = new LoroManager(schema, options);
  onCleanup(() => manager.dispose());
  return manager;
}

export function createLoroDoc(): LoroDoc {
  const doc = new LoroDoc();
  doc.setRecordTimestamp(true);
  return doc;
}

function createMirror<S extends GenericRootSchema>(
  doc: LoroDoc,
  schema: S
): Mirror<S> {
  return new Mirror({
    doc,
    schema,
  });
}