Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

16.9 KB
Raw Blame
import type { Span } from '@macro-inc/observability';
import { type DBSchema, type IDBPDatabase, openDB as idbOpen } from 'idb';
import { logSyncService, type WalContext } from './logger';
import { RecoverableDatabase } from './recoverable-database';
import type { RawUpdate } from './shared';
import type { LiveSyncSource } from './source';
import { telemetrySpan } from './telemetry';

export type WALEntry<T> = {
  id: number;
  update: T;
  /** True once the entry has been acked by its transport. Pruned at the next snapshot. */
  delivered: boolean;
  /** Epoch ms when the entry was appended. Used to drop stale undelivered edits. */
  createdAt: number;
};

function hasId<T>(
  entry: Omit<WALEntry<T>, 'id'> & { id?: number }
): entry is WALEntry<T> {
  return entry.id !== undefined;
}

export function hasExpired<T>(entry: WALEntry<T>, cutoff: number): boolean {
  return !entry.delivered && entry.createdAt < cutoff;
}

export interface WALStore<T> {
  append(update: T): Promise<void>;
  getAll(): Promise<WALEntry<T>[]>;
  /** Mark a set of entries as delivered (acked by transport). They remain in
   *  the store until pruneDelivered() is called. */
  markDelivered(ids: number[]): Promise<void>;
  /** Drop all delivered entries. Called by the snapshot tick after a save. */
  pruneDelivered(): Promise<void>;
  /** Drop entries older than `ttlMs`. Returns the number deleted. */
  pruneExpired(ttlMs: number): Promise<number>;
  count(): Promise<number>;
}

/** Undelivered entries older than this are dropped without replay. */
export const WAL_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week

/** DB name for the Loro doc-update WAL. Each WAL "purpose" (loro updates,
 *  offline comments, etc.) gets its own DB so schemas don't collide. */
export const LORO_WAL_DB_NAME = 'macro-document-wal';

const DB_VERSION = 1;

interface WALSchema<T> extends DBSchema {
  updates: {
    key: number;
    value: {
      id?: number;
      scopeId: string;
      update: T;
      delivered: boolean;
      createdAt: number;
    };
    indexes: { scopeId: string };
  };
}

function upgradeWAL<T>(db: IDBPDatabase<WALSchema<T>>): void {
  const store = db.createObjectStore('updates', {
    keyPath: 'id',
    autoIncrement: true,
  });
  store.createIndex('scopeId', 'scopeId');
}

export class BrowserWALStore<T> implements WALStore<T> {
  private readonly database: RecoverableDatabase<WALSchema<T>>;

  constructor(
    dbName: string,
    private readonly scopeId: string
  ) {
    this.database = new RecoverableDatabase(dbName, DB_VERSION, upgradeWAL);
  }

  /** List every scopeId that currently has at least one entry. Uses a
   *  unique-key cursor on the `scopeId` index, so it doesn't load entries. */
  static async listScopeIds(dbName: string): Promise<string[]> {
    const db = await idbOpen<WALSchema<unknown>>(dbName, DB_VERSION, {
      upgrade: upgradeWAL,
    });
    try {
      const scopeIds: string[] = [];
      let cursor = await db
        .transaction('updates')
        .store.index('scopeId')
        .openKeyCursor(null, 'nextunique');
      while (cursor) {
        scopeIds.push(cursor.key);
        cursor = await cursor.continue();
      }
      return scopeIds;
    } finally {
      db.close();
    }
  }

  public async append(update: T): Promise<void> {
    const entry = {
      scopeId: this.scopeId,
      update,
      delivered: false,
      createdAt: Date.now(),
    };
    for (let attempt = 0; ; attempt++) {
      const tx = await this.database.transaction('updates', 'readwrite');
      // A rejected request alone does not prove rollback. Wait for the native
      // transaction outcome before deciding whether replay is safe.
      const aborted = new Promise<boolean>((resolve) => {
        tx.addEventListener('abort', () => resolve(true), { once: true });
        tx.addEventListener('complete', () => resolve(false), { once: true });
      });
      try {
        // Wrap synchronous request errors too, so tx.done is always observed.
        const write = async () => tx.store.add(entry);
        await Promise.all([write(), tx.done]);
        return;
      } catch (error) {
        const rolledBack = await aborted;
        if (
          attempt === 1 ||
          !rolledBack ||
          (tx.error !== null && tx.error.name !== 'AbortError') ||
          !(error instanceof DOMException) ||
          error.name !== 'AbortError'
        ) {
          throw tx.error ?? error;
        }
      }
    }
  }

  public async getAll(): Promise<WALEntry<T>[]> {
    const tx = await this.database.transaction('updates', 'readonly');
    return tx.store.index('scopeId').getAll(this.scopeId) as Promise<
      WALEntry<T>[]
    >;
  }

  public async markDelivered(ids: number[]): Promise<void> {
    if (ids.length === 0) return;
    const tx = await this.database.transaction('updates', 'readwrite');
    const store = tx.objectStore('updates');
    for (const id of ids) {
      const row = await store.get(id);
      if (row) await store.put({ ...row, delivered: true });
    }
    await tx.done;
  }

  public async pruneDelivered(): Promise<void> {
    const entries = await this.getAll();
    const tx = await this.database.transaction('updates', 'readwrite');
    const store = tx.objectStore('updates');
    for (const row of entries) {
      if (row.delivered && row.id !== undefined) {
        await store.delete(row.id);
      }
    }
    await tx.done;
  }

  public async pruneExpired(ttlMs: number): Promise<number> {
    const entries = await this.getAll();
    const cutoff = Date.now() - ttlMs;
    const tx = await this.database.transaction('updates', 'readwrite');
    const store = tx.objectStore('updates');
    let deleted = 0;
    for (const row of entries) {
      if (hasId(row) && hasExpired(row, cutoff)) {
        await store.delete(row.id);
        deleted++;
      }
    }
    await tx.done;
    return deleted;
  }

  public async count(): Promise<number> {
    return (await this.getAll()).length;
  }
}

export class InMemoryWALStore<T> implements WALStore<T> {
  private entries: WALEntry<T>[] = [];
  private seq = 0;

  async append(update: T): Promise<void> {
    this.entries.push({
      id: ++this.seq,
      update,
      delivered: false,
      createdAt: Date.now(),
    });
  }

  async getAll(): Promise<WALEntry<T>[]> {
    return [...this.entries];
  }

  async markDelivered(ids: number[]): Promise<void> {
    const set = new Set(ids);
    for (const e of this.entries) if (set.has(e.id)) e.delivered = true;
  }

  async pruneDelivered(): Promise<void> {
    this.entries = this.entries.filter((e) => !e.delivered);
  }

  async pruneExpired(ttlMs: number): Promise<number> {
    const cutoff = Date.now() - ttlMs;
    const before = this.entries.length;
    this.entries = this.entries.filter((e) => !hasExpired(e, cutoff));
    return before - this.entries.length;
  }

  async count(): Promise<number> {
    return this.entries.length;
  }
}

/**
 * Append-only queue with retry semantics. Caller-supplied `push` does the
 * actual transport; the syncer handles persistence, batching, and dedupe.
 * Triggering flushes on transport reconnect (or other "try again" signals)
 * is the caller's responsibility.
 */
export class WALSyncer<T> {
  /** True while a flush is in progress — prevents concurrent flushes. */
  private isFlushing = false;
  /** True if append was called while a flush was in progress. Causes flush
   *  to re-run after completing so those entries aren't stranded. */
  private hasNewPending = false;
  private reportedFailures = new Map<'initialization' | 'flush', string>();
  /** Appends still writing to the store; see {@link flushPending}. */
  private readonly writes = new Set<Promise<void>>();
  public pendingFlush: Promise<void> = Promise.resolve();
  private cleanupFns: Array<() => void> = [];
  private deliveryOutage:
    | {
        id: string;
        span: Span;
        startedAt: number;
        failedAttempts: number;
        maxUndelivered: number;
      }
    | undefined;

  /**
   * Exists so that we can not "do stuff" until we have pruned expired entries
   * (like when snapshot loading occurs right after construction for example).
   */
  private readyPromise?: Promise<void>;

  constructor(
    private readonly store: WALStore<T>,
    private readonly push: (items: T[]) => Promise<boolean>,
    private readonly label?: string
  ) {
    void this.ready();
  }

  /* Right now just drops expired entries. */
  private async setup(): Promise<void> {
    const deleted = await this.store.pruneExpired(WAL_TTL_MS);
    if (deleted > 0 && this.label) {
      logSyncService({
        documentId: this.label,
        level: 'warn',
        context: { wal: await this.summary() },
        message: `WAL: dropped ${deleted} expired entries`,
      });
    }
  }

  public ready(): Promise<void> {
    if (!this.readyPromise) {
      this.readyPromise = this.setup();
      void this.observeSetup(this.readyPromise);
    }
    return this.readyPromise;
  }

  private async observeSetup(setup: Promise<void>): Promise<void> {
    try {
      await setup;
      this.reportedFailures.delete('initialization');
    } catch (error) {
      // Retry only on the next caller's request, sharing any in-flight setup.
      if (this.readyPromise === setup) this.readyPromise = undefined;
      this.reportFailure('initialization', error);
    }
  }

  public async append(item: T): Promise<void> {
    const appended = this.appendEntry(item);
    this.writes.add(appended);
    try {
      await appended;
    } finally {
      this.writes.delete(appended);
    }
  }

  /** Waits for appends still writing to the store, then flushes. */
  public async flushPending(): Promise<void> {
    await Promise.allSettled([...this.writes]);
    await this.flush();
  }

  private async appendEntry(item: T): Promise<void> {
    await this.ready();
    await this.store.append(item);
    if (this.label)
      logSyncService({
        documentId: this.label,
        level: 'debug',
        context: { wal: await this.summary() },
        message: 'WAL: wrote to IDB',
      });
    this.hasNewPending = true;
    void this.flush(); // unawaited
  }

  public flush(): Promise<void> {
    if (this.label)
      logSyncService({
        documentId: this.label,
        level: 'debug',
        context: {},
        message: `WAL flush: triggered (isFlushing=${this.isFlushing})`,
      });
    if (this.isFlushing) return this.pendingFlush;
    this.isFlushing = true;
    this.pendingFlush = this.doFlush();
    // Observe the same promise we return: awaited callers still see failures,
    // while timer/reconnect callers cannot create unhandled rejections.
    void this.observeFlush(this.pendingFlush);
    return this.pendingFlush;
  }

  private async observeFlush(flush: Promise<void>): Promise<void> {
    try {
      await flush;
      this.reportedFailures.delete('flush');
    } catch (error) {
      this.reportFailure('flush', error);
    }
  }

  private reportFailure(
    stage: 'initialization' | 'flush',
    error: unknown
  ): void {
    const details = {
      errName: error instanceof Error ? error.name : undefined,
      errMessage: error instanceof Error ? error.message : String(error),
      errStack: error instanceof Error ? error.stack : undefined,
    };
    const fingerprint = JSON.stringify(details);
    if (this.reportedFailures.get(stage) === fingerprint) return;
    this.reportedFailures.set(stage, fingerprint);
    logSyncService({
      documentId: this.label ?? 'unknown',
      level: 'error',
      context: { misc: { ...details, 'wal.stage': stage } },
      message: `WAL ${stage} failed; retrying on the next trigger`,
    });
  }

  public pruneDelivered(): Promise<void> {
    return this.store.pruneDelivered();
  }

  public async summary(): Promise<WalContext> {
    const entries = await this.store.getAll();
    const undelivered = entries.filter((e) => !e.delivered);
    return {
      count: entries.length,
      dirty: undelivered.length,
      mostRecentEdit: undelivered.at(-1)?.createdAt,
    };
  }

  public destroy(): void {
    if (this.deliveryOutage) {
      this.deliveryOutage.span.setAttr('outcome', 'abandoned');
      this.finishDeliveryOutage();
    }
    for (const fn of this.cleanupFns) fn();
    this.cleanupFns = [];
  }

  public addCleanup(fn: () => void): void {
    this.cleanupFns.push(fn);
  }

  private async doFlush(): Promise<void> {
    this.hasNewPending = false;
    let succeeded = true;

    try {
      await this.ready();
      const entries = await this.store.getAll();
      const undelivered = entries.filter((e) => !e.delivered);
      if (undelivered.length === 0) return; // nothing to do

      if (this.label)
        logSyncService({
          documentId: this.label,
          level: 'debug',
          context: { wal: await this.summary() },
          message: `WAL flush: pushing ${undelivered.length} entries`,
        });

      let delivered: boolean;
      try {
        delivered = await this.push(undelivered.map((e) => e.update));
      } catch (e) {
        this.recordDeliveryFailure('push_error', undelivered.length, e);
        throw e;
      }

      if (delivered) {
        await this.store.markDelivered(undelivered.map((e) => e.id));
        this.recordDeliveryRecovery(undelivered.length);
        if (this.label)
          logSyncService({
            documentId: this.label,
            level: 'debug',
            context: { wal: await this.summary() },
            message: `WAL flush: delivered ${undelivered.length} entries`,
          });
      } else {
        this.recordDeliveryFailure('not_acked', undelivered.length);
        succeeded = false;
      }
    } finally {
      this.isFlushing = false;
    }

    if (succeeded && this.hasNewPending) {
      this.isFlushing = true;
      this.pendingFlush = this.doFlush();
      return this.pendingFlush;
    }
  }

  private recordDeliveryFailure(
    kind: 'not_acked' | 'push_error',
    undelivered: number,
    error?: unknown
  ): void {
    if (!this.label) return;

    if (this.deliveryOutage) {
      this.deliveryOutage.failedAttempts++;
      this.deliveryOutage.maxUndelivered = Math.max(
        this.deliveryOutage.maxUndelivered,
        undelivered
      );
      this.deliveryOutage.span.setAttr(
        'wal.failed_attempts',
        this.deliveryOutage.failedAttempts
      );
      this.deliveryOutage.span.setAttr(
        'wal.max_undelivered',
        this.deliveryOutage.maxUndelivered
      );
      return;
    }

    const id = crypto.randomUUID();
    const span = telemetrySpan(this.label, 'wal.delivery_outage');
    span.setAttr('wal.outage.id', id);
    span.setAttr('wal.first_failure', kind);
    span.setAttr('wal.failed_attempts', 1);
    span.setAttr('wal.max_undelivered', undelivered);
    span.error(error ?? 'push not acked; entries remain undelivered');
    this.deliveryOutage = {
      id,
      span,
      startedAt: Date.now(),
      failedAttempts: 1,
      maxUndelivered: undelivered,
    };
    logSyncService({
      documentId: this.label,
      level: 'warn',
      context: {
        misc: {
          'wal.outage.id': id,
          'wal.first_failure': kind,
          'wal.undelivered': undelivered,
        },
      },
      message: `WAL delivery outage started (${undelivered} entries)`,
    });
  }

  private recordDeliveryRecovery(delivered: number): void {
    if (!this.deliveryOutage || !this.label) return;

    const { id, span, startedAt, failedAttempts, maxUndelivered } =
      this.deliveryOutage;
    const outageMs = Date.now() - startedAt;
    span.event('wal.ack.confirmed', { 'wal.delivered': delivered });
    span.setAttr('outcome', 'recovered');
    span.setAttr('wal.outage_ms', outageMs);
    span.setAttr('wal.failed_attempts', failedAttempts);
    span.setAttr('wal.max_undelivered', maxUndelivered);
    span.setAttr('wal.delivered', delivered);
    span.end();
    this.deliveryOutage = undefined;
    logSyncService({
      documentId: this.label,
      level: 'info',
      context: {
        misc: {
          'wal.outage.id': id,
          'wal.outage_ms': outageMs,
          'wal.failed_attempts': failedAttempts,
          'wal.max_undelivered': maxUndelivered,
          'wal.delivered': delivered,
        },
      },
      message: 'WAL delivery recovered',
    });
  }

  private finishDeliveryOutage(): void {
    if (!this.deliveryOutage) return;
    this.deliveryOutage.span.setAttr(
      'wal.outage_ms',
      Date.now() - this.deliveryOutage.startedAt
    );
    this.deliveryOutage.span.end();
    this.deliveryOutage = undefined;
  }
}

/** Build a WAL syncer wired to a Loro live sync source: BrowserWALStore for
 *  persistence, live.pushUpdate as the transport, and reconnect events
 *  trigger a re-flush. */
export function createWALSyncSource(
  live: LiveSyncSource
): WALSyncer<RawUpdate> {
  const store = new BrowserWALStore<RawUpdate>(
    LORO_WAL_DB_NAME,
    live.documentId
  );
  const syncer = new WALSyncer<RawUpdate>(
    store,
    (updates) => live.pushUpdate(updates),
    live.documentId
  );
  live.listen((event) => {
    if (event.type === 'reconnect') void syncer.flush();
  });
  return syncer;
}