Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

13.1 KB
Raw Blame
/**
 * Bidirectional projection between an embedded universe-canon entry and its
 * `catalog_ingredients` row — so the two NEVER diverge.
 *
 * Background. A universe canon entry (`universe.characters[]`, `places[]`,
 * `objects[]`) carries the FULL bible payload PLUS a stable `ingredientId`. The
 * matching `catalog_ingredients.payload` holds a parallel copy. Before this
 * module the two were copy-on-write mirrors: the Catalog detail page PATCHed
 * only the catalog row, and the Universe Builder canon surface PATCHed only
 * `universe.characters[]`, so an edit on either side silently went stale on the
 * other. This module collapses them to ONE store of record (the catalog row)
 * while keeping the embedded entry as a full projection CACHE (v1 decision —
 * many synchronous renderers / prompt-builders read `universe.characters[].x`).
 *
 * Two directions:
 *   projectToCanon(ingredientId, updatedIngredient)
 *     A catalog edit → fan the catalog row's name+payload into the embedded
 *     entry of every universe that references the ingredient. Best-effort /
 *     fire-and-forget at the call site (the catalog PATCH route already
 *     responded); failures are logged single-line and swallowed.
 *
 *   projectToCatalog(universeId, canonArrays, { guardToken })
 *     A canon edit → write each embedded entry's payload back into its catalog
 *     row. Called SYNCHRONOUSLY inside updateUniverse so the cache can never
 *     lag the authoritative row on the same request.
 *
 * Loop safety. projectToCanon calls updateUniverse, and updateUniverse calls
 * projectToCatalog — an A→B→A cycle. Each call carries its canonical id and
 * aliases through updateUniverse so only that operation's own echo is skipped.
 * Independent canon edits still run the normal freshness checks, even while a
 * forward projection for the same ingredient is pending.
 *
 * Conflict policy. LWW on `updatedAt` (no field-level merge). projectToCatalog
 * only writes when the embedded entry's `updatedAt` is newer-or-equal to the
 * catalog row's; projectToCanon accepts only the current catalog revision and
 * never replaces a newer embedded edit. The boot reconciler (reconcileCanonCatalog.js)
 * runs the same LWW comparison to collapse any pre-existing divergence.
 */

import { isDeepStrictEqual } from 'node:util';
import * as catalogDB from './catalogDB.js';
import { getCatalogIdentityAccess } from './catalogDB/identityAccess.js';
import { BIBLE_FIELD } from '../lib/storyBible.js';

// Canon array keys, derived from the bible registry so a future kind added
// there flows through without editing this file.
const CANON_ARRAY_KEYS = Object.freeze(Object.values(BIBLE_FIELD));

// Fields that live on the embedded canon entry but are NOT part of the catalog
// row's `payload` (the catalog row owns `id`/`name`/timestamps as columns, and
// `ingredientId` is the canon→catalog backlink, not payload content). Stripped
// both directions so a round-trip is shape-stable and we never smuggle a stale
// `schemaVersion` from one store into the other.
const NON_PAYLOAD_KEYS = ['id', 'ingredientId', 'createdAt', 'updatedAt', 'name', 'schemaVersion'];

/**
 * Build the catalog `payload` from an embedded canon entry (strip the control
 * keys the row owns as columns + the backlink + schemaVersion). Exported so the
 * boot reconciler (reconcileCanonCatalog.js) produces a byte-identical payload
 * — a divergent strip rule between the two would re-introduce the very drift
 * this module exists to kill.
 */
export function entryToPayload(entry) {
  const payload = { ...(entry && typeof entry === 'object' ? entry : {}) };
  for (const k of NON_PAYLOAD_KEYS) delete payload[k];
  return payload;
}

/**
 * Project a catalog edit into every universe that references the ingredient.
 *
 * Best-effort: returns a stats object `{ universes, written, skipped }` and
 * never throws on a per-universe failure (logged single-line). The caller (the
 * catalog PATCH route) invokes this AFTER responding, so a projection hiccup
 * must not turn a successful edit into an error.
 */
export async function projectToCanon(ingredientId, updatedIngredient, deps = {}) {
  // `deps.updateUniverse` is a test seam (the suite injects a spy to assert the
  // canon side is written exactly once without a live universe store). Prod
  // dynamic-imports the real one — a static import would form a require cycle
  // (universeBuilder imports this module).
  const updateUniverse = deps.updateUniverse || (await import('./universeBuilder.js')).updateUniverse;
  if (!ingredientId || !updatedIngredient) return { universes: 0, written: 0, skipped: 0 };

  ingredientId = updatedIngredient.id || ingredientId;
  // Cross-owner contextual canon stays an original snapshot. The catalog ID
  // resolves current canon without silently rewriting a peer-owned universe.
  if (await getCatalogIdentityAccess(ingredientId)) return { universes: 0, written: 0, skipped: 1 };
  const identityIds = new Set(await catalogDB.getIngredientIdentityIds(ingredientId));

  const stats = { universes: 0, written: 0, skipped: 0 };
  const refs = await catalogDB.listRefsForIngredient(ingredientId);
  const universeIds = [...new Set(
    refs.filter((r) => r.refKind === 'universe' && r.refId).map((r) => r.refId),
  )];
  if (universeIds.length === 0) return stats;            // no-universe-ref → silent no-op

  const name = updatedIngredient.name;
  const payload = entryToPayload(updatedIngredient.payload || {});
  const updatedAt = updatedIngredient.updatedAt;
  const sourceAt = Date.parse(updatedAt || '');

  for (const universeId of universeIds) {
    stats.universes++;
    // Mutator form: replace the mirrored payload on every embedded entry that
    // carries this ingredientId, across all canon arrays. Preserve only the
    // canon entry's control fields: merging onto the whole entry would retain
    // fields removed by an edit or revision restore and let stale canon data
    // leak back into Catalog on the next reverse projection. `silent: true`
    // suppresses the per-universe peer-sync fan-out so a catalog edit that
    // touches N universes doesn't emit N recordUpdated events. `guardToken`
    // threads the originating ingredientId so updateUniverse's synchronous
    // projectToCatalog skips it (loop break).
    let touched = false;
    const result = await updateUniverse(universeId, async (cur) => {
      // Read without taking the catalog write queue: reverse projection already
      // runs inside this universe queue. Waiting on both queues can deadlock.
      const current = await catalogDB.getIngredient(ingredientId);
      if (!current || current.id !== ingredientId || !Number.isFinite(sourceAt)
        || Date.parse(current.updatedAt || '') !== sourceAt
        || current.name !== name
        || !isDeepStrictEqual(entryToPayload(current.payload || {}), payload)) return null;
      const patch = {};
      for (const arrayKey of CANON_ARRAY_KEYS) {
        const list = Array.isArray(cur[arrayKey]) ? cur[arrayKey] : [];
        if (!list.some((e) => identityIds.has(e?.ingredientId))) continue;
        patch[arrayKey] = list.map((e) => {
          if (!identityIds.has(e?.ingredientId)) return e;
          // Alias-backed entries are archived context, not newer edits of
          // the canonical identity; an explicit catalog edit refreshes them.
          if (e.ingredientId === ingredientId && Date.parse(e.updatedAt || '') > sourceAt) return e;
          touched = true;
          const controlFields = Object.fromEntries(
            NON_PAYLOAD_KEYS.filter((key) => e[key] !== undefined).map((key) => [key, e[key]]),
          );
          return { ...controlFields, ...payload, name, ingredientId, updatedAt };
        });
      }
      return touched ? patch : null;
    }, { silent: true, canonProjectionGuard: ingredientId, canonProjectionIdentityIds: [...identityIds] }).catch((err) => {
      console.error(`🔁 projectToCanon: universe ${universeId} update failed: ${err.message}`);
      return null;
    });
    if (result && touched) stats.written++;
    else stats.skipped++;
  }
  return stats;
}

/**
 * Project a canon edit into the catalog row for every embedded entry that
 * carries an `ingredientId`. Called SYNCHRONOUSLY inside updateUniverse after
 * the canon arrays persist, so the catalog row never lags the embedded cache on
 * the same request.
 *
 *   canonArrays — the PERSISTED universe record (or `{ characters, places,
 *                 objects }` subset) read straight after the write.
 *   guardToken  — when set (a projectToCanon-originated write), the entry whose
 *                 ingredientId equals it is skipped (loop break).
 *   guardIdentityIds — aliases captured by that originating operation, also skipped.
 *
 * LWW: writes the catalog row only when the embedded entry's `updatedAt` is
 * newer-or-equal to the stored row's, re-checked under the row lock at write
 * time. Best-effort per entry (logged on failure); a write refused because the
 * row changed after the batched lookup counts as skipped. Returns
 * `{ written, skipped }`.
 */
export async function projectToCatalog(universeId, canonArrays, { guardToken = null, guardIdentityIds = [] } = {}) {
  const stats = { written: 0, skipped: 0 };
  if (!canonArrays || typeof canonArrays !== 'object') return stats;

  // Pass 1 — collect the candidate entries across every canon array and apply
  // the loop break up front, so the batched lookup below asks only for rows we
  // could actually write. A universe with N canon entries used to cost N
  // single-row SELECTs here (#3941); it now costs one.
  const guardedIds = new Set(guardIdentityIds);
  if (guardToken) guardedIds.add(guardToken);
  const candidates = [];
  for (const arrayKey of CANON_ARRAY_KEYS) {
    const list = Array.isArray(canonArrays[arrayKey]) ? canonArrays[arrayKey] : [];
    for (const entry of list) {
      const ingredientId = entry?.ingredientId;
      if (!ingredientId) continue;
      // Loop break: skip the ingredient the originating projectToCanon is
      // writing FROM (its catalog row is already authoritative this request).
      if (guardedIds.has(ingredientId)) {
        stats.skipped++;
        continue;
      }
      candidates.push({ ingredientId, entry });
    }
  }
  if (candidates.length === 0) return stats;

  // ONE query for every candidate's freshness. A lookup failure is fatal to the
  // whole projection pass (there is no per-entry state to fall back on), so it
  // is logged once and every candidate counts as skipped — same best-effort
  // contract the per-entry catch below keeps.
  const timestamps = await catalogDB.getIngredientTimestamps(
    candidates.map((c) => c.ingredientId),
    // A retained source cache is an archived contextual copy, not a fresh
    // authoritative edit. Only a catalog projection may refresh its identity.
    { excludeAliases: true },
  ).catch((err) => {
    console.error(`🔁 projectToCatalog: batched lookup of ${candidates.length} ingredients failed: ${err.message}`);
    return null;
  });
  if (!timestamps) {
    stats.skipped += candidates.length;
    return stats;
  }

  // Pass 2 — write only the rows the LWW comparison says are stale. Sequential
  // by design: each write is a multi-statement transaction (lock + sanitize +
  // update + revision), and firing N of them at once would trade the read
  // stampede this fix removed for a write one.
  //
  // The batched snapshot only prunes obvious losers. A catalog PATCH or another
  // universe save can commit while an earlier candidate's write is pending, so
  // updateIngredientIfFresh re-checks the clock under the row lock and reports
  // `stale` instead of overwriting the newer row (#12503).
  //
  // `ownWrites` keeps the row version and source clock of each write made in
  // this call, so a duplicate ingredientId compares against the entry written
  // earlier in the pass rather than the NOW() the trigger stamped (#4006).
  const ownWrites = new Map();
  for (const { ingredientId, entry } of candidates) {
    const rowUpdatedAt = timestamps.get(ingredientId);
    if (rowUpdatedAt === undefined) { stats.skipped++; continue; }   // no live row
    const priorWrite = ownWrites.get(ingredientId) || null;
    // LWW on updatedAt — only overwrite the catalog row when the embedded entry
    // is at-least-as-fresh. An older embedded snapshot (e.g. a concurrent
    // catalog edit won) is left alone.
    const entryAt = Date.parse(entry.updatedAt || '') || 0;
    const rowAt = Date.parse(priorWrite?.sourceUpdatedAt || rowUpdatedAt || '') || 0;
    if (entryAt < rowAt) { stats.skipped++; continue; }
    try {
      const sourceUpdatedAt = entry.updatedAt;
      const result = await catalogDB.updateIngredientIfFresh(
        ingredientId,
        { name: entry.name, payload: entryToPayload(entry) },
        { sourceUpdatedAt, priorWrite, source: 'sync', actor: 'canon-projection' },
      );
      if (result?.status !== 'written') { stats.skipped++; continue; }
      ownWrites.set(ingredientId, { syncSequence: result.ingredient.syncSequence, sourceUpdatedAt });
      stats.written++;
    } catch (err) {
      console.error(`🔁 projectToCatalog: ingredient ${ingredientId} update failed: ${err.message}`);
      stats.skipped++;
    }
  }
  return stats;
}