Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.6 KB
Raw Blame
import { getCatalogIdentityAccess } from '../services/catalogDB/identityAccess.js';
/**
 * One-time boot reconciliation: collapse any pre-existing divergence between an
 * embedded universe-canon entry and its `catalog_ingredients` row.
 *
 * Before the bidirectional projection (server/services/catalogCanonProjection.js)
 * landed, the Catalog detail page and the Universe Builder canon surface were
 * copy-on-write mirrors — an edit on either side went stale on the other. On
 * the first boot after the upgrade, those two stores can already disagree. This
 * repair walks every universe canon entry that carries an `ingredientId`,
 * LWW-merges the embedded payload vs the catalog payload on `updatedAt`, and
 * writes the WINNER to BOTH sides so they converge. From then on the live
 * projection keeps them in lockstep.
 *
 * Writes:
 *   - universe side: `updateUniverse(..., { silent })` — silent suppresses the
 *     per-universe peer-sync fan-out (this is a local data repair, not a user
 *     edit; peers reconcile on their own boot).
 *   - catalog side: `updateIngredientIfFresh(..., { source: 'sync' })` so the rewrite
 *     reads as a system reconciliation in the revision history, not a user edit.
 *
 * Idempotent + marker-gated (`data/catalog-canon-reconcile.applied.json`): once
 * a clean pass completes, every subsequent boot is a no-op. A second run over
 * already-converged rows is a no-op anyway (the LWW comparison finds them
 * equal-or-fresher and writes nothing material).
 *
 * The corresponding `scripts/migrations/055-reconcile-canon-catalog.js` is a
 * registration stub for the migration ledger — the actual repair runs here
 * (boot-time) because the `scripts/migrations/` runner executes before the DB
 * pool is up (same reason migrations 048–053 are boot-time + stubbed).
 *
 * Invoked from server/index.js at boot, after `ensureSchema()` + the bible→
 * catalog backfill + `migrateCatalogPayload`, so every promoted row exists and
 * is at its current payload-shape version before this reconciles content.
 */

import { readMarker, writeMarker } from '../lib/migrationMarker.js';
import { BIBLE_FIELD } from '../lib/storyBible.js';
import { listUniverses, updateUniverse } from '../services/universeBuilder.js';
import * as catalogDB from '../services/catalogDB.js';
// Reuse the live projection's canon-entry→payload strip so a reconcile and a
// live projection produce byte-identical catalog payloads (a divergent rule
// would re-introduce the very drift this repair exists to collapse).
import { entryToPayload } from '../services/catalogCanonProjection.js';

const MARKER_VERSION = 1;
const MARKER_FILENAME = 'catalog-canon-reconcile.applied.json';

const CANON_ARRAY_KEYS = Object.values(BIBLE_FIELD);     // ['characters','places','objects']

/**
 * Public entry point. Walk every universe canon entry with an `ingredientId`,
 * LWW-merge embedded vs catalog payload, write the winner to both sides.
 * `force` re-runs past the marker (tests / admin re-trigger).
 */
export async function reconcileCanonCatalog({ force = false } = {}) {
  const marker = await readMarker(MARKER_FILENAME);
  if (marker?.version === MARKER_VERSION && !force) {
    return { skipped: true, marker };
  }

  console.log('🔁 canon↔catalog reconcile: starting');
  const universes = await listUniverses({ includeDeleted: false });
  const totals = { universesScanned: 0, scanned: 0, catalogWon: 0, canonWon: 0, errors: 0 };

  // Stable snapshot of each catalog row's ORIGINAL `updatedAt` clock, keyed by
  // ingredientId, captured the first time we see the row. The same ingredient
  // can be embedded in multiple universes; writing a canon-winner bumps the
  // row's `updated_at` to NOW(), which — without this snapshot — would make a
  // LATER universe's genuinely-newer embedded copy lose to the artificially
  // fresh row and get clobbered. Comparing every universe against the pre-pass
  // clock keeps the winner determination stable across the whole reconcile.
  const rowClockAtStart = new Map();
  // The canon-winner already written to each catalog row this pass: its source
  // clock is the high-water mark (when the same ingredient is embedded in
  // several universes and more than one copy beats the ORIGINAL row, only the
  // NEWEST embedded copy may win), and its row version lets the next sibling
  // supersede our own write without mistaking it for a concurrent edit.
  const canonWinner = new Map();

  for (const universe of universes) {
    if (universe.deleted) continue;
    totals.universesScanned++;
    // Collect per-array, per-entry-id the winning payload to stamp back onto the
    // embedded entries in ONE silent updateUniverse below. The catalog side is
    // written inline (one updateIngredientIfFresh per catalog-loser).
    const canonPatches = {};                              // arrayKey -> { entryId -> winningEntryFields }
    for (const arrayKey of CANON_ARRAY_KEYS) {
      const list = Array.isArray(universe[arrayKey]) ? universe[arrayKey] : [];
      for (const entry of list) {
        const ingredientId = entry?.ingredientId;
        if (!ingredientId || !entry?.id) continue;          // entries without ingredientId untouched
        totals.scanned++;
        try {
          if (await getCatalogIdentityAccess(ingredientId)) continue; // retained contextual snapshot
          const row = await catalogDB.getIngredient(ingredientId);
          if (!row) continue;                               // dangling backlink — leave both sides alone
          const entryAt = Date.parse(entry.updatedAt || '') || 0;
          // Compare against the row's clock AS OF the start of this pass, not
          // its live value — a canon-winner write earlier in the pass bumped
          // `updated_at` to now and would otherwise wrongly win here.
          if (!rowClockAtStart.has(ingredientId)) {
            rowClockAtStart.set(ingredientId, Date.parse(row.updatedAt || '') || 0);
          }
          const rowAt = rowClockAtStart.get(ingredientId);
          if (rowAt > entryAt) {
            // Catalog row is newer → it wins. Stamp its name+payload onto the
            // embedded entry (silent universe write below).
            (canonPatches[arrayKey] ||= {})[entry.id] = {
              ...entryToPayload(row.payload || {}),
              name: row.name,
              ingredientId,
              updatedAt: row.updatedAt,
            };
            totals.catalogWon++;
          } else if (entryAt > rowAt && entryAt > (Date.parse(canonWinner.get(ingredientId)?.sourceUpdatedAt || '') || 0)) {
            // Embedded entry beats the original row AND any canon-winner already
            // written this pass → it wins. Write it into the catalog row and
            // raise the high-water mark so an older sibling copy can't clobber it.
            // The write re-checks the clock under the row lock, so a catalog
            // edit committed since the scan read the row survives (#12503).
            const result = await catalogDB.updateIngredientIfFresh(
              ingredientId,
              { name: entry.name, payload: entryToPayload(entry) },
              {
                sourceUpdatedAt: entry.updatedAt,
                priorWrite: canonWinner.get(ingredientId) || null,
                source: 'sync',
                actor: 'canon-catalog-reconcile',
              },
            );
            if (result?.status === 'written') {
              canonWinner.set(ingredientId, { syncSequence: result.ingredient.syncSequence, sourceUpdatedAt: entry.updatedAt });
              totals.canonWon++;
            } else {
              console.warn(`🔁 reconcile skipped ${universe.id}/${arrayKey}/${ingredientId}: catalog row changed during the pass`);
            }
          }
          // Equal timestamps → already converged; nothing to do.
        } catch (err) {
          console.error(`🔁 reconcile failed (${universe.id}/${arrayKey}/${ingredientId}): ${err.message}`);
          totals.errors++;
        }
      }
    }

    // Stamp catalog-winner payloads back onto the embedded entries in one silent
    // write per universe (skips when the catalog never won on this universe).
    const hasCanonPatch = Object.keys(canonPatches).some((k) => Object.keys(canonPatches[k]).length > 0);
    if (hasCanonPatch) {
      await updateUniverse(universe.id, (cur) => {
        const patch = {};
        for (const arrayKey of CANON_ARRAY_KEYS) {
          const byId = canonPatches[arrayKey];
          if (!byId || Object.keys(byId).length === 0) continue;
          const list = Array.isArray(cur[arrayKey]) ? cur[arrayKey] : [];
          patch[arrayKey] = list.map((e) => {
            const win = e?.id ? byId[e.id] : null;
            return win ? { ...e, ...win } : e;
          });
        }
        return Object.keys(patch).length > 0 ? patch : null;
      }, { silent: true }).catch((err) => {
        console.error(`🔁 reconcile: universe ${universe.id} canon stamp failed: ${err.message}`);
        totals.errors++;
      });
    }
  }

  const payload = { version: MARKER_VERSION, completedAt: new Date().toISOString(), stats: totals };
  // Only stamp the completion marker on a clean pass — a row-level failure
  // withholds it so the (idempotent) reconcile retries next boot.
  const wroteMarker = totals.errors === 0;
  if (wroteMarker) await writeMarker(MARKER_FILENAME, payload);

  console.log(
    `🔁 canon↔catalog reconcile: ${totals.universesScanned} universes, ${totals.scanned} entries scanned, ` +
    `${totals.catalogWon} catalog-won, ${totals.canonWon} canon-won, ${totals.errors} errors` +
    (wroteMarker ? '' : ' — marker NOT written (will retry next boot)'),
  );

  return { skipped: false, markerWritten: wroteMarker, ...payload };
}