Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Bidirectional projection between an embedded universe-canon entry and its
* `catalog_ingredients` row — so the two NEVER diverge.
*
* Background. A universe canon entry (`universe.characters[]`, `places[]`,
* `objects[]`) carries the FULL bible payload PLUS a stable `ingredientId`. The
* matching `catalog_ingredients.payload` holds a parallel copy. Before this
* module the two were copy-on-write mirrors: the Catalog detail page PATCHed
* only the catalog row, and the Universe Builder canon surface PATCHed only
* `universe.characters[]`, so an edit on either side silently went stale on the
* other. This module collapses them to ONE store of record (the catalog row)
* while keeping the embedded entry as a full projection CACHE (v1 decision —
* many synchronous renderers / prompt-builders read `universe.characters[].x`).
*
* Two directions:
* projectToCanon(ingredientId, updatedIngredient)
* A catalog edit → fan the catalog row's name+payload into the embedded
* entry of every universe that references the ingredient. Best-effort /
* fire-and-forget at the call site (the catalog PATCH route already
* responded); failures are logged single-line and swallowed.
*
* projectToCatalog(universeId, canonArrays, { guardToken })
* A canon edit → write each embedded entry's payload back into its catalog
* row. Called SYNCHRONOUSLY inside updateUniverse so the cache can never
* lag the authoritative row on the same request.
*
* Loop safety. projectToCanon calls updateUniverse, and updateUniverse calls
* projectToCatalog — an A→B→A cycle. Each call carries its canonical id and
* aliases through updateUniverse so only that operation's own echo is skipped.
* Independent canon edits still run the normal freshness checks, even while a
* forward projection for the same ingredient is pending.
*
* Conflict policy. LWW on `updatedAt` (no field-level merge). projectToCatalog
* only writes when the embedded entry's `updatedAt` is newer-or-equal to the
* catalog row's; projectToCanon accepts only the current catalog revision and
* never replaces a newer embedded edit. The boot reconciler (reconcileCanonCatalog.js)
* runs the same LWW comparison to collapse any pre-existing divergence.
*/
import { isDeepStrictEqual } from 'node:util';
import * as catalogDB from './catalogDB.js';
import { getCatalogIdentityAccess } from './catalogDB/identityAccess.js';
import { BIBLE_FIELD } from '../lib/storyBible.js';
// Canon array keys, derived from the bible registry so a future kind added
// there flows through without editing this file.
const CANON_ARRAY_KEYS = Object.freeze(Object.values(BIBLE_FIELD));
// Fields that live on the embedded canon entry but are NOT part of the catalog
// row's `payload` (the catalog row owns `id`/`name`/timestamps as columns, and
// `ingredientId` is the canon→catalog backlink, not payload content). Stripped
// both directions so a round-trip is shape-stable and we never smuggle a stale
// `schemaVersion` from one store into the other.
const NON_PAYLOAD_KEYS = ['id', 'ingredientId', 'createdAt', 'updatedAt', 'name', 'schemaVersion'];
/**
* Build the catalog `payload` from an embedded canon entry (strip the control
* keys the row owns as columns + the backlink + schemaVersion). Exported so the
* boot reconciler (reconcileCanonCatalog.js) produces a byte-identical payload
* — a divergent strip rule between the two would re-introduce the very drift
* this module exists to kill.
*/
export function entryToPayload(entry) {
const payload = { ...(entry && typeof entry === 'object' ? entry : {}) };
for (const k of NON_PAYLOAD_KEYS) delete payload[k];
return payload;
}
/**
* Project a catalog edit into every universe that references the ingredient.
*
* Best-effort: returns a stats object `{ universes, written, skipped }` and
* never throws on a per-universe failure (logged single-line). The caller (the
* catalog PATCH route) invokes this AFTER responding, so a projection hiccup
* must not turn a successful edit into an error.
*/
export async function projectToCanon(ingredientId, updatedIngredient, deps = {}) {
// `deps.updateUniverse` is a test seam (the suite injects a spy to assert the
// canon side is written exactly once without a live universe store). Prod
// dynamic-imports the real one — a static import would form a require cycle
// (universeBuilder imports this module).
const updateUniverse = deps.updateUniverse || (await import('./universeBuilder.js')).updateUniverse;
if (!ingredientId || !updatedIngredient) return { universes: 0, written: 0, skipped: 0 };
ingredientId = updatedIngredient.id || ingredientId;
// Cross-owner contextual canon stays an original snapshot. The catalog ID
// resolves current canon without silently rewriting a peer-owned universe.
if (await getCatalogIdentityAccess(ingredientId)) return { universes: 0, written: 0, skipped: 1 };
const identityIds = new Set(await catalogDB.getIngredientIdentityIds(ingredientId));
const stats = { universes: 0, written: 0, skipped: 0 };
const refs = await catalogDB.listRefsForIngredient(ingredientId);
const universeIds = [...new Set(
refs.filter((r) => r.refKind === 'universe' && r.refId).map((r) => r.refId),
)];
if (universeIds.length === 0) return stats; // no-universe-ref → silent no-op
const name = updatedIngredient.name;
const payload = entryToPayload(updatedIngredient.payload || {});
const updatedAt = updatedIngredient.updatedAt;
const sourceAt = Date.parse(updatedAt || '');
for (const universeId of universeIds) {
stats.universes++;
// Mutator form: replace the mirrored payload on every embedded entry that
// carries this ingredientId, across all canon arrays. Preserve only the
// canon entry's control fields: merging onto the whole entry would retain
// fields removed by an edit or revision restore and let stale canon data
// leak back into Catalog on the next reverse projection. `silent: true`
// suppresses the per-universe peer-sync fan-out so a catalog edit that
// touches N universes doesn't emit N recordUpdated events. `guardToken`
// threads the originating ingredientId so updateUniverse's synchronous
// projectToCatalog skips it (loop break).
let touched = false;
const result = await updateUniverse(universeId, async (cur) => {
// Read without taking the catalog write queue: reverse projection already
// runs inside this universe queue. Waiting on both queues can deadlock.
const current = await catalogDB.getIngredient(ingredientId);
if (!current || current.id !== ingredientId || !Number.isFinite(sourceAt)
|| Date.parse(current.updatedAt || '') !== sourceAt
|| current.name !== name
|| !isDeepStrictEqual(entryToPayload(current.payload || {}), payload)) return null;
const patch = {};
for (const arrayKey of CANON_ARRAY_KEYS) {
const list = Array.isArray(cur[arrayKey]) ? cur[arrayKey] : [];
if (!list.some((e) => identityIds.has(e?.ingredientId))) continue;
patch[arrayKey] = list.map((e) => {
if (!identityIds.has(e?.ingredientId)) return e;
// Alias-backed entries are archived context, not newer edits of
// the canonical identity; an explicit catalog edit refreshes them.
if (e.ingredientId === ingredientId && Date.parse(e.updatedAt || '') > sourceAt) return e;
touched = true;
const controlFields = Object.fromEntries(
NON_PAYLOAD_KEYS.filter((key) => e[key] !== undefined).map((key) => [key, e[key]]),
);
return { ...controlFields, ...payload, name, ingredientId, updatedAt };
});
}
return touched ? patch : null;
}, { silent: true, canonProjectionGuard: ingredientId, canonProjectionIdentityIds: [...identityIds] }).catch((err) => {
console.error(`🔁 projectToCanon: universe ${universeId} update failed: ${err.message}`);
return null;
});
if (result && touched) stats.written++;
else stats.skipped++;
}
return stats;
}
/**
* Project a canon edit into the catalog row for every embedded entry that
* carries an `ingredientId`. Called SYNCHRONOUSLY inside updateUniverse after
* the canon arrays persist, so the catalog row never lags the embedded cache on
* the same request.
*
* canonArrays — the PERSISTED universe record (or `{ characters, places,
* objects }` subset) read straight after the write.
* guardToken — when set (a projectToCanon-originated write), the entry whose
* ingredientId equals it is skipped (loop break).
* guardIdentityIds — aliases captured by that originating operation, also skipped.
*
* LWW: writes the catalog row only when the embedded entry's `updatedAt` is
* newer-or-equal to the stored row's, re-checked under the row lock at write
* time. Best-effort per entry (logged on failure); a write refused because the
* row changed after the batched lookup counts as skipped. Returns
* `{ written, skipped }`.
*/
export async function projectToCatalog(universeId, canonArrays, { guardToken = null, guardIdentityIds = [] } = {}) {
const stats = { written: 0, skipped: 0 };
if (!canonArrays || typeof canonArrays !== 'object') return stats;
// Pass 1 — collect the candidate entries across every canon array and apply
// the loop break up front, so the batched lookup below asks only for rows we
// could actually write. A universe with N canon entries used to cost N
// single-row SELECTs here (#3941); it now costs one.
const guardedIds = new Set(guardIdentityIds);
if (guardToken) guardedIds.add(guardToken);
const candidates = [];
for (const arrayKey of CANON_ARRAY_KEYS) {
const list = Array.isArray(canonArrays[arrayKey]) ? canonArrays[arrayKey] : [];
for (const entry of list) {
const ingredientId = entry?.ingredientId;
if (!ingredientId) continue;
// Loop break: skip the ingredient the originating projectToCanon is
// writing FROM (its catalog row is already authoritative this request).
if (guardedIds.has(ingredientId)) {
stats.skipped++;
continue;
}
candidates.push({ ingredientId, entry });
}
}
if (candidates.length === 0) return stats;
// ONE query for every candidate's freshness. A lookup failure is fatal to the
// whole projection pass (there is no per-entry state to fall back on), so it
// is logged once and every candidate counts as skipped — same best-effort
// contract the per-entry catch below keeps.
const timestamps = await catalogDB.getIngredientTimestamps(
candidates.map((c) => c.ingredientId),
// A retained source cache is an archived contextual copy, not a fresh
// authoritative edit. Only a catalog projection may refresh its identity.
{ excludeAliases: true },
).catch((err) => {
console.error(`🔁 projectToCatalog: batched lookup of ${candidates.length} ingredients failed: ${err.message}`);
return null;
});
if (!timestamps) {
stats.skipped += candidates.length;
return stats;
}
// Pass 2 — write only the rows the LWW comparison says are stale. Sequential
// by design: each write is a multi-statement transaction (lock + sanitize +
// update + revision), and firing N of them at once would trade the read
// stampede this fix removed for a write one.
//
// The batched snapshot only prunes obvious losers. A catalog PATCH or another
// universe save can commit while an earlier candidate's write is pending, so
// updateIngredientIfFresh re-checks the clock under the row lock and reports
// `stale` instead of overwriting the newer row (#12503).
//
// `ownWrites` keeps the row version and source clock of each write made in
// this call, so a duplicate ingredientId compares against the entry written
// earlier in the pass rather than the NOW() the trigger stamped (#4006).
const ownWrites = new Map();
for (const { ingredientId, entry } of candidates) {
const rowUpdatedAt = timestamps.get(ingredientId);
if (rowUpdatedAt === undefined) { stats.skipped++; continue; } // no live row
const priorWrite = ownWrites.get(ingredientId) || null;
// LWW on updatedAt — only overwrite the catalog row when the embedded entry
// is at-least-as-fresh. An older embedded snapshot (e.g. a concurrent
// catalog edit won) is left alone.
const entryAt = Date.parse(entry.updatedAt || '') || 0;
const rowAt = Date.parse(priorWrite?.sourceUpdatedAt || rowUpdatedAt || '') || 0;
if (entryAt < rowAt) { stats.skipped++; continue; }
try {
const sourceUpdatedAt = entry.updatedAt;
const result = await catalogDB.updateIngredientIfFresh(
ingredientId,
{ name: entry.name, payload: entryToPayload(entry) },
{ sourceUpdatedAt, priorWrite, source: 'sync', actor: 'canon-projection' },
);
if (result?.status !== 'written') { stats.skipped++; continue; }
ownWrites.set(ingredientId, { syncSequence: result.ingredient.syncSequence, sourceUpdatedAt });
stats.written++;
} catch (err) {
console.error(`🔁 projectToCatalog: ingredient ${ingredientId} update failed: ${err.message}`);
stats.skipped++;
}
}
return stats;
}