Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Give new universe canon entries a catalog record on save (#11909)
[._.]/1c4d74f94eb6
diff --git a/server/scripts/migrateBibleToCatalog.js b/server/scripts/migrateBibleToCatalog.jsindex b16fcbdff7..0c2a6fe768 100644--- a/server/scripts/migrateBibleToCatalog.js+++ b/server/scripts/migrateBibleToCatalog.js@@ -22,10 +22,12 @@import { createHash } from 'crypto';import { readMarker, writeMarker } from '../lib/migrationMarker.js';import { BIBLE_KINDS, BIBLE_FIELD } from '../lib/storyBible.js';-import { listUniverses, updateUniverse } from '../services/universeBuilder.js';+import { listUniverses, getUniverse, updateUniverse, ERR_NOT_FOUND } from '../services/universeBuilder.js';import * as catalogDB from '../services/catalogDB.js';-const MARKER_VERSION = 1;+// v2 re-walks every install once: before canon entries were promoted on save,+// an entry added after the first walk stayed out of the catalog.+const MARKER_VERSION = 2;const MARKER_FILENAME = 'catalog-backfill.applied.json';// Derived from BIBLE_KINDS so a future kind added there flows through here@@ -199,6 +201,60 @@ async function migrateUniverse(universe) {return stats;}+/**+ * True when a canon entry has no catalog row yet. Exported so the universe+ * write path can decide, without a DB round-trip, whether to promote.+ */+export function hasUnpromotedCanon(universe) {+ return KINDS.some(({ array }) => (Array.isArray(universe?.[array]) ? universe[array] : [])+ .some((entry) => entry?.id && entry?.name && !entry.ingredientId));+}++// One promotion per universe at a time; a save landing mid-walk re-runs once+// the current walk finishes so its new entries are not missed. The boot walk+// goes through the same map so a save during it can't race the same+// deterministic catalog id.+const promoting = new Map();++// A deleted or missing universe has nothing left to promote.+async function loadLiveUniverse(universeId) {+ const universe = await getUniverse(universeId, { includeDeleted: true }).catch((err) => {+ if (err?.code === ERR_NOT_FOUND) return null;+ throw err;+ });+ return universe && !universe.deleted ? universe : null;+}++function startPromotion(universeId, firstPass = null) {+ const state = { again: !firstPass, promise: null };+ state.promise = (async () => {+ let stats = firstPass ? await firstPass() : null;+ while (state.again) {+ state.again = false;+ const universe = await loadLiveUniverse(universeId);+ if (!universe || !hasUnpromotedCanon(universe)) break;+ stats = await migrateUniverse(universe);+ }+ return stats;+ })().finally(() => promoting.delete(universeId));+ promoting.set(universeId, state);+ return state.promise;+}++/**+ * Promote one universe's not-yet-promoted canon entries to catalog rows and+ * stamp the ids back. Called after a universe save adds canon entries, so the+ * Casting / Environments / Objects pages see them without a manual sync.+ */+export function promoteUniverseCanon(universeId) {+ const running = promoting.get(universeId);+ if (running) {+ running.again = true;+ return running.promise;+ }+ return startPromotion(universeId);+}+/*** Public entry point. Runs the migration once, then no-ops on every* subsequent boot. Wired into server/index.js after ensureSchema().@@ -215,7 +271,9 @@ export async function migrateBibleToCatalog({ force = false } = {}) {const totals = { universesScanned: 0, promoted: 0, peerReconciled: 0, skipped: 0, errors: 0 };for (const universe of universes) {if (universe.deleted) continue;- const result = await migrateUniverse(universe);+ while (promoting.has(universe.id)) await promoting.get(universe.id).promise.catch(() => {});+ let result;+ await startPromotion(universe.id, async () => { result = await migrateUniverse(universe); });totals.universesScanned++;totals.promoted += result.promoted;totals.peerReconciled += result.peerReconciled;diff --git a/server/scripts/migrateBibleToCatalog.test.js b/server/scripts/migrateBibleToCatalog.test.jsindex fc48ca55cf..0ac36d76f4 100644--- a/server/scripts/migrateBibleToCatalog.test.js+++ b/server/scripts/migrateBibleToCatalog.test.js@@ -25,6 +25,13 @@ vi.mock('../services/universeBuilder.js', () => ({// guard can be exercised independently by overriding this mock per-test.listUniverses: vi.fn(async ({ includeDeleted = true } = {}) =>(includeDeleted ? universes : universes.filter((u) => !u.deleted))),+ ERR_NOT_FOUND: 'NOT_FOUND',+ // Throws NOT_FOUND for a missing or soft-deleted universe, like the real one.+ getUniverse: vi.fn(async (id, { includeDeleted = false } = {}) => {+ const u = universes.find((x) => x.id === id);+ if (!u || (u.deleted && !includeDeleted)) throw Object.assign(new Error(`Universe not found: ${id}`), { code: 'NOT_FOUND' });+ return u;+ }),updateUniverse: vi.fn(async (id, patchOrMutator) => {const u = universes.find((x) => x.id === id);if (!u) return null;@@ -86,7 +93,7 @@ vi.mock('../lib/migrationMarker.js', () => ({writeMarker: vi.fn(async (_name, payload) => { markerBlob = JSON.stringify(payload); }),}));-import { migrateBibleToCatalog } from './migrateBibleToCatalog.js';+import { migrateBibleToCatalog, promoteUniverseCanon, hasUnpromotedCanon } from './migrateBibleToCatalog.js';import { listUniverses } from '../services/universeBuilder.js';function makeUniverse(overrides = {}) {@@ -152,7 +159,14 @@ describe('migrateBibleToCatalog', () => {// Marker is now written; a plain (non-forced) re-run must no-op.const rerun = await migrateBibleToCatalog();expect(rerun.skipped).toBe(true);- expect(rerun.marker.version).toBe(1);+ expect(rerun.marker.version).toBe(2);+ });++ it('re-walks once on an install whose marker predates save-time promotion', async () => {+ markerBlob = JSON.stringify({ version: 1, completedAt: '2026-01-01T00:00:00.000Z' });+ const result = await migrateBibleToCatalog();+ expect(result.skipped).toBe(false);+ expect(result.stats.promoted).toBe(4);});it('re-links an already-promoted entry that lost its ref (idempotent linking)', async () => {@@ -191,3 +205,53 @@ describe('migrateBibleToCatalog', () => {expect(result.stats.promoted).toBe(4);});});++describe('promoteUniverseCanon', () => {+ it('promotes only the entries a save added, under deterministic ids', async () => {+ await migrateBibleToCatalog();+ universes[0].places.push({ id: 'p-2', name: 'Night Ferry' });+ expect(hasUnpromotedCanon(universes[0])).toBe(true);++ const stats = await promoteUniverseCanon('u-1');++ expect(stats.promoted).toBe(1);+ expect(ingredients.size).toBe(5);+ const ferry = universes[0].places.find((p) => p.id === 'p-2');+ expect(ferry.ingredientId).toMatch(/^cat-plc-bible-/);+ expect(refLinks).toContainEqual({ ingredientId: ferry.ingredientId, refKind: 'universe', refId: 'u-1', role: 'canon-place' });+ expect(hasUnpromotedCanon(universes[0])).toBe(false);+ });++ it('is a no-op when every entry already has a catalog row', async () => {+ await migrateBibleToCatalog();+ createIngredient.mockClear();+ expect(await promoteUniverseCanon('u-1')).toBeNull();+ expect(createIngredient).not.toHaveBeenCalled();+ });++ it('coalesces overlapping saves into one walk plus one follow-up', async () => {+ const first = promoteUniverseCanon('u-1');+ const second = promoteUniverseCanon('u-1');+ expect(second).toBe(first);+ await first;+ expect(ingredients.size).toBe(4);+ expect(createIngredient).toHaveBeenCalledTimes(4);+ });++ it('quietly does nothing for a universe deleted or removed before it runs', async () => {+ universes[0].deleted = true;+ expect(await promoteUniverseCanon('u-1')).toBeNull();+ expect(await promoteUniverseCanon('u-missing')).toBeNull();+ expect(createIngredient).not.toHaveBeenCalled();+ });++ it('waits for the boot walk instead of racing it for the same catalog ids', async () => {+ const boot = migrateBibleToCatalog({ force: true });+ universes[0].places.push({ id: 'p-3', name: 'Rooftop' });+ const save = promoteUniverseCanon('u-1');+ const [bootResult] = await Promise.all([boot, save]);+ expect(bootResult.stats.errors).toBe(0);+ expect(universes[0].places.every((p) => p.ingredientId)).toBe(true);+ expect(new Set(createIngredient.mock.calls.map(([input]) => input.id)).size).toBe(createIngredient.mock.calls.length);+ });+});diff --git a/server/services/universeBuilder/crud.js b/server/services/universeBuilder/crud.jsindex b38f52d381..73ff5e4750 100644--- a/server/services/universeBuilder/crud.js+++ b/server/services/universeBuilder/crud.js@@ -35,6 +35,7 @@ import {import { listSeries } from '../pipeline/series.js';import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';import { codedError } from '../../lib/codedError.js';+import { isFileBackend } from '../../lib/pgFileFacade.js';// Once-per-process flag for the canon-backfill log — readState() runs in both// the queue and from un-queued readers, and the in-memory migration is cheap@@ -313,6 +314,7 @@ export async function createUniverse(input = {}) {console.log(`⚠️ universe: auto-subscribe after create failed: ${err.message}`);});}+ promoteNewCanonEntries(created);return created;}@@ -739,9 +741,27 @@ export async function updateUniverse(id, patchOrMutator = {}, options = {}) {if (!silent) {emitRecordUpdated('universe', merged.id);}+ promoteNewCanonEntries(merged);return merged;}+// A save that adds a canon character/place/object gives it a catalog row (and+// stamps the ingredientId back) so Casting / Environments / Objects list it+// without a manual "Sync from Universes". Deterministic ids keep two peers that+// promote the same entry on the same row. Fire-and-forget: the save already+// succeeded, and the promotion re-reads the universe inside its own write.+// Ephemeral scratch universes and the file/test backend stay out of the catalog.+function promoteNewCanonEntries(universe) {+ if (!universe || universe.ephemeral || isFileBackend()) return;+ import('../../scripts/migrateBibleToCatalog.js')+ .then(({ hasUnpromotedCanon, promoteUniverseCanon }) => (+ hasUnpromotedCanon(universe) ? promoteUniverseCanon(universe.id) : null+ ))+ .catch((err) => {+ console.error(`❌ canon→catalog promotion failed for ${universe.id}: ${err.message}`);+ });+}+/*** Append one analyzed art reference to a universe's `styleReferences`, and* optionally adopt the style guidance the same analysis proposed — as a DELTA,