Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Give new universe canon entries a catalog record on save (#11909)

[._.]/1c4d74f94eb6
diff --git a/server/scripts/migrateBibleToCatalog.js b/server/scripts/migrateBibleToCatalog.js
index b16fcbdff7..0c2a6fe768 100644
--- a/server/scripts/migrateBibleToCatalog.js
+++ b/server/scripts/migrateBibleToCatalog.js
@@ -22,10 +22,12 @@
import { createHash } from 'crypto';
import { readMarker, writeMarker } from '../lib/migrationMarker.js';
import { BIBLE_KINDS, BIBLE_FIELD } from '../lib/storyBible.js';
-import { listUniverses, updateUniverse } from '../services/universeBuilder.js';
+import { listUniverses, getUniverse, updateUniverse, ERR_NOT_FOUND } from '../services/universeBuilder.js';
import * as catalogDB from '../services/catalogDB.js';
-const MARKER_VERSION = 1;
+// v2 re-walks every install once: before canon entries were promoted on save,
+// an entry added after the first walk stayed out of the catalog.
+const MARKER_VERSION = 2;
const MARKER_FILENAME = 'catalog-backfill.applied.json';
// Derived from BIBLE_KINDS so a future kind added there flows through here
@@ -199,6 +201,60 @@ async function migrateUniverse(universe) {
return stats;
}
+/**
+ * True when a canon entry has no catalog row yet. Exported so the universe
+ * write path can decide, without a DB round-trip, whether to promote.
+ */
+export function hasUnpromotedCanon(universe) {
+ return KINDS.some(({ array }) => (Array.isArray(universe?.[array]) ? universe[array] : [])
+ .some((entry) => entry?.id && entry?.name && !entry.ingredientId));
+}
+
+// One promotion per universe at a time; a save landing mid-walk re-runs once
+// the current walk finishes so its new entries are not missed. The boot walk
+// goes through the same map so a save during it can't race the same
+// deterministic catalog id.
+const promoting = new Map();
+
+// A deleted or missing universe has nothing left to promote.
+async function loadLiveUniverse(universeId) {
+ const universe = await getUniverse(universeId, { includeDeleted: true }).catch((err) => {
+ if (err?.code === ERR_NOT_FOUND) return null;
+ throw err;
+ });
+ return universe && !universe.deleted ? universe : null;
+}
+
+function startPromotion(universeId, firstPass = null) {
+ const state = { again: !firstPass, promise: null };
+ state.promise = (async () => {
+ let stats = firstPass ? await firstPass() : null;
+ while (state.again) {
+ state.again = false;
+ const universe = await loadLiveUniverse(universeId);
+ if (!universe || !hasUnpromotedCanon(universe)) break;
+ stats = await migrateUniverse(universe);
+ }
+ return stats;
+ })().finally(() => promoting.delete(universeId));
+ promoting.set(universeId, state);
+ return state.promise;
+}
+
+/**
+ * Promote one universe's not-yet-promoted canon entries to catalog rows and
+ * stamp the ids back. Called after a universe save adds canon entries, so the
+ * Casting / Environments / Objects pages see them without a manual sync.
+ */
+export function promoteUniverseCanon(universeId) {
+ const running = promoting.get(universeId);
+ if (running) {
+ running.again = true;
+ return running.promise;
+ }
+ return startPromotion(universeId);
+}
+
/**
* Public entry point. Runs the migration once, then no-ops on every
* subsequent boot. Wired into server/index.js after ensureSchema().
@@ -215,7 +271,9 @@ export async function migrateBibleToCatalog({ force = false } = {}) {
const totals = { universesScanned: 0, promoted: 0, peerReconciled: 0, skipped: 0, errors: 0 };
for (const universe of universes) {
if (universe.deleted) continue;
- const result = await migrateUniverse(universe);
+ while (promoting.has(universe.id)) await promoting.get(universe.id).promise.catch(() => {});
+ let result;
+ await startPromotion(universe.id, async () => { result = await migrateUniverse(universe); });
totals.universesScanned++;
totals.promoted += result.promoted;
totals.peerReconciled += result.peerReconciled;
diff --git a/server/scripts/migrateBibleToCatalog.test.js b/server/scripts/migrateBibleToCatalog.test.js
index fc48ca55cf..0ac36d76f4 100644
--- a/server/scripts/migrateBibleToCatalog.test.js
+++ b/server/scripts/migrateBibleToCatalog.test.js
@@ -25,6 +25,13 @@ vi.mock('../services/universeBuilder.js', () => ({
// guard can be exercised independently by overriding this mock per-test.
listUniverses: vi.fn(async ({ includeDeleted = true } = {}) =>
(includeDeleted ? universes : universes.filter((u) => !u.deleted))),
+ ERR_NOT_FOUND: 'NOT_FOUND',
+ // Throws NOT_FOUND for a missing or soft-deleted universe, like the real one.
+ getUniverse: vi.fn(async (id, { includeDeleted = false } = {}) => {
+ const u = universes.find((x) => x.id === id);
+ if (!u || (u.deleted && !includeDeleted)) throw Object.assign(new Error(`Universe not found: ${id}`), { code: 'NOT_FOUND' });
+ return u;
+ }),
updateUniverse: vi.fn(async (id, patchOrMutator) => {
const u = universes.find((x) => x.id === id);
if (!u) return null;
@@ -86,7 +93,7 @@ vi.mock('../lib/migrationMarker.js', () => ({
writeMarker: vi.fn(async (_name, payload) => { markerBlob = JSON.stringify(payload); }),
}));
-import { migrateBibleToCatalog } from './migrateBibleToCatalog.js';
+import { migrateBibleToCatalog, promoteUniverseCanon, hasUnpromotedCanon } from './migrateBibleToCatalog.js';
import { listUniverses } from '../services/universeBuilder.js';
function makeUniverse(overrides = {}) {
@@ -152,7 +159,14 @@ describe('migrateBibleToCatalog', () => {
// Marker is now written; a plain (non-forced) re-run must no-op.
const rerun = await migrateBibleToCatalog();
expect(rerun.skipped).toBe(true);
- expect(rerun.marker.version).toBe(1);
+ expect(rerun.marker.version).toBe(2);
+ });
+
+ it('re-walks once on an install whose marker predates save-time promotion', async () => {
+ markerBlob = JSON.stringify({ version: 1, completedAt: '2026-01-01T00:00:00.000Z' });
+ const result = await migrateBibleToCatalog();
+ expect(result.skipped).toBe(false);
+ expect(result.stats.promoted).toBe(4);
});
it('re-links an already-promoted entry that lost its ref (idempotent linking)', async () => {
@@ -191,3 +205,53 @@ describe('migrateBibleToCatalog', () => {
expect(result.stats.promoted).toBe(4);
});
});
+
+describe('promoteUniverseCanon', () => {
+ it('promotes only the entries a save added, under deterministic ids', async () => {
+ await migrateBibleToCatalog();
+ universes[0].places.push({ id: 'p-2', name: 'Night Ferry' });
+ expect(hasUnpromotedCanon(universes[0])).toBe(true);
+
+ const stats = await promoteUniverseCanon('u-1');
+
+ expect(stats.promoted).toBe(1);
+ expect(ingredients.size).toBe(5);
+ const ferry = universes[0].places.find((p) => p.id === 'p-2');
+ expect(ferry.ingredientId).toMatch(/^cat-plc-bible-/);
+ expect(refLinks).toContainEqual({ ingredientId: ferry.ingredientId, refKind: 'universe', refId: 'u-1', role: 'canon-place' });
+ expect(hasUnpromotedCanon(universes[0])).toBe(false);
+ });
+
+ it('is a no-op when every entry already has a catalog row', async () => {
+ await migrateBibleToCatalog();
+ createIngredient.mockClear();
+ expect(await promoteUniverseCanon('u-1')).toBeNull();
+ expect(createIngredient).not.toHaveBeenCalled();
+ });
+
+ it('coalesces overlapping saves into one walk plus one follow-up', async () => {
+ const first = promoteUniverseCanon('u-1');
+ const second = promoteUniverseCanon('u-1');
+ expect(second).toBe(first);
+ await first;
+ expect(ingredients.size).toBe(4);
+ expect(createIngredient).toHaveBeenCalledTimes(4);
+ });
+
+ it('quietly does nothing for a universe deleted or removed before it runs', async () => {
+ universes[0].deleted = true;
+ expect(await promoteUniverseCanon('u-1')).toBeNull();
+ expect(await promoteUniverseCanon('u-missing')).toBeNull();
+ expect(createIngredient).not.toHaveBeenCalled();
+ });
+
+ it('waits for the boot walk instead of racing it for the same catalog ids', async () => {
+ const boot = migrateBibleToCatalog({ force: true });
+ universes[0].places.push({ id: 'p-3', name: 'Rooftop' });
+ const save = promoteUniverseCanon('u-1');
+ const [bootResult] = await Promise.all([boot, save]);
+ expect(bootResult.stats.errors).toBe(0);
+ expect(universes[0].places.every((p) => p.ingredientId)).toBe(true);
+ expect(new Set(createIngredient.mock.calls.map(([input]) => input.id)).size).toBe(createIngredient.mock.calls.length);
+ });
+});
diff --git a/server/services/universeBuilder/crud.js b/server/services/universeBuilder/crud.js
index b38f52d381..73ff5e4750 100644
--- a/server/services/universeBuilder/crud.js
+++ b/server/services/universeBuilder/crud.js
@@ -35,6 +35,7 @@ import {
import { listSeries } from '../pipeline/series.js';
import { isNonBlankStr, isStr, trimTo } from '../../lib/textUtils.js';
import { codedError } from '../../lib/codedError.js';
+import { isFileBackend } from '../../lib/pgFileFacade.js';
// Once-per-process flag for the canon-backfill log — readState() runs in both
// the queue and from un-queued readers, and the in-memory migration is cheap
@@ -313,6 +314,7 @@ export async function createUniverse(input = {}) {
console.log(`⚠️ universe: auto-subscribe after create failed: ${err.message}`);
});
}
+ promoteNewCanonEntries(created);
return created;
}
@@ -739,9 +741,27 @@ export async function updateUniverse(id, patchOrMutator = {}, options = {}) {
if (!silent) {
emitRecordUpdated('universe', merged.id);
}
+ promoteNewCanonEntries(merged);
return merged;
}
+// A save that adds a canon character/place/object gives it a catalog row (and
+// stamps the ingredientId back) so Casting / Environments / Objects list it
+// without a manual "Sync from Universes". Deterministic ids keep two peers that
+// promote the same entry on the same row. Fire-and-forget: the save already
+// succeeded, and the promotion re-reads the universe inside its own write.
+// Ephemeral scratch universes and the file/test backend stay out of the catalog.
+function promoteNewCanonEntries(universe) {
+ if (!universe || universe.ephemeral || isFileBackend()) return;
+ import('../../scripts/migrateBibleToCatalog.js')
+ .then(({ hasUnpromotedCanon, promoteUniverseCanon }) => (
+ hasUnpromotedCanon(universe) ? promoteUniverseCanon(universe.id) : null
+ ))
+ .catch((err) => {
+ console.error(`❌ canon→catalog promotion failed for ${universe.id}: ${err.message}`);
+ });
+}
+
/**
* Append one analyzed art reference to a universe's `styleReferences`, and
* optionally adopt the style guidance the same analysis proposed — as a DELTA,