Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(catalog): reject stale universe canon projections (#12501)
[._.]/ Adam Eivyf86cb161cdd4
diff --git a/server/services/catalogCanonProjection.js b/server/services/catalogCanonProjection.jsindex d099245b7e..6c1b9c7fe4 100644--- a/server/services/catalogCanonProjection.js+++ b/server/services/catalogCanonProjection.js@@ -34,11 +34,12 @@** Conflict policy. LWW on `updatedAt` (no field-level merge). projectToCatalog* only writes when the embedded entry's `updatedAt` is newer-or-equal to the- * catalog row's; projectToCanon always wins (the catalog row IS the store of- * record after a catalog edit). The boot reconciler (reconcileCanonCatalog.js)+ * catalog row's; projectToCanon accepts only the current catalog revision and+ * never replaces a newer embedded edit. The boot reconciler (reconcileCanonCatalog.js)* runs the same LWW comparison to collapse any pre-existing divergence.*/+import { isDeepStrictEqual } from 'node:util';import * as catalogDB from './catalogDB.js';import { getCatalogIdentityAccess } from './catalogDB/identityAccess.js';import { BIBLE_FIELD } from '../lib/storyBible.js';@@ -96,8 +97,6 @@ export async function projectToCanon(ingredientId, updatedIngredient, deps = {})if (await getCatalogIdentityAccess(ingredientId)) return { universes: 0, written: 0, skipped: 1 };const identityIds = new Set(await catalogDB.getIngredientIdentityIds(ingredientId));- // Guard the originating ingredient so the updateUniverse → projectToCatalog- // echo inside this call can't write the catalog row a second time.for (const id of identityIds) inFlight.add(id);const stats = { universes: 0, written: 0, skipped: 0 };try {@@ -109,7 +108,8 @@ export async function projectToCanon(ingredientId, updatedIngredient, deps = {})const name = updatedIngredient.name;const payload = entryToPayload(updatedIngredient.payload || {});- const updatedAt = updatedIngredient.updatedAt || new Date().toISOString();+ const updatedAt = updatedIngredient.updatedAt;+ const sourceAt = Date.parse(updatedAt || '');for (const universeId of universeIds) {stats.universes++;@@ -122,14 +122,24 @@ export async function projectToCanon(ingredientId, updatedIngredient, deps = {})// touches N universes doesn't emit N recordUpdated events. `guardToken`// threads the originating ingredientId so updateUniverse's synchronous// projectToCatalog skips it (loop break).- const result = await updateUniverse(universeId, (cur) => {- let touched = false;+ let touched = false;+ const result = await updateUniverse(universeId, async (cur) => {+ // Read without taking the catalog write queue: reverse projection already+ // runs inside this universe queue. Waiting on both queues can deadlock.+ const current = await catalogDB.getIngredient(ingredientId);+ if (!current || current.id !== ingredientId || !Number.isFinite(sourceAt)+ || Date.parse(current.updatedAt || '') !== sourceAt+ || current.name !== name+ || !isDeepStrictEqual(entryToPayload(current.payload || {}), payload)) return null;const patch = {};for (const arrayKey of CANON_ARRAY_KEYS) {const list = Array.isArray(cur[arrayKey]) ? cur[arrayKey] : [];if (!list.some((e) => identityIds.has(e?.ingredientId))) continue;patch[arrayKey] = list.map((e) => {if (!identityIds.has(e?.ingredientId)) return e;+ // Alias-backed entries are archived context, not newer edits of+ // the canonical identity; an explicit catalog edit refreshes them.+ if (e.ingredientId === ingredientId && Date.parse(e.updatedAt || '') > sourceAt) return e;touched = true;const controlFields = Object.fromEntries(NON_PAYLOAD_KEYS.filter((key) => e[key] !== undefined).map((key) => [key, e[key]]),@@ -142,7 +152,7 @@ export async function projectToCanon(ingredientId, updatedIngredient, deps = {})console.error(`🔁 projectToCanon: universe ${universeId} update failed: ${err.message}`);return null;});- if (result) stats.written++;+ if (result && touched) stats.written++;else stats.skipped++;}return stats;diff --git a/server/services/catalogCanonProjection.ordering.test.js b/server/services/catalogCanonProjection.ordering.test.jsnew file mode 100644index 0000000000..07d0619a17--- /dev/null+++ b/server/services/catalogCanonProjection.ordering.test.js@@ -0,0 +1,163 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';+import { request } from '../lib/testHelper.js';+import { readFile } from 'node:fs/promises';+import { join } from 'node:path';+import {+ cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy, mockNoPeers, mockNoPeerSync,+} from '../lib/mockPathsDataRoot.js';++vi.mock('../lib/fileUtils.js', async (importOriginal) =>+ makePathsProxy(await importOriginal(), { dataRoot: () => lazyTempDataRoot('canon-ordering-') }));+vi.mock('./instances.js', () => mockNoPeers());+vi.mock('./sharing/peerSync.js', () => mockNoPeerSync());+vi.mock('./catalogDB/identityAccess.js', () => ({ getCatalogIdentityAccess: vi.fn(async () => null) }));+vi.mock('./catalogDB.js', () => ({+ getIngredient: vi.fn(),+ getIngredientIdentityIds: vi.fn(async (id) => [id]),+ listRefsForIngredient: vi.fn(),+ getIngredientTimestamps: vi.fn(),+ updateIngredient: vi.fn(),+}));++const catalog = await import('./catalogDB.js');+const { projectToCanon } = await import('./catalogCanonProjection.js');+const { updateUniverse, getUniverse } = await import('./universeBuilder/crud.js');+const { store } = await import('./universeBuilder/storeFacade.js');+const { sanitizeTemplate } = await import('./universeBuilder/sanitize.js');+const { default: router } = await import('../routes/universeBuilder/crud.js');+const app = express().use('/universes', router);+const universeId = 'example-universe';+const ingredientId = 'example-ingredient';+const BASE = '2026-01-01T00:00:00.000Z';+const OLD = '2026-01-02T00:00:00.000Z';+const NEW = '2026-01-03T00:00:00.000Z';+let row;++function barrier() {+ let resolve;+ const promise = new Promise((done) => { resolve = done; });+ return { promise, resolve };+}+function commit(name, updatedAt, payload = {}) {+ row = { id: ingredientId, name, updatedAt, payload };+ return structuredClone(row);+}+async function persistedEntry() {+ const disk = JSON.parse(await readFile(join(lazyTempDataRoot('canon-ordering-'), 'universes', universeId, 'index.json'), 'utf8'));+ const response = await request(app).get(`/universes/${universeId}`);+ expect(response.status).toBe(200);+ expect(response.body.characters[0]).toEqual(disk.characters[0]);+ return disk.characters[0];+}+async function holdFirstRefRead() {+ const entered = barrier();+ const release = barrier();+ catalog.listRefsForIngredient.mockImplementationOnce(async () => {+ entered.resolve();+ await release.promise;+ return [{ refKind: 'universe', refId: universeId }];+ });+ return { entered, release };+}++beforeEach(async () => {+ vi.clearAllMocks();+ commit('Initial', BASE);+ catalog.getIngredient.mockImplementation(async () => structuredClone(row));+ catalog.getIngredientTimestamps.mockImplementation(async () => new Map([[ingredientId, row.updatedAt]]));+ catalog.updateIngredient.mockImplementation(async (_id, patch) => {+ row = { ...row, ...structuredClone(patch), updatedAt: new Date().toISOString() };+ return structuredClone(row);+ });+ catalog.listRefsForIngredient.mockResolvedValue([{ refKind: 'universe', refId: universeId }]);+ await store().writeRecord(universeId, sanitizeTemplate({+ id: universeId, name: 'Example Universe', createdAt: BASE, updatedAt: BASE,+ characters: [{ id: 'example-character', ingredientId, name: 'Initial',+ createdAt: BASE, updatedAt: BASE, physicalDescription: 'Remove this description' }],+ }));+});+afterAll(cleanupTempDataRoots);++describe('catalog projection ordering through persisted universe writes', () => {+ it.each([NEW, OLD])('keeps New when Old completes last (New clock %s)', async (newAt) => {+ const { entered, release } = await holdFirstRefRead();+ const oldProjection = projectToCanon(ingredientId, commit('Old', OLD));+ try {+ await entered.promise;+ expect(await projectToCanon(ingredientId, commit('New', newAt))).toMatchObject({ written: 1 });+ } finally {+ release.resolve();+ await oldProjection;+ }+ expect(row.name).toBe('New');+ expect(await persistedEntry()).toMatchObject({ name: 'New', updatedAt: newAt });+ expect(catalog.updateIngredient).not.toHaveBeenCalled();+ });++ it('rechecks the catalog after entering the universe queue', async () => {+ const entered = barrier();+ const release = barrier();+ const queued = barrier();+ const blocker = updateUniverse(universeId, async () => {+ entered.resolve();+ await release.promise;+ return null;+ });+ await entered.promise;+ const projection = projectToCanon(ingredientId, commit('Old', OLD), {+ updateUniverse: (...args) => {+ const pending = updateUniverse(...args);+ queued.resolve();+ return pending;+ },+ });+ try {+ await queued.promise;+ commit('New', NEW);+ } finally {+ release.resolve();+ await blocker;+ }+ expect(await projection).toEqual({ universes: 1, written: 0, skipped: 1 });+ expect(await persistedEntry()).toMatchObject({ name: 'Initial', updatedAt: BASE });+ expect(row.name).toBe('New');+ });++ it('preserves a genuine newer embedded edit while a forward projection is delayed', async () => {+ const { entered, release } = await holdFirstRefRead();+ const projection = projectToCanon(ingredientId, commit('Catalog edit', OLD));+ let edited;+ try {+ await entered.promise;+ edited = await updateUniverse(universeId, (cur) => ({+ characters: cur.characters.map((entry) => ({ ...entry, name: 'Universe edit' })),+ }));+ } finally {+ release.resolve();+ await projection;+ }+ expect(await persistedEntry()).toMatchObject({ name: 'Universe edit', updatedAt: edited.characters[0].updatedAt });+ // The independent in-flight reverse-sync suppression remains tracked in #12502.+ // Once that projection ends, an ordinary canon save still carries the edit.+ await updateUniverse(universeId, (cur) => ({ characters: cur.characters }));+ expect(row.name).toBe('Universe edit');+ });++ it('keeps the source clock and field removals so a later save cannot resurrect stale catalog content', async () => {+ const projected = commit('Catalog edit', OLD, { role: 'Hero' });+ expect(await projectToCanon(ingredientId, projected)).toMatchObject({ written: 1 });+ const entry = await persistedEntry();+ expect(entry).toMatchObject({ id: 'example-character', ingredientId, createdAt: BASE,+ updatedAt: OLD, name: 'Catalog edit', role: 'Hero', physicalDescription: '' });+ expect(catalog.updateIngredient).not.toHaveBeenCalled();+ commit('Later catalog edit', NEW);+ await updateUniverse(universeId, (cur) => ({+ characters: [...cur.characters, { id: 'example-other', name: 'Other character', updatedAt: BASE }],+ }));+ expect((await getUniverse(universeId)).characters[1].updatedAt > BASE).toBe(true);+ expect(await persistedEntry()).toMatchObject({ updatedAt: OLD, name: 'Catalog edit' });+ expect(row.name).toBe('Later catalog edit');+ expect(catalog.updateIngredient).not.toHaveBeenCalled();+ });+});diff --git a/server/services/catalogCanonProjection.test.js b/server/services/catalogCanonProjection.test.jsindex c37e1ac80e..86572568ba 100644--- a/server/services/catalogCanonProjection.test.js+++ b/server/services/catalogCanonProjection.test.js@@ -1,8 +1,8 @@/*** Bidirectional catalog↔canon projection — the highest-risk file in this PR.*- * Needs a live Postgres (the same one `npm start` uses) for the catalog rows +- * refs; SKIPS cleanly when no DB is reachable. The universe side is INJECTED as+ * Uses the isolated portos_test database for catalog rows and refs; skips+ * when the test DB is unavailable. The universe side is INJECTED as* a spy (`updateUniverse` dep) so the suite asserts the projection's fan-out /* loop-break / LWW behavior without standing up a real universe store.*@@ -68,7 +68,7 @@ describe.skipIf(!runDb)('catalogCanonProjection', () => {seenUniverses.push(universeId);// Simulate the embedded entry carrying this ingredientId so the mutator// produces a patch (returns null otherwise → counted as skipped).- const patch = mutator({+ const patch = await mutator({characters: [{id: 'e1', ingredientId: ing.id, name: 'old',createdAt: '2026-01-01T00:00:00.000Z', staleField: 'remove me',@@ -78,7 +78,8 @@ describe.skipIf(!runDb)('catalogCanonProjection', () => {return patch; // truthy → counted as "written"});- const stats = await projectToCanon(ing.id, { ...ing, name: 'Renamed', payload: { role: 'Hero' } }, { updateUniverse });+ const updated = await catalogDB.updateIngredient(ing.id, { name: 'Renamed', payload: { role: 'Hero' } });+ const stats = await projectToCanon(ing.id, updated, { updateUniverse });expect(stats.universes).toBe(2);expect(stats.written).toBe(2);expect(seenUniverses.sort()).toEqual([uA, uB].sort());@@ -87,8 +88,6 @@ describe.skipIf(!runDb)('catalogCanonProjection', () => {createdAt: '2026-01-01T00:00:00.000Z',});expect(projected[0]).not.toHaveProperty('staleField');- // Guard set drains — no leaked in-flight token.- expect(_inFlightSize()).toBe(0);});it('projectToCanon is a silent no-op for an ingredient with no universe ref', async () => {@@ -148,13 +147,16 @@ describe.skipIf(!runDb)('catalogCanonProjection', () => {// would persist canon then synchronously project back to catalog.const canon = {characters: [{- id: 'e1', ingredientId: ing.id, name: 'Loop Guard', role: 'after',- updatedAt: new Date(Date.now() + 60_000).toISOString(),+ id: 'e1', ingredientId: ing.id, name: 'Loop Guard', role: 'before',+ updatedAt: ing.updatedAt,}],};- await projectToCatalog(universeId, canon, { guardToken: opts?.canonProjectionGuard });+ const patch = await mutator(canon);+ expect(patch).not.toBeNull();+ const persisted = { ...canon, ...patch };+ await projectToCatalog(universeId, persisted, { guardToken: opts?.canonProjectionGuard });catalogWritesInsideEcho = updSpy.mock.calls.length - before;- return mutator(canon); // truthy+ return persisted;});const updated = await catalogDB.updateIngredient(ing.id, { payload: { role: 'after' } });diff --git a/server/services/catalogDB.merge.db.test.js b/server/services/catalogDB.merge.db.test.jsindex 7ac8ef5fa3..337e40f6bc 100644--- a/server/services/catalogDB.merge.db.test.js+++ b/server/services/catalogDB.merge.db.test.js@@ -167,7 +167,7 @@ describe.skipIf(!ready)('recoverable local catalog merge', () => {const updated = await getIngredient(input.sourceId);let projected;const stats = await projectToCanon(input.sourceId, updated, { updateUniverse: async (_id, mutate) => {- projected = mutate({ characters: [oldEntry] });+ projected = await mutate({ characters: [oldEntry] });// The synchronous echo cannot race the source alias against the survivor.expect(await projectToCatalog('example-universe', projected)).toEqual({ written: 0, skipped: 1 });return projected;diff --git a/server/services/universeBuilder/crud.js b/server/services/universeBuilder/crud.jsindex 73ff5e4750..a513ea2077 100644--- a/server/services/universeBuilder/crud.js+++ b/server/services/universeBuilder/crud.js@@ -376,7 +376,7 @@ function canonEntryContentEqual(a, b) {// the bible sanitizer otherwise preserves the prior timestamp and the// canon→catalog projection would skip a genuine change. Untouched entries keep// their old timestamp so this never manufactures projection churn.-function stampChangedCanonEntries(prev, next) {+function stampChangedCanonEntries(prev, next, canonProjectionGuard) {const nowMs = Date.now();for (const key of BIBLE_KEYS) {const nextList = Array.isArray(next[key]) ? next[key] : null;@@ -385,7 +385,8 @@ function stampChangedCanonEntries(prev, next) {(Array.isArray(prev?.[key]) ? prev[key] : []).map((e) => [e?.id, e]),);next[key] = nextList.map((entry) => {- if (!entry?.id) return entry;+ // A mirrored catalog revision keeps its source clock; it is not a new edit.+ if (!entry?.id || (canonProjectionGuard && entry.ingredientId === canonProjectionGuard)) return entry;const before = prevById.get(entry.id);if (before && canonEntryContentEqual(before, entry)) return entry;// Monotonic bump: a content change MUST advance the LWW clock past the@@ -644,7 +645,7 @@ export async function updateUniverse(id, patchOrMutator = {}, options = {}) {// on genuine content changes makes the embedded entry's timestamp truthful// so projectToCatalog correctly carries the edit across. Unchanged entries// keep their old timestamp (no spurious revisions / no projection churn).- stampChangedCanonEntries(cur, mergedRecord);+ stampChangedCanonEntries(cur, mergedRecord, canonProjectionGuard);// Persist the stale-reference-sheet null at write time so the on-disk// record catches up with what the GET-route pruner shows. Otherwise a// PATCH that omits `characters` (e.g. rename) merges from `cur` and