Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep catalog conflict clocks monotonic at wire precision (#12500)
[._.]/ Adam Eivyf416d31799f9
diff --git a/scripts/lib/dbTestFiles.js b/scripts/lib/dbTestFiles.jsindex 0936560678..fd678d2553 100644--- a/scripts/lib/dbTestFiles.js+++ b/scripts/lib/dbTestFiles.js@@ -33,6 +33,7 @@ export const DB_TEST_INCLUDE = ['services/catalogDB.facets.db.test.js','services/catalogDB.media.db.test.js','services/catalogSync.tombstoneRevival.db.test.js',+ 'services/catalogSync.changeClock.db.test.js','services/catalogSync.pendingParents.db.test.js','services/humanActivity.db.test.js','services/postRunDb.db.test.js',diff --git a/server/lib/db.ddlParity.test.js b/server/lib/db.ddlParity.test.jsindex 446e48d154..dacca02e92 100644--- a/server/lib/db.ddlParity.test.js+++ b/server/lib/db.ddlParity.test.js@@ -279,6 +279,21 @@ describe('DDL parity (init-db.sql ↔ db/schema ensureSchema)', () => {// ── Targeted pins: contracts a two-sided set comparison cannot see ──────────+ // #12500: function bodies are not compared above, so a fresh install could+ // keep a transaction-start NOW() clock that the upgrade path no longer has.+ it('every catalog LWW trigger stamps the monotonic change clock in both files', () => {+ const fns = ['update_catalog_ingredient_timestamp', 'update_catalog_scrap_timestamp', 'update_catalog_ref_sync_seq',+ 'update_catalog_relation_sync_seq', 'update_catalog_media_sync_seq', 'update_catalog_tag_timestamp'];+ for (const [label, source] of [['init-db.sql', INIT_SQL], ['db/schema', DB_JS]]) {+ expect(source, `${label} missing catalog_next_change_clock`).toMatch(/CREATE OR REPLACE FUNCTION catalog_next_change_clock\(previous TIMESTAMPTZ\)/);+ for (const fn of fns) {+ const body = new RegExp(`CREATE OR REPLACE FUNCTION ${fn}\\(\\)([\\s\\S]*?)\\$\\$ LANGUAGE plpgsql`).exec(source)?.[1];+ expect(body, `${label} ${fn}`).toContain('NEW.updated_at := catalog_next_change_clock(OLD.updated_at)');+ expect(body, `${label} ${fn}`).not.toMatch(/NEW\.updated_at := NOW\(\)/);+ }+ }+ });+// memories is SQL_ONLY_TOLERATED, so the shared-table comparison cannot// catch fresh-install drift in its generated expression or matching index.it('stores the same memory search lexemes on fresh installs and upgrades', () => {diff --git a/server/lib/db/schema/catalog.js b/server/lib/db/schema/catalog.jsindex 20264af78c..502ee5a1b7 100644--- a/server/lib/db/schema/catalog.js+++ b/server/lib/db/schema/catalog.js@@ -325,6 +325,21 @@ export const catalogDdl = [`CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_key ON catalog_ingredient_media (media_key)`,`CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_sync_seq ON catalog_ingredient_media (sync_sequence)`,+ // #12500: the LWW clock every catalog trigger stamps on a local change.+ // Transaction-start NOW() let a transaction that began before a+ // concurrent, already-committed edit stamp an OLDER clock than the row it+ // overwrote, and peers' strict `EXCLUDED.updated_at >` guard then dropped+ // it for good. The wire carries milliseconds (Date#toISOString), so the+ // clock is millisecond-representable and strictly above the previous one+ // at that precision; a future (skewed peer) clock is never moved backward.+ `CREATE OR REPLACE FUNCTION catalog_next_change_clock(previous TIMESTAMPTZ)+ RETURNS TIMESTAMPTZ AS $$+ SELECT GREATEST(+ date_trunc('milliseconds', clock_timestamp()),+ date_trunc('milliseconds', previous) + INTERVAL '1 millisecond'+ )+ $$ LANGUAGE sql VOLATILE`,+`CREATE OR REPLACE FUNCTION update_catalog_ingredient_timestamp()RETURNS TRIGGER AS $$DECLARE@@ -342,7 +357,7 @@ export const catalogDdl = [);IF NOT content_changed THEN RETURN NEW; END IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredients', 'sync_sequence'));RETURN NEW;@@ -371,7 +386,7 @@ export const catalogDdl = [);IF NOT content_changed THEN RETURN NEW; END IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_scraps', 'sync_sequence'));RETURN NEW;@@ -416,7 +431,7 @@ export const catalogDdl = [IF NEW.deleted IS DISTINCT FROM OLD.deletedOR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_refs', 'sync_sequence'));END IF;@@ -438,7 +453,7 @@ export const catalogDdl = [IF NEW.deleted IS DISTINCT FROM OLD.deletedOR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_relations', 'sync_sequence'));END IF;@@ -464,7 +479,7 @@ export const catalogDdl = [OR NEW.caption IS DISTINCT FROM OLD.captionOR NEW.metadata IS DISTINCT FROM OLD.metadata THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_media', 'sync_sequence'));END IF;@@ -494,7 +509,7 @@ export const catalogDdl = [);IF NOT content_changed THEN RETURN NEW; END IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_tags', 'sync_sequence'));RETURN NEW;diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sqlindex e19c124271..bdd8b4891d 100644--- a/server/scripts/init-db.sql+++ b/server/scripts/init-db.sql@@ -660,6 +660,18 @@ CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_ingredient ON catalog_ingredienCREATE INDEX IF NOT EXISTS idx_catalog_ing_media_key ON catalog_ingredient_media (media_key);CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_sync_seq ON catalog_ingredient_media (sync_sequence);+-- #12500: the LWW clock every catalog trigger stamps on a local change —+-- millisecond-representable (the wire precision) and strictly above the+-- previous clock, so a transaction that began before a concurrent committed+-- edit cannot stamp an older clock, and a future clock never moves backward.+CREATE OR REPLACE FUNCTION catalog_next_change_clock(previous TIMESTAMPTZ)+RETURNS TIMESTAMPTZ AS $$+ SELECT GREATEST(+ date_trunc('milliseconds', clock_timestamp()),+ date_trunc('milliseconds', previous) + INTERVAL '1 millisecond'+ )+$$ LANGUAGE sql VOLATILE;+-- Auto-update updated_at and bump sync_sequence on content/metadata changes.-- Mirrors update_memory_timestamp's pattern: skip the bump on no-content-change-- so cosmetic touches don't trigger sync. Respects explicit updated_at (used by@@ -685,7 +697,7 @@ BEGINEND IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredients', 'sync_sequence'));RETURN NEW;@@ -719,7 +731,7 @@ BEGINEND IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_scraps', 'sync_sequence'));RETURN NEW;@@ -766,7 +778,7 @@ BEGINIF NEW.deleted IS DISTINCT FROM OLD.deletedOR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_refs', 'sync_sequence'));END IF;@@ -790,7 +802,7 @@ BEGINIF NEW.deleted IS DISTINCT FROM OLD.deletedOR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_relations', 'sync_sequence'));END IF;@@ -818,7 +830,7 @@ BEGINOR NEW.caption IS DISTINCT FROM OLD.captionOR NEW.metadata IS DISTINCT FROM OLD.metadata THENIF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_media', 'sync_sequence'));END IF;@@ -854,7 +866,7 @@ BEGINEND IF;IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN- NEW.updated_at := NOW();+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);END IF;NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_tags', 'sync_sequence'));RETURN NEW;diff --git a/server/services/catalogDB/ingredients.js b/server/services/catalogDB/ingredients.jsindex 5b07ad4476..ba716d9e8b 100644--- a/server/services/catalogDB/ingredients.js+++ b/server/services/catalogDB/ingredients.js@@ -309,8 +309,7 @@ export async function reviveDeletedIngredient(id, { type, name, payload = {}, taconst result = await query(`UPDATE catalog_ingredientsSET deleted = false, deleted_at = NULL,- type = $2, name = $3, payload = $4::jsonb, tags = $5,- updated_at = NOW()+ type = $2, name = $3, payload = $4::jsonb, tags = $5WHERE id = $1 AND deleted = trueRETURNING *`,[id, type, String(name).trim(), JSON.stringify(storedPayload), tags || []],diff --git a/server/services/catalogSync.changeClock.db.test.js b/server/services/catalogSync.changeClock.db.test.jsnew file mode 100644index 0000000000..7f7b62015f--- /dev/null+++ b/server/services/catalogSync.changeClock.db.test.js@@ -0,0 +1,154 @@+/**+ * DB-backed regressions for #12500: the catalog LWW clock (`updated_at`) a+ * local change stamps must be strictly newer than the row it overwrites at+ * wire (millisecond) precision. Peers apply only `EXCLUDED.updated_at >`+ * the stored clock, so a regressed or same-millisecond clock is a version the+ * peer drops permanently once its cursor moves past it.+ *+ * Needs a live Postgres: the clock is assigned by the catalog triggers and+ * judged by the peer-apply SQL guard, neither of which a mocked query() runs.+ */++import { randomUUID } from 'node:crypto';+import { describe, it, expect, afterAll } from 'vitest';+import { checkHealth, ensureSchema, close, query, withTransaction } from '../lib/db.js';+import { requireDbOrSkip } from '../lib/dbTestGate.js';+import {+ createIngredient, updateIngredient, getIngredient, deleteIngredient, reviveDeletedIngredient,+} from './catalogDB/ingredients.js';+import { linkIngredientToRef } from './catalogDB/refs.js';+import { upsertIngredientFromPeer } from './catalogDB/sync.js';++let dbReady = false;+let skipReason = '';+{+ const health = await checkHealth().catch((e) => ({ connected: false, error: e?.message }));+ if (!health.connected) {+ skipReason = `Postgres not reachable (${health.error || 'no connection'})`;+ } else {+ await ensureSchema().catch(() => {});+ const recheck = await checkHealth().catch(() => ({ hasCatalogSchema: false }));+ if (recheck.hasCatalogSchema) dbReady = true;+ else skipReason = 'catalog schema not present (ensureSchema did not create catalog tables)';+ }+}+const runDb = requireDbOrSkip('services/catalogSync.changeClock.db.test', dbReady, skipReason);++const createdIds = new Set();++afterAll(async () => {+ if (!dbReady) return;+ for (const id of createdIds) {+ await deleteIngredient(id, { hard: true }).catch(() => {});+ }+ await close();+});++async function makeIngredient(name) {+ const ing = await createIngredient({ type: 'idea', name, payload: {}, tags: [] });+ createdIds.add(ing.id);+ return ing;+}++// The receiving install: a separate row fed only through the peer-apply path.+function replicaOf(wire) {+ const id = `${wire.id}-replica-${randomUUID()}`;+ createdIds.add(id);+ return { id, apply: (version) => upsertIngredientFromPeer({ ...version, id }) };+}++const wireMs = (iso) => Date.parse(iso);++describe.skipIf(!runDb)('catalog conflict clock is monotonic at wire precision (#12500)', () => {+ it('a transaction that began before a committed edit still ships a newer clock, so the peer keeps both edits', async () => {+ const ing = await makeIngredient('Clock Subject');++ let resumeA;+ const gate = new Promise((resolve) => { resumeA = resolve; });+ let markStarted;+ const started = new Promise((resolve) => { markStarted = resolve; });+ // Transaction A opens first (fixing its NOW()), then waits while B commits.+ const transactionA = withTransaction(async (tx) => {+ await tx.query('SELECT 1');+ markStarted();+ await gate;+ return updateIngredient(ing.id, { payload: { summary: 'late disjoint payload' } }, { client: tx });+ });+ await started;++ const versionB = await updateIngredient(ing.id, { name: 'Renamed By B' });+ const replica = replicaOf(versionB);+ expect(await replica.apply(versionB)).toMatchObject({ applied: true });++ resumeA();+ const versionA = await transactionA;+ expect(versionA.name).toBe('Renamed By B');+ expect(versionA.payload.summary).toBe('late disjoint payload');+ expect(wireMs(versionA.updatedAt)).toBeGreaterThan(wireMs(versionB.updatedAt));++ expect(await replica.apply(versionA)).toMatchObject({ applied: true });+ const onPeer = await getIngredient(replica.id);+ expect(onPeer.name).toBe('Renamed By B');+ expect(onPeer.payload.summary).toBe('late disjoint payload');+ });++ it('local changes inside one transaction (one NOW()) get distinct, ordered wire clocks, including tombstone and revival', async () => {+ const ing = await makeIngredient('Same Instant Subject');+ await linkIngredientToRef(ing.id, 'universe', 'u-test-12500', 'canon-character');++ const { ingredientClocks, refClocks } = await withTransaction(async (tx) => {+ const ingredientClocks = [];+ for (const set of [`name = 'Second Name'`, `payload = '{"summary":"x"}'::jsonb`, 'deleted = true, deleted_at = NOW()', 'deleted = false, deleted_at = NULL']) {+ const { rows } = await tx.query(`UPDATE catalog_ingredients SET ${set} WHERE id = $1 RETURNING updated_at`, [ing.id]);+ ingredientClocks.push(rows[0].updated_at.toISOString());+ }+ const refClocks = [];+ for (const set of ['deleted = true, deleted_at = NOW()', 'deleted = false, deleted_at = NULL']) {+ const { rows } = await tx.query(+ `UPDATE catalog_ingredient_refs SET ${set}+ WHERE ingredient_id = $1 AND ref_kind = 'universe' AND ref_id = 'u-test-12500' RETURNING updated_at`,+ [ing.id],+ );+ refClocks.push(rows[0].updated_at.toISOString());+ }+ return { ingredientClocks, refClocks };+ });++ const strictlyIncreasing = (clocks) => clocks.every((c, i) => i === 0 || wireMs(c) > wireMs(clocks[i - 1]));+ expect(wireMs(ingredientClocks[0])).toBeGreaterThan(wireMs(ing.updatedAt));+ expect(strictlyIncreasing(ingredientClocks)).toBe(true);+ expect(strictlyIncreasing(refClocks)).toBe(true);+ });++ it('a local edit, soft-delete and revival never move a future peer clock backward', async () => {+ const ing = await makeIngredient('Future Clock Subject');+ const futureClock = new Date(Date.now() + 60_000).toISOString();+ expect(await upsertIngredientFromPeer({ ...ing, name: 'From Skewed Peer', updatedAt: futureClock })).toMatchObject({ applied: true });++ const edited = await updateIngredient(ing.id, { name: 'Local After Future' });+ expect(wireMs(edited.updatedAt)).toBeGreaterThan(wireMs(futureClock));++ await deleteIngredient(ing.id);+ const { rows: [tombstone] } = await query('SELECT updated_at FROM catalog_ingredients WHERE id = $1', [ing.id]);+ expect(tombstone.updated_at.getTime()).toBeGreaterThan(wireMs(edited.updatedAt));++ const revived = await reviveDeletedIngredient(ing.id, { type: 'idea', name: 'Revived' });+ expect(wireMs(revived.updatedAt)).toBeGreaterThan(tombstone.updated_at.getTime());+ });++ it('no-op updates and a repeated inbound envelope create no new feed work', async () => {+ const ing = await makeIngredient('No-op Subject');+ const seq = async () => (await query('SELECT sync_sequence, updated_at FROM catalog_ingredients WHERE id = $1', [ing.id])).rows[0];++ const before = await seq();+ await query(`UPDATE catalog_ingredients SET name = name WHERE id = $1`, [ing.id]);+ expect(await seq()).toEqual(before);++ const envelope = { ...ing, name: 'Inbound Once', updatedAt: new Date(Date.now() + 60_000).toISOString() };+ expect(await upsertIngredientFromPeer(envelope)).toMatchObject({ applied: true });+ const afterApply = await seq();+ expect(afterApply.updated_at.toISOString()).toBe(envelope.updatedAt);+ expect(await upsertIngredientFromPeer(envelope)).toMatchObject({ applied: false });+ expect(await seq()).toEqual(afterApply);+ });+});