Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep catalog conflict clocks monotonic at wire precision (#12500)

[._.]/ Adam Eivyf416d31799f9
diff --git a/scripts/lib/dbTestFiles.js b/scripts/lib/dbTestFiles.js
index 0936560678..fd678d2553 100644
--- a/scripts/lib/dbTestFiles.js
+++ b/scripts/lib/dbTestFiles.js
@@ -33,6 +33,7 @@ export const DB_TEST_INCLUDE = [
'services/catalogDB.facets.db.test.js',
'services/catalogDB.media.db.test.js',
'services/catalogSync.tombstoneRevival.db.test.js',
+ 'services/catalogSync.changeClock.db.test.js',
'services/catalogSync.pendingParents.db.test.js',
'services/humanActivity.db.test.js',
'services/postRunDb.db.test.js',
diff --git a/server/lib/db.ddlParity.test.js b/server/lib/db.ddlParity.test.js
index 446e48d154..dacca02e92 100644
--- a/server/lib/db.ddlParity.test.js
+++ b/server/lib/db.ddlParity.test.js
@@ -279,6 +279,21 @@ describe('DDL parity (init-db.sql ↔ db/schema ensureSchema)', () => {
// ── Targeted pins: contracts a two-sided set comparison cannot see ──────────
+ // #12500: function bodies are not compared above, so a fresh install could
+ // keep a transaction-start NOW() clock that the upgrade path no longer has.
+ it('every catalog LWW trigger stamps the monotonic change clock in both files', () => {
+ const fns = ['update_catalog_ingredient_timestamp', 'update_catalog_scrap_timestamp', 'update_catalog_ref_sync_seq',
+ 'update_catalog_relation_sync_seq', 'update_catalog_media_sync_seq', 'update_catalog_tag_timestamp'];
+ for (const [label, source] of [['init-db.sql', INIT_SQL], ['db/schema', DB_JS]]) {
+ expect(source, `${label} missing catalog_next_change_clock`).toMatch(/CREATE OR REPLACE FUNCTION catalog_next_change_clock\(previous TIMESTAMPTZ\)/);
+ for (const fn of fns) {
+ const body = new RegExp(`CREATE OR REPLACE FUNCTION ${fn}\\(\\)([\\s\\S]*?)\\$\\$ LANGUAGE plpgsql`).exec(source)?.[1];
+ expect(body, `${label} ${fn}`).toContain('NEW.updated_at := catalog_next_change_clock(OLD.updated_at)');
+ expect(body, `${label} ${fn}`).not.toMatch(/NEW\.updated_at := NOW\(\)/);
+ }
+ }
+ });
+
// memories is SQL_ONLY_TOLERATED, so the shared-table comparison cannot
// catch fresh-install drift in its generated expression or matching index.
it('stores the same memory search lexemes on fresh installs and upgrades', () => {
diff --git a/server/lib/db/schema/catalog.js b/server/lib/db/schema/catalog.js
index 20264af78c..502ee5a1b7 100644
--- a/server/lib/db/schema/catalog.js
+++ b/server/lib/db/schema/catalog.js
@@ -325,6 +325,21 @@ export const catalogDdl = [
`CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_key ON catalog_ingredient_media (media_key)`,
`CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_sync_seq ON catalog_ingredient_media (sync_sequence)`,
+ // #12500: the LWW clock every catalog trigger stamps on a local change.
+ // Transaction-start NOW() let a transaction that began before a
+ // concurrent, already-committed edit stamp an OLDER clock than the row it
+ // overwrote, and peers' strict `EXCLUDED.updated_at >` guard then dropped
+ // it for good. The wire carries milliseconds (Date#toISOString), so the
+ // clock is millisecond-representable and strictly above the previous one
+ // at that precision; a future (skewed peer) clock is never moved backward.
+ `CREATE OR REPLACE FUNCTION catalog_next_change_clock(previous TIMESTAMPTZ)
+ RETURNS TIMESTAMPTZ AS $$
+ SELECT GREATEST(
+ date_trunc('milliseconds', clock_timestamp()),
+ date_trunc('milliseconds', previous) + INTERVAL '1 millisecond'
+ )
+ $$ LANGUAGE sql VOLATILE`,
+
`CREATE OR REPLACE FUNCTION update_catalog_ingredient_timestamp()
RETURNS TRIGGER AS $$
DECLARE
@@ -342,7 +357,7 @@ export const catalogDdl = [
);
IF NOT content_changed THEN RETURN NEW; END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredients', 'sync_sequence'));
RETURN NEW;
@@ -371,7 +386,7 @@ export const catalogDdl = [
);
IF NOT content_changed THEN RETURN NEW; END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_scraps', 'sync_sequence'));
RETURN NEW;
@@ -416,7 +431,7 @@ export const catalogDdl = [
IF NEW.deleted IS DISTINCT FROM OLD.deleted
OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_refs', 'sync_sequence'));
END IF;
@@ -438,7 +453,7 @@ export const catalogDdl = [
IF NEW.deleted IS DISTINCT FROM OLD.deleted
OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_relations', 'sync_sequence'));
END IF;
@@ -464,7 +479,7 @@ export const catalogDdl = [
OR NEW.caption IS DISTINCT FROM OLD.caption
OR NEW.metadata IS DISTINCT FROM OLD.metadata THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_media', 'sync_sequence'));
END IF;
@@ -494,7 +509,7 @@ export const catalogDdl = [
);
IF NOT content_changed THEN RETURN NEW; END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_tags', 'sync_sequence'));
RETURN NEW;
diff --git a/server/scripts/init-db.sql b/server/scripts/init-db.sql
index e19c124271..bdd8b4891d 100644
--- a/server/scripts/init-db.sql
+++ b/server/scripts/init-db.sql
@@ -660,6 +660,18 @@ CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_ingredient ON catalog_ingredien
CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_key ON catalog_ingredient_media (media_key);
CREATE INDEX IF NOT EXISTS idx_catalog_ing_media_sync_seq ON catalog_ingredient_media (sync_sequence);
+-- #12500: the LWW clock every catalog trigger stamps on a local change —
+-- millisecond-representable (the wire precision) and strictly above the
+-- previous clock, so a transaction that began before a concurrent committed
+-- edit cannot stamp an older clock, and a future clock never moves backward.
+CREATE OR REPLACE FUNCTION catalog_next_change_clock(previous TIMESTAMPTZ)
+RETURNS TIMESTAMPTZ AS $$
+ SELECT GREATEST(
+ date_trunc('milliseconds', clock_timestamp()),
+ date_trunc('milliseconds', previous) + INTERVAL '1 millisecond'
+ )
+$$ LANGUAGE sql VOLATILE;
+
-- Auto-update updated_at and bump sync_sequence on content/metadata changes.
-- Mirrors update_memory_timestamp's pattern: skip the bump on no-content-change
-- so cosmetic touches don't trigger sync. Respects explicit updated_at (used by
@@ -685,7 +697,7 @@ BEGIN
END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredients', 'sync_sequence'));
RETURN NEW;
@@ -719,7 +731,7 @@ BEGIN
END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_scraps', 'sync_sequence'));
RETURN NEW;
@@ -766,7 +778,7 @@ BEGIN
IF NEW.deleted IS DISTINCT FROM OLD.deleted
OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_refs', 'sync_sequence'));
END IF;
@@ -790,7 +802,7 @@ BEGIN
IF NEW.deleted IS DISTINCT FROM OLD.deleted
OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_relations', 'sync_sequence'));
END IF;
@@ -818,7 +830,7 @@ BEGIN
OR NEW.caption IS DISTINCT FROM OLD.caption
OR NEW.metadata IS DISTINCT FROM OLD.metadata THEN
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_ingredient_media', 'sync_sequence'));
END IF;
@@ -854,7 +866,7 @@ BEGIN
END IF;
IF NEW.updated_at IS NULL OR NEW.updated_at = OLD.updated_at THEN
- NEW.updated_at := NOW();
+ NEW.updated_at := catalog_next_change_clock(OLD.updated_at);
END IF;
NEW.sync_sequence := nextval(pg_get_serial_sequence('catalog_tags', 'sync_sequence'));
RETURN NEW;
diff --git a/server/services/catalogDB/ingredients.js b/server/services/catalogDB/ingredients.js
index 5b07ad4476..ba716d9e8b 100644
--- a/server/services/catalogDB/ingredients.js
+++ b/server/services/catalogDB/ingredients.js
@@ -309,8 +309,7 @@ export async function reviveDeletedIngredient(id, { type, name, payload = {}, ta
const result = await query(
`UPDATE catalog_ingredients
SET deleted = false, deleted_at = NULL,
- type = $2, name = $3, payload = $4::jsonb, tags = $5,
- updated_at = NOW()
+ type = $2, name = $3, payload = $4::jsonb, tags = $5
WHERE id = $1 AND deleted = true
RETURNING *`,
[id, type, String(name).trim(), JSON.stringify(storedPayload), tags || []],
diff --git a/server/services/catalogSync.changeClock.db.test.js b/server/services/catalogSync.changeClock.db.test.js
new file mode 100644
index 0000000000..7f7b62015f
--- /dev/null
+++ b/server/services/catalogSync.changeClock.db.test.js
@@ -0,0 +1,154 @@
+/**
+ * DB-backed regressions for #12500: the catalog LWW clock (`updated_at`) a
+ * local change stamps must be strictly newer than the row it overwrites at
+ * wire (millisecond) precision. Peers apply only `EXCLUDED.updated_at >`
+ * the stored clock, so a regressed or same-millisecond clock is a version the
+ * peer drops permanently once its cursor moves past it.
+ *
+ * Needs a live Postgres: the clock is assigned by the catalog triggers and
+ * judged by the peer-apply SQL guard, neither of which a mocked query() runs.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { describe, it, expect, afterAll } from 'vitest';
+import { checkHealth, ensureSchema, close, query, withTransaction } from '../lib/db.js';
+import { requireDbOrSkip } from '../lib/dbTestGate.js';
+import {
+ createIngredient, updateIngredient, getIngredient, deleteIngredient, reviveDeletedIngredient,
+} from './catalogDB/ingredients.js';
+import { linkIngredientToRef } from './catalogDB/refs.js';
+import { upsertIngredientFromPeer } from './catalogDB/sync.js';
+
+let dbReady = false;
+let skipReason = '';
+{
+ const health = await checkHealth().catch((e) => ({ connected: false, error: e?.message }));
+ if (!health.connected) {
+ skipReason = `Postgres not reachable (${health.error || 'no connection'})`;
+ } else {
+ await ensureSchema().catch(() => {});
+ const recheck = await checkHealth().catch(() => ({ hasCatalogSchema: false }));
+ if (recheck.hasCatalogSchema) dbReady = true;
+ else skipReason = 'catalog schema not present (ensureSchema did not create catalog tables)';
+ }
+}
+const runDb = requireDbOrSkip('services/catalogSync.changeClock.db.test', dbReady, skipReason);
+
+const createdIds = new Set();
+
+afterAll(async () => {
+ if (!dbReady) return;
+ for (const id of createdIds) {
+ await deleteIngredient(id, { hard: true }).catch(() => {});
+ }
+ await close();
+});
+
+async function makeIngredient(name) {
+ const ing = await createIngredient({ type: 'idea', name, payload: {}, tags: [] });
+ createdIds.add(ing.id);
+ return ing;
+}
+
+// The receiving install: a separate row fed only through the peer-apply path.
+function replicaOf(wire) {
+ const id = `${wire.id}-replica-${randomUUID()}`;
+ createdIds.add(id);
+ return { id, apply: (version) => upsertIngredientFromPeer({ ...version, id }) };
+}
+
+const wireMs = (iso) => Date.parse(iso);
+
+describe.skipIf(!runDb)('catalog conflict clock is monotonic at wire precision (#12500)', () => {
+ it('a transaction that began before a committed edit still ships a newer clock, so the peer keeps both edits', async () => {
+ const ing = await makeIngredient('Clock Subject');
+
+ let resumeA;
+ const gate = new Promise((resolve) => { resumeA = resolve; });
+ let markStarted;
+ const started = new Promise((resolve) => { markStarted = resolve; });
+ // Transaction A opens first (fixing its NOW()), then waits while B commits.
+ const transactionA = withTransaction(async (tx) => {
+ await tx.query('SELECT 1');
+ markStarted();
+ await gate;
+ return updateIngredient(ing.id, { payload: { summary: 'late disjoint payload' } }, { client: tx });
+ });
+ await started;
+
+ const versionB = await updateIngredient(ing.id, { name: 'Renamed By B' });
+ const replica = replicaOf(versionB);
+ expect(await replica.apply(versionB)).toMatchObject({ applied: true });
+
+ resumeA();
+ const versionA = await transactionA;
+ expect(versionA.name).toBe('Renamed By B');
+ expect(versionA.payload.summary).toBe('late disjoint payload');
+ expect(wireMs(versionA.updatedAt)).toBeGreaterThan(wireMs(versionB.updatedAt));
+
+ expect(await replica.apply(versionA)).toMatchObject({ applied: true });
+ const onPeer = await getIngredient(replica.id);
+ expect(onPeer.name).toBe('Renamed By B');
+ expect(onPeer.payload.summary).toBe('late disjoint payload');
+ });
+
+ it('local changes inside one transaction (one NOW()) get distinct, ordered wire clocks, including tombstone and revival', async () => {
+ const ing = await makeIngredient('Same Instant Subject');
+ await linkIngredientToRef(ing.id, 'universe', 'u-test-12500', 'canon-character');
+
+ const { ingredientClocks, refClocks } = await withTransaction(async (tx) => {
+ const ingredientClocks = [];
+ for (const set of [`name = 'Second Name'`, `payload = '{"summary":"x"}'::jsonb`, 'deleted = true, deleted_at = NOW()', 'deleted = false, deleted_at = NULL']) {
+ const { rows } = await tx.query(`UPDATE catalog_ingredients SET ${set} WHERE id = $1 RETURNING updated_at`, [ing.id]);
+ ingredientClocks.push(rows[0].updated_at.toISOString());
+ }
+ const refClocks = [];
+ for (const set of ['deleted = true, deleted_at = NOW()', 'deleted = false, deleted_at = NULL']) {
+ const { rows } = await tx.query(
+ `UPDATE catalog_ingredient_refs SET ${set}
+ WHERE ingredient_id = $1 AND ref_kind = 'universe' AND ref_id = 'u-test-12500' RETURNING updated_at`,
+ [ing.id],
+ );
+ refClocks.push(rows[0].updated_at.toISOString());
+ }
+ return { ingredientClocks, refClocks };
+ });
+
+ const strictlyIncreasing = (clocks) => clocks.every((c, i) => i === 0 || wireMs(c) > wireMs(clocks[i - 1]));
+ expect(wireMs(ingredientClocks[0])).toBeGreaterThan(wireMs(ing.updatedAt));
+ expect(strictlyIncreasing(ingredientClocks)).toBe(true);
+ expect(strictlyIncreasing(refClocks)).toBe(true);
+ });
+
+ it('a local edit, soft-delete and revival never move a future peer clock backward', async () => {
+ const ing = await makeIngredient('Future Clock Subject');
+ const futureClock = new Date(Date.now() + 60_000).toISOString();
+ expect(await upsertIngredientFromPeer({ ...ing, name: 'From Skewed Peer', updatedAt: futureClock })).toMatchObject({ applied: true });
+
+ const edited = await updateIngredient(ing.id, { name: 'Local After Future' });
+ expect(wireMs(edited.updatedAt)).toBeGreaterThan(wireMs(futureClock));
+
+ await deleteIngredient(ing.id);
+ const { rows: [tombstone] } = await query('SELECT updated_at FROM catalog_ingredients WHERE id = $1', [ing.id]);
+ expect(tombstone.updated_at.getTime()).toBeGreaterThan(wireMs(edited.updatedAt));
+
+ const revived = await reviveDeletedIngredient(ing.id, { type: 'idea', name: 'Revived' });
+ expect(wireMs(revived.updatedAt)).toBeGreaterThan(tombstone.updated_at.getTime());
+ });
+
+ it('no-op updates and a repeated inbound envelope create no new feed work', async () => {
+ const ing = await makeIngredient('No-op Subject');
+ const seq = async () => (await query('SELECT sync_sequence, updated_at FROM catalog_ingredients WHERE id = $1', [ing.id])).rows[0];
+
+ const before = await seq();
+ await query(`UPDATE catalog_ingredients SET name = name WHERE id = $1`, [ing.id]);
+ expect(await seq()).toEqual(before);
+
+ const envelope = { ...ing, name: 'Inbound Once', updatedAt: new Date(Date.now() + 60_000).toISOString() };
+ expect(await upsertIngredientFromPeer(envelope)).toMatchObject({ applied: true });
+ const afterApply = await seq();
+ expect(afterApply.updated_at.toISOString()).toBe(envelope.updatedAt);
+ expect(await upsertIngredientFromPeer(envelope)).toMatchObject({ applied: false });
+ expect(await seq()).toEqual(afterApply);
+ });
+});