Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: validate Brain sync clocks and order conflicts by time
Adam Eivy13f88ab87757
diff --git a/server/lib/brainValidation.js b/server/lib/brainValidation.jsindex d955cae79e..a74502701b 100644--- a/server/lib/brainValidation.js+++ b/server/lib/brainValidation.js@@ -5,6 +5,7 @@ import { EFFORT_LEVELS } from './providerModels.js';import { canonicalThreadRefKind } from './threadRefKinds.js';import { THREAD_STATUSES, THREAD_PRIORITIES } from './brainThreads.js';import { MAX_QUALITY } from './spacedRepetition.js';+import { parseTsMs } from './lwwTimestamp.js';// Destination enum. `links` is reachable only from the bare-URL capture// short-circuit (a pasted URL is filed straight to the links collection) — the@@ -590,7 +591,11 @@ const brainSyncChangeSchema = z.object({op: z.enum(['create', 'update', 'delete']),type: z.string(),id: z.string(),- record: z.record(z.unknown()).nullable().optional(),+ // Missing records/clocks retain the legacy skipped-change behavior. A clock+ // that IS supplied must be parseable; keep other record fields for newer peers.+ record: z.object({+ updatedAt: z.string().refine(value => parseTsMs(value) !== null, 'Invalid record timestamp').optional(),+ }).passthrough().nullable().optional(),originInstanceId: z.string().optional(),ts: z.string()});diff --git a/server/routes/brainSync.test.js b/server/routes/brainSync.test.jsindex 338bd06738..a379f6393d 100644--- a/server/routes/brainSync.test.js+++ b/server/routes/brainSync.test.js@@ -26,6 +26,7 @@ vi.mock('../services/brainParity.js', () => ({}));import * as brainParity from '../services/brainParity.js';+import * as brainSync from '../services/brainSync.js';import brainSyncRoutes from './brainSync.js';const buildApp = () => {@@ -40,6 +41,32 @@ beforeEach(() => {vi.clearAllMocks();});+describe('POST /api/brain/sync', () => {+ it.each(['not-a-date', 42, { value: '2026-10-15' }])('rejects malformed record clocks before applying changes: %j', async (updatedAt) => {+ const res = await request(buildApp()).post('/api/brain/sync').send({ changes: [{+ seq: 1, op: 'update', type: 'people', id: 'peer-person',+ record: { name: 'Remote person', updatedAt }, ts: '2026-10-15T00:00:00Z',+ }] });++ expect(res.status).toBe(400);+ expect(brainSync.applyRemoteChanges).not.toHaveBeenCalled();+ });++ it('preserves legacy clock formats and unknown record fields', async () => {+ brainSync.applyRemoteChanges.mockResolvedValue({ updated: 1 });+ const change = {+ seq: 1, op: 'update', type: 'people', id: 'peer-person',+ record: { name: 'Remote person', updatedAt: '2026-10-15T12:00:00+02:00', futureField: { keep: true } },+ ts: '2026-10-15T10:00:00Z',+ };++ const res = await request(buildApp()).post('/api/brain/sync').send({ changes: [change] });++ expect(res.status).toBe(200);+ expect(brainSync.applyRemoteChanges).toHaveBeenCalledWith([change]);+ });+});+describe('GET /api/brain/reconcile/manifest', () => {it('serves the parity manifest a peer audits against', async () => {brainParity.buildBrainManifest.mockResolvedValue({ types: { people: [{ id: 'p1', updatedAt: 'T1', deleted: false }] } });diff --git a/server/services/brainStorage.js b/server/services/brainStorage.jsindex 68cd6412b3..2c6fdf4c14 100644--- a/server/services/brainStorage.js+++ b/server/services/brainStorage.js@@ -43,6 +43,7 @@ import * as brainSyncLog from './brainSyncLog.js';import { createCollectionStore } from '../lib/collectionStore.js';import { linkIsRepo, normalizeRepoLinkFields } from '../lib/repoLinkFields.js';import { supersedingTimestamp } from '../lib/tombstones.js';+import { parseTsMs, compareNewerWins } from '../lib/lwwTimestamp.js';const DATA_DIR = PATHS.brain;@@ -1378,18 +1379,23 @@ export async function applyRemoteRecord(type, id, record, op) {if (!store.isValidId(id)) {return { applied: false, reason: 'invalid_id' };}+ // Both delta pulls and snapshots bypass the HTTP push schema. Reject invalid+ // clocks here too, before any write or relay, while preserving missing-clock+ // skips for legacy peers. Parseable offsets/legacy formats remain supported.+ if (!record?.updatedAt) {+ return { applied: false, reason: 'missing_timestamp' };+ }+ if (parseTsMs(record.updatedAt) === null) {+ return { applied: false, reason: 'invalid_timestamp' };+ }return store.queueRecordWrite(id, async () => {const existing = await store.loadOne(id);if (op === 'delete') {- // Require updatedAt on delete operations for last-writer-wins conflict resolution- if (!record?.updatedAt) {- return { applied: false, reason: 'missing_timestamp' };- }// LWW: skip if our copy (live record OR existing tombstone) is at least as// new as the incoming delete. The tombstone-vs-tombstone case makes a// repeated delete idempotent → not relayed → the echo loop converges.- if (existing && existing.updatedAt >= record.updatedAt) {+ if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {return rejectedAgainst(existing, record, true);}// Tombstone in place even when no local record exists. A delete that@@ -1400,18 +1406,11 @@ export async function applyRemoteRecord(type, id, record, op) {record.originInstanceId ?? existing?.originInstanceId));} else {- // A create/update with no updatedAt has no LWW clock — `existing.updatedAt- // >= undefined` is always false, which would let it silently overwrite a- // tombstone and resurrect a deleted record. Reject it (mirrors the delete- // path) so the loop-breaker can't be defeated by a timestamp-less create.- if (!record?.updatedAt) {- return { applied: false, reason: 'missing_timestamp' };- }// Guard also fires when `existing` is a tombstone — a stale create (older// updatedAt than the recorded delete) is rejected, breaking the// resurrection loop. A genuinely newer create (later updatedAt than the// tombstone) still wins and legitimately revives the record.- if (existing && existing.updatedAt >= record.updatedAt) {+ if (existing && !compareNewerWins(record.updatedAt, existing.updatedAt)) {return rejectedAgainst(existing, record, record._deleted === true);}// Defense-in-depth: a create carrying `_deleted` (a future peer, or adiff --git a/server/services/brainStorage.test.js b/server/services/brainStorage.test.jsindex d8efe09c1e..cfcadb9787 100644--- a/server/services/brainStorage.test.js+++ b/server/services/brainStorage.test.js@@ -157,6 +157,62 @@ describe('brainStorage tombstones', () => {expect(await brainStorage.getById('people', 'no-ts')).toBeNull();});+ it('rejects malformed live and delete clocks through delta and snapshot sync without relaying them', async () => {+ const created = await brainStorage.create('people', { name: 'Keep local' });+ const since = brainSyncLog.getCurrentSeq();+ const changes = [+ { op: 'update', type: 'people', id: created.id, record: { name: 'Bad', updatedAt: 'not-a-date' } },+ { op: 'delete', type: 'people', id: 'invalid-clock-delete', record: { updatedAt: 42 } },+ ];++ expect(await applyRemoteChanges(changes)).toMatchObject({ updated: 0, deleted: 0, skipped: 2 });+ expect(await applyBrainSnapshot({ records: { people: {+ [created.id]: { name: 'Bad snapshot', updatedAt: 'not-a-date' },+ 'invalid-clock-snapshot': { _deleted: true, updatedAt: { invalid: true } },+ } } })).toMatchObject({ updated: 0, deleted: 0, skipped: 2 });+ expect(await brainStorage.getById('people', created.id)).toMatchObject({ name: 'Keep local', updatedAt: created.updatedAt });+ const raw = await brainStorage.getRawRecords('people');+ expect(raw['invalid-clock-delete']).toBeUndefined();+ expect(raw['invalid-clock-snapshot']).toBeUndefined();+ expect((await brainSyncLog.getChangesSince(since)).changes).toEqual([]);+ });++ it('orders offset clocks by instant across live updates, deletes and snapshot reconciliation', async () => {+ const id = 'offset-clock-order';+ await applyRemoteChanges([{ op: 'create', type: 'people', id,+ record: { name: 'Current', updatedAt: '2026-10-15T11:00:00.000Z' } }]);++ // 12:00 +02 is 10:00 UTC, even though its text sorts after 11:00Z.+ expect(await applyRemoteChanges([+ { op: 'update', type: 'people', id, record: { name: 'Stale', updatedAt: '2026-10-15T12:00:00+02:00' } },+ { op: 'delete', type: 'people', id, record: { updatedAt: '2026-10-15T12:00:00+02:00' } },+ ])).toMatchObject({ updated: 0, deleted: 0, skipped: 2 });+ expect(await brainStorage.getById('people', id)).toMatchObject({ name: 'Current' });++ // 10:00 -02 is 12:00 UTC: this newer snapshot tombstone must win.+ expect(await applyBrainSnapshot({ records: { people: {+ [id]: { _deleted: true, updatedAt: '2026-10-15T10:00:00-02:00' },+ } } })).toMatchObject({ deleted: 1 });+ expect(await applyRemoteChanges([{ op: 'create', type: 'people', id,+ record: { name: 'Stale revival', updatedAt: '2026-10-15T11:30:00Z' } }])).toMatchObject({ skipped: 1 });+ expect(await brainStorage.getById('people', id)).toBeNull();+ expect(await applyBrainSnapshot({ records: { people: {+ [id]: { name: 'Newer revival', updatedAt: '2026-10-15T13:00:00Z' },+ } } })).toMatchObject({ updated: 1 });+ expect(await brainStorage.getById('people', id)).toMatchObject({ name: 'Newer revival' });+ });++ it('allows a valid peer clock to recover an already-persisted malformed incumbent', async () => {+ const id = 'legacy-invalid-clock';+ const dir = join(getTempRoot(), 'brain', 'people', id);+ mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, 'index.json'), JSON.stringify({ name: 'Broken legacy', updatedAt: 'not-a-date' }));++ expect(await applyRemoteChanges([{ op: 'update', type: 'people', id,+ record: { name: 'Recovered', updatedAt: '2026-10-15T00:00:00Z' } }])).toMatchObject({ updated: 1 });+ expect(await brainStorage.getById('people', id)).toMatchObject({ name: 'Recovered' });+ });+it('persists a _deleted create as a proper tombstone (defense-in-depth)', async () => {const res = await brainStorage.applyRemoteRecord('projects', 'fwd-1',diff --git a/server/services/brainSyncLog.js b/server/services/brainSyncLog.jsindex d99de895bb..47ab26e31e 100644--- a/server/services/brainSyncLog.js+++ b/server/services/brainSyncLog.js@@ -15,6 +15,7 @@ import { existsSync, createReadStream } from 'fs';import { join } from 'path';import { createMutex } from '../lib/asyncMutex.js';import { appendFileGuarded, ensureDir, safeJSONParse, PATHS, atomicWrite } from '../lib/fileUtils.js';+import { compareNewerWins } from '../lib/lwwTimestamp.js';const DATA_DIR = PATHS.brain;const SYNC_LOG_FILE = join(DATA_DIR, 'sync_log.jsonl');@@ -360,14 +361,13 @@ export async function getChangesSince(sinceSeq, limit = 100) {* returning the surviving terminal entry.** Incumbent wins ties (same timestamp) matching applyRemoteRecord.- * Entries missing updatedAt are skipped.+ * Missing/unparseable clocks lose; offsets compare by instant, as at apply time.*/function replayTerminal(entries) {let accepted = null;for (const e of [...entries].sort((a, b) => a.seq - b.seq)) {const ts = e?.record?.updatedAt;- if (ts == null) continue;- if (accepted == null || ts > accepted.record.updatedAt) accepted = e;+ if (compareNewerWins(ts, accepted?.record.updatedAt)) accepted = e;}return accepted;}@@ -475,7 +475,7 @@ export async function compactLog(minSeq = 0, { force = false } = {}) {if (tailItems) {const supersededByTail = tailItems.some(i => {const tailTs = i.entry?.record?.updatedAt;- return tailTs != null && olderWinner.record?.updatedAt != null && tailTs > olderWinner.record.updatedAt;+ return compareNewerWins(tailTs, olderWinner.record?.updatedAt);});if (supersededByTail) {continue;diff --git a/server/services/brainSyncLog.test.js b/server/services/brainSyncLog.test.jsindex 408b8825b6..257611ddb3 100644--- a/server/services/brainSyncLog.test.js+++ b/server/services/brainSyncLog.test.js@@ -566,6 +566,24 @@ describe('brainSyncLog', () => {expect(lines[0].seq).toBe(99); // max seq preserved});+ it.each([0, 50])('preserves the valid newest instant across malformed and offset clocks at floor %i', async (floor) => {+ const entries = [+ { seq: 1, op: 'create', type: 'people', id: 'clock', record: { updatedAt: '2026-10-15T09:00:00Z' } },+ { seq: 2, op: 'delete', type: 'people', id: 'clock', record: { updatedAt: '2026-10-15T11:00:00Z' } },+ { seq: 3, op: 'update', type: 'people', id: 'clock', record: { updatedAt: 'not-a-date' } },+ { seq: 99, op: 'create', type: 'people', id: 'clock', record: { updatedAt: '2026-10-15T12:00:00+02:00' } },+ ];+ writeLog(entries.map(entry => JSON.stringify(entry)).join('\n') + '\n');+ await initSyncLog();++ await compactLog(floor);++ const { changes } = await getChangesSince(0);+ expect(changes[0]).toMatchObject({ op: 'delete', record: { updatedAt: '2026-10-15T11:00:00Z' } });+ expect(changes).toHaveLength(floor === 0 ? 1 : 2);+ expect(getCurrentSeq()).toBe(99);+ });+it('preserves pre-floor LWW winner before verbatim tail when tail is stale under positive floor', async () => {// seq 1: create links x (Jan-01)// seq 2: delete links x (Jan-02) -> winning delete