Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(persistence): unwedge the sync head a chained write held back; repair the convergence kill list
garrytan2548227cf201
diff --git a/src/core/persistence/fault-points.ts b/src/core/persistence/fault-points.tsindex 5d8c1bcdc6..6c42b1a267 100644--- a/src/core/persistence/fault-points.ts+++ b/src/core/persistence/fault-points.ts@@ -36,6 +36,9 @@ export type FaultPoint = `publication:${PublicationBoundary}` | `effect:${Effect// #6712 PR 1 (L2): after the cursor step that holds a failed entry and settles its index commits (`holdUnderGate`), and after// the cursor step that defers an entry whose file changed after admission past the manifest end (`deferChangedEntry`).| 'sync:held_settled' | 'sync:retry_deferred'+ // #6712 PR 4b: after the cursor step that passes over a pass's entry whose file changed after admission (the level moved; the+ // next round classifies the new level) commits.+ | 'sync:passed_over'// #6712 PR 1: after a pass took the source's drain lease (before it reads the cursor); after each cursor the fan-in// superseded is retired (before the next one), and after a pass stamped its authority on a cursor another writer started.| 'sync:lease_acquired' | 'sync:cursor_adopted'diff --git a/src/core/persistence/journal.ts b/src/core/persistence/journal.tsindex 566d8a1e61..6338620246 100644--- a/src/core/persistence/journal.ts+++ b/src/core/persistence/journal.ts@@ -734,7 +734,10 @@ const LEASE_HELD = `EXISTS (SELECT 1 FROM gbrain_cycle_locks l WHERE ${SYNC_KIND* the worktree after sequence `seen` (the newest sequence when this process last claimed a sync head there) earns the sync* side one head, so a stream of writes cannot starve it. Uncorrelated with the candidate row, so a claim reads it once.* The FIFO claim holds back only sync rows the write would pass (sequence below it): an earlier unfinished write is the- * head-of-root rules' business, and one stalled in another process must not stop the sync behind it.+ * head-of-root rules' business, and one stalled in another process must not stop the sync behind it. A write queued+ * behind an earlier queued write that waits on a sync row naming its page cannot go first either (a write never passes+ * an earlier one), so it holds back nothing: counting it held back the very sync row the earlier write waited on, and+ * the three rows waited on each other with no foreground commit left to earn the sync side a head.*/const FOREGROUND_WAITING_ON = (nonce: string, seen: string, worktree = 'TRUE') => `SELECT f.worktree_id, f.sequence FROM persistence_requests fWHERE ${worktree} AND f.state IN ('queued','running','recovering') AND (f.state<>'running' OR (NOT f.publication_started@@ -742,6 +745,10 @@ const FOREGROUND_WAITING_ON = (nonce: string, seen: string, worktree = 'TRUE') =AND f.recovery IS NULL AND NOT ${SYNC_KIND('f')}AND NOT EXISTS (SELECT 1 FROM persistence_requests named WHERE named.worktree_id=f.worktree_id AND named.sequence<f.sequenceAND named.state IN ('queued','running','recovering') AND ${SYNC_KIND('named')} AND ${NAMES_PAGE('named', 'f')})+ AND NOT EXISTS (SELECT 1 FROM persistence_requests ahead WHERE ahead.worktree_id=f.worktree_id AND ahead.sequence<f.sequence+ AND ahead.state='queued' AND NOT ${SYNC_KIND('ahead')}+ AND EXISTS (SELECT 1 FROM persistence_requests named WHERE named.worktree_id=ahead.worktree_id AND named.sequence<ahead.sequence+ AND named.state IN ('queued','running','recovering') AND ${SYNC_KIND('named')} AND ${NAMES_PAGE('named', 'ahead')}))AND NOT EXISTS (SELECT 1 FROM persistence_requests c WHERE c.sequence>COALESCE(${seen},0) AND c.worktree_id=f.worktree_idAND c.state='committed' AND NOT ${SYNC_KIND('c')})`;diff --git a/src/core/persistence/sync-run.ts b/src/core/persistence/sync-run.tsindex 1e9e19c083..772d94495d 100644--- a/src/core/persistence/sync-run.ts+++ b/src/core/persistence/sync-run.ts@@ -1559,6 +1559,7 @@ async function runManagedSync(engine: BrainEngine, opts: SyncOpts, slice: { maxPif (cursor.passId && ((fileChangedAfterAdmission(done) && !passedOverBefore(cursor.passId, pending.requestId)) || (done.error_code === 'page_identity_changed'&& await liveRevision(engine, context.sourceId, pending) !== (pending.intent.expected_revision == null ? null : String(pending.intent.expected_revision))))) {cursor = await saveCursor(engine, key, cursor, committedPast({ ...cursor, counts: { ...cursor.counts } }, pending));+ await faultPoint('sync:passed_over', { sourceId: cursor.sourceId, path: pending.intent.path ?? undefined });assertActive();continue;}diff --git a/test/claim-fifo-foreground-waiting.test.ts b/test/claim-fifo-foreground-waiting.test.tsindex dd923863ed..6b241386cb 100644--- a/test/claim-fifo-foreground-waiting.test.ts+++ b/test/claim-fifo-foreground-waiting.test.ts@@ -136,3 +136,22 @@ test('a write another process claimed and has not begun to publish holds back thexpect(await s.claim()).toBe(head);}}, 60_000);++// The convergence nightly's hour-long wedge: a remember queued behind a put that waits on the drain's sync row naming its+// page counted as a write that may go first, so that sync row (and every sync row below the remember) was held back, the+// put stayed behind the sync row and the remember behind the put. No row of the three could move and no foreground commit+// came to earn the sync side a head.+test('a write queued behind an earlier write that waits on a sync row naming its page holds back no sync head', async () => {+ for (const engine of engines) {+ const s = await scene(engine);+ const head = await s.sync('notes/synced-d');+ const named = await s.add('notes/synced-d');+ const behind = await s.add('notes/foreground-d');+ s.seen[s.worktree] = await s.maxSequence();+ expect(await s.claim()).toBe(head);+ await s.commit(head);+ expect(await s.claim()).toBe(named);+ await s.commit(named);+ expect(await s.claim()).toBe(behind);+ }+}, 60_000);diff --git a/test/sync-convergence.serial.test.ts b/test/sync-convergence.serial.test.tsindex 5cbdcfc387..46764984a3 100644--- a/test/sync-convergence.serial.test.ts+++ b/test/sync-convergence.serial.test.ts@@ -58,7 +58,7 @@import { afterAll, beforeAll, describe, expect, test } from 'bun:test';import { execFileSync, spawn, type ChildProcess } from 'node:child_process';import { randomUUID } from 'node:crypto';-import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { dirname, join, relative, resolve } from 'node:path';import postgres from '#postgres';@@ -128,18 +128,20 @@ const ENTITY = 'people/alice-example';* file-publication pair (`before_publication`/`after_publication`) never fires for it. Per-page seams are counted* per submit_job request (nth inside the first 60% of a round); `lane:applied` and `sync:before_group_admission`* fire once per bulk group, and a small round may form one group, so they kill at the first; `sync:mid_checkpoint`- * fires on every cursor save (the first or second). #6712's `sync:retry_deferred` and `sync:held_settled` need a file- * that changes after its freeze: their round primes one with an uncommitted edit at `sync:after_freeze`, which is- * deferred once (the first) and, when its retry finds the same newer bytes, held `worktree_dirty` (the hold); the- * harness commits the edit after the round.+ * fires on every cursor save (the first or second); `sync:batch_entries` fires once per pass round, after the level+ * read and before any admission. `sync:passed_over` and `sync:held_settled` need a file that changes after its freeze:+ * their round primes one with an uncommitted edit at `sync:after_freeze`, which the pass passes over once (the level+ * moved) and, when the next round reads the same uncommitted bytes, holds `worktree_dirty` (the hold); the harness+ * commits the edit after the round. Every point here must be one a pass crosses: the cursor engine's+ * `sync:retry_deferred` and `sync:prescreened` are company-source only since #6712 PR 4b, so a non-company kill loop+ * never reaches them.*/const KILL_POINTS = [{ point: 'publication:before_commit', per: 'page' }, { point: 'publication:after_commit', per: 'page' },{ point: 'consumer:preparing', per: 'page' }, { point: 'lane:applied', per: 'group' },{ point: 'sync:before_group_admission', per: 'group' }, { point: 'sync:mid_checkpoint', per: 'save' },- { point: 'sync:held_settled', per: 'held' }, { point: 'sync:retry_deferred', per: 'deferred' },- // #6712 PR 1: inside the guarded kernel run of prescreened entries (its round also commits mode-only changes, so it has some).- { point: 'sync:prescreened', per: 'group' },+ { point: 'sync:held_settled', per: 'held' }, { point: 'sync:passed_over', per: 'deferred' },+ { point: 'sync:batch_entries', per: 'group' },// #6712 PR 2 (L5): after the checkpoint committed and the lease was released, before the owed tail runs (the next sync runs it).{ point: 'sync:tail_started', per: 'group' },] as const satisfies ReadonlyArray<{ point: FaultPoint; per: 'page' | 'group' | 'save' | 'held' | 'deferred' }>;@@ -359,12 +361,6 @@ class Brain {const ack = performance.now();for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });}- /** #6712 PR 1: a mode-only commit of these notes (same blobs), which the discovery prescreen passes without an import. */- chmodNotes(indexes: number[], message: string): void {- const paths = indexes.map(i => `${noteSlug(i)}.md`);- for (const path of paths) chmodSync(join(this.repo, path), 0o755);- this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);- }writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);}@@ -875,7 +871,6 @@ scenario(['a', 'b', 'c'], 'managed sync converges after a seeded kill loop of clconst tag = `round-${p.round}`;const first = (p.round * perRound) % NOTES;const indexes = Array.from({ length: perRound }, (_, i) => (first + i) % NOTES);- if (p.point === 'sync:prescreened') b.chmodNotes(Array.from({ length: 3 }, (_, i) => (first + perRound + 1 + 2 * i) % NOTES), `round ${p.round} modes`);const edited = p.primed ? indexes[Math.floor(perRound / 2)]! : null;const fault: Fault = { point: p.point, nth: p.nth, marker: b.marker(tag), operation: p.operation,...(edited !== null ? { prime: { point: 'sync:after_freeze', nth: 1, path: `${noteSlug(edited)}.md`,@@ -899,7 +894,10 @@ scenario(['a', 'b', 'c'], 'managed sync converges after a seeded kill loop of clif (plan[p.round - 2]?.kind === 'client') afterClientKill.push(tag);await serve.stop();}- if (edited !== null) {+ // A prime that never fired (the round's sync never froze the entry) left nothing to commit: that is a fault that did+ // not fire, named by the oracle with the rest of the run's evidence, not a harness crash that discards it.+ if (edited !== null && !b.git('status', '--porcelain', '--', `${noteSlug(edited)}.md`)) b.fired[`round ${p.round} prime sync:after_freeze ${noteSlug(edited)}.md`] = null;+ else if (edited !== null) {const editStart = performance.now();b.git('add', '--', `${noteSlug(edited)}.md`); b.git('commit', '-qm', `round ${p.round}: the editor commits`, '--', `${noteSlug(edited)}.md`);b.ledger.record({ kind: 'commit', slug: noteSlug(edited), marker: noteMarker(edited, `Round ${p.round} edit`), touches: ['db', 'file'], start: editStart, ack: performance.now() });