Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(persistence): unwedge the sync head a chained write held back; repair the convergence kill list

garrytan2548227cf201
diff --git a/src/core/persistence/fault-points.ts b/src/core/persistence/fault-points.ts
index 5d8c1bcdc6..6c42b1a267 100644
--- a/src/core/persistence/fault-points.ts
+++ b/src/core/persistence/fault-points.ts
@@ -36,6 +36,9 @@ export type FaultPoint = `publication:${PublicationBoundary}` | `effect:${Effect
// #6712 PR 1 (L2): after the cursor step that holds a failed entry and settles its index commits (`holdUnderGate`), and after
// the cursor step that defers an entry whose file changed after admission past the manifest end (`deferChangedEntry`).
| 'sync:held_settled' | 'sync:retry_deferred'
+ // #6712 PR 4b: after the cursor step that passes over a pass's entry whose file changed after admission (the level moved; the
+ // next round classifies the new level) commits.
+ | 'sync:passed_over'
// #6712 PR 1: after a pass took the source's drain lease (before it reads the cursor); after each cursor the fan-in
// superseded is retired (before the next one), and after a pass stamped its authority on a cursor another writer started.
| 'sync:lease_acquired' | 'sync:cursor_adopted'
diff --git a/src/core/persistence/journal.ts b/src/core/persistence/journal.ts
index 566d8a1e61..6338620246 100644
--- a/src/core/persistence/journal.ts
+++ b/src/core/persistence/journal.ts
@@ -734,7 +734,10 @@ const LEASE_HELD = `EXISTS (SELECT 1 FROM gbrain_cycle_locks l WHERE ${SYNC_KIND
* the worktree after sequence `seen` (the newest sequence when this process last claimed a sync head there) earns the sync
* side one head, so a stream of writes cannot starve it. Uncorrelated with the candidate row, so a claim reads it once.
* The FIFO claim holds back only sync rows the write would pass (sequence below it): an earlier unfinished write is the
- * head-of-root rules' business, and one stalled in another process must not stop the sync behind it.
+ * head-of-root rules' business, and one stalled in another process must not stop the sync behind it. A write queued
+ * behind an earlier queued write that waits on a sync row naming its page cannot go first either (a write never passes
+ * an earlier one), so it holds back nothing: counting it held back the very sync row the earlier write waited on, and
+ * the three rows waited on each other with no foreground commit left to earn the sync side a head.
*/
const FOREGROUND_WAITING_ON = (nonce: string, seen: string, worktree = 'TRUE') => `SELECT f.worktree_id, f.sequence FROM persistence_requests f
WHERE ${worktree} AND f.state IN ('queued','running','recovering') AND (f.state<>'running' OR (NOT f.publication_started
@@ -742,6 +745,10 @@ const FOREGROUND_WAITING_ON = (nonce: string, seen: string, worktree = 'TRUE') =
AND f.recovery IS NULL AND NOT ${SYNC_KIND('f')}
AND NOT EXISTS (SELECT 1 FROM persistence_requests named WHERE named.worktree_id=f.worktree_id AND named.sequence<f.sequence
AND named.state IN ('queued','running','recovering') AND ${SYNC_KIND('named')} AND ${NAMES_PAGE('named', 'f')})
+ AND NOT EXISTS (SELECT 1 FROM persistence_requests ahead WHERE ahead.worktree_id=f.worktree_id AND ahead.sequence<f.sequence
+ AND ahead.state='queued' AND NOT ${SYNC_KIND('ahead')}
+ AND EXISTS (SELECT 1 FROM persistence_requests named WHERE named.worktree_id=ahead.worktree_id AND named.sequence<ahead.sequence
+ AND named.state IN ('queued','running','recovering') AND ${SYNC_KIND('named')} AND ${NAMES_PAGE('named', 'ahead')}))
AND NOT EXISTS (SELECT 1 FROM persistence_requests c WHERE c.sequence>COALESCE(${seen},0) AND c.worktree_id=f.worktree_id
AND c.state='committed' AND NOT ${SYNC_KIND('c')})`;
diff --git a/src/core/persistence/sync-run.ts b/src/core/persistence/sync-run.ts
index 1e9e19c083..772d94495d 100644
--- a/src/core/persistence/sync-run.ts
+++ b/src/core/persistence/sync-run.ts
@@ -1559,6 +1559,7 @@ async function runManagedSync(engine: BrainEngine, opts: SyncOpts, slice: { maxP
if (cursor.passId && ((fileChangedAfterAdmission(done) && !passedOverBefore(cursor.passId, pending.requestId)) || (done.error_code === 'page_identity_changed'
&& await liveRevision(engine, context.sourceId, pending) !== (pending.intent.expected_revision == null ? null : String(pending.intent.expected_revision))))) {
cursor = await saveCursor(engine, key, cursor, committedPast({ ...cursor, counts: { ...cursor.counts } }, pending));
+ await faultPoint('sync:passed_over', { sourceId: cursor.sourceId, path: pending.intent.path ?? undefined });
assertActive();
continue;
}
diff --git a/test/claim-fifo-foreground-waiting.test.ts b/test/claim-fifo-foreground-waiting.test.ts
index dd923863ed..6b241386cb 100644
--- a/test/claim-fifo-foreground-waiting.test.ts
+++ b/test/claim-fifo-foreground-waiting.test.ts
@@ -136,3 +136,22 @@ test('a write another process claimed and has not begun to publish holds back th
expect(await s.claim()).toBe(head);
}
}, 60_000);
+
+// The convergence nightly's hour-long wedge: a remember queued behind a put that waits on the drain's sync row naming its
+// page counted as a write that may go first, so that sync row (and every sync row below the remember) was held back, the
+// put stayed behind the sync row and the remember behind the put. No row of the three could move and no foreground commit
+// came to earn the sync side a head.
+test('a write queued behind an earlier write that waits on a sync row naming its page holds back no sync head', async () => {
+ for (const engine of engines) {
+ const s = await scene(engine);
+ const head = await s.sync('notes/synced-d');
+ const named = await s.add('notes/synced-d');
+ const behind = await s.add('notes/foreground-d');
+ s.seen[s.worktree] = await s.maxSequence();
+ expect(await s.claim()).toBe(head);
+ await s.commit(head);
+ expect(await s.claim()).toBe(named);
+ await s.commit(named);
+ expect(await s.claim()).toBe(behind);
+ }
+}, 60_000);
diff --git a/test/sync-convergence.serial.test.ts b/test/sync-convergence.serial.test.ts
index 5cbdcfc387..46764984a3 100644
--- a/test/sync-convergence.serial.test.ts
+++ b/test/sync-convergence.serial.test.ts
@@ -58,7 +58,7 @@
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
import { randomUUID } from 'node:crypto';
-import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, relative, resolve } from 'node:path';
import postgres from '#postgres';
@@ -128,18 +128,20 @@ const ENTITY = 'people/alice-example';
* file-publication pair (`before_publication`/`after_publication`) never fires for it. Per-page seams are counted
* per submit_job request (nth inside the first 60% of a round); `lane:applied` and `sync:before_group_admission`
* fire once per bulk group, and a small round may form one group, so they kill at the first; `sync:mid_checkpoint`
- * fires on every cursor save (the first or second). #6712's `sync:retry_deferred` and `sync:held_settled` need a file
- * that changes after its freeze: their round primes one with an uncommitted edit at `sync:after_freeze`, which is
- * deferred once (the first) and, when its retry finds the same newer bytes, held `worktree_dirty` (the hold); the
- * harness commits the edit after the round.
+ * fires on every cursor save (the first or second); `sync:batch_entries` fires once per pass round, after the level
+ * read and before any admission. `sync:passed_over` and `sync:held_settled` need a file that changes after its freeze:
+ * their round primes one with an uncommitted edit at `sync:after_freeze`, which the pass passes over once (the level
+ * moved) and, when the next round reads the same uncommitted bytes, holds `worktree_dirty` (the hold); the harness
+ * commits the edit after the round. Every point here must be one a pass crosses: the cursor engine's
+ * `sync:retry_deferred` and `sync:prescreened` are company-source only since #6712 PR 4b, so a non-company kill loop
+ * never reaches them.
*/
const KILL_POINTS = [
{ point: 'publication:before_commit', per: 'page' }, { point: 'publication:after_commit', per: 'page' },
{ point: 'consumer:preparing', per: 'page' }, { point: 'lane:applied', per: 'group' },
{ point: 'sync:before_group_admission', per: 'group' }, { point: 'sync:mid_checkpoint', per: 'save' },
- { point: 'sync:held_settled', per: 'held' }, { point: 'sync:retry_deferred', per: 'deferred' },
- // #6712 PR 1: inside the guarded kernel run of prescreened entries (its round also commits mode-only changes, so it has some).
- { point: 'sync:prescreened', per: 'group' },
+ { point: 'sync:held_settled', per: 'held' }, { point: 'sync:passed_over', per: 'deferred' },
+ { point: 'sync:batch_entries', per: 'group' },
// #6712 PR 2 (L5): after the checkpoint committed and the lease was released, before the owed tail runs (the next sync runs it).
{ point: 'sync:tail_started', per: 'group' },
] as const satisfies ReadonlyArray<{ point: FaultPoint; per: 'page' | 'group' | 'save' | 'held' | 'deferred' }>;
@@ -359,12 +361,6 @@ class Brain {
const ack = performance.now();
for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });
}
- /** #6712 PR 1: a mode-only commit of these notes (same blobs), which the discovery prescreen passes without an import. */
- chmodNotes(indexes: number[], message: string): void {
- const paths = indexes.map(i => `${noteSlug(i)}.md`);
- for (const path of paths) chmodSync(join(this.repo, path), 0o755);
- this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);
- }
writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {
this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);
}
@@ -875,7 +871,6 @@ scenario(['a', 'b', 'c'], 'managed sync converges after a seeded kill loop of cl
const tag = `round-${p.round}`;
const first = (p.round * perRound) % NOTES;
const indexes = Array.from({ length: perRound }, (_, i) => (first + i) % NOTES);
- if (p.point === 'sync:prescreened') b.chmodNotes(Array.from({ length: 3 }, (_, i) => (first + perRound + 1 + 2 * i) % NOTES), `round ${p.round} modes`);
const edited = p.primed ? indexes[Math.floor(perRound / 2)]! : null;
const fault: Fault = { point: p.point, nth: p.nth, marker: b.marker(tag), operation: p.operation,
...(edited !== null ? { prime: { point: 'sync:after_freeze', nth: 1, path: `${noteSlug(edited)}.md`,
@@ -899,7 +894,10 @@ scenario(['a', 'b', 'c'], 'managed sync converges after a seeded kill loop of cl
if (plan[p.round - 2]?.kind === 'client') afterClientKill.push(tag);
await serve.stop();
}
- if (edited !== null) {
+ // A prime that never fired (the round's sync never froze the entry) left nothing to commit: that is a fault that did
+ // not fire, named by the oracle with the rest of the run's evidence, not a harness crash that discards it.
+ if (edited !== null && !b.git('status', '--porcelain', '--', `${noteSlug(edited)}.md`)) b.fired[`round ${p.round} prime sync:after_freeze ${noteSlug(edited)}.md`] = null;
+ else if (edited !== null) {
const editStart = performance.now();
b.git('add', '--', `${noteSlug(edited)}.md`); b.git('commit', '-qm', `round ${p.round}: the editor commits`, '--', `${noteSlug(edited)}.md`);
b.ledger.record({ kind: 'commit', slug: noteSlug(edited), marker: noteMarker(edited, `Round ${p.round} edit`), touches: ['db', 'file'], start: editStart, ack: performance.now() });