Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(sync): a stop never rewrites a pass row its pass already ended; a failed open-row re-read decides nothing
garrytanb3a4a172e267
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ad7f0a1a15..44d6d83fd7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -26,7 +26,7 @@ gbrain doctor --only managed_sync_blocked### Itemized changes- Sync pass and lease readers (`sync-status`, doctor `managed_sync_blocked`): one observation per decision, database-clock lock liveness, `openPassRowsStill`, one `inspectLocks` query for every open-pass source (#7611, #7614).-- `resumableRowStop`: transient and lease-lost stops end the pass row `resumable`, while a permission refusal keeps the row open so it still reads interrupted, then abandoned; a lost-lease row is written under its own lease token only (#7614).+- `resumableRowStop`: transient and lease-lost stops end the pass row `resumable`, while a permission refusal keeps the row open so it still reads interrupted, then abandoned; a lost-lease row is written under its own lease token only, and a stop never rewrites a row its pass already ended (a folded stamp's `synced` row keeps its watermark) (#7614).- `readLiveDrainHolder`: the notifier block, the on-commit notifier and `sources status` `writer_heartbeat` read the drain holder only while it renews (#7614).- Lean read-only Postgres serve: a source-keyed read socket; hooks try source, then read, then legacy; delegated sync never targets the read socket (#7612).- `EffectWorkerOptions.lockWaitMs` (test seam, default unchanged) cuts the w5 contention scenario from 37 s to 2.5 s (#7782).diff --git a/src/core/persistence/sync-blocked.ts b/src/core/persistence/sync-blocked.tsindex 1475134b74..ef80d96be3 100644--- a/src/core/persistence/sync-blocked.ts+++ b/src/core/persistence/sync-blocked.ts@@ -68,7 +68,8 @@ export async function readBlockedManagedSyncs(engine: BrainEngine, sourceIds?: sreturn { source_id, row, interrupted, abandoned: abandonedPasses(row, now, interrupted) >= ABANDONED_REPORT_STREAK };}).filter(entry => entry.interrupted || entry.abandoned);// #7614: a pass that ended (releasing its lease) after the pass read changed its row, so it is not listed; nor is an archived source.- const still = candidates.length ? await openPassRowsStill(engine, candidates) : new Set<string>();+ // A re-read that failed lists nothing new.+ const still = candidates.length ? await openPassRowsStill(engine, candidates).catch(() => new Set<string>()) : new Set<string>();for (const { source_id, row, abandoned } of candidates) {if (still.has(source_id)) listed.push({ source_id, code: abandoned ? 'pass_abandoned' : 'pass_interrupted', run_id: row.value.telemetry.pass_id, blocked_since: abandoned ? passStuckSince(row) : row.updated_at });}diff --git a/src/core/persistence/sync-pass-row.ts b/src/core/persistence/sync-pass-row.tsindex 40d8040172..da23cd26a1 100644--- a/src/core/persistence/sync-pass-row.ts+++ b/src/core/persistence/sync-pass-row.ts@@ -166,14 +166,16 @@ export async function readLivePassRows(engine: Exec, sourceIds?: readonly string* Writes the pass row inside `tx` (the caller has run `assertDrainLease` in the same transaction). `pin`: the pass that* just took the lease replaces whatever row a former holder left; otherwise the write lands only while the row still* carries `leaseToken`. Returns false when it did not land (another pass pinned since). `clock`: `telemetry.started_at` is the- * transaction's clock (`now()`, as an ISO string), so the write needs no earlier read of it.+ * transaction's clock (`now()`, as an ISO string), so the write needs no earlier read of it. `onlyOpen`: the write lands only+ * on a row still `open`, so a stop after a folded stamp ended the row `synced` never rewrites the stamp's obligations.*/-export async function writePassRow(tx: Exec, value: ManagedSyncPassValue, opts: { pin?: boolean; clock?: boolean } = {}): Promise<boolean> {+export async function writePassRow(tx: Exec, value: ManagedSyncPassValue, opts: { pin?: boolean; clock?: boolean; onlyOpen?: boolean } = {}): Promise<boolean> {const json = opts.clock ? `jsonb_set($3::text::jsonb,'{telemetry,started_at}',to_jsonb(to_char(now() AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')))` : '$3::text::jsonb';const rows = await tx.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES($1,$2,jsonb_build_array(${json}),now())ON CONFLICT (op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys, updated_at=now()- WHERE $4::boolean OR op_checkpoints.completed_keys->0->>'lease_token'=$5 RETURNING 1`,- [MANAGED_SYNC_PASS_OP, passFingerprint(value.source_id, value.incarnation), JSON.stringify(value), opts.pin === true, value.lease_token]);+ WHERE ($4::boolean OR op_checkpoints.completed_keys->0->>'lease_token'=$5)+ AND (NOT $6::boolean OR op_checkpoints.completed_keys->0->'telemetry'->>'outcome'='open') RETURNING 1`,+ [MANAGED_SYNC_PASS_OP, passFingerprint(value.source_id, value.incarnation), JSON.stringify(value), opts.pin === true, value.lease_token, opts.onlyOpen === true]);return rows.length > 0;}diff --git a/src/core/persistence/sync-pass-run.ts b/src/core/persistence/sync-pass-run.tsindex 17deeb9594..1fd45ff4cf 100644--- a/src/core/persistence/sync-pass-run.ts+++ b/src/core/persistence/sync-pass-run.ts@@ -760,7 +760,7 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Prconst stampFor = (level: ShadowLevel) => passStampFor(engine, { context, head, slugMode: levelSlugMode, episode, selection, configured, mode, leaseToken: lease.token, retry, level, selectionFp, parserFp,pack, full: kind === 'full', counts, extract: !processingOptions.noExtract, since: value.telemetry.started_at, rescreen: legacyWaived.size > 0, closures });- const save = async (outcome: PassOutcome, stopReason: string | null, pin = false, fenced = true) => {+ const save = async (outcome: PassOutcome, stopReason: string | null, pin = false, fenced = true, onlyOpen = false) => {await beat.pending; const ended = outcome !== 'open';if (ended && outcome !== 'synced') {const open = await openPassRequests(engine, { sourceId, passId, carried: value.obligations.pending_requests ?? [] });@@ -770,7 +770,7 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Prclasses: applied.classes, ...(applied.summary ? { summary: applied.summary } : {}),...(ended ? { ended_at: new Date().toISOString(), duration_ms: Date.now() - started } : {}) };if (outcome === 'blocked' || stopReason === 'not_converged') value.obligations.stuck_since ??= new Date().toISOString();- await engine.transaction(async tx => { if (fenced) await assertDrainLease(tx, sourceId); await writePassRow(tx, value, { pin }); });+ await engine.transaction(async tx => { if (fenced) await assertDrainLease(tx, sourceId); await writePassRow(tx, value, { pin, onlyOpen }); });};// GBRA-108: a committed legacy-sweep slice is progress the row and the drain's governor see; #7355: so is a level read still working (`levelBeat`).const onSwept = async () => { assertActive(); await save('open', null); opts.onProgress?.({ phase: 'managed_sync.legacy_swept', bankedFiles: sliceBase.pages }); }, beat = levelBeat(opts, sliceBase, () => save('open', null));@@ -998,11 +998,11 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Prif (input.signal?.aborted && error instanceof Error && error.name === 'AbortError') {// A folded stamp with nothing since ended the pass: the abort stops nothing it owes.if (folded?.done === doneKey()) return synced(true);- await save('resumable', 'timeout').catch(() => undefined); return result('partial', 'timeout');+ await save('resumable', 'timeout', false, true, true).catch(() => undefined); return result('partial', 'timeout');}// A Git listing that did not finish cleanly classifies nothing (design v3.2 §9): the pass stops blocked, having written nothing for that round.if (error instanceof SyncTreeReadIncomplete) { await save('blocked', 'reconcile_read_incomplete').catch(() => undefined); return result('blocked_by_failures', 'reconcile_read_incomplete'); }- const stop = resumableRowStop(error); if (stop) await save('resumable', stop, false, stop !== 'lease_lost').catch(() => undefined); throw error;+ const stop = resumableRowStop(error); if (stop) await save('resumable', stop, false, stop !== 'lease_lost', true).catch(() => undefined); throw error;} finally {unregisterPassCost(passId);}diff --git a/src/core/persistence/sync-status.ts b/src/core/persistence/sync-status.tsindex 4eab3232b9..a962f4e027 100644--- a/src/core/persistence/sync-status.ts+++ b/src/core/persistence/sync-status.ts@@ -414,7 +414,9 @@ export async function readSyncStatus(engine: BrainEngine, sourceId: string, optsconst lease = source.drain_lock ? await readDrainLease(engine, sourceId).catch(() => undefined) : null;const holder = lease?.live ? lease.holder : null;// #7614: a pass that ended and released its lease between the source read and the lease read is not interrupted: the open row must still be the one read above.- const raced = source.drain_lock && lease !== undefined && !holder && passRow?.value.telemetry.outcome === 'open' && !(await openPassRowsStill(engine, [{ source_id: sourceId, row: passRow }])).has(sourceId);+ // A re-read that failed decides nothing either: the pass keeps the quiet-window rule.+ const raced = source.drain_lock && lease !== undefined && !holder && passRow?.value.telemetry.outcome === 'open'+ && !(await openPassRowsStill(engine, [{ source_id: sourceId, row: passRow }]).catch(() => undefined))?.has(sourceId);const interrupted = lease === undefined || raced || passRow?.value.telemetry.outcome !== 'open' ? undefined : !holder;const passState = passRow ? passStatusState(passRow, withheld, source.writeback_failed, Date.now(), interrupted) : null;const resume = !header && passRow ? passResumeArgs(sourceId, passRow) : managedSyncResumeArgs({ sourceId, processingOptions: header?.processingOptions, syncOptions: header?.syncOptions });diff --git a/test/e2e/serve-read-only-session-lane-postgres.test.ts b/test/e2e/serve-read-only-session-lane-postgres.test.tsindex 0a13977238..5866d3ccd8 100644--- a/test/e2e/serve-read-only-session-lane-postgres.test.ts+++ b/test/e2e/serve-read-only-session-lane-postgres.test.ts@@ -122,7 +122,7 @@ describe.skipIf(!DATABASE_URL)('#7599 serve --access read-only on the session laheld();await new Promise<void>(resolve => { release = resolve; });});- await brainHeld;+ await Promise.race([brainHeld, holding]);let n = 0;let s: Awaited<ReturnType<typeof startServe>>;try {@@ -140,7 +140,7 @@ describe.skipIf(!DATABASE_URL)('#7599 serve --access read-only on the session lafor (let i = 0; i < 60 && !s.log().includes("owner of this host's notifier"); i++) await Bun.sleep(500);expect(s.log()).toContain("owner of this host's notifier");await stopServe();- }, 120_000);+ }, 150_000);test('#7612: a read-write serve started after a read-only one for the same source answers delegated sync, not unsupported_kind', async () => {const cfg = { engine: 'postgres' as const, database_url: directBase };diff --git a/test/sync-pass-fold-stop-7652.test.ts b/test/sync-pass-fold-stop-7652.test.tsnew file mode 100644index 0000000000..9f8f0dccf7--- /dev/null+++ b/test/sync-pass-fold-stop-7652.test.ts@@ -0,0 +1,50 @@+/**+ * #7652 review: a folded stamp commits inside the round's last import and ends the pass row `synced` with the new+ * `converged` watermark. A transient failure after that commit, before the pass adopts the stamp, must not end the row+ * again from the stale in-memory value (`resumable`, the previous target, the writeback list gone).+ */+import { expect, spyOn, test } from 'bun:test';+import { execFileSync } from 'node:child_process';+import { writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { performManagedSync } from '../src/core/persistence/sync-run.ts';+import * as passRowModule from '../src/core/persistence/sync-pass-row.ts';+import { readSyncStatus } from '../src/core/persistence/sync-status.ts';+import { managedBrain } from './helpers/managed-brain.ts';++const readPassRow = passRowModule.readPassRow;+const git = (root: string, args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args],+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();+const commitFile = (root: string, name: string) => {+ writeFileSync(join(root, name), `---\ntype: concept\ntitle: ${name}\n---\n\nA note about ${name}.\n`);+ git(root, ['add', '-A']); git(root, ['commit', '-qm', name]);+};++test('a transient stop after a folded stamp committed leaves the synced pass row and its watermark', () => managedBrain(async ({ engine, root }) => {+ const sync = () => performManagedSync(engine, { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true });+ await sync();+ const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation::text AS incarnation FROM sources WHERE id='default'");+ const firstPass = (await readPassRow(engine, 'default', incarnation!))!.value.telemetry.pass_id;+ commitFile(root, 'b-example.md');+ const head = git(root, ['rev-parse', 'HEAD']);+ let thrown = false;+ const spy = spyOn(passRowModule, 'readPassRow').mockImplementation((async (...args: Parameters<typeof readPassRow>) => {+ const row = await readPassRow(...args);+ if (!thrown && row?.value.telemetry.outcome === 'synced' && row.value.telemetry.pass_id !== firstPass) {+ thrown = true;+ throw Object.assign(new Error('Connection terminated unexpectedly'), { code: '08006' });+ }+ return row;+ }) as typeof readPassRow);+ try {+ await expect(sync()).rejects.toThrow('Connection terminated unexpectedly');+ } finally {+ spy.mockRestore();+ }+ expect(thrown).toBe(true);+ const after = (await readPassRow(engine, 'default', incarnation!))!.value;+ expect(after.telemetry.outcome).toBe('synced');+ expect(after.obligations.converged?.target).toBe(head);+ const status = await readSyncStatus(engine, 'default', { detail: false });+ expect([status.state, status.reason]).toEqual(['synced', 'nothing_pending']);+}, { setup: ({ root }) => { git(root, ['init', '-q']); commitFile(root, 'a-example.md'); } }), 120_000);diff --git a/test/sync-pass-readers-7614.test.ts b/test/sync-pass-readers-7614.test.tsindex 05e2ca3883..3e8b5e0bac 100644--- a/test/sync-pass-readers-7614.test.ts+++ b/test/sync-pass-readers-7614.test.ts@@ -25,6 +25,7 @@ import { readAppliedWatermark } from '../src/core/persistence/applied-watermark.import { installFaultHook } from '../src/core/persistence/fault-points.ts';import { readBlockedManagedSyncs } from '../src/core/persistence/sync-blocked.ts';import { readSourceMovement } from '../src/core/persistence/sync-movement.ts';+import * as passStatus from '../src/core/persistence/sync-pass-status.ts';import { abandonedPasses, passStatusState } from '../src/core/persistence/sync-pass-status.ts';import { MANAGED_SYNC_PASS_OP, type PassRow, readPassRow } from '../src/core/persistence/sync-pass-row.ts';import { performManagedSync } from '../src/core/persistence/sync-run.ts';@@ -133,6 +134,15 @@ for (const backend of testBackends()) {} finally { for (const spy of failed) spy.mockRestore(); }});+ synced('#7652 review: a failed re-read of the open pass row decides nothing in status and lists nothing in doctor', async ({ engine, status, blocked }) => {+ await setPass(engine, 'open', 0); await dead(engine);+ const failed = spyOn(passStatus, 'openPassRowsStill').mockRejectedValue(new Error('pass row read failed'));+ try {+ expect((await status()).reason).toBe('pass_unfinished');+ expect(await blocked()).toEqual([]);+ } finally { failed.mockRestore(); }+ });+synced('#7614: doctor\'s message for killed passes only, the human INTERRUPTED line and the sync_blocked health band', async ({ engine }) => {// Killed passes only: doctor does not say freshness jobs end dead (a resume over a killed pass completes).await setPass(engine, 'open', 1); await unlock(engine);