Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(sync): a stop never rewrites a pass row its pass already ended; a failed open-row re-read decides nothing

garrytanb3a4a172e267
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ad7f0a1a15..44d6d83fd7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,7 +26,7 @@ gbrain doctor --only managed_sync_blocked
### Itemized changes
- Sync pass and lease readers (`sync-status`, doctor `managed_sync_blocked`): one observation per decision, database-clock lock liveness, `openPassRowsStill`, one `inspectLocks` query for every open-pass source (#7611, #7614).
-- `resumableRowStop`: transient and lease-lost stops end the pass row `resumable`, while a permission refusal keeps the row open so it still reads interrupted, then abandoned; a lost-lease row is written under its own lease token only (#7614).
+- `resumableRowStop`: transient and lease-lost stops end the pass row `resumable`, while a permission refusal keeps the row open so it still reads interrupted, then abandoned; a lost-lease row is written under its own lease token only, and a stop never rewrites a row its pass already ended (a folded stamp's `synced` row keeps its watermark) (#7614).
- `readLiveDrainHolder`: the notifier block, the on-commit notifier and `sources status` `writer_heartbeat` read the drain holder only while it renews (#7614).
- Lean read-only Postgres serve: a source-keyed read socket; hooks try source, then read, then legacy; delegated sync never targets the read socket (#7612).
- `EffectWorkerOptions.lockWaitMs` (test seam, default unchanged) cuts the w5 contention scenario from 37 s to 2.5 s (#7782).
diff --git a/src/core/persistence/sync-blocked.ts b/src/core/persistence/sync-blocked.ts
index 1475134b74..ef80d96be3 100644
--- a/src/core/persistence/sync-blocked.ts
+++ b/src/core/persistence/sync-blocked.ts
@@ -68,7 +68,8 @@ export async function readBlockedManagedSyncs(engine: BrainEngine, sourceIds?: s
return { source_id, row, interrupted, abandoned: abandonedPasses(row, now, interrupted) >= ABANDONED_REPORT_STREAK };
}).filter(entry => entry.interrupted || entry.abandoned);
// #7614: a pass that ended (releasing its lease) after the pass read changed its row, so it is not listed; nor is an archived source.
- const still = candidates.length ? await openPassRowsStill(engine, candidates) : new Set<string>();
+ // A re-read that failed lists nothing new.
+ const still = candidates.length ? await openPassRowsStill(engine, candidates).catch(() => new Set<string>()) : new Set<string>();
for (const { source_id, row, abandoned } of candidates) {
if (still.has(source_id)) listed.push({ source_id, code: abandoned ? 'pass_abandoned' : 'pass_interrupted', run_id: row.value.telemetry.pass_id, blocked_since: abandoned ? passStuckSince(row) : row.updated_at });
}
diff --git a/src/core/persistence/sync-pass-row.ts b/src/core/persistence/sync-pass-row.ts
index 40d8040172..da23cd26a1 100644
--- a/src/core/persistence/sync-pass-row.ts
+++ b/src/core/persistence/sync-pass-row.ts
@@ -166,14 +166,16 @@ export async function readLivePassRows(engine: Exec, sourceIds?: readonly string
* Writes the pass row inside `tx` (the caller has run `assertDrainLease` in the same transaction). `pin`: the pass that
* just took the lease replaces whatever row a former holder left; otherwise the write lands only while the row still
* carries `leaseToken`. Returns false when it did not land (another pass pinned since). `clock`: `telemetry.started_at` is the
- * transaction's clock (`now()`, as an ISO string), so the write needs no earlier read of it.
+ * transaction's clock (`now()`, as an ISO string), so the write needs no earlier read of it. `onlyOpen`: the write lands only
+ * on a row still `open`, so a stop after a folded stamp ended the row `synced` never rewrites the stamp's obligations.
*/
-export async function writePassRow(tx: Exec, value: ManagedSyncPassValue, opts: { pin?: boolean; clock?: boolean } = {}): Promise<boolean> {
+export async function writePassRow(tx: Exec, value: ManagedSyncPassValue, opts: { pin?: boolean; clock?: boolean; onlyOpen?: boolean } = {}): Promise<boolean> {
const json = opts.clock ? `jsonb_set($3::text::jsonb,'{telemetry,started_at}',to_jsonb(to_char(now() AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')))` : '$3::text::jsonb';
const rows = await tx.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES($1,$2,jsonb_build_array(${json}),now())
ON CONFLICT (op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys, updated_at=now()
- WHERE $4::boolean OR op_checkpoints.completed_keys->0->>'lease_token'=$5 RETURNING 1`,
- [MANAGED_SYNC_PASS_OP, passFingerprint(value.source_id, value.incarnation), JSON.stringify(value), opts.pin === true, value.lease_token]);
+ WHERE ($4::boolean OR op_checkpoints.completed_keys->0->>'lease_token'=$5)
+ AND (NOT $6::boolean OR op_checkpoints.completed_keys->0->'telemetry'->>'outcome'='open') RETURNING 1`,
+ [MANAGED_SYNC_PASS_OP, passFingerprint(value.source_id, value.incarnation), JSON.stringify(value), opts.pin === true, value.lease_token, opts.onlyOpen === true]);
return rows.length > 0;
}
diff --git a/src/core/persistence/sync-pass-run.ts b/src/core/persistence/sync-pass-run.ts
index 17deeb9594..1fd45ff4cf 100644
--- a/src/core/persistence/sync-pass-run.ts
+++ b/src/core/persistence/sync-pass-run.ts
@@ -760,7 +760,7 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Pr
const stampFor = (level: ShadowLevel) => passStampFor(engine, { context, head, slugMode: levelSlugMode, episode, selection, configured, mode, leaseToken: lease.token, retry, level, selectionFp, parserFp,
pack, full: kind === 'full', counts, extract: !processingOptions.noExtract, since: value.telemetry.started_at, rescreen: legacyWaived.size > 0, closures });
- const save = async (outcome: PassOutcome, stopReason: string | null, pin = false, fenced = true) => {
+ const save = async (outcome: PassOutcome, stopReason: string | null, pin = false, fenced = true, onlyOpen = false) => {
await beat.pending; const ended = outcome !== 'open';
if (ended && outcome !== 'synced') {
const open = await openPassRequests(engine, { sourceId, passId, carried: value.obligations.pending_requests ?? [] });
@@ -770,7 +770,7 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Pr
classes: applied.classes, ...(applied.summary ? { summary: applied.summary } : {}),
...(ended ? { ended_at: new Date().toISOString(), duration_ms: Date.now() - started } : {}) };
if (outcome === 'blocked' || stopReason === 'not_converged') value.obligations.stuck_since ??= new Date().toISOString();
- await engine.transaction(async tx => { if (fenced) await assertDrainLease(tx, sourceId); await writePassRow(tx, value, { pin }); });
+ await engine.transaction(async tx => { if (fenced) await assertDrainLease(tx, sourceId); await writePassRow(tx, value, { pin, onlyOpen }); });
};
// GBRA-108: a committed legacy-sweep slice is progress the row and the drain's governor see; #7355: so is a level read still working (`levelBeat`).
const onSwept = async () => { assertActive(); await save('open', null); opts.onProgress?.({ phase: 'managed_sync.legacy_swept', bankedFiles: sliceBase.pages }); }, beat = levelBeat(opts, sliceBase, () => save('open', null));
@@ -998,11 +998,11 @@ export async function runSyncPass(engine: BrainEngine, input: SyncPassInput): Pr
if (input.signal?.aborted && error instanceof Error && error.name === 'AbortError') {
// A folded stamp with nothing since ended the pass: the abort stops nothing it owes.
if (folded?.done === doneKey()) return synced(true);
- await save('resumable', 'timeout').catch(() => undefined); return result('partial', 'timeout');
+ await save('resumable', 'timeout', false, true, true).catch(() => undefined); return result('partial', 'timeout');
}
// A Git listing that did not finish cleanly classifies nothing (design v3.2 §9): the pass stops blocked, having written nothing for that round.
if (error instanceof SyncTreeReadIncomplete) { await save('blocked', 'reconcile_read_incomplete').catch(() => undefined); return result('blocked_by_failures', 'reconcile_read_incomplete'); }
- const stop = resumableRowStop(error); if (stop) await save('resumable', stop, false, stop !== 'lease_lost').catch(() => undefined); throw error;
+ const stop = resumableRowStop(error); if (stop) await save('resumable', stop, false, stop !== 'lease_lost', true).catch(() => undefined); throw error;
} finally {
unregisterPassCost(passId);
}
diff --git a/src/core/persistence/sync-status.ts b/src/core/persistence/sync-status.ts
index 4eab3232b9..a962f4e027 100644
--- a/src/core/persistence/sync-status.ts
+++ b/src/core/persistence/sync-status.ts
@@ -414,7 +414,9 @@ export async function readSyncStatus(engine: BrainEngine, sourceId: string, opts
const lease = source.drain_lock ? await readDrainLease(engine, sourceId).catch(() => undefined) : null;
const holder = lease?.live ? lease.holder : null;
// #7614: a pass that ended and released its lease between the source read and the lease read is not interrupted: the open row must still be the one read above.
- const raced = source.drain_lock && lease !== undefined && !holder && passRow?.value.telemetry.outcome === 'open' && !(await openPassRowsStill(engine, [{ source_id: sourceId, row: passRow }])).has(sourceId);
+ // A re-read that failed decides nothing either: the pass keeps the quiet-window rule.
+ const raced = source.drain_lock && lease !== undefined && !holder && passRow?.value.telemetry.outcome === 'open'
+ && !(await openPassRowsStill(engine, [{ source_id: sourceId, row: passRow }]).catch(() => undefined))?.has(sourceId);
const interrupted = lease === undefined || raced || passRow?.value.telemetry.outcome !== 'open' ? undefined : !holder;
const passState = passRow ? passStatusState(passRow, withheld, source.writeback_failed, Date.now(), interrupted) : null;
const resume = !header && passRow ? passResumeArgs(sourceId, passRow) : managedSyncResumeArgs({ sourceId, processingOptions: header?.processingOptions, syncOptions: header?.syncOptions });
diff --git a/test/e2e/serve-read-only-session-lane-postgres.test.ts b/test/e2e/serve-read-only-session-lane-postgres.test.ts
index 0a13977238..5866d3ccd8 100644
--- a/test/e2e/serve-read-only-session-lane-postgres.test.ts
+++ b/test/e2e/serve-read-only-session-lane-postgres.test.ts
@@ -122,7 +122,7 @@ describe.skipIf(!DATABASE_URL)('#7599 serve --access read-only on the session la
held();
await new Promise<void>(resolve => { release = resolve; });
});
- await brainHeld;
+ await Promise.race([brainHeld, holding]);
let n = 0;
let s: Awaited<ReturnType<typeof startServe>>;
try {
@@ -140,7 +140,7 @@ describe.skipIf(!DATABASE_URL)('#7599 serve --access read-only on the session la
for (let i = 0; i < 60 && !s.log().includes("owner of this host's notifier"); i++) await Bun.sleep(500);
expect(s.log()).toContain("owner of this host's notifier");
await stopServe();
- }, 120_000);
+ }, 150_000);
test('#7612: a read-write serve started after a read-only one for the same source answers delegated sync, not unsupported_kind', async () => {
const cfg = { engine: 'postgres' as const, database_url: directBase };
diff --git a/test/sync-pass-fold-stop-7652.test.ts b/test/sync-pass-fold-stop-7652.test.ts
new file mode 100644
index 0000000000..9f8f0dccf7
--- /dev/null
+++ b/test/sync-pass-fold-stop-7652.test.ts
@@ -0,0 +1,50 @@
+/**
+ * #7652 review: a folded stamp commits inside the round's last import and ends the pass row `synced` with the new
+ * `converged` watermark. A transient failure after that commit, before the pass adopts the stamp, must not end the row
+ * again from the stale in-memory value (`resumable`, the previous target, the writeback list gone).
+ */
+import { expect, spyOn, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { performManagedSync } from '../src/core/persistence/sync-run.ts';
+import * as passRowModule from '../src/core/persistence/sync-pass-row.ts';
+import { readSyncStatus } from '../src/core/persistence/sync-status.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+
+const readPassRow = passRowModule.readPassRow;
+const git = (root: string, args: string[]) => execFileSync('git', ['-C', root, '-c', 'user.name=Example', '-c', '[email protected]', '-c', 'commit.gpgsign=false', ...args],
+ { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+const commitFile = (root: string, name: string) => {
+ writeFileSync(join(root, name), `---\ntype: concept\ntitle: ${name}\n---\n\nA note about ${name}.\n`);
+ git(root, ['add', '-A']); git(root, ['commit', '-qm', name]);
+};
+
+test('a transient stop after a folded stamp committed leaves the synced pass row and its watermark', () => managedBrain(async ({ engine, root }) => {
+ const sync = () => performManagedSync(engine, { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true });
+ await sync();
+ const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>("SELECT incarnation::text AS incarnation FROM sources WHERE id='default'");
+ const firstPass = (await readPassRow(engine, 'default', incarnation!))!.value.telemetry.pass_id;
+ commitFile(root, 'b-example.md');
+ const head = git(root, ['rev-parse', 'HEAD']);
+ let thrown = false;
+ const spy = spyOn(passRowModule, 'readPassRow').mockImplementation((async (...args: Parameters<typeof readPassRow>) => {
+ const row = await readPassRow(...args);
+ if (!thrown && row?.value.telemetry.outcome === 'synced' && row.value.telemetry.pass_id !== firstPass) {
+ thrown = true;
+ throw Object.assign(new Error('Connection terminated unexpectedly'), { code: '08006' });
+ }
+ return row;
+ }) as typeof readPassRow);
+ try {
+ await expect(sync()).rejects.toThrow('Connection terminated unexpectedly');
+ } finally {
+ spy.mockRestore();
+ }
+ expect(thrown).toBe(true);
+ const after = (await readPassRow(engine, 'default', incarnation!))!.value;
+ expect(after.telemetry.outcome).toBe('synced');
+ expect(after.obligations.converged?.target).toBe(head);
+ const status = await readSyncStatus(engine, 'default', { detail: false });
+ expect([status.state, status.reason]).toEqual(['synced', 'nothing_pending']);
+}, { setup: ({ root }) => { git(root, ['init', '-q']); commitFile(root, 'a-example.md'); } }), 120_000);
diff --git a/test/sync-pass-readers-7614.test.ts b/test/sync-pass-readers-7614.test.ts
index 05e2ca3883..3e8b5e0bac 100644
--- a/test/sync-pass-readers-7614.test.ts
+++ b/test/sync-pass-readers-7614.test.ts
@@ -25,6 +25,7 @@ import { readAppliedWatermark } from '../src/core/persistence/applied-watermark.
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
import { readBlockedManagedSyncs } from '../src/core/persistence/sync-blocked.ts';
import { readSourceMovement } from '../src/core/persistence/sync-movement.ts';
+import * as passStatus from '../src/core/persistence/sync-pass-status.ts';
import { abandonedPasses, passStatusState } from '../src/core/persistence/sync-pass-status.ts';
import { MANAGED_SYNC_PASS_OP, type PassRow, readPassRow } from '../src/core/persistence/sync-pass-row.ts';
import { performManagedSync } from '../src/core/persistence/sync-run.ts';
@@ -133,6 +134,15 @@ for (const backend of testBackends()) {
} finally { for (const spy of failed) spy.mockRestore(); }
});
+ synced('#7652 review: a failed re-read of the open pass row decides nothing in status and lists nothing in doctor', async ({ engine, status, blocked }) => {
+ await setPass(engine, 'open', 0); await dead(engine);
+ const failed = spyOn(passStatus, 'openPassRowsStill').mockRejectedValue(new Error('pass row read failed'));
+ try {
+ expect((await status()).reason).toBe('pass_unfinished');
+ expect(await blocked()).toEqual([]);
+ } finally { failed.mockRestore(); }
+ });
+
synced('#7614: doctor\'s message for killed passes only, the human INTERRUPTED line and the sync_blocked health band', async ({ engine }) => {
// Killed passes only: doctor does not say freshness jobs end dead (a resume over a killed pass completes).
await setPass(engine, 'open', 1); await unlock(engine);