Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6518: incarnation-scoped readers versus incarnation-blind writers. A managed
* source re-pointed at a new checkout gets a new incarnation; the cursor and
* failure rows written under the old one stay in `op_checkpoints`, every
* incarnation-joined reader drops them (`readManagedSyncFailures`, the orphan
* classifier, the clear paths), and the local ledger keeps their mirror rows.
*
* Contract pinned here (reviewed plan G3.2, [R18], CEO C05):
* - `classifyManagedSyncOrphans` names such rows `incarnation_retired`, and
* `retireManagedSyncOrphan` removes cursor + failure + manifest + mirror row
* even when a live local writer with `syncOptions` exists (that veto applies
* to live incarnations only); a manifest another cursor shares is kept.
* - a ledger row whose `managed_cursor_key` has no durable failure row under
* any incarnation is `mirror_only`: listed by the preview, pruned on
* `--apply`, and pruned by the successful-sync clear path for its source.
* - the orphan diagnostic is reachable from doctor with no visible managed
* failure (`managed.length === 0`), locally with the repair command and
* remotely without keys or paths.
* - `doctor --fast` (no engine) reports mirror rows as `warn` "could not be
* verified", never `fail`, while an old unmanaged failure keeps `fail`.
* - `readSyncStatus` filters the cursor by the current incarnation: an
* old-only cursor is `cursor: null` + `cursor_incarnation_retired` whose
* `next` is the repair preview; beside a current cursor it is invisible.
* - a failed durable read never reads as "no failures" and never authorises a
* mirror deletion; archived sources are excluded and refused at apply.
* Both engines where a database is involved.
*/
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { checkSyncFailures } from '../src/commands/doctor/checks/sync-failures.ts';
import { classifyManagedSyncOrphans, retireManagedSyncOrphan } from '../src/core/persistence/managed-sync-orphans.ts';
import { clearSettledManagedSyncFailures, readManagedSyncFailures, recordManagedSyncFailure } from '../src/core/persistence/sync-failures.ts';
import { readSyncStatus } from '../src/core/persistence/sync-status.ts';
import { registerLocalWriter, readLocalWriter } from '../src/core/persistence/identity.ts';
import { managedSyncOrphansRepair } from '../src/core/repair/managed-sync-orphans.ts';
import { runRepair } from '../src/core/repair/core.ts';
import { repairSpec } from '../src/core/repair/registry.ts';
import { loadSyncFailures, mirrorManagedSyncFailure, recordFailures } from '../src/core/sync-failure-ledger.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { withEnv } from './helpers/with-env.ts';
const home = mkdtempSync(join(tmpdir(), 'gbrain-6518-'));
const env = { GBRAIN_SYNC_FAILURES_DIR: home, GBRAIN_HOME: home, GBRAIN_BRAIN_ID: 'host' };
const engines: BrainEngine[] = [];
let closePostgres: (() => Promise<void>) | undefined;
const ctx = (engine: BrainEngine) => ({ engine, config: { engine: engine.kind }, logger: { info() {}, warn() {}, error() {} }, dryRun: false, remote: false } as unknown as OperationContext);
const spec = repairSpec('managed-sync-orphans');
beforeAll(async () => {
const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engines.push(lite);
if (process.env.DATABASE_URL) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL); engines.push(pg.engine); closePostgres = pg.close; }
await withEnv(env, async () => { for (const engine of engines) await registerLocalWriter(engine, 'cli'); });
}, 120_000);
afterAll(async () => { for (const engine of engines) await engine.disconnect(); await closePostgres?.(); rmSync(home, { recursive: true, force: true }); });
const each = (fn: (engine: BrainEngine) => Promise<void>) => withEnv(env, async () => { for (const engine of engines) { rmSync(join(home, 'sync-failures.jsonl'), { force: true }); await fn(engine); } });
const keys = async (engine: BrainEngine, op: string, like: string) => (await engine.executeRaw<{ fingerprint: string }>('SELECT fingerprint FROM op_checkpoints WHERE op=$1 AND fingerprint LIKE $2 ORDER BY fingerprint', [op, `${like}%`])).map(row => row.fingerprint);
const mirrors = (prefix: string) => loadSyncFailures().filter(row => row.managed_cursor_key?.startsWith(prefix)).map(row => row.managed_cursor_key);
/** A live source plus bookkeeping written under its previous incarnation (the issue's re-pointed checkout). */
async function fixture(engine: BrainEngine) {
const id = `s6518-${randomUUID().slice(0, 8)}`, prefix = `ck-${id}-`;
await engine.executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [id]);
const [{ incarnation: old }] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [id]);
const liveWriterId = (await readLocalWriter(engine, 'cli'))!.id;
const principal = { authority: { writer: { principal: { kind: 'local_cli', id: liveWriterId } } } };
const cursor = (key: string, incarnation: string, body: Record<string, unknown>) => engine.executeRaw(
"INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-sync',$1,$2::text::jsonb)",
[key, JSON.stringify([{ sourceId: id, incarnation, index: 0, total: 1, target: 'deadbeef', syncOptions: { full: false }, ...principal, ...body }])]);
const failure = (key: string, incarnation: string, runId: string, path: string) => recordManagedSyncFailure(engine, { source_id: id, source_incarnation: incarnation, path, code: 'recovery_required',
message: 'The private physical-checkout identity cannot be verified.', request_id: null, run_id: runId, target: 'deadbeef', cursor_key: key, phase: 'resume', state: 'failed', observation_id: `${runId}:0` });
// The probe's pair: an unfinished cursor of the live writer (with recorded options) and its same-key failure row, plus the manifest.
await cursor(`${prefix}old`, old, { runId: `${prefix}run-old` });
await failure(`${prefix}old`, old, `${prefix}run-old`, 'notes/2026-09-30.md');
await engine.executeRaw("INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-sync-manifest',$1,'[{\"path\":\"notes/2026-09-30.md\"}]'::jsonb)", [`${prefix}run-old`]);
// A failure row under the old incarnation with no cursor under its key.
await failure(`${prefix}oldfail`, old, `${prefix}run-oldfail`, 'notes/2026-09-29.md');
// Re-point: the source gets a new incarnation, as `sources set-path` to a new checkout does.
await engine.executeRaw('UPDATE sources SET incarnation=gen_random_uuid() WHERE id=$1', [id]);
const [{ incarnation: current }] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [id]);
expect(current).not.toBe(old);
return { id, prefix, old, current, cursor, failure, principal };
}
test('an old-incarnation cursor and failure are incarnation_retired; doctor reaches the diagnostic with no visible managed failure; sync status does not name the dead cursor', async () => each(async engine => {
const f = await fixture(engine);
// The issue's symptoms on master: every incarnation-joined reader drops the rows while the mirror keeps them.
expect((await readManagedSyncFailures(engine, [f.id])).length).toBe(0);
expect(mirrors(f.prefix).sort()).toEqual([`${f.prefix}old`, `${f.prefix}oldfail`]);
const before = await classifyManagedSyncOrphans(engine, [f.id]);
expect(before.orphans.map(orphan => [orphan.kind, orphan.cursor_key, orphan.reason, orphan.source_incarnation]).sort()).toEqual([
['failure_row', `${f.prefix}oldfail`, 'incarnation_retired', f.old], ['unfinished_cursor', `${f.prefix}old`, 'incarnation_retired', f.old]]);
expect(before.retained).toEqual([]);
// (ii) reachable with managed.length === 0: locally with the repair, remotely without keys or paths.
const local = await checkSyncFailures(engine, { remote: false, sourceIds: [f.id] });
expect(local?.status).toBe('warn');
expect(local!.message).toContain('incarnation_retired');
expect(local!.message).toContain(`gbrain repair managed-sync-orphans --source ${f.id}`);
expect(local!.message).toContain('says nothing about whether the current source\'s files are imported');
expect((local!.details as { orphans: unknown[] }).orphans).toHaveLength(2);
const remote = await checkSyncFailures(engine, { remote: true, sourceIds: [f.id] });
expect(remote?.status).toBe('warn');
expect(remote!.message).not.toContain(f.prefix);
expect(remote!.message).not.toContain('notes/');
expect(remote!.details).toBeUndefined();
// (4) status: the dead cursor is not reported as resumable; the next step is the repair preview.
const status = await readSyncStatus(engine, f.id);
expect(status.cursor).toBeNull();
expect(status.run_id).toBeNull();
expect(status.last_error).toMatchObject({ code: 'cursor_incarnation_retired', needs_human: false, safe_actions: ['repair'] });
expect(status.next?.argv).toEqual(['gbrain', 'repair', 'managed-sync-orphans', '--source', f.id]);
expect(JSON.stringify(status)).not.toContain('deadbeef');
// An old-only cursor beside a current one: the current cursor is the one shown.
await f.cursor(`${f.prefix}current`, f.current, { runId: `${f.prefix}run-current`, index: 0, total: 3 });
await engine.executeRaw("INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-sync-manifest',$1,'[{\"path\":\"a.md\"},{\"path\":\"b.md\"},{\"path\":\"c.md\"}]'::jsonb)", [`${f.prefix}run-current`]);
const beside = await readSyncStatus(engine, f.id);
expect(beside.cursor).toMatchObject({ index: 0, total: 3, done: false });
expect(beside.run_id).toBe(`${f.prefix}run-current`);
expect(beside.last_error?.code).not.toBe('cursor_incarnation_retired');
await engine.executeRaw("DELETE FROM op_checkpoints WHERE fingerprint IN ($1,$2)", [`${f.prefix}current`, `${f.prefix}run-current`]);
// The repair: preview lists both with the class; apply retires them although the recording principal is the live local writer with syncOptions.
const scope = { brain_id: 'host', source_ids: [f.id] };
const preview = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: false, explicit: true, spec });
expect(preview.affected).toBe(2);
const applied = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: true, explicit: true, expect: preview.preview_hash, spec });
expect(applied.applied).toBe(2);
expect(await keys(engine, 'managed-sync', f.prefix)).toEqual([]);
expect(await keys(engine, 'managed-sync-failure', f.prefix)).toEqual([]);
expect(await keys(engine, 'managed-sync-manifest', f.prefix)).toEqual([]);
expect(mirrors(f.prefix)).toEqual([]);
expect((await classifyManagedSyncOrphans(engine, [f.id])).orphans).toEqual([]);
expect(await checkSyncFailures(engine, { remote: false, sourceIds: [f.id] })).toBeNull();
const after = await readSyncStatus(engine, f.id);
expect(after.cursor).toBeNull();
expect(after.last_error).toBeNull();
expect(after.next).toBeNull();
}), 120_000);
test('a mirror row with no durable failure is mirror_only: listed, pruned on apply, pruned by the successful-sync clear; a durable row appearing before apply keeps it', async () => each(async engine => {
const id = `s6518m-${randomUUID().slice(0, 8)}`;
await engine.executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [id]);
const [{ incarnation }] = await engine.executeRaw<{ incarnation: string }>('SELECT incarnation::text AS incarnation FROM sources WHERE id=$1', [id]);
const mirror = (key: string) => mirrorManagedSyncFailure({ source_id: id, path: 'notes/a.md', code: 'recovery_required', message: 'stale', target: 'deadbeef', cursor_key: key,
request_id: null, run_id: `run-${key}`, observation_id: `run-${key}:0`, first_seen: new Date().toISOString(), attempts: 1 });
mirror(`${id}-m1`); mirror(`${id}-m2`); mirror(`${id}-m3`);
// A mirror whose durable row exists under the current incarnation is not mirror_only.
await recordManagedSyncFailure(engine, { source_id: id, source_incarnation: incarnation, path: 'notes/live.md', code: 'storage_error', message: 'durable', request_id: null,
run_id: 'run-live', target: 'deadbeef', cursor_key: `${id}-live`, phase: 'receipt', state: 'failed', observation_id: 'run-live:0' });
const classified = await classifyManagedSyncOrphans(engine, [id]);
expect(classified.orphans.filter(orphan => orphan.reason === 'mirror_only').map(orphan => orphan.cursor_key).sort()).toEqual([`${id}-m1`, `${id}-m2`, `${id}-m3`]);
expect(classified.orphans.find(orphan => orphan.cursor_key === `${id}-m1`)).toMatchObject({ kind: 'mirror_row', source_id: id });
const scope = { brain_id: 'host', source_ids: [id] };
const preview = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: false, explicit: true, spec });
expect(preview.affected).toBe(4); // three mirrors + the durable no_cursor failure row
// A durable failure row appears under m2 between preview and apply: its mirror is no longer mirror-only and is kept
// (a `<checkpoint>` path, which no imported commit settles, so the #6288 clear below leaves the durable row alone).
await recordManagedSyncFailure(engine, { source_id: id, source_incarnation: incarnation, path: '<checkpoint>', code: 'storage_error', message: 'late', request_id: null,
run_id: `run-${id}-m2`, target: 'deadbeef', cursor_key: `${id}-m2`, phase: 'receipt', state: 'failed', observation_id: `run-${id}-m2:1` });
const applied = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: true, explicit: true, expect: preview.preview_hash, spec });
expect(applied.applied).toBe(3);
expect(mirrors(`${id}-`).sort()).toEqual([`${id}-m2`]);
expect(await keys(engine, 'managed-sync-failure', `${id}-`)).toEqual([`${id}-m2`]);
// The successful-sync clear path prunes this source's mirror-only rows and leaves another source's alone.
mirror(`${id}-m4`);
const otherId = `${id}-other`;
mirrorManagedSyncFailure({ source_id: otherId, path: 'x.md', code: 'storage_error', message: 'other', target: null, cursor_key: `${otherId}-m`, request_id: null, run_id: 'r', observation_id: 'r:0', first_seen: new Date().toISOString(), attempts: 1 });
await clearSettledManagedSyncFailures(engine, { sourceId: id, incarnation, root: home, gitRoot: home, target: 'deadbeef' });
expect(mirrors(`${id}-`).sort()).toEqual([`${id}-m2`, `${otherId}-m`]);
}), 120_000);
test('doctor --fast (no engine) warns on mirror rows it cannot verify and keeps fail for an old unmanaged failure', async () => each(async () => {
const id = `s6518f-${randomUUID().slice(0, 8)}`;
mirrorManagedSyncFailure({ source_id: id, path: 'notes/a.md', code: 'recovery_required', message: 'stale', target: 'deadbeef', cursor_key: `${id}-m`, request_id: null, run_id: 'r',
observation_id: 'r:0', first_seen: new Date(Date.now() - 10 * 86_400_000).toISOString(), attempts: 1 });
// Age the mirror row past the 72 h fail cadence: before, it alone turned the fast check `fail`.
const aged = loadSyncFailures().map(row => row.managed_cursor_key === `${id}-m` ? { ...row, ts: new Date(Date.now() - 10 * 86_400_000).toISOString() } : row);
const { writeFileSync } = await import('node:fs');
writeFileSync(join(home, 'sync-failures.jsonl'), aged.map(row => JSON.stringify(row)).join('\n') + '\n');
const mirrorOnly = await checkSyncFailures(null, { remote: false, sourceIds: [id] });
expect(mirrorOnly?.status).toBe('warn');
expect(mirrorOnly!.message).toContain('could not be verified without a database connection');
expect(mirrorOnly!.message).toContain('gbrain doctor --only sync_failures');
expect(mirrorOnly!.message).toMatch(/less certainty/);
expect(mirrorOnly!.remediation).toBeUndefined();
// A genuinely old unmanaged failure in the same ledger keeps its authoritative severity.
recordFailures(id, [{ path: 'legacy.md', error: 'YAML broke' }], 'c1');
const agedBoth = loadSyncFailures().map(row => row.path === 'legacy.md' ? { ...row, ts: new Date(Date.now() - 10 * 86_400_000).toISOString() } : row);
writeFileSync(join(home, 'sync-failures.jsonl'), agedBoth.map(row => JSON.stringify(row)).join('\n') + '\n');
const mixed = await checkSyncFailures(null, { remote: false, sourceIds: [id] });
expect(mixed?.status).toBe('fail');
expect(mixed!.message).toContain('1 unresolved sync failure(s)');
expect(mixed!.message).toContain('could not be verified');
const remote = await checkSyncFailures(null, { remote: true, sourceIds: [id] });
expect(remote?.status).toBe('fail');
expect(remote!.message).not.toContain('legacy.md');
}));
test('a failing durable read is not "no failures" and never deletes a mirror; archived sources are excluded and refused at apply; a recreated source retires its old rows', async () => each(async engine => {
const f = await fixture(engine);
const executeRaw = engine.executeRaw;
const failReads = () => { engine.executeRaw = (async (sql: string, params?: unknown[]) => {
// The classifier's mirror read (the durable rows under the ledger's keys) fails; the current-incarnation read still answers.
if (/managed-sync-failure/.test(sql) && (/fingerprint=ANY/.test(sql) || sql.startsWith("SELECT 1 FROM op_checkpoints WHERE op='managed-sync-failure'"))) throw Object.assign(new Error('connection reset'), { code: 'ECONNRESET' });
return executeRaw.call(engine, sql, params);
}) as BrainEngine['executeRaw']; };
const restore = () => { engine.executeRaw = executeRaw; };
// Preview with a healthy read, then the apply's own re-check fails: the mirror and the rows stay.
const scope = { brain_id: 'host', source_ids: [f.id] };
mirrorManagedSyncFailure({ source_id: f.id, path: 'notes/m.md', code: 'storage_error', message: 'm', target: null, cursor_key: `${f.prefix}mirror`, request_id: null, run_id: 'rm', observation_id: 'rm:0', first_seen: new Date().toISOString(), attempts: 1 });
const preview = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: false, explicit: true, spec });
expect(preview.affected).toBe(3);
failReads();
try {
await expect(classifyManagedSyncOrphans(engine, [f.id])).rejects.toMatchObject({ code: 'ECONNRESET' });
const check = await checkSyncFailures(engine, { remote: false, sourceIds: [f.id] });
expect(check?.status).toBe('warn');
expect(check!.message).toContain('could not be classified (ECONNRESET)');
expect(check!.message).toContain('not proof');
await expect(retireManagedSyncOrphan(engine, { kind: 'mirror_row', cursor_key: `${f.prefix}mirror`, run_id: null, source_id: f.id, source_incarnation: f.current, reason: 'mirror_only' })).rejects.toMatchObject({ code: 'ECONNRESET' });
} finally { restore(); }
expect(mirrors(f.prefix).sort()).toEqual([`${f.prefix}mirror`, `${f.prefix}old`, `${f.prefix}oldfail`]);
expect(await keys(engine, 'managed-sync', f.prefix)).toEqual([`${f.prefix}old`]);
// Archived: not classified; an approved set from before the archive is refused row by row (kept), nothing deleted.
await engine.executeRaw('UPDATE sources SET archived=true, archived_at=now(), archive_expires_at=now()+interval \'72 hours\' WHERE id=$1', [f.id]);
expect((await classifyManagedSyncOrphans(engine, null)).orphans.filter(orphan => orphan.source_id === f.id)).toEqual([]);
for (const orphan of (await classifyManagedSyncOrphans(engine, null)).orphans) expect(orphan.source_id).not.toBe(f.id);
const keptCursor = await retireManagedSyncOrphan(engine, { kind: 'unfinished_cursor', cursor_key: `${f.prefix}old`, run_id: `${f.prefix}run-old`, source_id: f.id, source_incarnation: f.old, reason: 'incarnation_retired' });
expect(keptCursor).toEqual({ cursor: false, failure: false, manifest: false, mirror: false });
expect(await keys(engine, 'managed-sync', f.prefix)).toEqual([`${f.prefix}old`]);
expect(await keys(engine, 'managed-sync-failure', f.prefix).then(rows => rows.length)).toBe(2);
await engine.executeRaw('UPDATE sources SET archived=false, archived_at=NULL, archive_expires_at=NULL WHERE id=$1', [f.id]);
// Recreated: the source is removed and added again (a third incarnation); rows under both earlier incarnations are retired.
await f.cursor(`${f.prefix}second`, f.current, { runId: `${f.prefix}run-second` });
await engine.executeRaw('DELETE FROM sources WHERE id=$1', [f.id]);
await engine.executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [f.id]);
const retired = (await classifyManagedSyncOrphans(engine, [f.id])).orphans;
expect(retired.map(orphan => [orphan.cursor_key, orphan.reason]).sort()).toEqual([
[`${f.prefix}mirror`, 'mirror_only'], [`${f.prefix}old`, 'incarnation_retired'], [`${f.prefix}oldfail`, 'incarnation_retired'], [`${f.prefix}second`, 'incarnation_retired']]);
// A manifest another cursor still references survives the retirement of one of them.
await f.cursor(`${f.prefix}twin`, f.old, { runId: `${f.prefix}run-old` });
const twinRetired = await retireManagedSyncOrphan(engine, { kind: 'unfinished_cursor', cursor_key: `${f.prefix}old`, run_id: `${f.prefix}run-old`, source_id: f.id, source_incarnation: f.old, reason: 'incarnation_retired' });
expect(twinRetired).toEqual({ cursor: true, failure: true, manifest: false, mirror: true });
expect(await keys(engine, 'managed-sync-manifest', f.prefix)).toEqual([`${f.prefix}run-old`]);
expect(mirrors(f.prefix).sort()).toEqual([`${f.prefix}mirror`, `${f.prefix}oldfail`]);
// Preview/apply race: a row retired by another apply in between is reported kept, and the rest of the set still applies.
const race = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: false, explicit: true, spec });
await engine.executeRaw("DELETE FROM op_checkpoints WHERE fingerprint=$1", [`${f.prefix}second`]);
const raced = await runRepair(ctx(engine), managedSyncOrphansRepair, scope, { apply: true, explicit: true, expect: race.preview_hash, spec });
expect(raced.applied).toBe(race.affected - 1);
expect(raced.skipped).toBe(1);
expect(await keys(engine, 'managed-sync', f.prefix)).toEqual([]);
expect(await keys(engine, 'managed-sync-failure', f.prefix)).toEqual([]);
expect(mirrors(f.prefix)).toEqual([]);
}), 120_000);