Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
40.8 KB
Raw Blame
/**
 * The real-process brain harness shared by the managed-sync convergence gate (test/sync-convergence.serial.test.ts)
 * and the busy-agent soak cell (test/busy-agent-soak.serial.test.ts): one isolated Postgres database behind
 * transaction-mode PgBouncer, reached by every child process through latency proxies, a Git worktree bound to one
 * managed source, real `src/cli.ts` child processes (optionally under the fault preload
 * test/fixtures/convergence-fault-kill.ts), a resident stdio serve, the acknowledged-write ledger and the oracle reads.
 * Each suite passes its own knobs (seed, latency, log directory, repro line). Setup: scripts/persistence/README.md.
 */
import { test } from 'bun:test';
import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, relative, resolve } from 'node:path';
import postgres from '#postgres';
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { claimWorktree } from '../../src/core/persistence/ownership.ts';
import { parseMarkdown } from '../../src/core/markdown.ts';
import { parseFactsFence } from '../../src/core/facts-fence.ts';
import { contentHash } from '../../src/core/utils.ts';
import { CODES } from '../../src/core/error-registry.ts';
import type { FaultPoint } from '../../src/core/persistence/fault-points.ts';
import { assertSafeE2eDatabaseUrl } from './db-guard.ts';
import { keylessBrainEnv } from './provider-env.ts';
import { withEnv } from './with-env.ts';
import { LatencyProxy } from './latency-proxy.ts';
import type { ViolationClass } from '../../scripts/persistence/model.ts';

const REPO_ROOT = resolve(import.meta.dir, '../..');
const FAULT_PRELOAD = join(REPO_ROOT, 'test/fixtures/convergence-fault-kill.ts');
export const SOURCE = 'workspace';
export const BIG = 'companies/acme-example';
export const ENTITY = 'people/alice-example';
export const MAX_RECOVERY_RUNS = 2;

/** What a suite passes to every brain: the knobs its failures print, where its logs go, and the targets behind the proxies. */
export interface HarnessKnobs {
  /** Prefixes every progress line (`[<tag> <scenario>]`) and names the log file `<tag>.log` in logDir. */
  tag: string;
  seed: number;
  latencyMs: number;
  /** Per child process (a sync or a writer call) unless a call passes its own. */
  syncTimeoutMs: number;
  logDir: string | undefined;
  pooledTarget: string;
  directTarget: string;
  /** The suite's other knobs, printed after the seed and modes in every failure. */
  summary: string;
  /** The local command that reruns these modes, printed in every failure. */
  repro: (modes: string[]) => string;
}

/** mulberry32: a fixed seed gives the same fault plan and writer targets on every run. */
export function prng(seed: number): () => number {
  let a = seed >>> 0;
  return () => {
    a = (a + 0x6d2b79f5) >>> 0;
    let t = a;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

// ---------------------------------------------------------------- the status oracle

/** Every `gbrain sync status --json` field the operator loop reads (sync-status.ts `SyncStatus`). */
export const STATUS_FIELDS = ['source_id', 'run_id', 'cursor', 'committed_last_10m', 'live_job_id', 'holds', 'last_error', 'needs_human',
  'resume_argv', 'checkpoint_blockers', 'cursors', 'terminal_failures', 'next'] as const;
export interface StatusExpectation { holds?: Array<{ code: string; path: string }>; needsHuman?: boolean }
/**
 * What a converged source's status must say. Field-present assertions first: a renderer that drops a field fails
 * here, not in an agent's loop. `checkpoint_blockers` and `cursors` are arrays on every status (#6712 PR 0 made
 * `checkpoint_blockers` always present). The harness reads `sync status --json --detail`: since #6712 PR 2 the
 * terminal summary, blocker list and cursor join run only behind the flag.
 */
/** The default (no `--detail`) status after the chaos: every skipped heavy read says so and names the `--detail` command (#6712 PR 2, DX F1). */
export function fastStatusProblems(status: Record<string, unknown> | null): string[] {
  if (!status) return ['sync status --json printed no JSON document'];
  return (['checkpoint_blockers_read', 'cursors_read', 'terminal_failures_read'] as const).flatMap(field => {
    const read = status[field] as { status?: string; fix?: { argv?: string[] } } | undefined;
    return read?.status === 'not_read' && read.fix?.argv?.includes('--detail') ? [] : [`sync status --json: ${field} is ${JSON.stringify(read)}, expected not_read with a --detail fix`];
  });
}

export function statusProblems(status: Record<string, unknown> | null, expected: StatusExpectation = {}): string[] {
  if (!status) return ['sync status --json printed no JSON document'];
  const out = STATUS_FIELDS.filter(field => !(field in status)).map(field => `sync status --json: field ${field} missing`);
  if (out.length) return out;
  const cursor = status.cursor as { done?: boolean } | null;
  if (!Array.isArray(status.cursors)) out.push('sync status --json: cursors is not an array');
  if (!Array.isArray(status.checkpoint_blockers)) out.push('sync status --json: checkpoint_blockers is not an array');
  else if (status.checkpoint_blockers.length) out.push(`sync status --json: ${status.checkpoint_blockers.length} checkpoint blocker(s)`);
  // A finished run's cursor is retired, so a converged source shows none (null) or a done one.
  if (cursor && cursor.done !== true) out.push(`sync status --json: cursor not done (${JSON.stringify(cursor)})`);
  const terminal = status.terminal_failures as { status?: string; active_blockers?: unknown[] } | null;
  if (terminal?.status !== 'ok') out.push(`sync status --json: terminal_failures.status is ${terminal?.status}`);
  else if (!Array.isArray(terminal.active_blockers)) out.push('sync status --json: terminal_failures.active_blockers is not an array');
  else if (terminal.active_blockers.length) out.push(`sync status --json: ${terminal.active_blockers.length} active terminal blocker(s)`);
  if (status.needs_human !== (expected.needsHuman ?? false)) out.push(`sync status --json: needs_human is ${status.needs_human}, expected ${expected.needsHuman ?? false}`);
  const holds = Array.isArray(status.holds) ? (status.holds as Array<{ code: string; path: string }>).map(h => `${h.code}:${h.path}`).sort() : ['holds is not an array'];
  const want = (expected.holds ?? []).map(h => `${h.code}:${h.path}`).sort();
  if (JSON.stringify(holds) !== JSON.stringify(want)) out.push(`sync status --json: holds [${holds.join(', ')}], expected [${want.join(', ')}]`);
  if (status.last_error !== null) out.push(`sync status --json: last_error ${JSON.stringify(status.last_error)}`);
  if (!expected.holds?.length && status.next !== null) out.push(`sync status --json: next ${JSON.stringify(status.next)}`);
  return out;
}

// ---------------------------------------------------------------- the acknowledged-write ledger

/**
 * One writer operation that returned success, recorded by the harness at the moment it saw the success (never read
 * back from the brain), or a write the brain accepted (exit 10) that had not settled when its writer stopped, recorded
 * when its receipt, read after recovery, said committed (`Brain.settlePending`). `touches` is where the write landed: the page in the database, its file, or (for `remember`)
 * an additive fact row. `marker` is text that only this write put there; `null` on a page location is a deletion.
 */
export interface Ack { kind: 'commit' | 'put' | 'remember' | 'maintenance' | 'db_only' | 'git_mv' | 'git_rm' | 'unlink'; slug: string;
  marker: string | null; touches: Array<'db' | 'file'> | 'fact'; start: number; ack: number }
export class Ledger {
  acks: Ack[] = [];
  record(ack: Ack): void { this.acks.push(ack); }
  /**
   * For every slug and location, the writes no later acknowledged write causally supersedes (none started after it
   * was acknowledged); the final state must show one of them. An additive fact survives unless a page write on its
   * slug may have been ordered after it.
   */
  problems(disk: Map<string, string>, db: Map<string, { body: string } | null>, facts: Set<string>): string[] {
    const out: string[] = [];
    const pageWrites = this.acks.filter(a => a.touches !== 'fact' && a.kind !== 'maintenance');
    for (const slug of new Set(pageWrites.map(a => a.slug))) {
      for (const where of ['db', 'file'] as const) {
        const writes = pageWrites.filter(a => a.slug === slug && (a.touches as string[]).includes(where));
        if (!writes.length) continue;
        const maximal = writes.filter(w => !writes.some(later => later.start > w.ack));
        const seen = where === 'file' ? disk.get(slug) ?? null : db.get(slug)?.body ?? null;
        if (maximal.some(w => w.marker === null ? seen === null : seen?.includes(w.marker))) continue;
        out.push(`${where} ${slug}: lost acknowledged write; has ${seen === null ? 'nothing' : JSON.stringify(seen.slice(0, 80))}, expected one of ${maximal.map(w => `${w.kind}@${Math.round(w.ack)}:${w.marker === null ? '(absent)' : JSON.stringify(w.marker)}`).join(' | ')}`);
      }
    }
    for (const fact of this.acks.filter(a => a.touches === 'fact')) {
      const excused = pageWrites.some(w => w.slug === fact.slug && (w.touches as string[]).includes('db') && w.ack > fact.start);
      if (!excused && !facts.has(fact.marker!)) out.push(`fact on ${fact.slug}: lost acknowledged remember ${JSON.stringify(fact.marker)}`);
    }
    return out;
  }
}

// ---------------------------------------------------------------- the fixture

export function noteText(i: number, version: string): string {
  return `---\ntype: concept\ntitle: Note ${i} Example\n---\n\n${version} body for note ${i}, an observation about widget-co and acme-example.\n\n${'Supporting detail sentence. '.repeat(8 + (i % 5))}\n`;
}
export const noteMarker = (i: number, version: string) => `${version} body for note ${i},`;
export const noteSlug = (i: number) => `notes/note-${String(i).padStart(4, '0')}`;
export function bigPage(facts: number, timeline: number, version: string): string {
  const day = (n: number) => new Date(Date.UTC(2018, 0, 1) + n * 86_400_000).toISOString().slice(0, 10);
  const paragraphs = Array.from({ length: 30 }, (_, i) => `Section ${i} (${version}): acme-example ${'expanded with a steady stream of synthetic operating detail. '.repeat(6)}`);
  const rows = Array.from({ length: facts }, (_, i) =>
    `| ${i + 1} | Synthetic claim ${i + 1} about acme-example | fact | 0.9 | world | medium | ${day(i)} |  | fixture |  |`);
  const bullets = Array.from({ length: timeline }, (_, i) => `- **${day(i)}** | Synthetic event ${i + 1} at acme-example`);
  return [
    '---', 'type: company', 'title: Acme Example', '---', '', '# Acme Example', '', ...paragraphs.flatMap(p => [p, '']),
    '## Facts', '', '<!--- gbrain:facts:begin -->',
    '| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |',
    '|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|',
    ...rows, '<!--- gbrain:facts:end -->', '', '<!-- timeline -->', '', ...bullets, '',
  ].join('\n');
}
export const entityText = '---\ntype: person\ntitle: Alice Example\n---\n\nAlice Example works with acme-example.\n';

// ---------------------------------------------------------------- one brain

export interface Fault { point: FaultPoint; nth: number; marker: string; operation?: string; path?: string; action?: 'kill' | 'edit' | 'stall' | 'throw';
  edit?: { file: string; content: string }; release?: string; ms?: number; sqlstate?: string;
  prime?: { point: FaultPoint; nth: number; path?: string; edit: { file: string; content: string } } }
export interface Run { code: number; out: string; err: string; ms: number }
export interface SyncRun { proc: ReturnType<typeof Bun.spawn>; result: Promise<Run> }

export async function until(check: () => boolean | Promise<boolean>, label: string, timeoutMs = 300_000): Promise<void> {
  const deadline = performance.now() + timeoutMs;
  while (performance.now() < deadline) {
    if (await check()) return;
    await Bun.sleep(100);
  }
  throw new Error(`timed out waiting for ${label}`);
}
export function lastJson(out: string): Record<string, unknown> | null {
  try { return JSON.parse(out) as Record<string, unknown>; } catch { /* several documents */ }
  for (const line of out.trim().split('\n').reverse()) { try { return JSON.parse(line) as Record<string, unknown>; } catch { /* not the envelope */ } }
  return null;
}
/** The longest `stalled Ns` progress line a sync printed: how long it waited on a predecessor's leftovers. */
export function worstStall(err: string): number { return Math.max(0, ...[...err.matchAll(/stalled (\d+)s/g)].map(m => Number(m[1]))); }

/**
 * One scenario's brain: its database, home, Git worktree, proxies, child processes, ledger and log. `context()` is
 * what every failure prints: seed, modes, the fault plan and the one-line repro.
 */
export class Brain {
  home = ''; repo = ''; database = '';
  admin: ReturnType<typeof postgres> | undefined;
  engine: PostgresEngine | undefined;
  pooledProxy: LatencyProxy | undefined; directProxy: LatencyProxy | undefined;
  pooledUrl = ''; directUrl = '';
  serves: StdioServe[] = [];
  ledger = new Ledger();
  /** Writes the brain accepted (exit 10) that had not settled when their writer stopped; `settlePending` reads their receipts after recovery. */
  pending: Array<{ write: Omit<Ack, 'ack'>; requestId: string }> = [];
  plan: string[] = [];
  fired: Record<string, string | null> = {};
  killedPids = new Set<number>();
  recoveryRuns: Array<Run & { envelope: Record<string, unknown> | null }> = [];
  elapsed: Record<string, number> = {};
  calibration: string[] = [];
  readonly rand: () => number;
  constructor(readonly name: string, readonly modes: string[], readonly knobs: HarnessKnobs) { this.rand = prng(knobs.seed ^ [...name].reduce((h, c) => Math.imul(h, 31) + c.charCodeAt(0), 7)); }

  between(lo: number, hi: number): number { return lo + Math.floor(this.rand() * (hi - lo + 1)); }
  note(line: string): void {
    console.error(`[${this.knobs.tag} ${this.name}] ${line}`);
    if (this.knobs.logDir) { mkdirSync(this.knobs.logDir, { recursive: true }); appendFileSync(join(this.knobs.logDir, `${this.knobs.tag}.log`), `[${this.name}] ${line}\n`); }
  }
  keepLog(name: string, text: string): void { if (this.knobs.logDir) { mkdirSync(join(this.knobs.logDir, this.name), { recursive: true }); writeFileSync(join(this.knobs.logDir, this.name, name), text); } }
  context(): string {
    return [`seed ${this.knobs.seed}; modes ${this.modes.join(',')}; ${this.knobs.summary}`,
      `fault plan: ${this.plan.join('; ') || 'none'}`,
      `fired: ${Object.entries(this.fired).map(([k, v]) => `${k} => ${v ?? 'did not fire'}`).join('; ') || 'none'}`,
      `repro (Postgres + PgBouncer from scripts/persistence/README.md): ${this.knobs.repro(this.modes)}`].join('\n');
  }
  /** Throws with the problems and the failure context; a passing check prints nothing. */
  verdict(problems: string[]): void {
    if (problems.length) throw new Error(`${problems.length} problem(s):\n  ${problems.slice(0, 40).join('\n  ')}\n${this.context()}`);
  }
  marker(tag: string): string { return join(this.home, `fault-${tag}.json`); }
  recordFault(tag: string, label: string): string | null {
    const at = existsSync(this.marker(tag)) ? readFileSync(this.marker(tag), 'utf8') : null;
    this.fired[label] = at;
    if (at) { const fired = JSON.parse(at) as { pid: number; action: string }; if (fired.action === 'kill') this.killedPids.add(fired.pid); }
    return at;
  }

  git(...args: string[]): string { return execFileSync('git', ['-C', this.repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); }
  /**
   * Writes files and commits exactly those paths, as the shared-checkout rules require of every committer; each write
   * is acknowledged when the commit returns. Sweeping the worktree (`git add -A`) would commit a writer's in-flight
   * write-through bytes as an out-of-band Git change of a page that writer is still writing.
   */
  commit(message: string, files: Array<{ slug: string; content: string; marker: string }>): void {
    const start = performance.now();
    for (const f of files) { mkdirSync(dirname(join(this.repo, `${f.slug}.md`)), { recursive: true }); writeFileSync(join(this.repo, `${f.slug}.md`), f.content); }
    const paths = files.map(f => `${f.slug}.md`);
    this.git('add', '--', ...paths); this.git('commit', '-qm', message, '--', ...paths);
    const ack = performance.now();
    for (const f of files) this.ledger.record({ kind: 'commit', slug: f.slug, marker: f.marker, touches: ['db', 'file'], start, ack });
  }
  writeNotes(indexes: number[], version: string, message: string, extra: Array<{ slug: string; content: string; marker: string }> = []): void {
    this.commit(message, [...indexes.map(i => ({ slug: noteSlug(i), content: noteText(i, version), marker: noteMarker(i, version) })), ...extra]);
  }
  bigFile(facts: number, timeline: number, version: string) { return { slug: BIG, content: bigPage(facts, timeline, version), marker: `Section 0 (${version})` }; }

  childEnv(extra: Record<string, string | undefined> = {}): Record<string, string> {
    return keylessBrainEnv(process.env, this.home, {
      DATABASE_URL: this.pooledUrl, GBRAIN_DATABASE_URL: undefined, GBRAIN_DIRECT_DATABASE_URL: this.directUrl,
      GBRAIN_BRAIN_ID: 'host', GBRAIN_SOURCE: SOURCE, GBRAIN_SWEEP: '0', GBRAIN_HOOKS: undefined,
      GBRAIN_SKIP_STARTUP_HOOKS: '1', GBRAIN_NO_SKILL_NAG: '1', GBRAIN_SERVE_BOOT_TIMEOUT_SECONDS: '300',
      GBRAIN_SYNC_NO_DELEGATE: undefined, GBRAIN_TEST_CONVERGENCE_FAULT: undefined,
      ...extra,
    });
  }
  faultArgs(fault?: Fault): { preload: string[]; env: Record<string, string | undefined> } {
    if (!fault) return { preload: [], env: {} };
    return { preload: ['--preload', FAULT_PRELOAD], env: { GBRAIN_TEST_CONVERGENCE_FAULT: JSON.stringify(fault) } };
  }
  spawnCli(args: string[], opts: { fault?: Fault; stdin?: string; timeoutMs?: number } = {}): SyncRun {
    const { preload, env } = this.faultArgs(opts.fault);
    const started = performance.now();
    const proc = Bun.spawn([process.execPath, '--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), ...args], {
      cwd: REPO_ROOT, env: this.childEnv(env), stdin: opts.stdin === undefined ? 'ignore' : new Blob([opts.stdin]), stdout: 'pipe', stderr: 'pipe',
    });
    const timer = setTimeout(() => proc.kill('SIGKILL'), opts.timeoutMs ?? this.knobs.syncTimeoutMs);
    const result = Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited])
      .then(([out, err, code]) => ({ code, out, err, ms: performance.now() - started }))
      .finally(() => clearTimeout(timer));
    return { proc, result };
  }
  sync(args: string[], fault?: Fault): SyncRun { return this.spawnCli(['sync', '--source', SOURCE, ...args], { fault }); }
  async committedSince(at: string): Promise<number> {
    return (await this.engine!.executeRaw<{ n: number }>(`SELECT count(*)::int AS n FROM persistence_requests WHERE state='committed' AND completed_at > $1::timestamptz AND operation='submit_job'`, [at]))[0].n;
  }
  async now(): Promise<string> { return (await this.engine!.executeRaw<{ at: string }>('SELECT now()::text AS at'))[0].at; }

  async setup(seed: (b: Brain) => void, config: Record<string, string> = {}): Promise<void> {
    assertSafeE2eDatabaseUrl(this.knobs.directTarget);
    this.home = mkdtempSync(join(tmpdir(), 'gb-conv-'));
    mkdirSync(join(this.home, '.gbrain'));
    this.database = `gbrain_test_${this.knobs.tag.replaceAll('-', '_')}_${randomUUID().replaceAll('-', '')}`;
    this.admin = postgres(this.knobs.directTarget, { max: 1, prepare: false });
    await this.admin.unsafe(`CREATE DATABASE ${this.database}`);
    const direct = new URL(this.knobs.directTarget); direct.pathname = `/${this.database}`;
    const pooled = new URL(this.knobs.pooledTarget); pooled.pathname = `/${this.database}`;
    this.pooledProxy = new LatencyProxy({ host: pooled.hostname, port: Number(pooled.port) || 5432 }, this.knobs.latencyMs / 2);
    this.directProxy = new LatencyProxy({ host: direct.hostname, port: Number(direct.port) || 5432 }, this.knobs.latencyMs / 2);
    await Promise.all([this.pooledProxy.listen(), this.directProxy.listen()]);
    const viaPooled = new URL(pooled); viaPooled.hostname = '127.0.0.1'; viaPooled.port = String(this.pooledProxy.port); viaPooled.searchParams.set('prepare', 'false');
    const viaDirect = new URL(direct); viaDirect.hostname = '127.0.0.1'; viaDirect.port = String(this.directProxy.port);
    this.pooledUrl = viaPooled.toString(); this.directUrl = viaDirect.toString();
    this.engine = new PostgresEngine();
    await withEnv({ GBRAIN_HOME: this.home, GBRAIN_DIRECT_DATABASE_URL: undefined }, async () => {
      await this.engine!.connect({ database_url: direct.toString(), poolSize: 4 });
      await this.engine!.initSchema();
    });
    writeFileSync(join(this.home, '.gbrain', 'config.json'), JSON.stringify({ engine: 'postgres', database_url: this.pooledUrl, embedding_disabled: true }));
    this.repo = mkdtempSync(join(tmpdir(), 'gb-conv-repo-'));
    this.git('init', '-q'); this.git('config', 'user.email', '[email protected]'); this.git('config', 'user.name', 'Example');
    seed(this);
    await withEnv({ GBRAIN_HOME: this.home }, async () => {
      await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
      await this.engine!.executeRaw('INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,\'{}\')', [SOURCE, this.repo]);
      await claimWorktree(this.engine!, SOURCE, this.repo);
      for (const [key, value] of Object.entries(config)) await this.engine!.setConfig(key, value);
      await this.engine!.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
    });
    this.note(`seed=${this.knobs.seed} latency=${this.knobs.latencyMs}ms round trip via proxies (pooled -> ${pooled.host}, direct -> ${direct.host}); modes ${this.modes.join(',')}`);
    // Phase 0: a fault-free import, so the oracle is calibrated before any chaos.
    const started = performance.now();
    const baseline = await this.sync(['--no-pull', '--no-delegate', '--json']).result;
    this.elapsed.baseline_import_ms = performance.now() - started;
    if (baseline.code !== 0) throw new Error(`baseline sync exited ${baseline.code}\n${baseline.out}\n${baseline.err}\n${this.context()}`);
    this.calibration = await this.diskDbMismatches(new Set());
    if (existsSync(join(this.repo, `${BIG}.md`))) {
      const rows = await this.childRowProblems();
      this.calibration.push(...rows.problems, ...(rows.timelineDuplicates || rows.timelineStrays ? [`timeline: ${rows.timelineDuplicates} duplicate and ${rows.timelineStrays} stray rows after a fault-free import`] : []));
    }
    this.note(`phase 0: fault-free import in ${(this.elapsed.baseline_import_ms / 1000).toFixed(1)}s; oracle calibration problems: ${this.calibration.length}`);
  }

  /** A fresh resident serve and at most MAX_RECOVERY_RUNS plain syncs, no operator step; a synced run that HEAD has moved past since (a queued write's Git commit landing during or after it) is followed by the next. */
  async recover(): Promise<void> {
    const started = performance.now();
    const serve = new StdioServe(this);
    await serve.start();
    for (let run = 1; run <= MAX_RECOVERY_RUNS; run++) {
      const r = await this.sync(['--no-pull', '--json']).result;
      const envelope = lastJson(r.out);
      this.recoveryRuns.push({ ...r, envelope });
      this.keepLog(`recovery-${run}-client.log`, `${r.out}\n${r.err}`);
      const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);
      const head = this.git('rev-parse', 'HEAD');
      const behind = source.last_commit && source.last_commit !== head ? this.git('log', '--format=%h %s', `${source.last_commit}..HEAD`).split('\n').filter(Boolean) : [];
      this.note(`recovery run ${run}: exit ${r.code} in ${(r.ms / 1000).toFixed(1)}s outcome=${String(envelope?.outcome ?? envelope?.sync_status ?? envelope?.error ?? 'none')}${behind.length ? `; ${behind.length} commit(s) landed after it: ${behind.join('; ')}` : ''}`);
      if (r.code === 0 && envelope?.outcome === 'synced' && !behind.length) break;
    }
    this.elapsed.recovery_ms = performance.now() - started;
    this.keepLog('recovery-serve.log', serve.stderr);
    this.note(`elapsed: ${Object.entries(this.elapsed).map(([k, ms]) => `${k.replace(/_ms$/, '')} ${(ms / 1000).toFixed(1)}s`).join(', ')}; recovery took ${this.recoveryRuns.length} sync run(s)`);
  }

  /**
   * #6841: an accepted write still queued when its writer gave up may commit later, after a write the ledger holds.
   * Once the brain has converged, each one's receipt is read the way the write protocol tells a client to after exit
   * 10 (get_write_request). A committed write is acknowledged then: it may have landed any time since its start, so it
   * is concurrent with every write in that window. Any other state means it never landed and stays out of the ledger.
   */
  async settlePending(receiptState = async (requestId: string) =>
    lastJson((await this.spawnCli(['call', '--source', SOURCE, 'get_write_request', JSON.stringify({ request_id: requestId })]).result).out)?.state): Promise<void> {
    for (const { write, requestId } of this.pending) {
      const state = await receiptState(requestId);
      if (state === 'committed') this.ledger.record({ ...write, ack: performance.now() });
      this.note(`accepted ${write.kind} of ${write.slug} (request ${requestId}) unsettled when its writer stopped: receipt ${String(state ?? 'unreadable')}`);
    }
  }

  async teardown(): Promise<void> {
    for (const serve of this.serves) await serve.stop().catch(() => {});
    try { await this.engine?.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1'); } catch { /* best effort */ }
    await this.engine?.disconnect().catch(() => {});
    await this.pooledProxy?.close().catch(() => {});
    await this.directProxy?.close().catch(() => {});
    if (this.admin) {
      try { if (this.database) await this.admin.unsafe(`DROP DATABASE IF EXISTS ${this.database} WITH (FORCE)`); } finally { await this.admin.end(); }
    }
    for (const dir of [this.home, this.repo]) if (dir) rmSync(dir, { recursive: true, force: true });
  }

  // ------------------------------------------------------------ oracle reads

  diskPages(): Map<string, string> {
    const out = new Map<string, string>();
    const walk = (dir: string) => {
      for (const entry of readdirSync(dir, { withFileTypes: true })) {
        if (entry.name.startsWith('.')) continue;
        const full = join(dir, entry.name);
        if (entry.isDirectory()) walk(full);
        else if (entry.name.endsWith('.md')) out.set(relative(this.repo, full).replace(/\.md$/, ''), readFileSync(full, 'utf8'));
      }
    };
    walk(this.repo);
    return out;
  }
  async dbPages(): Promise<Map<string, { hash: string | null; body: string }>> {
    const rows = await this.engine!.executeRaw<{ slug: string; content_hash: string | null; body: string }>(
      `SELECT slug, content_hash, coalesce(compiled_truth,'') || E'\\n' || coalesce(timeline,'') AS body FROM pages WHERE source_id=$1 AND deleted_at IS NULL`, [SOURCE]);
    return new Map(rows.map(r => [r.slug, { hash: r.content_hash, body: r.body }]));
  }
  async diskDbMismatches(exclude: Set<string>): Promise<string[]> {
    const disk = this.diskPages(), db = await this.dbPages(), out: string[] = [];
    for (const [slug, content] of disk) {
      if (exclude.has(slug)) continue;
      if (!db.has(slug)) out.push(`${slug}: file exists, no live page`);
      else if (db.get(slug)!.hash !== fileHash(slug, content)) out.push(`${slug}: page content_hash differs from the file`);
    }
    for (const slug of db.keys()) if (!disk.has(slug) && !exclude.has(slug)) out.push(`${slug}: live page, no file`);
    return out;
  }
  async ledgerProblems(): Promise<string[]> {
    const facts = new Set((await this.engine!.executeRaw<{ fact: string }>('SELECT fact FROM facts WHERE source_id=$1 AND expired_at IS NULL', [SOURCE])).map(r => r.fact));
    return this.ledger.problems(this.diskPages(), await this.dbPages(), facts);
  }
  async openRequests(): Promise<unknown[]> {
    const open = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; recovery: boolean; blocked_reason: string | null }>(
      `SELECT request_id::text, slug, state, recovery IS NOT NULL AS recovery, blocked_reason FROM persistence_requests
        WHERE state IN ('queued','deferred','running','recovering') OR recovery IS NOT NULL ORDER BY sequence`);
    const worktrees = await this.engine!.executeRaw<{ id: string; state: string }>(`SELECT id::text, state FROM persistence_worktrees WHERE state<>'active'`);
    return [...open.map(r => `open request ${JSON.stringify(r)}`), ...worktrees.map(w => `worktree ${w.id} is ${w.state}`)];
  }
  async unexplainedTerminal(exclude: Set<string>): Promise<string[]> {
    const rows = await this.engine!.executeRaw<{ request_id: string; slug: string; state: string; error_code: string | null; operation: string }>(
      `SELECT request_id::text, slug, state, error_code, operation FROM persistence_requests WHERE state IN ('conflict','failed','cancelled') ORDER BY sequence`);
    const disk = this.diskPages(), db = await this.dbPages();
    this.note(`terminal non-committed requests: ${rows.length} (${[...new Set(rows.map(r => `${r.state}:${r.error_code}`))].join(', ') || 'none'})`);
    return rows.filter(row => {
      const content = disk.get(row.slug);
      const superseded = exclude.has(row.slug) || (content !== undefined && db.get(row.slug)?.hash === fileHash(row.slug, content));
      return !superseded && !(row.error_code && row.error_code in CODES);
    }).map(row => `unexplained ${row.state} request ${row.request_id} (${row.slug}, ${row.error_code ?? 'no code'})`);
  }
  async cursorProblems(): Promise<string[]> {
    const [source] = await this.engine!.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1', [SOURCE]);
    const head = this.git('rev-parse', 'HEAD');
    return source.last_commit === head ? [] : [`sources.last_commit ${source.last_commit} is not HEAD ${head}`];
  }
  async status(detail = true): Promise<Record<string, unknown> | null> {
    const r = await this.spawnCli(['sync', 'status', '--source', SOURCE, '--json', ...(detail ? ['--detail'] : [])]).result;
    this.keepLog(detail ? 'status.json' : 'status-fast.json', `${r.out}\n${r.err}`);
    return lastJson(r.out);
  }
  recoveryProblems(): string[] {
    const last = this.recoveryRuns.at(-1);
    return last?.code === 0 && last.envelope?.outcome === 'synced' ? []
      : [`no plain sync succeeded within ${MAX_RECOVERY_RUNS} runs: ${this.recoveryRuns.map(r => `exit ${r.code} outcome ${String(r.envelope?.outcome ?? r.envelope?.error ?? 'none')}`).join('; ')}`];
  }
  /**
   * The large page's child rows against its file. Facts are keyed by fence row, so
   * loss, duplication and strays are all problems. Timeline rows are judged for
   * loss only; duplicate and stray timeline rows come back as a measured count,
   * because derived timeline rows are never pruned (a known defect outside #6650).
   */
  async childRowProblems(): Promise<{ problems: string[]; timelineDuplicates: number; timelineStrays: number }> {
    const content = this.diskPages().get(BIG);
    if (!content) return { problems: [`${BIG}: no file`], timelineDuplicates: 0, timelineStrays: 0 };
    const out: string[] = [];
    const expectedTimeline = [...content.matchAll(/^- \*\*(\d{4}-\d{2}-\d{2})\*\* \| (.+)$/gm)].map(m => `${m[1]}|${m[2]}`);
    const timeline = await this.engine!.executeRaw<{ key: string; n: number }>(
      `SELECT t.date::text || '|' || t.summary AS key, count(*)::int AS n FROM timeline_entries t JOIN pages p ON p.id=t.page_id
        WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL AND t.event_page_id IS NULL GROUP BY 1`, [SOURCE, BIG]);
    const have = new Map(timeline.map(r => [r.key, r.n]));
    const lostTimeline = expectedTimeline.filter(k => !have.has(k));
    const dupTimeline = timeline.filter(r => r.n > 1);
    const extraTimeline = timeline.filter(r => !expectedTimeline.includes(r.key));
    if (lostTimeline.length) out.push(`timeline: ${lostTimeline.length} of ${expectedTimeline.length} file entries missing (first: ${lostTimeline[0]})`);
    const expectedFacts = parseFactsFence(content).facts.map(f => `${f.rowNum}|${f.claim}`);
    const facts = await this.engine!.executeRaw<{ key: string; n: number }>(
      `SELECT row_num::text || '|' || fact AS key, count(*)::int AS n FROM facts
        WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL GROUP BY 1`, [SOURCE, BIG]);
    const haveFacts = new Map(facts.map(r => [r.key, r.n]));
    const lostFacts = expectedFacts.filter(k => !haveFacts.has(k));
    const dupFacts = facts.filter(r => r.n > 1);
    const extraFacts = facts.filter(r => !expectedFacts.includes(r.key));
    if (lostFacts.length) out.push(`facts: ${lostFacts.length} of ${expectedFacts.length} fence rows missing (first: ${lostFacts[0]})`);
    if (dupFacts.length) out.push(`facts: ${dupFacts.length} rows duplicated (first: ${dupFacts[0].key} x${dupFacts[0].n})`);
    if (extraFacts.length) out.push(`facts: ${extraFacts.length} rows not in the fence (first: ${extraFacts[0].key})`);
    return { problems: out, timelineDuplicates: dupTimeline.reduce((n, r) => n + r.n - 1, 0), timelineStrays: extraTimeline.length };
  }
}

export function fileHash(slug: string, content: string): string {
  const parsed = parseMarkdown(content, `${slug}.md`);
  return contentHash({ title: parsed.title, type: parsed.type, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline, frontmatter: parsed.frontmatter, tags: [...parsed.tags].sort() });
}

/** One stdio MCP session against a spawned `gbrain serve`; the serve is the host's resident persistence owner. */
export class StdioServe {
  readonly child: ChildProcess;
  stderr = '';
  exited: Promise<number | null>;
  private nextId = 1;
  private pending = new Map<number, (msg: Record<string, unknown>) => void>();
  private buf = '';
  constructor(private brain: Brain, fault?: Fault) {
    const { preload, env } = brain.faultArgs(fault);
    this.child = spawn(process.execPath, ['--no-env-file', ...preload, join(REPO_ROOT, 'src/cli.ts'), 'serve'], { cwd: REPO_ROOT, env: brain.childEnv(env), stdio: ['pipe', 'pipe', 'pipe'] });
    this.exited = new Promise(done => this.child.once('exit', code => done(code)));
    this.child.stderr!.on('data', (d: Buffer) => { this.stderr += d.toString('utf8'); });
    this.child.stdin!.on('error', () => {});
    this.child.stdout!.on('data', (d: Buffer) => {
      this.buf += d.toString('utf8');
      let idx: number;
      while ((idx = this.buf.indexOf('\n')) !== -1) {
        const line = this.buf.slice(0, idx).trim(); this.buf = this.buf.slice(idx + 1);
        if (!line) continue;
        try {
          const msg = JSON.parse(line) as Record<string, unknown>;
          const id = msg.id as number | undefined;
          if (id !== undefined && this.pending.has(id)) { this.pending.get(id)!(msg); this.pending.delete(id); }
        } catch { /* serve chatter */ }
      }
    });
    brain.serves.push(this);
  }
  get pid(): number { return this.child.pid!; }
  get alive(): boolean { return this.child.exitCode === null && this.child.signalCode === null; }
  request(method: string, params: Record<string, unknown>, timeoutMs = 240_000): Promise<Record<string, unknown>> {
    const id = this.nextId++;
    return new Promise((done, reject) => {
      const t = setTimeout(() => { this.pending.delete(id); reject(new Error(`rpc ${method} timed out`)); }, timeoutMs);
      this.pending.set(id, msg => { clearTimeout(t); done(msg); });
      this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n');
    });
  }
  async start(): Promise<void> {
    await this.request('initialize', { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 'convergence', version: '1' } });
    this.child.stdin!.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
    await until(async () => (await this.brain.engine!.executeRaw<{ n: number }>(
      `SELECT count(*)::int AS n FROM persistence_consumers WHERE pid=$1 AND kind='serve' AND renewed_at > now() - interval '30 seconds'`, [this.pid]))[0].n > 0,
      `serve ${this.pid} heartbeat`, 120_000);
  }
  async stop(): Promise<void> {
    if (!this.alive) return;
    this.child.kill('SIGTERM');
    const timer = setTimeout(() => this.child.kill('SIGKILL'), 10_000);
    await this.exited; clearTimeout(timer);
  }
}

/**
 * A writer call that retries an unsettled attempt (exit 10, a lost owner) with the same request id and a terminal refusal with a new one.
 * When the attempts or the chaos run out on an unsettled attempt, the request it names may still commit, so it comes back `pending`.
 */
export async function acknowledged(b: Brain, label: string, call: (requestId: string) => SyncRun, stop: () => boolean): Promise<{ start: number; ack: number } | { pending: string } | null> {
  let requestId = randomUUID();
  let unsettled = false;
  for (let attempt = 1; attempt <= 20 && !stop(); attempt++) {
    const start = performance.now();
    const r = await call(requestId).result;
    b.keepLog(`writer-${label.replaceAll('/', '_')}-${attempt}.log`, `exit ${r.code}\n${r.out}\n${r.err}`);
    if (r.code === 0) return { start, ack: performance.now() };
    const terminal = /\((conflict|failed|cancelled)\)/.test(r.err);
    b.note(`writer ${label} attempt ${attempt} exit ${r.code}${terminal ? ' (terminal; next attempt uses a new request id)' : ''}: ${r.err.trim().split('\n').slice(-1)[0] ?? ''}`);
    unsettled = !terminal;
    if (terminal) requestId = randomUUID();
    await Bun.sleep(3000);
  }
  return unsettled ? { pending: requestId } : null;
}
export const outcome = (ok: { ack: number } | { pending: string } | null) => !ok ? 'unacknowledged' : 'pending' in ok ? `pending ${ok.pending}` : 'ok';

/** Oracle tests are named by the persistence gate's violation class (scripts/persistence/model.ts) they would report. */
export const as = (violation: ViolationClass, title: string): string => `[${violation}] ${title}`;

// ---------------------------------------------------------------- the shared oracle

export interface EndState { exclude?: Set<string>; status?: StatusExpectation; bigPage?: boolean }
/** The oracle tests every scenario ends with; `end()` is read when the tests run (after the scenario's beforeAll). */
export function oracleTests(b: Brain, end: () => EndState): void {
  test('the oracle is calibrated: a fault-free import satisfies it', () => b.verdict(b.calibration));
  test('every planned fault fired, so the chaos actually happened', () => b.verdict(Object.entries(b.fired).filter(([, at]) => at === null).map(([fault]) => `${fault} did not fire`)));
  test(as('wedge', `a plain sync succeeds within ${MAX_RECOVERY_RUNS} runs after the chaos, with no operator step`), () => b.verdict(b.recoveryProblems()));
  test(as('projection_drift', 'disk == database for every page the end state does not name'), async () => b.verdict(await b.diskDbMismatches(end().exclude ?? new Set())));
  test(as('wedge', 'no request blocks the checkpoint or a root'), async () => b.verdict((await b.openRequests()).map(String)));
  test(as('untyped_error', 'every non-committed terminal request is superseded or carries a registered error code'), async () => b.verdict(await b.unexplainedTerminal(end().exclude ?? new Set())));
  test(as('wedge', 'the cursor reached HEAD'), async () => b.verdict(await b.cursorProblems()));
  test(as('lost_write', 'every acknowledged write survives or is causally superseded by a later acknowledged write'), async () => {
    b.note(`ledger: ${b.ledger.acks.length} acknowledged writes (${[...new Set(b.ledger.acks.map(a => a.kind))].map(kind => `${kind} ${b.ledger.acks.filter(a => a.kind === kind).length}`).join(', ')})`);
    b.verdict(await b.ledgerProblems());
  });
  test(as('wedge', 'sync status --json reports the converged state with every field present'), async () => b.verdict(statusProblems(await b.status(), end().status)));
  test(as('wedge', 'plain sync status --json names every heavy read it skipped instead of reading as all-clear'), async () => b.verdict(fastStatusProblems(await b.status(false))));
  if (end().bigPage !== false) test(as('lost_write', 'the large page keeps every fence fact exactly once and loses no timeline entry'), async () => {
    const { problems, timelineDuplicates, timelineStrays } = await b.childRowProblems();
    b.note(`large page timeline (measured, not asserted): ${timelineDuplicates} duplicate rows, ${timelineStrays} rows not in the file`);
    b.verdict(problems);
  });
}