Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
27.0 KB
Raw Blame
/**
 * Upgrade matrix (GBRA-224 W2): one cell. Install a published old release the
 * way the README tells a user to, build a small brain on it, run serve (and a
 * jobs supervisor on Postgres), upgrade through the old release's own updater
 * while the GitHub fixture shows only the candidate, then check the candidate
 * on that brain. docs/ci-upgrade-matrix.md lists every step and what it
 * guards.
 */
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { dirname, join } from 'node:path';
import { startGithubFixture, type FixtureHandle } from './github-fixture.ts';
import { StdioMcp } from './mcp-probe.ts';
import { diffCatalog, diffContent, type Difference, type Snapshot } from './snapshot.ts';
import { DOCTOR_ARGV, doctorVerdict, judgeCell, type Check, type Failure, type KnownException } from './verdict.ts';
import type { Candidate } from './versions.ts';
import { startMockProviders, type MockServer } from '../bench/retrieval-scale/mock-providers.ts';
import postgres from '#postgres';

export type Engine = 'pglite' | 'postgres';
export type Keys = 'keyless' | 'keyed';
export type BrainShape = 'repo-less' | 'repo';
export type Fault = 'corrupt-after-upgrade';

export interface CellSpec {
  from: { tag: string; version: string; sha: string; roles: string[] };
  candidate: Candidate;
  engine: Engine;
  keys: Keys;
  brain: BrainShape;
  harnessRoot: string;
  harnessSha: string;
  workDir: string;
  cacheDir: string;
  /** Superuser URL of the Postgres server for Postgres cells; each cell creates its own database. */
  postgresUrl?: string;
  fault?: Fault | null;
  exceptions: KnownException[];
}

export interface Step { name: string; ok: boolean; ms: number; exit?: number | null; detail?: string; log?: string }

/**
 * Every cell process follows the alpha channel (every published release). The fixture publishes the candidate as the
 * newest release and never a stable evidence record, so a from-version that defaults to stable (v0.60.606.0 on, #8726)
 * would refuse with stable_invalid instead of upgrading to the build under test. Releases before that read no
 * GBRAIN_RELEASE_CHANNEL and already follow every release; it outranks self_upgrade.channel on the ones that read it.
 */
export const CELL_RELEASE_CHANNEL_ENV: Readonly<Record<string, string>> = { GBRAIN_RELEASE_CHANNEL: 'alpha' };

/** The refs the fixture serves while the old release upgrades: the candidate everywhere, the from-version's own tag kept. */
export function upgradeRefs(from: { tag: string; sha: string }, candidate: { version: string; sha: string }): Record<string, string> {
  return { HEAD: candidate.sha, 'latest-stable': candidate.sha, stable: candidate.sha, [from.tag]: from.sha, ['v' + candidate.version]: candidate.sha };
}

export const cellId = (s: { from: { tag: string }; engine: string; keys: string; brain: string }) => [s.from.tag, s.engine, s.keys, s.brain].join('__');

const SEED_WORD = 'zephyrquartz';
const NEW_WORD = 'obsidianlantern';
const SEED_PAGES: Array<{ slug: string; body: string }> = [
  { slug: 'concepts/upgrade-matrix-alpha', body: '# Upgrade matrix alpha\n\nThe alpha page mentions ' + SEED_WORD + ' and links to [[concepts/upgrade-matrix-beta]].\n' },
  { slug: 'concepts/upgrade-matrix-beta', body: '# Upgrade matrix beta\n\nThe beta page is about lighthouse maintenance schedules and tidal charts.\n' },
  { slug: 'people/alice-example', body: '# Alice Example\n\nAlice Example works at acme-example on storage engines.\n\n<!--- gbrain:facts:begin -->\n| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |\n|---|---|---|---|---|---|---|---|---|---|\n| 1 | Prefers written design reviews | preference | 0.9 | world | medium | 2026-01-05 | | upgrade-matrix seed | |\n| 2 | Leads the storage team at acme-example | fact | 1.0 | world | high | 2025-11-01 | | upgrade-matrix seed | |\n<!--- gbrain:facts:end -->\n' },
];
const DELETED_SLUG = 'notes/upgrade-matrix-deleted';

function stubBin(dir: string, log: string): void {
  mkdirSync(dir, { recursive: true });
  const write = (name: string, body: string) => { writeFileSync(join(dir, name), '#!/bin/sh\necho "' + name + ' $*" >> "' + log + '"\n' + body + '\n', { mode: 0o755 }); };
  write('crontab', 'case "$1" in -l) echo "no crontab for matrix" >&2; exit 1;; esac\ncat > /dev/null\nexit 0');
  write('systemctl', 'echo "Failed to connect to bus: No medium found" >&2\nexit 1');
  write('launchctl', 'exit 1');
  write('loginctl', 'exit 1');
}

export async function runCell(spec: CellSpec): Promise<Record<string, unknown>> {
  const id = cellId(spec);
  const dir = join(spec.workDir, id);
  rmSync(dir, { recursive: true, force: true });
  const home = join(dir, 'home'), bunDir = join(dir, 'bun'), logs = join(dir, 'logs'), tmp = join(dir, 'tmp');
  for (const d of [home, bunDir, logs, tmp]) mkdirSync(d, { recursive: true });
  const serviceLog = join(dir, 'service-calls.log');
  stubBin(join(dir, 'stub-bin'), serviceLog);

  const steps: Step[] = [];
  const failures: Failure[] = [];
  const fail = (check: Check, detail: string) => { failures.push({ check, detail: detail.slice(0, 2000) }); };
  const started = Date.now();
  const receipt: Record<string, unknown> = {
    schema: 'gbrain-upgrade-matrix-cell/1', id, harness_sha: spec.harnessSha,
    candidate: spec.candidate, from: spec.from, shape: { engine: spec.engine, keys: spec.keys, brain: spec.brain }, fault: spec.fault ?? null,
  };

  const mock: MockServer | null = spec.keys === 'keyed' ? startMockProviders({ manifest: { seed: 224, chunks: 64, pages: 16, dims: 1024 } }) : null;
  const fx: FixtureHandle = await startGithubFixture({
    source: spec.harnessRoot, mirrorDir: join(dir, 'mirror.git'), archiveDir: join(spec.cacheDir, 'archives'), certDir: join(spec.cacheDir, 'certs'),
    forward: mock ? { 'api.voyageai.com': mock.url } : {},
  });
  writeFileSync(join(dir, 'gitconfig'), '[url "' + fx.mirror + '"]\n\tinsteadOf = https://github.com/garrytan/gbrain\n[user]\n\tname = upgrade-matrix\n\temail = [email protected]\n[init]\n\tdefaultBranch = main\n');

  let databaseUrl: string | undefined;
  const dbName = 'um_' + createHash('sha1').update(id + process.pid + started).digest('hex').slice(0, 16);
  const env: Record<string, string> = {
    PATH: [join(dir, 'stub-bin'), join(bunDir, 'bin'), dirname(process.execPath), '/usr/local/bin', '/usr/bin', '/bin'].join(':'),
    HOME: home, GBRAIN_HOME: home, BUN_INSTALL: bunDir, BUN_INSTALL_CACHE_DIR: join(spec.cacheDir, 'bun-install-cache'), TMPDIR: tmp,
    LANG: 'C.UTF-8', NO_COLOR: '1', TERM: 'dumb', USER: process.env.USER ?? 'runner',
    GIT_CONFIG_GLOBAL: join(dir, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1',
    HTTPS_PROXY: fx.proxyUrl, HTTP_PROXY: fx.proxyUrl, https_proxy: fx.proxyUrl, http_proxy: fx.proxyUrl,
    NO_PROXY: '127.0.0.1,localhost', no_proxy: '127.0.0.1,localhost', NODE_EXTRA_CA_CERTS: fx.caFile, GITHUB_API_URL: fx.apiUrl,
    ...CELL_RELEASE_CHANNEL_ENV,
    ...(mock ? { VOYAGE_API_KEY: 'pa-upgrade-matrix-mock-not-a-key' } : {}),
  };

  let logSeq = 0;
  const run = async (name: string, cmd: string[], opts: { stdin?: string; timeoutMs?: number; check?: Check; extraEnv?: Record<string, string> } = {}) => {
    const t = Date.now();
    const log = join(logs, String(++logSeq).padStart(2, '0') + '-' + name.replace(/[^a-z0-9-]+/gi, '_') + '.log');
    const proc = Bun.spawn(cmd, { cwd: home, env: { ...env, ...opts.extraEnv }, stdin: opts.stdin !== undefined ? new Blob([opts.stdin]) : 'ignore', stdout: 'pipe', stderr: 'pipe', timeout: opts.timeoutMs ?? 300_000, killSignal: 'SIGKILL' });
    const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
    writeFileSync(log, '$ ' + cmd.join(' ') + '\n[exit ' + code + ' after ' + (Date.now() - t) + ' ms]\n--- stdout\n' + stdout + '\n--- stderr\n' + stderr);
    const step: Step = { name, ok: code === 0, ms: Date.now() - t, exit: code, log: log.slice(dir.length + 1) };
    if (code !== 0) step.detail = (stderr.trim() || stdout.trim()).split('\n').slice(-6).join('\n').slice(0, 800);
    steps.push(step);
    if (code !== 0 && opts.check) fail(opts.check, name + ' exited ' + code + ': ' + (step.detail ?? ''));
    return { code, stdout, stderr };
  };
  const note = (name: string, ok: boolean, detail?: string) => { steps.push({ name, ok, ms: 0, ...(detail ? { detail: detail.slice(0, 800) } : {}) }); };
  const gbrain = (name: string, args: string[], opts: Parameters<typeof run>[2] = {}) => run(name, ['gbrain', ...args], opts);
  const installDir = (root = join(bunDir, 'install', 'global')) => join(root, 'node_modules', 'gbrain');

  const brainChild = (name: string, extra: string[], install = installDir()) => run(name, [
    process.execPath, join(spec.harnessRoot, 'scripts', 'upgrade-matrix', 'snapshot-child.ts'),
    ...(spec.engine === 'pglite' ? ['--pglite', JSON.parse(readFileSync(join(home, '.gbrain', 'config.json'), 'utf8')).database_path, '--install', install] : ['--url', databaseUrl!]),
    ...extra,
  ], { timeoutMs: 180_000 });
  const snapshot = async (name: string, catalog: boolean): Promise<Snapshot | null> => {
    const r = await brainChild('snapshot-' + name, catalog ? ['--catalog'] : []);
    if (r.code !== 0) return null;
    writeFileSync(join(dir, 'snapshot-' + name + '.json'), r.stdout);
    return JSON.parse(r.stdout) as Snapshot;
  };

  const processes: { serve: StdioMcp | null; supervisor: boolean } = { serve: null, supervisor: false };
  const startServe = async (label: string, cli: string[] = ['gbrain']): Promise<string | null> => {
    const proc = Bun.spawn([...cli, 'serve'], { cwd: home, env, stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' });
    const client = new StdioMcp(proc as never);
    processes.serve = client;
    const t = Date.now();
    try {
      const version = await client.initialize();
      steps.push({ name: 'serve-' + label, ok: true, ms: Date.now() - t, detail: 'serverInfo.version ' + version });
      return version;
    } catch (e) {
      steps.push({ name: 'serve-' + label, ok: false, ms: Date.now() - t, detail: String(e) + '\n' + client.stderr.join('').slice(-600) });
      return null;
    }
  };
  const stopServe = async () => { if (processes.serve) { await processes.serve.close(); processes.serve = null; } };
  const stopAll = async () => {
    await stopServe();
    if (processes.supervisor) { await gbrain('supervisor-stop', ['jobs', 'supervisor', 'stop'], { timeoutMs: 60_000 }); processes.supervisor = false; }
  };

  try {
    // ── 1. old release, installed and seeded as a user would ──────────────
    fx.setRefs({ HEAD: spec.from.sha, 'latest-stable': spec.from.sha, stable: spec.from.sha, [spec.from.tag]: spec.from.sha });
    const install = await run('install-old', ['bun', 'install', '-g', 'github:garrytan/gbrain'], { timeoutMs: 600_000, check: 'setup' });
    const oldVersion = install.code === 0 ? (await gbrain('version-old', ['--version'], { check: 'setup' })).stdout.trim() : '';
    if (install.code === 0 && !oldVersion.endsWith(spec.from.version)) fail('setup', 'installed ' + oldVersion + ', expected ' + spec.from.version);
    if (failures.length) throw new SetupError();

    if (spec.engine === 'postgres') {
      if (!spec.postgresUrl) { fail('setup', 'a Postgres cell needs --postgres-url'); throw new SetupError(); }
      const admin = new URL(spec.postgresUrl);
      await adminSql(spec.postgresUrl, 'CREATE DATABASE ' + dbName);
      admin.pathname = '/' + dbName;
      databaseUrl = admin.toString();
    }
    const initArgs = ['init', '--non-interactive',
      ...(spec.engine === 'pglite' ? ['--pglite'] : ['--url', databaseUrl!]),
      ...(spec.keys === 'keyless' ? ['--no-embedding'] : []),
      ...(spec.brain === 'repo' ? ['--git'] : [])];
    await gbrain('init', initArgs, { check: 'setup', timeoutMs: 600_000 });
    if (failures.length) throw new SetupError();

    for (const page of SEED_PAGES) await gbrain('seed-put-' + page.slug.split('/')[1], ['put', page.slug], { stdin: page.body, check: 'setup' });
    await gbrain('seed-remember', ['remember', 'Alice Example reviews storage designs on Tuesdays', '--entity', 'people/alice-example', '--provenance', 'upgrade-matrix seed'], { check: 'setup' });
    const doomed = await gbrain('seed-put-deleted', ['put', DELETED_SLUG], { stdin: '# To be deleted\n\nThis page is soft-deleted before the upgrade.\n', check: 'setup' });
    const revision = parseJson(doomed.stdout)?.revision;
    await gbrain('seed-delete', ['delete', DELETED_SLUG, ...(typeof revision === 'string' ? ['--expected-revision', revision] : [])], { check: 'setup' });
    if (spec.keys === 'keyed') await gbrain('seed-embed', ['embed', '--stale', '--yes'], { check: 'setup' });
    const seedDream = await gbrain('seed-dream', ['dream', '--json'], { timeoutMs: 600_000 });
    receipt.seed_dream = seedDream.code === 0 ? 'ok' : 'failed (exit ' + seedDream.code + '; recorded, not judged: the old release ran it)';
    if (failures.length) throw new SetupError();

    const before = await snapshot('before', false);
    if (!before) { fail('setup', 'could not read the seeded brain'); throw new SetupError(); }
    const oldGlobal = join(dir, 'old-global');
    cpSync(join(bunDir, 'install', 'global'), oldGlobal, { recursive: true, verbatimSymlinks: true });

    const oldServeVersion = await startServe('old');
    if (oldServeVersion !== spec.from.version) { fail('setup', 'the old serve did not answer MCP as ' + spec.from.version + ' (got ' + oldServeVersion + ')'); throw new SetupError(); }
    if (spec.engine === 'postgres') {
      const sup = await gbrain('supervisor-start-old', ['jobs', 'supervisor', 'start', '--detach', '--json'], { timeoutMs: 120_000, check: 'setup' });
      processes.supervisor = sup.code === 0;
      if (failures.length) throw new SetupError();
    }
    const oldServe = processes.serve!;

    // ── 2. the old release's own updater, seeing only the candidate ───────
    fx.setRefs(upgradeRefs(spec.from, spec.candidate));
    const switchedAt = fx.requests.length;
    let up = await gbrain('upgrade', ['upgrade'], { timeoutMs: 900_000 });
    if (up.code === 3 && /restart_refused|--stop-mcp/.test(up.stdout + up.stderr)) {
      receipt.upgrade_needed_stop_mcp = true;
      up = await gbrain('upgrade-stop-mcp', ['upgrade', '--stop-mcp'], { timeoutMs: 900_000 });
    }
    if (up.code !== 0) fail('upgrade', 'gbrain upgrade exited ' + up.code + ': ' + (up.stderr || up.stdout).trim().split('\n').slice(-8).join('\n'));

    const served = fx.requests.slice(switchedAt);
    const foreign = served.filter(r => r.sha && r.sha !== spec.candidate.sha && r.status === 200);
    receipt.interposition = { after_switch: served, proxy: dedupe(fx.proxy), denied: dedupe(fx.proxy.filter(p => p.action === 'denied')) };
    if (foreign.length) fail('interposition', 'the updater fetched non-candidate commits: ' + foreign.map(r => r.host + r.path + '@' + r.sha!.slice(0, 7)).join(', '));
    if (!served.some(r => r.sha === spec.candidate.sha && r.status === 200 && /tarball/.test(r.path))) fail('interposition', 'the updater never fetched the candidate tarball from the fixture');

    // ── 3. the installed build is the candidate commit ────────────────────
    const newVersion = (await gbrain('version-new', ['--version'])).stdout.trim();
    const tree = compareInstalledTree(spec.harnessRoot, spec.candidate.sha, installDir());
    const lock = readFileSync(join(bunDir, 'install', 'global', 'bun.lock'), 'utf8');
    const lockSha = /"gbrain@github:garrytan\/gbrain#([0-9a-f]{7,40})"/.exec(lock)?.[1] ?? null;
    receipt.installed = { old_version: oldVersion, new_version: newVersion, lock_sha: lockSha, tree_files: tree.checked, tree_mismatches: tree.mismatches.slice(0, 20), tree_missing: tree.missing.slice(0, 20) };
    if (!newVersion.endsWith(spec.candidate.version)) fail('installed_build', 'gbrain --version says ' + newVersion + ', expected ' + spec.candidate.version);
    if (!lockSha || !spec.candidate.sha.startsWith(lockSha)) fail('installed_build', 'bun.lock pins ' + lockSha + ', expected ' + spec.candidate.sha.slice(0, 7));
    if (tree.mismatches.length || tree.missing.length) fail('installed_build', tree.mismatches.length + ' installed files differ from the candidate tree and ' + tree.missing.length + ' are missing: ' + [...tree.mismatches, ...tree.missing].slice(0, 5).join(', '));

    // ── 4. one build: nothing from before the upgrade still runs ──────────
    if (oldServe.alive) {
      receipt.restart_fix_needed = true;
      const fix = await gbrain('restart-only', ['upgrade', '--restart-only', '--stop-mcp'], { timeoutMs: 900_000 });
      if (fix.code !== 0) fail('one_build', 'gbrain upgrade --restart-only --stop-mcp exited ' + fix.code);
      await Promise.race([oldServe.proc.exited, new Promise(r => setTimeout(r, 20_000))]);
      if (oldServe.alive) fail('one_build', 'the serve started on ' + spec.from.version + ' is still running after the upgrade and restart');
      await oldServe.close();
    }
    processes.serve = null;
    const newServeVersion = await startServe('new');
    if (newServeVersion !== spec.candidate.version) fail('serve', 'the restarted serve reports ' + newServeVersion + ', expected ' + spec.candidate.version);
    else {
      try {
        const hits = await processes.serve!.callTool('search', { query: SEED_WORD });
        if (!hits.includes('upgrade-matrix-alpha')) fail('serve', 'MCP search for the seed word did not return the seeded page');
        note('serve-search', true);
      } catch (e) { fail('serve', String(e)); }
    }
    if (spec.engine === 'postgres') {
      if (processes.supervisor) await waitForWorkerHeartbeat();
      const doc = await gbrain('doctor-processes', ['doctor', '--json'], { timeoutMs: 300_000 });
      const skew = parseJson(doc.stdout)?.checks?.find((c: any) => c.name === 'runtime_build_skew');
      receipt.runtime_build_skew = skew ?? null;
      if (!skew || skew.status !== 'ok') fail('one_build', 'runtime_build_skew is ' + (skew?.status ?? 'missing') + ': ' + String(skew?.message ?? '').slice(0, 300));
    }
    await stopAll();
    const leftovers = gbrainProcessesFor(home);
    if (leftovers.length) fail('one_build', 'gbrain processes still running for this brain after stopping serve and the supervisor: ' + leftovers.join('; '));

    // ── 5. migrations, schema head, doctor ─────────────────────────────────
    await gbrain('apply-migrations', ['apply-migrations', '--yes', '--non-interactive'], { timeoutMs: 900_000, check: 'apply_migrations' });
    const doctor = await gbrain('doctor', DOCTOR_ARGV, { timeoutMs: 300_000 });
    const report = parseJson(doctor.stdout);
    const schema = report?.checks?.find((c: any) => c.name === 'schema_version');
    const m = /Version (\d+) \(latest: (\d+)\)/.exec(String(schema?.message ?? ''));
    receipt.schema_version = schema?.message ?? null;
    if (!m || m[1] !== m[2] || schema.status !== 'ok') fail('schema_head', 'schema_version check: ' + (schema ? schema.status + ' ' + schema.message : 'missing from doctor --json'));
    const dv = doctorVerdict(report);
    receipt.doctor = { source: dv.source, status: report?.status ?? null, health_score: report?.health_score ?? null, failing: dv.failing };
    if (!dv.ok) fail('doctor', dv.failing.length ? dv.failing.join('; ') : 'doctor printed no JSON report (exit ' + doctor.code + ')');

    // ── 6. preservation: canonical content equal before and after ─────────
    if (spec.fault === 'corrupt-after-upgrade') {
      const injected = await brainChild('fault-corrupt', ['--exec', "SET session_replication_role = replica; UPDATE pages SET title = title || ' (corrupted by the upgrade-matrix fault)' WHERE slug = 'concepts/upgrade-matrix-beta'; UPDATE pages SET deleted_at = NULL WHERE slug = '" + DELETED_SLUG + "'"]);
      if (injected.code !== 0) { fail('setup', 'the corruption fault could not be injected'); throw new SetupError(); }
    }
    const after = await snapshot('after', true);
    if (!after) fail('preservation', 'could not read the upgraded brain');
    else {
      const diffs = diffContent(before, after);
      receipt.preservation = { differences: diffs.slice(0, 50), compared: Object.fromEntries(Object.entries(before.content).map(([t, v]) => [t, v.rows.length])) };
      if (diffs.length) fail('preservation', diffs.length + ' canonical differences: ' + diffs.slice(0, 5).map(d => d.kind + ' ' + d.where).join('; '));
    }

    // ── 7. the upgraded brain works ────────────────────────────────────────
    const newSlug = 'concepts/upgrade-matrix-after';
    await gbrain('write', ['put', newSlug], { stdin: '# After the upgrade\n\nWritten by the candidate; it mentions ' + NEW_WORD + '.\n', check: 'write' });
    for (const [word, slug] of [[SEED_WORD, 'upgrade-matrix-alpha'], [NEW_WORD, 'upgrade-matrix-after']] as const) {
      const s = await gbrain('search-' + word, ['search', word], { check: 'keyword_search' });
      if (s.code === 0 && !s.stdout.includes(slug)) fail('keyword_search', 'search ' + word + ' did not return ' + slug);
    }
    if (mock) {
      const calls = mock.stats.embed_inputs;
      await gbrain('embed-new', ['embed', newSlug, '--yes'], { check: 'vector_search' });
      if (mock.stats.embed_inputs <= calls) fail('vector_search', 'embedding the new page made no provider call');
      const queried = mock.stats.embed_calls;
      const q = await gbrain('vector-query', ['query', 'lighthouse maintenance schedules and tidal charts'], { check: 'vector_search' });
      if (mock.stats.embed_calls <= queried) fail('vector_search', 'query did not embed the question (no vector arm)');
      if (q.code === 0 && !q.stdout.includes('upgrade-matrix-beta')) fail('vector_search', 'query did not return the matching seed page');
    }
    await gbrain('dream', ['dream', '--json'], { timeoutMs: 600_000, check: 'dream' });

    // ── 8. mixed versions: the old binary on the upgraded brain ───────────
    const pre = await snapshot('mixed-before', true);
    const oldCli = [process.execPath, join(installDir(oldGlobal), 'src', 'cli.ts')];
    await run('old-version', [...oldCli, '--version']);
    await run('old-search', [...oldCli, 'search', SEED_WORD]);
    await run('old-doctor', [...oldCli, 'doctor', '--json'], { timeoutMs: 300_000 });
    const oldAgain = await startServe('old-again', oldCli);
    receipt.old_serve_on_upgraded_brain = oldAgain ?? 'did not answer MCP (recorded, not judged)';
    if (oldAgain) { try { await processes.serve!.callTool('search', { query: SEED_WORD }); } catch { /* the old build may refuse; only mutation is judged */ } }
    await stopAll();
    const post = pre ? await snapshot('mixed-after', true) : null;
    if (!pre || !post) fail('mixed_version', 'could not snapshot the brain around the old binary');
    else {
      const diffs: Difference[] = [...diffCatalog(pre, post), ...diffContent(pre, post)];
      receipt.mixed_version = { differences: diffs.slice(0, 50) };
      if (diffs.length) fail('mixed_version', diffs.length + ' schema or content changes by ' + spec.from.version + ': ' + diffs.slice(0, 5).map(d => d.kind + ' ' + d.where).join('; '));
    }
  } catch (e) {
    if (!(e instanceof SetupError)) fail('setup', 'harness error: ' + (e instanceof Error ? e.stack ?? e.message : String(e)));
  } finally {
    await stopAll().catch(() => {});
    for (const pid of gbrainPids(home)) { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }
    fx.stop();
    mock?.stop();
    if (databaseUrl && spec.postgresUrl) await adminSql(spec.postgresUrl, 'DROP DATABASE IF EXISTS ' + dbName + ' WITH (FORCE)').catch(() => {});
  }

  const judged = judgeCell(spec.from.tag, failures, spec.exceptions);
  receipt.verdict = spec.fault && judged.verdict === 'pass' ? 'fail' : judged.verdict;
  receipt.failures = judged.open;
  receipt.known_exceptions = judged.excepted;
  receipt.service_calls = existsSync(serviceLog) ? readFileSync(serviceLog, 'utf8').trim().split('\n').filter(Boolean) : [];
  receipt.steps = steps;
  receipt.elapsed_s = Math.round((Date.now() - started) / 1000);
  writeFileSync(join(dir, 'receipt.json'), JSON.stringify(receipt, null, 2) + '\n');
  return receipt;


  async function waitForWorkerHeartbeat(): Promise<void> {
    await new Promise(r => setTimeout(r, 15_000));
  }
}

class SetupError extends Error {}

async function adminSql(url: string, statement: string): Promise<void> {
  const sql = postgres(url, { max: 1, onnotice: () => {} });
  try { await sql.unsafe(statement); } finally { await sql.end({ timeout: 5 }); }
}

function parseJson(text: string): any {
  try { return JSON.parse(text); } catch {
    const start = text.indexOf('{');
    try { return start >= 0 ? JSON.parse(text.slice(start)) : null; } catch { return null; }
  }
}

function dedupe<T extends { host: string; action: string }>(rows: T[]): T[] {
  return rows.filter((r, i) => rows.findIndex(o => o.host === r.host && o.action === r.action) === i);
}

/** Every blob in the candidate's tree compared with the installed package (git hash-object), skipping paths Bun does not install. */
export function compareInstalledTree(repo: string, sha: string, installed: string): { checked: number; mismatches: string[]; missing: string[] } {
  const listing = execFileSync('git', ['-C', repo, 'ls-tree', '-r', '--full-tree', sha], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
  const mismatches: string[] = [], missing: string[] = [];
  const paths: string[] = [], expected: string[] = [];
  for (const line of listing.split('\n')) {
    const m = /^(\d+) blob ([0-9a-f]+)\t(.+)$/.exec(line);
    if (!m || m[1] === '120000' || m[1] === '160000') continue;
    const path = m[3]!;
    if (path.startsWith('.git/') || /(^|\/)node_modules\//.test(path)) continue;
    if (!existsSync(join(installed, path))) { missing.push(path); continue; }
    paths.push(path); expected.push(m[2]!);
  }
  for (let i = 0; i < paths.length; i += 2000) {
    const chunk = paths.slice(i, i + 2000);
    const got = execFileSync('git', ['hash-object', '--no-filters', '--stdin-paths'], { cwd: installed, input: chunk.join('\n') + '\n', encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).trim().split('\n');
    chunk.forEach((p, j) => { if (got[j] !== expected[i + j]) mismatches.push(p); });
  }
  return { checked: paths.length, mismatches, missing };
}

function gbrainPids(home: string): number[] {
  const out: number[] = [];
  for (const entry of readdirSync('/proc')) {
    if (!/^\d+$/.test(entry) || Number(entry) === process.pid) continue;
    try {
      const environ = readFileSync('/proc/' + entry + '/environ', 'utf8');
      if (environ.split('\0').includes('GBRAIN_HOME=' + home)) out.push(Number(entry));
    } catch { /* gone or not ours */ }
  }
  return out;
}

function gbrainProcessesFor(home: string): string[] {
  return gbrainPids(home).map(pid => {
    try { return pid + ' ' + readFileSync('/proc/' + pid + '/cmdline', 'utf8').split('\0').join(' ').trim().slice(0, 200); } catch { return String(pid); }
  });
}