Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
41.9 KB
Raw Blame
/**
 * gbrain claw-test — end-to-end "fresh user" test harness.
 *
 * Two tiers:
 *   gbrain claw-test                              — scripted (no LLM, CI gate)
 *   gbrain claw-test --live --agent openclaw      — real agent, friction discovery
 *
 * Phases (scripted mode):
 *   setup → install_brain → import → query → extract → verify → render
 *
 * The harness sets GBRAIN_HOME=<tempdir> so the run is hermetic. Each child
 * gbrain invocation runs with --progress-json and the harness captures stderr
 * to assert expected_phases from scenario.json fired.
 *
 * See ~/.claude/plans/system-instruction-you-are-working-noble-biscuit.md
 * for the full design rationale (D1–D23 decisions).
 */

import { spawn } from '../core/spawn.ts';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, existsSync, readFileSync, appendFileSync, chmodSync, cpSync, lstatSync } from 'fs';
import { join, resolve, basename, dirname } from 'path';
import { fileURLToPath } from 'url';
import { tmpdir } from 'os';
import { randomBytes } from 'crypto';
import { registerCleanup } from '../core/process-cleanup.ts';
import { logFriction, frictionDir, frictionFile } from '../core/friction.ts';
import { loadScenario, listScenarios, readBrief, type ScenarioConfig } from '../core/claw-test/scenarios.ts';
import { parseProgressEvents, verifyExpectedPhases } from '../core/claw-test/progress-tail.ts';
import { resolveAgentRunner, listRegisteredAgents, registerAgentRunner, validateBinPathEnv } from '../core/claw-test/agent-runner.ts';
import { OpenClawRunner } from '../core/claw-test/runners/openclaw.ts';
import { HermesRunner } from '../core/claw-test/runners/hermes.ts';
import { GrokRunner } from '../core/claw-test/runners/grok.ts';
import { OpencodeRunner } from '../core/claw-test/runners/opencode.ts';
import { createTranscriptSink } from '../core/claw-test/transcript-capture.ts';

// Ensure built-in runners are registered.
registerAgentRunner('openclaw', () => new OpenClawRunner());
registerAgentRunner('hermes', () => new HermesRunner());
registerAgentRunner('grok', () => new GrokRunner());
registerAgentRunner('opencode', () => new OpencodeRunner());

interface HarnessOpts {
  scenario: string;
  live: boolean;
  agent: string;
  keepTempdir: boolean;
  listAgents: boolean;
  help: boolean;
  /** Path to the gbrain binary used to invoke child commands (always set by
   *  parseArgs: GBRAIN_BIN_OVERRIDE when valid; else the compiled gbrain
   *  binary, or a synthesized launcher when running under the bun runtime —
   *  see resolveGbrainBin). */
  gbrainBin: string;
}

interface PhaseOutcome {
  phase: string;
  exitCode: number;
  durationMs: number;
  stderrEvents: number;
  stdoutTail: string;
  stderrTail: string;
  /** Full stdout, only populated when invokeGbrain is asked to capture it. */
  stdoutFull?: string;
}

const TAIL_BYTES = 4_096;
/** Per-phase cap for the harness's own gbrain children (staging, scripted
 *  phases, oracle probes). Env override is a test/incident escape hatch. */
const SUBPROCESS_TIMEOUT_MS = envTimeoutMs('GBRAIN_CLAW_PHASE_TIMEOUT_MS', 5 * 60_000);
/** Wall clock for the live agent turn — real fresh-install turns run long
 *  (help text promises "5 to 10 min"), so the agent gets double the phase cap. */
const LIVE_AGENT_TIMEOUT_MS = envTimeoutMs('GBRAIN_CLAW_AGENT_TIMEOUT_MS', 10 * 60_000);

function envTimeoutMs(name: string, fallback: number): number {
  const raw = process.env[name];
  if (!raw) return fallback;
  const n = Number(raw);
  return Number.isFinite(n) && n > 0 ? n : fallback;
}

export async function runClawTest(args: string[]): Promise<number> {
  const opts = parseArgs(args);

  if (opts.help) {
    printHelp();
    return 0;
  }

  if (opts.listAgents) {
    return cmdListAgents();
  }

  // Charset guard: both values flow into filesystem paths (scenario → the
  // fixtures root join, agent → the run-id → the tempdir template), so a
  // traversal-shaped value would either escape the fixtures root or crash
  // sanitizeRunId mid-run. Usage error, exit 2.
  const NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
  for (const [flag, value] of [['scenario', opts.scenario], ['agent', opts.agent]] as const) {
    if (!NAME_RE.test(value)) {
      console.error(`invalid --${flag} value ${JSON.stringify(value)}: letters, digits, dot, dash, underscore only`);
      return 2;
    }
  }

  let scenario: ScenarioConfig;
  try {
    scenario = loadScenario(opts.scenario);
  } catch (e) {
    console.error(`scenario load failed: ${e instanceof Error ? e.message : String(e)}`);
    const available = listScenarios();
    if (available.length) console.error(`available scenarios: ${available.join(', ')}`);
    return 2;
  }

  const runId = newRunId(opts.agent);
  const runRoot = mkdtempSync(join(tmpdir(), `claw-test-${runId}-`));
  const gbrainHome = runRoot; // configDir() appends '.gbrain' itself
  const transcriptPath = join(runRoot, 'transcript.jsonl');
  console.log(`run-id: ${runId}`);
  console.log(`tempdir: ${runRoot}`);

  // Run-start meta record. Agent-name resolution in `gbrain friction diff`
  // depends on this: a fully clean run otherwise writes zero agent-stamped
  // entries and could never be resolved by agent name. Uses the existing
  // phase-marker kind (no new FrictionKind) + additive scenario/harness_schema
  // fields.
  const agentLabel = opts.live ? opts.agent : 'scripted';
  try {
    logFriction({
      runId,
      phase: 'harness',
      kind: 'phase-marker',
      marker: 'start',
      message: `run start: scenario=${scenario.name} agent=${agentLabel}`,
      source: 'harness',
      agent: agentLabel,
      scenario: scenario.name,
      harnessSchema: 1,
    });
  } catch { /* best effort */ }

  // SIGINT/SIGTERM finalization (D11)
  let interrupted = false;
  const onSignal = () => {
    interrupted = true;
    try {
      logFriction({
        runId,
        phase: 'harness',
        message: 'run interrupted by signal',
        kind: 'interrupted',
        source: 'harness',
        agent: agentLabel,
      });
    } catch { /* best effort */ }
  };
  process.once('SIGINT', onSignal);
  process.once('SIGTERM', onSignal);

  let exitCode = 0;
  try {
    if (opts.live) {
      exitCode = await runLive(opts, scenario, { runId, runRoot, gbrainHome, transcriptPath });
    } else {
      exitCode = await runScripted(opts, scenario, { runId, runRoot, gbrainHome });
    }
  } catch (e) {
    // Without this, a thrown run (spawn failure, runner detect race) would
    // reach the finally block with exitCode still 0 and stamp a
    // `run complete … exit=0` meta record — the friction log (diff/render's
    // input) silently recording success for a crashed run.
    exitCode = 1;
    const msg = e instanceof Error ? e.message : String(e);
    console.error(`claw-test: run crashed: ${msg}`);
    if (e instanceof Error && e.stack) console.error(e.stack);
    try {
      logFriction({
        runId,
        phase: 'harness',
        message: `harness crashed: ${msg}`,
        severity: 'blocker',
        source: 'harness',
        agent: agentLabel,
      });
    } catch { /* best effort */ }
  } finally {
    process.off('SIGINT', onSignal);
    process.off('SIGTERM', onSignal);
    // Run-completion meta record (pairs with the start marker above).
    try {
      logFriction({
        runId,
        phase: 'harness',
        kind: 'phase-marker',
        marker: 'end',
        message: `run complete: scenario=${scenario.name} agent=${agentLabel} exit=${exitCode}`,
        source: 'harness',
        agent: agentLabel,
        scenario: scenario.name,
      });
    } catch { /* best effort */ }
    // Persist agent/child-side friction BEFORE the tempdir is deleted. The
    // children run with GBRAIN_HOME=<runRoot>, so their friction lands under
    // <runRoot>/.gbrain/friction/<runId>.jsonl — rmSync below would silently
    // destroy it on every run, leaving `friction render`/`diff` with only the
    // harness's half of the story. Merge into the parent's friction file
    // (same runId; the two sides write disjoint entries).
    mergeChildFriction(runRoot, runId);
    if (!opts.keepTempdir && !interrupted) {
      try { rmSync(runRoot, { recursive: true, force: true }); } catch { /* best effort */ }
    } else {
      console.log(`tempdir kept at: ${runRoot}`);
    }
  }

  // Always render at the end so the operator can immediately see the report.
  console.log('---');
  console.log(`friction log:    ${join(frictionDir(), runId + '.jsonl')}`);
  console.log(`render report:   gbrain friction render --run-id ${runId}`);

  if (interrupted) return 130;
  return exitCode;
}

// ---------------------------------------------------------------------------
// Scripted mode
// ---------------------------------------------------------------------------

/**
 * v0.32.x: env vars that, when inherited from the parent process, would
 * break the claw-test scripted harness's "hermetic PGLite tempdir"
 * contract. The harness runs `gbrain init --pglite` then a sequence of
 * subsequent phases that ALL must hit the same tempdir brain. If the
 * parent process (e.g. a CI runner with DATABASE_URL set for OTHER e2e
 * tests) leaks DATABASE_URL into the children, loadConfig sees the env
 * var and silently flips inferredEngine to 'postgres' (config.ts:143-145),
 * forcing every phase to hit the parent's test Postgres instead of the
 * hermetic PGLite. Result: phases race against each other (multi-tenant
 * effects on the shared DB) and the harness hangs.
 *
 * Strip these on entry so the child env carries no Postgres-pointing
 * variable. PGLite-only by design.
 */
const POSTGRES_POLLUTION_ENV_VARS = ['DATABASE_URL', 'GBRAIN_DATABASE_URL'];

/**
 * Child env for gbrain invocations (scripted phases AND live-mode staging /
 * oracle probes): parent env minus Postgres-pointing vars AND minus every
 * other GBRAIN_* routing/tuning var — a stray operator GBRAIN_BRAIN_ID /
 * GBRAIN_SOURCE / threshold override would misroute the staging and oracle
 * probes and produce false verify verdicts (the same class scripts/run-e2e.sh
 * scrubs for e2e hermeticity). The two vars the harness owns are re-applied
 * last so a parent override can't win.
 */
function buildChildEnv(ctx: { runId: string; gbrainHome: string }): Record<string, string> {
  const parentEnv = process.env as Record<string, string | undefined>;
  const childEnv: Record<string, string> = {};
  for (const [k, v] of Object.entries(parentEnv)) {
    if (v === undefined) continue;
    if (POSTGRES_POLLUTION_ENV_VARS.includes(k)) continue;
    if (k.startsWith('GBRAIN_')) continue;
    childEnv[k] = v;
  }
  childEnv.GBRAIN_HOME = ctx.gbrainHome;
  childEnv.GBRAIN_FRICTION_RUN_ID = ctx.runId;
  return childEnv;
}

/** The hermetic run's PGLite path (configDir appends '.gbrain'). One spelling
 *  for all probe/seed sites — a drifted copy would silently probe a
 *  nonexistent db and fail the upgrade oracle as "unreadable". */
function pgliteDbPath(gbrainHome: string): string {
  return join(gbrainHome, '.gbrain', 'brain.pglite');
}

async function runScripted(
  opts: HarnessOpts,
  scenario: ScenarioConfig,
  ctx: { runId: string; runRoot: string; gbrainHome: string },
): Promise<number> {
  // Filter out Postgres-pointing env vars before forwarding to children.
  // The harness is PGLite-only by design; an inherited DATABASE_URL
  // would force loadConfig() to flip the engine to 'postgres' at the
  // next phase boundary and break the hermetic-tempdir contract.
  const childEnv = buildChildEnv(ctx);

  const phases: { name: string; argv: string[] }[] = [];
  // Phase 2: install_brain. `--no-embedding` defers embedding setup so the
  // claw-test runs without API keys (v0.37.10.0+ requires an embedding
  // provider OR the deferral flag); this harness exercises CLI ergonomics,
  // not embedding pipelines.
  phases.push({ name: 'install_brain', argv: ['init', '--pglite', '--no-embedding'] });

  // Phase 3: import (only when scenario has a brain dir)
  // Capture brainDir for downstream phases that need an explicit --dir
  // (extract requires it post-#688 — defaults to configured source, and
  // gbrain init --pglite doesn't register a fs source).
  let brainDir: string | undefined;
  if (scenario.brainRelative) {
    brainDir = join(scenario.dir, scenario.brainRelative);
    phases.push({ name: 'import', argv: ['import', brainDir, '--no-embed', '--progress-json'] });
  }

  // Phase 4: query (best-effort sanity)
  phases.push({ name: 'query', argv: ['query', 'the'] });

  // Phase 5: extract (positional argument is required: 'all' covers links + timeline).
  // Pass --dir explicitly because the install_brain phase doesn't register
  // a fs source; without --dir, post-#688 extract refuses with "No brain
  // directory configured." When the scenario has no brain dir, skip.
  if (brainDir) {
    phases.push({ name: 'extract', argv: ['extract', 'all', '--source', 'fs', '--dir', brainDir, '--progress-json'] });
  }

  // Phase 6: verify
  phases.push({ name: 'verify', argv: ['doctor', '--json', '--progress-json'] });

  // Pre-phase: upgrade scenario seeds the database. A missing dump is a LOUD
  // failure, not a skip: skipping would init a current-version database and
  // report a false-green "upgrade" that never exercised a migration.
  if (scenario.kind === 'upgrade') {
    const seedSql = scenario.seedRelative ? join(scenario.dir, scenario.seedRelative, 'dump.sql') : null;
    if (!seedSql || !existsSync(seedSql)) {
      const msg = seedSql
        ? `upgrade scenario has no seed dump at ${seedSql}`
        : 'upgrade scenario declares no seed dir';
      logFriction({
        runId: ctx.runId,
        phase: 'seed',
        message: msg,
        severity: 'blocker',
        hint: 'upgrade runs need a real dump.sql to measure the migration (see the TODOS entry for the v0.18 seed dump)',
        source: 'harness',
        agent: 'scripted',
      });
      console.error(`[seed] ${msg}`);
      return 1;
    }
    const dbPath = pgliteDbPath(ctx.gbrainHome);
    mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
    const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
    try {
      await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
      console.log(`[seed] replayed ${seedSql} → ${dbPath}`);
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      logFriction({
        runId: ctx.runId,
        phase: 'seed',
        message: `seed replay failed: ${msg}`,
        severity: 'blocker',
        source: 'harness',
        agent: 'scripted',
      });
      return 1;
    }
  }

  const allStderr: string[] = [];
  const outcomes: PhaseOutcome[] = [];
  for (const phase of phases) {
    const outcome = await invokeGbrain(opts.gbrainBin, phase.argv, ctx.runRoot, childEnv);
    outcome.phase = phase.name;
    outcomes.push(outcome);
    allStderr.push(outcome.stderrTail);
    if (outcome.exitCode !== 0) {
      logFriction({
        runId: ctx.runId,
        phase: phase.name,
        message: `command failed (exit ${outcome.exitCode}): gbrain ${phase.argv.join(' ')}`,
        severity: 'error',
        hint: outcome.stderrTail.trim().slice(0, 500),
        source: 'harness',
        agent: 'scripted',
      });
      return 1;
    } else {
      logFriction({
        runId: ctx.runId,
        phase: phase.name,
        message: `phase complete in ${outcome.durationMs}ms`,
        kind: 'phase-marker',
        marker: 'end',
        source: 'harness',
        agent: 'scripted',
      });
    }
  }

  // Phase verification: collect all events from every captured stderr and assert coverage.
  const events = allStderr.flatMap(parseProgressEvents);
  const missing = verifyExpectedPhases(events, scenario.expectedPhases);
  if (missing.length) {
    for (const phaseName of missing) {
      logFriction({
        runId: ctx.runId,
        phase: phaseName,
        message: `expected progress event for "${phaseName}" never fired`,
        severity: 'blocker',
        hint: 'either the command did not run or it did not emit progress events; check phase log above',
        source: 'harness',
        agent: 'scripted',
      });
    }
    return 1;
  }

  return 0;
}

// ---------------------------------------------------------------------------
// Live mode
// ---------------------------------------------------------------------------

/** Per-agent install hints for the agent_detect blocker. */
const AGENT_INSTALL_HINTS: Record<string, string> = {
  openclaw: 'install openclaw or set OPENCLAW_BIN',
  hermes: 'install hermes (https://hermes-agent.nousresearch.com) or set HERMES_BIN',
  // Official xAI CLI only — the community superagent-ai grok-cli ships a
  // colliding `grok` binary (docs/mcp/GROK-CLI-PIN.md).
  grok: 'install grok (npm: @xai-official/grok, or https://x.ai/cli/install.sh) or set GROK_BIN',
  // SST terminal agent — not OpenClaw, and not the renamed-to-Crush ancestor
  // that shares the binary name (docs/mcp/OPENCODE-CLI-PIN.md).
  opencode: 'install opencode (npm: opencode-ai, or https://opencode.ai/install) or set OPENCODE_BIN',
};

/**
 * Live mode. Hermeticity posture (deliberate, documented): live mode runs the
 * OPERATOR's configured agent — the real agent home (~/.openclaw, ~/.hermes,
 * model settings, skills) is inherited — against a HERMETIC BRAIN
 * (GBRAIN_HOME=tempdir). The fully hermetic lane is the door e2e
 * (install-real-hermes.serial.test.ts), which isolates the agent home too.
 */
async function runLive(
  opts: HarnessOpts,
  scenario: ScenarioConfig,
  ctx: { runId: string; runRoot: string; gbrainHome: string; transcriptPath: string },
): Promise<number> {
  let runner;
  try {
    runner = resolveAgentRunner(opts.agent);
  } catch (e) {
    console.error(e instanceof Error ? e.message : String(e));
    return 2;
  }

  const detected = await runner.detect();
  if (!detected.available) {
    console.error(`agent "${opts.agent}" not available: ${detected.reason ?? 'unknown'}`);
    logFriction({
      runId: ctx.runId,
      phase: 'agent_detect',
      message: `agent ${opts.agent} not available: ${detected.reason ?? 'unknown'}`,
      severity: 'blocker',
      hint: AGENT_INSTALL_HINTS[opts.agent],
      source: 'harness',
      agent: opts.agent,
    });
    return 2;
  }

  // ---- Stage the scenario (scenario-driven; mirrors the scripted branch) ----
  // The BRIEF's preconditions must actually exist before the agent reads it:
  // fresh-install promises "workspace already has an AGENTS.md", "3 small
  // markdown pages already there" (./brain), and "user just ran gbrain init".
  // Without staging, every live run starts in an empty tempdir and the run
  // measures recovery-from-broken-fixture, not gbrain friction.
  const childEnv = buildChildEnv(ctx);
  const gbrainBin = opts.gbrainBin;
  const stageFailed = await stageLiveScenario(opts, scenario, ctx, childEnv, gbrainBin);
  if (stageFailed !== 0) return stageFailed;

  // Upgrade oracle needs the pre-turn schema version (non-mutating probe —
  // any gbrain CLI connect would auto-apply migrations and do the agent's
  // work for it).
  const dbPath = pgliteDbPath(ctx.gbrainHome);
  let preVersion: number | null = null;
  if (scenario.kind === 'upgrade') {
    const { readPgliteSchemaVersion } = await import('../core/claw-test/seed-pglite.ts');
    preVersion = await readPgliteSchemaVersion(dbPath);
  }

  // ---- PATH shim: the BRIEF says `gbrain …`; make bare `gbrain` resolve to
  // THIS harness's binary (operator PATH may have none, or a stale global). ----
  const shimDir = join(ctx.runRoot, '.harness-bin');
  mkdirSync(shimDir, { recursive: true });
  const shimPath = join(shimDir, 'gbrain');
  // Single-quoted: validateBinPathEnv rejects quote/metacharacter values, so
  // the interpolation cannot break out of the quoting.
  writeFileSync(shimPath, `#!/bin/sh\nexec '${gbrainBin}' "$@"\n`, 'utf-8');
  chmodSync(shimPath, 0o755);

  const sink = createTranscriptSink(ctx.transcriptPath);
  const env: Record<string, string> = {
    GBRAIN_HOME: ctx.gbrainHome,
    GBRAIN_FRICTION_RUN_ID: ctx.runId,
    PATH: `${shimDir}:${process.env.PATH ?? ''}`,
  };

  const brief = readBrief(scenario);
  let result;
  try {
    result = await runner.invoke({
      cwd: ctx.runRoot,
      brief,
      env,
      timeoutMs: LIVE_AGENT_TIMEOUT_MS,
      transcriptSink: sink,
    });
  } finally {
    await sink.close();
  }

  if (result.exitCode !== 0) {
    logFriction({
      runId: ctx.runId,
      phase: 'agent_invoke',
      message: `agent exited with code ${result.exitCode} after ${result.durationMs}ms`,
      severity: 'error',
      source: 'harness',
      agent: opts.agent,
    });
    return result.exitCode;
  }

  // ---- Success oracle: exit code alone passes an agent that did nothing. ----
  return verifyLiveOutcome(opts, scenario, ctx, childEnv, gbrainBin, preVersion);
}

/** Stage the workspace per scenario.kind before the agent turn. Returns 0 or a failing exit code. */
async function stageLiveScenario(
  opts: HarnessOpts,
  scenario: ScenarioConfig,
  ctx: { runId: string; runRoot: string; gbrainHome: string },
  childEnv: Record<string, string>,
  gbrainBin: string,
): Promise<number> {
  const failStage = (message: string, hint?: string): number => {
    logFriction({
      runId: ctx.runId,
      phase: 'stage',
      message,
      severity: 'blocker',
      hint,
      source: 'harness',
      agent: opts.agent,
    });
    console.error(`[stage] ${message}`);
    return 1;
  };

  if (scenario.kind === 'upgrade') {
    // Seed ONLY — running init here would walk the migration chain forward
    // and do the very upgrade the agent turn is supposed to perform (any
    // gbrain connect auto-migrates). Same seed-first order as scripted mode.
    if (scenario.seedRelative) {
      const seedSql = join(scenario.dir, scenario.seedRelative, 'dump.sql');
      if (existsSync(seedSql)) {
        const dbPath = pgliteDbPath(ctx.gbrainHome);
        mkdirSync(join(ctx.gbrainHome, '.gbrain'), { recursive: true });
        const { seedPgliteFromFile } = await import('../core/claw-test/seed-pglite.ts');
        try {
          await seedPgliteFromFile({ dbPath, sqlPath: seedSql });
          console.log(`[stage] replayed ${seedSql} → ${dbPath}`);
        } catch (e) {
          return failStage(`seed replay failed: ${e instanceof Error ? e.message : String(e)}`);
        }
      } else {
        return failStage(`upgrade scenario has no seed dump at ${seedSql}`, 'upgrade runs need a real dump.sql to measure the migration (see the TODOS entry for the v0.18 seed dump)');
      }
    } else {
      return failStage('upgrade scenario declares no seed dir');
    }
    return 0;
  }

  // fresh-install: copy the scenario's brain pages + an AGENTS.md stub, then
  // init the brain (the BRIEF says the user "just ran gbrain init").
  if (scenario.brainRelative) {
    const src = join(scenario.dir, scenario.brainRelative);
    if (!existsSync(src)) {
      // Fail loudly, matching the upgrade branch's missing-seed blocker: a
      // silent skip here would fail the query oracle later with the
      // misleading hint "the agent likely skipped the import step" when the
      // real cause is a broken fixture.
      return failStage(`fresh-install scenario declares brain dir ${scenario.brainRelative} but it does not exist at ${src}`);
    }
    cpSync(src, join(ctx.runRoot, 'brain'), { recursive: true });
  }
  const agentsMd = join(ctx.runRoot, 'AGENTS.md');
  if (!existsSync(agentsMd)) {
    // Deliberately references NO skill files: staging creates none, and a row
    // pointing at a missing SKILL.md flips doctor's resolver_health to fail
    // (caught in rehearsal). Post-v0.33 scaffolded skills route via their own
    // frontmatter triggers, so a prose stub satisfies the BRIEF's
    // "workspace already has an AGENTS.md routing file" precondition.
    writeFileSync(
      agentsMd,
      '# Workspace routing\n\nSkills scaffolded under `skills/` route via their frontmatter `triggers:`.\n',
      'utf-8',
    );
  }
  const init = await invokeGbrain(gbrainBin, ['init', '--pglite', '--no-embedding'], ctx.runRoot, childEnv);
  if (init.exitCode !== 0) {
    return failStage(`gbrain init failed during staging (exit ${init.exitCode})`, init.stderrTail.trim().slice(0, 500));
  }
  console.log('[stage] fresh-install workspace staged (brain pages + AGENTS.md + init)');
  return 0;
}

/** Post-turn verification. Logs friction (phase 'verify') and returns 1 on any failure. */
async function verifyLiveOutcome(
  opts: HarnessOpts,
  scenario: ScenarioConfig,
  ctx: { runId: string; runRoot: string; gbrainHome: string },
  childEnv: Record<string, string>,
  gbrainBin: string,
  preVersion: number | null,
): Promise<number> {
  const failures: { message: string; hint?: string }[] = [];

  if (scenario.kind === 'upgrade') {
    // The upgrade oracle is the schema version reaching LATEST during the
    // agent turn, read via the non-mutating direct-PGLite probe (doctor/any
    // CLI connect would apply the migrations itself and mask a do-nothing
    // agent). MUST run before any declared query oracle below — the query's
    // own CLI connect migrates, which would corrupt a later version read.
    const { readPgliteSchemaVersion } = await import('../core/claw-test/seed-pglite.ts');
    const { LATEST_VERSION } = await import('../core/migrate.ts');
    const dbPath = pgliteDbPath(ctx.gbrainHome);
    const postVersion = await readPgliteSchemaVersion(dbPath);
    if (preVersion === null || postVersion === null) {
      failures.push({ message: `upgrade oracle: schema version unreadable (pre=${preVersion} post=${postVersion})` });
    } else if (postVersion <= preVersion) {
      failures.push({
        message: `upgrade oracle: schema version did not advance during the agent turn (pre=${preVersion} post=${postVersion})`,
        hint: 'the agent never ran a gbrain command that walks the migration chain',
      });
    } else if (postVersion < LATEST_VERSION) {
      // Advancing one step is not an upgrade: any gbrain connect migrates to
      // latest, so a partial version means the agent's run died mid-chain.
      failures.push({
        message: `upgrade oracle: schema version advanced but stopped short of latest (pre=${preVersion} post=${postVersion} latest=${LATEST_VERSION})`,
        hint: 'a gbrain command started the migration chain but did not complete it',
      });
    }
  } else {
    // doctor: fresh --no-embedding brains report status "warnings" (observed
    // — embedding setup is deferred by design), so healthy AND warnings both
    // pass. Anything else — including output the harness cannot parse — is a
    // failure: an oracle that shrugs at unparsable output fails open.
    const doc = await invokeGbrain(gbrainBin, ['doctor', '--json'], ctx.runRoot, childEnv, { captureFullStdout: true });
    if (doc.exitCode !== 0) {
      failures.push({ message: `verify: doctor exited ${doc.exitCode}`, hint: doc.stderrTail.trim().slice(0, 300) });
    } else {
      const report = parseLastJson(doc.stdoutFull ?? doc.stdoutTail);
      const status = report && typeof report === 'object' ? (report as Record<string, unknown>).status : undefined;
      if (report === null || typeof status !== 'string') {
        failures.push({ message: 'verify: doctor exited 0 but its JSON output was unparsable' });
      } else if (status !== 'healthy' && status !== 'warnings') {
        failures.push({ message: `verify: doctor reports status ${JSON.stringify(status)}` });
      }
    }
  }

  // A declared oracle is enforced for EVERY kind (loadScenario validates it
  // for every kind — accepting config it never enforces would be a silent
  // no-op; an upgrade agent that migrates the schema but loses the seeded
  // data must still fail a declared query oracle).
  const oracle = scenario.oracle;
  if (oracle?.query) {
    const q = await invokeGbrain(gbrainBin, ['query', oracle.query, '--json'], ctx.runRoot, childEnv, { captureFullStdout: true });
    const parsed = q.exitCode === 0 ? parseLastJson(q.stdoutFull ?? q.stdoutTail) : null;
    const min = oracle.minResults ?? 1;
    // query --json emits a bare array; anything else on a zero exit means the
    // command's contract broke — fail even when min_results is 0, because
    // "0 results required" never licenses unparsable output.
    if (q.exitCode === 0 && !Array.isArray(parsed)) {
      failures.push({
        message: `verify: query ${JSON.stringify(oracle.query)} exited 0 but its JSON output was unparsable`,
      });
    } else {
      const count = Array.isArray(parsed) ? parsed.length : 0;
      if (q.exitCode !== 0 || count < min) {
        failures.push({
          message: `verify: query ${JSON.stringify(oracle.query)} returned ${count} result(s), expected >= ${min} (exit ${q.exitCode})`,
          hint: 'the agent likely skipped the import step from the brief',
        });
      }
    }
  }
  for (const rel of oracle?.filesExist ?? []) {
    if (!existsSync(join(ctx.runRoot, rel))) {
      failures.push({
        message: `verify: expected file missing after run: ${rel}`,
        hint: 'the agent likely skipped a brief step that produces this file',
      });
    }
  }

  for (const f of failures) {
    logFriction({
      runId: ctx.runId,
      phase: 'verify',
      message: f.message,
      severity: 'error',
      hint: f.hint,
      source: 'harness',
      agent: opts.agent,
    });
    console.error(`[verify] ${f.message}`);
  }
  return failures.length ? 1 : 0;
}

/**
 * Parse the trailing JSON document from CLI stdout (defensive: banners or
 * notices may precede the payload).
 */
function parseLastJson(stdout: string): unknown {
  const text = stdout.trim();
  try {
    return JSON.parse(text);
  } catch { /* fall through */ }
  const starts = ['{', '['];
  for (let i = 0; i < text.length; i++) {
    if (starts.includes(text[i])) {
      try {
        return JSON.parse(text.slice(i));
      } catch { /* keep scanning */ }
    }
  }
  return null;
}

/** Refuse to import child friction files larger than this — the file lives in
 *  a workspace the AGENT writes to, so its size is untrusted. */
const CHILD_FRICTION_MAX_BYTES = 4 * 1024 * 1024;

/**
 * E0: merge the child-side friction file (written under the run's hermetic
 * GBRAIN_HOME) into the parent process's friction dir so it survives tempdir
 * cleanup. Best-effort — a merge failure never fails the run.
 *
 * The child file is UNTRUSTED input (in live mode the agent can write
 * arbitrary bytes at that path), and the destination is the operator's
 * permanent friction log: require a regular file (no symlink — an agent-
 * dropped link could import any readable file on the box), cap the size, and
 * append only lines that parse as JSON objects so the log stays valid JSONL.
 * Exported for tests.
 */
export function mergeChildFriction(runRoot: string, runId: string): void {
  try {
    const childFile = join(runRoot, '.gbrain', 'friction', `${runId}.jsonl`);
    const st = lstatSync(childFile, { throwIfNoEntry: false });
    if (!st) return;
    if (!st.isFile()) {
      console.error(`[friction] skipping child friction merge: ${childFile} is not a regular file`);
      return;
    }
    if (st.size > CHILD_FRICTION_MAX_BYTES) {
      console.error(`[friction] skipping child friction merge: ${childFile} is ${st.size} bytes (cap ${CHILD_FRICTION_MAX_BYTES})`);
      return;
    }
    const parentFile = frictionFile(runId);
    if (resolve(childFile) === resolve(parentFile)) return;
    const raw = readFileSync(childFile, 'utf-8');
    if (!raw.trim()) return;
    const kept: string[] = [];
    let skipped = 0;
    for (const line of raw.split('\n')) {
      if (!line.trim()) continue;
      try {
        const parsed = JSON.parse(line);
        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) kept.push(line);
        else skipped++;
      } catch {
        skipped++;
      }
    }
    if (skipped) console.error(`[friction] child friction merge skipped ${skipped} non-JSONL line(s)`);
    if (!kept.length) return;
    const dir = frictionDir();
    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
    appendFileSync(parentFile, kept.join('\n') + '\n', 'utf-8');
  } catch { /* best effort */ }
}

// ---------------------------------------------------------------------------
// Subprocess helpers
// ---------------------------------------------------------------------------

function invokeGbrain(
  bin: string,
  argv: string[],
  cwd: string,
  env: Record<string, string>,
  invokeOpts?: { captureFullStdout?: boolean },
): Promise<PhaseOutcome> {
  return new Promise((resolvePromise) => {
    const start = Date.now();
    // #6775: on POSIX the phase leads its own process group, so every stop
    // (timeout, interrupt, settle) reaches what it started too: a grandchild
    // that ignores SIGTERM, or one that outlives the leader holding the pipes.
    const grouped = process.platform !== 'win32';
    const child = spawn(bin, argv, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], shell: false, detached: grouped });
    const signalPhase = (signal: NodeJS.Signals | 0): boolean => {
      if (!child.pid) return false;
      try { process.kill(grouped ? -child.pid : child.pid, signal); return true; } catch { return false; }
    };
    const stdout: Buffer[] = [];
    const stderr: Buffer[] = [];
    child.stdout?.on('data', (b: Buffer) => stdout.push(b));
    child.stderr?.on('data', (b: Buffer) => stderr.push(b));

    // A hung child (e.g. a leaked PGLite lock holder from the agent turn)
    // must not wedge the harness/CI job forever: SIGTERM at the phase cap,
    // SIGKILL if it lingers.
    let timedOut = false;
    let killTimer: ReturnType<typeof setTimeout> | null = null;
    const escalate = () => { if (!killTimer) killTimer = setTimeout(() => signalPhase('SIGKILL'), 10_000); };
    const wallClockTimer = setTimeout(() => {
      timedOut = true;
      signalPhase('SIGTERM');
      escalate();
    }, SUBPROCESS_TIMEOUT_MS);
    // A group of its own no longer receives the terminal's Ctrl-C, and
    // process-cleanup deliberately leaves SIGINT to commands, so relay it: the
    // first SIGINT is forwarded (SIGKILL after the grace), a second kills.
    let interrupts = 0;
    const onSigint = () => {
      interrupts++;
      if (interrupts > 1) { signalPhase('SIGKILL'); return; }
      signalPhase('SIGINT');
      escalate();
    };
    process.on('SIGINT', onSigint);
    // SIGTERM/SIGHUP and crashes exit through process-cleanup's pass (3 s
    // deadline): stop the group there before the harness exits.
    const releaseCleanup = registerCleanup('claw-test phase process group', async () => {
      if (!signalPhase('SIGTERM')) return;
      for (let waited = 0; waited < 1_500 && signalPhase(0); waited += 100) {
        await new Promise((r) => setTimeout(r, 100));
      }
      if (signalPhase(0)) signalPhase('SIGKILL');
    });
    const clearTimers = () => {
      clearTimeout(wallClockTimer);
      if (killTimer) clearTimeout(killTimer);
      process.off('SIGINT', onSigint);
      releaseCleanup();
    };

    let settled = false;
    const settle = (code: number | null) => {
      if (settled) return;
      settled = true;
      clearTimers();
      // The leader is gone; whatever it left in its group goes with it.
      if (grouped) signalPhase('SIGKILL');
      let stderrText = Buffer.concat(stderr).toString('utf-8');
      if (timedOut) stderrText += `\nharness: killed after ${SUBPROCESS_TIMEOUT_MS}ms phase timeout`;
      const stdoutText = Buffer.concat(stdout).toString('utf-8');
      resolvePromise({
        phase: '',
        exitCode: typeof code === 'number' ? code : (timedOut ? 124 : 1),
        durationMs: Date.now() - start,
        stderrEvents: parseProgressEvents(stderrText).length,
        stdoutTail: tailOf(stdoutText),
        stderrTail: stderrText,
        // The 4KB tail is fine for logging but NOT for parsing JSON payloads
        // (doctor --json exceeds it and would lose its opening brace).
        ...(invokeOpts?.captureFullStdout ? { stdoutFull: stdoutText } : {}),
      });
    };

    child.on('error', (err) => {
      if (settled) return;
      settled = true;
      clearTimers();
      const stderrJoined = Buffer.concat(stderr).toString('utf-8') + '\nspawn error: ' + err.message;
      resolvePromise({
        phase: '',
        exitCode: 127,
        durationMs: Date.now() - start,
        stderrEvents: 0,
        stdoutTail: tailOf(Buffer.concat(stdout).toString('utf-8')),
        stderrTail: tailOf(stderrJoined),
      });
    });
    // 'close' (pipes drained) is the clean path; 'exit' + grace covers a
    // grandchild that inherits the pipes and outlives the kill — without it a
    // timed-out phase whose child leaked a subprocess would wedge forever.
    child.on('close', (code) => settle(code));
    child.on('exit', (code) => {
      const t = setTimeout(() => settle(code), 2_000);
      t.unref?.();
    });
  });
}

function tailOf(s: string): string {
  if (s.length <= TAIL_BYTES) return s;
  return s.slice(-TAIL_BYTES);
}

// ---------------------------------------------------------------------------
// Argv parsing + helpers
// ---------------------------------------------------------------------------

function parseArgs(args: string[]): HarnessOpts {
  const out: HarnessOpts = {
    scenario: 'fresh-install',
    live: false,
    agent: 'openclaw',
    keepTempdir: false,
    listAgents: false,
    help: args.includes('--help') || args.includes('-h'),
    gbrainBin: resolveGbrainBin(),
  };
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--live') out.live = true;
    else if (a === '--keep-tempdir') out.keepTempdir = true;
    else if (a === '--list-agents') out.listAgents = true;
    else if (a === '--scenario') out.scenario = args[++i] ?? out.scenario;
    else if (a === '--agent') out.agent = args[++i] ?? out.agent;
  }
  return out;
}

/**
 * Resolve the gbrain binary child invocations use. GBRAIN_BIN_OVERRIDE goes
 * through the same absolute/no-dotdot/no-metacharacter validation as the
 * *_BIN runner overrides — the value is interpolated into a generated
 * PATH-shim script in live mode, so a relative value would re-resolve through
 * the shimmed PATH and self-exec forever, and quoting-hostile characters
 * would become code. Invalid overrides are rejected loudly (stderr) and the
 * harness falls back to the current executable.
 *
 * The fallback is NOT bare process.execPath: under `bun run src/cli.ts` (the
 * canonical source install) or a bun-global launcher, execPath is the Bun
 * RUNTIME, and children would run `bun init` / `bun import` instead of gbrain
 * — scaffolding a Bun project in the hermetic workspace and failing the rest
 * of the run. When execPath looks like bun, synthesize a launcher shim that
 * re-enters this checkout's cli.ts; only a compiled gbrain binary returns
 * execPath directly.
 */
let cachedGbrainBin: string | null = null;

function resolveGbrainBin(): string {
  if (cachedGbrainBin) return cachedGbrainBin;
  cachedGbrainBin = resolveGbrainBinUncached();
  return cachedGbrainBin;
}

function resolveGbrainBinUncached(): string {
  const override = process.env.GBRAIN_BIN_OVERRIDE?.trim();
  if (override) {
    const invalid = validateBinPathEnv('GBRAIN_BIN_OVERRIDE', override);
    if (!invalid) return override;
    console.error(`ignoring ${invalid}; falling back to the current executable`);
  }
  const exe = process.execPath;
  if (/^bun(-profile)?(\.exe)?$/i.test(basename(exe))) {
    // src/commands/claw-test.ts → ../cli.ts. Under a compiled binary this
    // branch never fires (execPath is the gbrain binary itself); under bun
    // (dev checkout or bun-global install) import.meta resolves to the real
    // source file next to cli.ts.
    const cliTs = resolve(dirname(fileURLToPath(import.meta.url)), '..', 'cli.ts');
    if (existsSync(cliTs) && !/['\n\r]/.test(exe) && !/['\n\r]/.test(cliTs)) {
      const dir = mkdtempSync(join(tmpdir(), 'gbrain-launcher-'));
      const launcher = join(dir, 'gbrain');
      writeFileSync(launcher, `#!/bin/sh\nexec '${exe}' '${cliTs}' "$@"\n`, 'utf-8');
      chmodSync(launcher, 0o755);
      return launcher;
    }
    console.error(
      'claw-test: running under the bun runtime but the gbrain CLI entrypoint could not be located — ' +
      'child gbrain invocations would run bun itself. Set GBRAIN_BIN_OVERRIDE to a gbrain binary.',
    );
  }
  return exe;
}

function newRunId(agent: string): string {
  const now = new Date();
  const ts = now.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-');
  const suf = randomBytes(4).toString('hex');
  return `claw-test-${ts}-${agent}-${suf}`;
}

async function cmdListAgents(): Promise<number> {
  const names = listRegisteredAgents();
  if (!names.length) {
    console.log('no agents registered');
    return 0;
  }
  // Detect concurrently but AWAIT all of them, then print in
  // listRegisteredAgents() order (the accessor sorts alphabetically).
  // The prior fire-and-forget .then() version returned before any detection
  // resolved, so output could vanish in CLI teardown.
  const lines = await Promise.all(names.map(async (name) => {
    try {
      const runner = resolveAgentRunner(name);
      try {
        const d = await runner.detect();
        return `${name}: ${d.available ? `available at ${d.binPath}` : `unavailable: ${d.reason}`}`;
      } catch {
        return `${name}: (detect error)`;
      }
    } catch {
      return `${name}: (factory error)`;
    }
  }));
  for (const line of lines) console.log(line);
  return 0;
}

function printHelp() {
  console.log(`gbrain claw-test — end-to-end claw-setup friction harness

Usage:
  gbrain claw-test [--scenario <name>] [--live --agent <name>] [--keep-tempdir]
  gbrain claw-test --list-agents

Defaults:
  --scenario fresh-install
  --agent openclaw (live mode only)

Scripted mode runs canonical commands without an LLM (CI gate).
Live mode spawns a real agent and lets it drive (~5–10 min, costs tokens).

Live mode runs YOUR configured agent (it may read/write your real agent home,
e.g. ~/.openclaw or ~/.hermes) against a hermetic brain. The door e2e suite is
the fully hermetic lane.

Examples:
  gbrain claw-test --scenario fresh-install
  gbrain claw-test --scenario upgrade-from-v0.18 --keep-tempdir
  gbrain claw-test --live --agent openclaw
  gbrain claw-test --live --agent opencode
  gbrain claw-test --live --agent hermes
  gbrain claw-test --live --agent grok`);
}