Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.6 KB
Raw Blame
/**
 * BrainBench orchestrator.
 *
 * Engine economy (eng-review D9): ONE in-memory PGLite for the whole run,
 * `resetTables()` between fixtures — the longmemeval lesson; per-fixture WASM
 * cold boots would blow the <2 min CI budget. Read-only suites (know-to-ask,
 * push) seed once and run ALL adapters against the same brain; mutating work
 * (write-back) runs after the replays, and the next fixture starts from a
 * reset. A sentinel-slug test pins that sharing leaks nothing.
 *
 * Continuity pairs (writer fixture → production write-back → reader fixture)
 * run on a shared brain; scores land on the READER's cell. The write path is
 * gbrain's pipeline — harness-INDEPENDENT in v1 (disclosed in
 * docs/eval/BRAINBENCH.md) — so each pair preps ONCE (reset + seed +
 * write-back) and every requested harness replays the read-only reader against
 * that shared state. The per-(writer×reader)-ordering loop this replaces
 * rebuilt byte-identical brains 6x per pair and replayed a writer whose state
 * could not outlive the call (review findings: dead work, identical scores).
 * The writer axis activates when harness-specific write paths actually land.
 *
 * Sealed gold: adapters only ever receive AdapterFixtureView + PublicTurn
 * (toPublicTurn picks fields; gold never crosses).
 */

import { BudgetTracker } from '../../core/budget/budget-tracker.ts';
import { createBenchmarkBrain, resetTables } from '../longmemeval/harness.ts';
import type { BrainEngine } from '../../core/engine.ts';
import { ClaudeCodeAdapter } from './adapters/claude-code.ts';
import { CodexAdapter } from './adapters/codex.ts';
import { OpenClawAdapter } from './adapters/openclaw.ts';
import { scoreKnowToAsk } from './metrics/know-to-ask.ts';
import { scorePush } from './metrics/push.ts';
import { runWriteBack, type WriteBackScore } from './metrics/write-back.ts';
import { scoreContinuityPair } from './metrics/continuity.ts';
import { SeedError, seedBrain, type SeedOutcome } from './seed.ts';
import { assembleTrustCell, runTrustSuites } from './trust-suites.ts';
import { configureDecideBrain, decideRowReceipt, decideSpendTotals, spendDelta, type DecideEvalRun } from '../decide-eval-flags.ts';
import {
  isTrustSuite,
  round4,
  toPublicTurn,
  type AdapterFixtureView,
  type BrainBenchSuite,
  type HarnessAdapter,
  type HarnessName,
  type LoadedCorpus,
  type LoadedFixture,
  type SuiteMetrics,
  type TurnRow,
} from './types.ts';

export interface RunBrainBenchOpts {
  harnesses: HarnessName[];
  suites: BrainBenchSuite[];
  includeHoldout: boolean;
  /** Run the real LLM extractor for write-back (budget-guarded upstream). */
  llm: boolean;
  /** Spend cap for --llm mode (threaded to the extraction pipeline's tracker). */
  budgetUsd?: number;
  /** Progress note sink (CLI wires the shared stderr reporter). */
  onProgress?: (note: string) => void;
  /** System One arm (`--decide`, src/eval/decide-eval-flags.ts): configures the benchmark brain and adds per-turn receipts. */
  decide?: DecideEvalRun | null;
}

export interface RunBrainBenchOutput {
  cells: SuiteMetrics[];
  turn_rows: TurnRow[];
  seed_failures: Array<{ fixture_id: string; error: string }>;
  fixtures_run: number;
}

function makeAdapter(name: HarnessName): HarnessAdapter {
  switch (name) {
    case 'openclaw':
      return new OpenClawAdapter();
    case 'claude-code':
      return new ClaudeCodeAdapter();
    case 'codex':
      return new CodexAdapter();
  }
}

const SEAM: Record<HarnessName, 'production' | 'contract'> = {
  openclaw: 'production',
  // v0.46.15 (TODOS:556 P1): the claude-code row now drives the REAL
  // UserPromptSubmit hook over the real IPC socket — see the adapter header.
  'claude-code': 'production',
  codex: 'contract',
};

/** The SEAM map and the adapters' own seam fields must agree — pinned by
 * test/brainbench-adapters.test.ts (outside-voice F4: the scoreboard reads
 * THIS map, so a flipped adapter field alone changes nothing). */
export function seamFor(name: HarnessName): 'production' | 'contract' {
  return SEAM[name];
}

function adapterView(lf: LoadedFixture): AdapterFixtureView {
  return {
    fixture_id: lf.fixture.fixture_id,
    active_source: lf.fixture.active_source ?? 'default',
    turns: lf.fixture.turns.map(toPublicTurn),
  };
}

function crossSourceSlugs(
  injected: string[],
  slugSource: Map<string, Set<string>>,
  activeSource: string,
): string[] {
  return injected.filter((s) => {
    const set = slugSource.get(s);
    return set !== undefined && !set.has(activeSource);
  });
}

/**
 * Replay every USER turn of a fixture through one adapter. Assistant turns
 * feed prior context only (the production reflex fires on user messages).
 * Returns one TurnRow per (user turn × suite) so per-suite scorers and
 * external re-scorers see self-contained rows.
 */
async function replayFixture(
  engine: BrainEngine,
  adapter: HarnessAdapter,
  lf: LoadedFixture,
  seed: SeedOutcome,
  rowSuites: BrainBenchSuite[],
  decide?: DecideEvalRun | null,
): Promise<TurnRow[]> {
  const view = adapterView(lf);
  const activeSource = view.active_source;
  await adapter.beginConversation(engine, view);
  const rows: TurnRow[] = [];
  let priorContext = '';
  try {
    for (const turn of view.turns) {
      if (turn.role !== 'user') {
        priorContext += `\n${turn.text}`;
        continue;
      }
      const spendBefore = decide ? await decideSpendTotals(engine) : null;
      const result = await adapter.replayTurn(turn, priorContext);
      const decideReceipt = decide && spendBefore ? decideRowReceipt(decide, result.decide, spendDelta(spendBefore, await decideSpendTotals(engine))) : null;
      const gold = lf.gold.turns[String(turn.turn_id)] ?? null;
      for (const suite of rowSuites) {
        rows.push({
          fixture_id: lf.fixture.fixture_id,
          turn_id: turn.turn_id,
          harness: adapter.name,
          suite,
          injected_slugs: result.injectedSlugs,
          injected_tokens: result.injectedTokens,
          gold,
          cross_source_slugs: crossSourceSlugs(result.injectedSlugs, seed.slugSource, activeSource),
          latency_ms: Math.round(result.latencyMs * 1000) / 1000,
          ...(decideReceipt ? { decide: decideReceipt } : {}),
        });
      }
      priorContext += `\n${turn.text}`;
      if (result.injectedText) priorContext += `\n${result.injectedText}`;
    }
  } finally {
    await adapter.endConversation();
  }
  return rows;
}

interface WriteBackAgg {
  gold_total: number;
  gold_failed: number;
  survived: number;
  provenance_ok: number;
  stored_rows: number;
  matched_any_gold: number;
  fixtures: string[];
  failed_items: string[];
}

interface ContinuityAgg {
  gold_total: number;
  gold_failed: number;
  fixtures: string[];
  failed_items: string[];
}

export async function runBrainBench(
  corpus: LoadedCorpus,
  opts: RunBrainBenchOpts,
): Promise<RunBrainBenchOutput> {
  const progress = opts.onProgress ?? (() => {});
  const turnRows: TurnRow[] = [];
  const seedFailures: Array<{ fixture_id: string; error: string }> = [];

  // v0.46.15 (F4/R2-5): ONE adapter per harness per RUN, torn down in the
  // finally — a production seam owns long-lived infrastructure (the
  // claude-code IPC server) across fixtures instead of paying per-fixture
  // setup (the ~15s gate stays fast). setupRun is LAZY on first replay
  // (codex ship-review): a `--suite write-back` run replays nothing, so it
  // must not require the IPC substrate; and a setup that only happens inside
  // the try below can never leak a live socket server past the finally.
  // Input dedupe: a duplicated harness name would construct two instances
  // and orphan the first (Map overwrite).
  const harnessList = [...new Set(opts.harnesses)];
  const adapters = new Map<HarnessName, HarnessAdapter>();
  for (const h of harnessList) adapters.set(h, makeAdapter(h));
  const setUp = new Set<HarnessName>();
  const adapterFor = async (h: HarnessName): Promise<HarnessAdapter> => {
    const a = adapters.get(h)!;
    if (!setUp.has(h)) {
      setUp.add(h); // before the await — teardown must cover a FAILED setup too
      await a.setupRun?.();
    }
    return a;
  };

  const wantedSuites = new Set(opts.suites);
  const eligible = corpus.fixtures.filter((lf) => {
    if (lf.fixture.holdout && !opts.includeHoldout) return false;
    return lf.fixture.suites.some((s) => wantedSuites.has(s));
  });

  // Continuity pairs are orchestrated separately from regular fixtures.
  const pairFixtures = new Map<string, { writer?: LoadedFixture; reader?: LoadedFixture }>();
  const regular: LoadedFixture[] = [];
  // Memory-trust fixtures run on their own persistence-enabled brain after everything else (trust-scenario.ts).
  const trustFixtures: LoadedFixture[] = [];
  for (const lf of eligible) {
    if (lf.fixture.suites.some(isTrustSuite)) {
      trustFixtures.push(lf);
      continue;
    }
    const cont = lf.fixture.continuity;
    if (cont && wantedSuites.has('continuity')) {
      const p = pairFixtures.get(cont.pair_id) ?? {};
      p[cont.pair_role] = lf;
      pairFixtures.set(cont.pair_id, p);
      // A writer that also declares know-to-ask/push runs as a regular fixture
      // too (its retrieval gold is independent of the pair).
      if (lf.fixture.suites.some((s) => s !== 'continuity' && s !== 'write-back' && wantedSuites.has(s))) {
        regular.push(lf);
      }
    } else {
      regular.push(lf);
    }
  }

  const writeBackAgg: WriteBackAgg = {
    gold_total: 0, gold_failed: 0, survived: 0, provenance_ok: 0,
    stored_rows: 0, matched_any_gold: 0, fixtures: [], failed_items: [],
  };
  const continuityByReader = new Map<HarnessName, ContinuityAgg>();
  let trustAgg: Awaited<ReturnType<typeof runTrustSuites>> | null = null;

  // RUN-scoped --llm budget (review finding: a per-invocation cap would
  // multiply by fixture count — ~$550 worst case on the committed corpus).
  const llmTracker = opts.llm
    ? new BudgetTracker({ maxCostUsd: opts.budgetUsd ?? 5, label: 'brainbench:llm' })
    : undefined;

  const engine = await createBenchmarkBrain();
  let fixturesRun = 0;
  try {
    if (opts.decide) await configureDecideBrain(engine, opts.decide, { includeSearchPins: true });
    // ---- regular fixtures: seed once, replay all adapters, then mutate ----
    for (const lf of regular) {
      const id = lf.fixture.fixture_id;
      progress(`fixture ${id}`);
      await resetTables(engine);
      let seed: SeedOutcome;
      try {
        seed = await seedBrain(engine, lf.fixture);
      } catch (err) {
        if (err instanceof SeedError) {
          seedFailures.push({ fixture_id: id, error: err.message });
          continue;
        }
        throw err;
      }
      fixturesRun++;

      const retrievalSuites = lf.fixture.suites.filter(
        (s): s is BrainBenchSuite => (s === 'know-to-ask' || s === 'push') && wantedSuites.has(s),
      );
      if (retrievalSuites.length > 0) {
        for (const harness of harnessList) {
          const adapter = await adapterFor(harness);
          const rows = await replayFixture(engine, adapter, lf, seed, retrievalSuites, opts.decide);
          turnRows.push(...rows);
        }
      }

      // Exactly ONE owner per writer's write-back score: the pair loop owns it
      // when continuity is being run (adversarial finding: a hybrid writer
      // double-counted); THIS loop owns it when continuity is filtered out
      // (codex P2: `--suite write-back` alone silently dropped the writers'
      // 12 gold items — 46 vs 58 between filtered and default runs).
      if (
        lf.fixture.suites.includes('write-back') &&
        wantedSuites.has('write-back') &&
        !(lf.fixture.continuity && wantedSuites.has('continuity'))
      ) {
        const score = await runWriteBack(engine, lf.fixture, lf.gold, {
          llm: opts.llm,
          budgetUsd: opts.budgetUsd,
          budgetTracker: llmTracker,
        });
        accumulateWriteBack(writeBackAgg, id, score);
      }
    }

    // ---- continuity pairs: ONE prep per pair, every harness reads it ----
    for (const [pairId, pair] of pairFixtures) {
      if (!pair.writer || !pair.reader) continue; // loader validates; belt+suspenders
      const writer = pair.writer;
      const reader = pair.reader;
      const decisions = reader.gold.continuity?.decisions ?? [];
      if (!decisions.length) continue;

      progress(`continuity ${pairId}`);
      await resetTables(engine);
      let writerSeed: SeedOutcome;
      let readerSeed: SeedOutcome;
      try {
        writerSeed = await seedBrain(engine, writer.fixture);
        readerSeed = await seedBrain(engine, reader.fixture);
      } catch (err) {
        if (err instanceof SeedError) {
          seedFailures.push({ fixture_id: pairId, error: err.message });
          continue;
        }
        throw err;
      }
      fixturesRun += 2;
      // The reader replays against BOTH fixtures' seeded pages — merge the
      // slug→source maps or cross-source detection is structurally blind to
      // writer-seeded slugs (red-team finding: the zero-gate was vacuous).
      const mergedSeed: SeedOutcome = {
        slugSource: new Map(writerSeed.slugSource),
        pages: writerSeed.pages + readerSeed.pages,
        facts: writerSeed.facts + readerSeed.facts,
      };
      for (const [slug, sources] of readerSeed.slugSource) {
        const set = mergedSeed.slugSource.get(slug) ?? new Set<string>();
        for (const s of sources) set.add(s);
        mergedSeed.slugSource.set(slug, set);
      }

      // The writer's decisions persist through the PRODUCTION pipeline —
      // harness-independent in v1, so it runs once per pair. Writers that
      // declare the write-back suite are SCORED here too (red-team finding:
      // the score was computed and dropped, leaving 15 writers' provenance
      // gold unmeasured in the default run and making the write-back cell
      // composition flag-dependent).
      const writerWb = await runWriteBack(engine, writer.fixture, writer.gold, {
        llm: opts.llm,
        budgetUsd: opts.budgetUsd,
        budgetTracker: llmTracker,
      });
      if (writer.fixture.suites.includes('write-back') && wantedSuites.has('write-back')) {
        accumulateWriteBack(writeBackAgg, writer.fixture.fixture_id, writerWb);
      }

      // Every requested harness reads the SAME persisted state (read-only).
      for (const readerHarness of harnessList) {
        const readerAdapter = await adapterFor(readerHarness);
        const readerRows = await replayFixture(engine, readerAdapter, reader, mergedSeed, ['continuity'], opts.decide);
        turnRows.push(...readerRows);

        const activeSource = reader.fixture.active_source ?? 'default';
        const score = await scoreContinuityPair(engine, activeSource, pairId, readerRows, decisions);

        const agg = continuityByReader.get(readerHarness) ?? {
          gold_total: 0, gold_failed: 0, fixtures: [], failed_items: [],
        };
        agg.gold_total += score.gold_total;
        agg.gold_failed += score.gold_failed;
        if (!agg.fixtures.includes(reader.fixture.fixture_id)) agg.fixtures.push(reader.fixture.fixture_id);
        agg.failed_items.push(...score.failed_items.map((f) => `${f} [reader: ${readerHarness}]`));
        continuityByReader.set(readerHarness, agg);
      }
    }

    // ---- memory-trust suites: real tiered writes, one source per fixture (trust-suites.ts) ----
    if (trustFixtures.length > 0) {
      trustAgg = await runTrustSuites(trustFixtures, { harnesses: harnessList, adapterFor, progress });
      turnRows.push(...trustAgg.turnRows);
      fixturesRun += trustFixtures.length;
    }
  } finally {
    for (const a of adapters.values()) {
      try {
        await a.teardownRun?.();
      } catch {
        /* teardown is best-effort — a leaked temp dir must not fail the run */
      }
    }
    await engine.disconnect();
  }

  // ---- assemble cells ----
  const cells: SuiteMetrics[] = [];
  for (const harness of harnessList) {
    for (const suite of opts.suites) {
      const cell = isTrustSuite(suite)
        ? (trustAgg ? assembleTrustCell(harness, suite, trustAgg, SEAM[harness]) : null)
        : assembleCell(harness, suite, turnRows, writeBackAgg, continuityByReader, opts.llm);
      if (cell) cells.push(cell);
    }
  }

  return { cells, turn_rows: turnRows, seed_failures: seedFailures, fixtures_run: fixturesRun };
}

function accumulateWriteBack(agg: WriteBackAgg, fixtureId: string, score: WriteBackScore): void {
  agg.gold_total += score.gold_total;
  agg.gold_failed += score.gold_failed;
  agg.survived += score.survived;
  agg.provenance_ok += score.provenance_ok;
  agg.stored_rows += score.stored_rows;
  agg.matched_any_gold += score.matched_any_gold;
  agg.fixtures.push(fixtureId);
  agg.failed_items.push(...score.failed_items);
}

function assembleCell(
  harness: HarnessName,
  suite: BrainBenchSuite,
  turnRows: TurnRow[],
  writeBackAgg: WriteBackAgg,
  continuityByReader: Map<HarnessName, ContinuityAgg>,
  llm: boolean,
): SuiteMetrics | null {
  if (suite === 'write-back') {
    if (writeBackAgg.fixtures.length === 0) return null;
    // The write path is gbrain's pipeline — identical for every harness seam
    // in v1, so each harness cell carries the same (once-computed) numbers.
    // When harness-specific write paths land, this is where they diverge.
    const metrics: Record<string, number> = {
      write_back_fidelity: round4(
        writeBackAgg.gold_total > 0 ? writeBackAgg.survived / writeBackAgg.gold_total : 1,
      ),
      provenance_accuracy: round4(
        writeBackAgg.survived > 0 ? writeBackAgg.provenance_ok / writeBackAgg.survived : 1,
      ),
    };
    if (llm) {
      // Σ-aggregated extraction quality (review finding: per-fixture values
      // were computed but never reached any cell).
      metrics.extraction_recall = round4(
        writeBackAgg.gold_total > 0 ? writeBackAgg.survived / writeBackAgg.gold_total : 1,
      );
      metrics.extraction_precision = round4(
        writeBackAgg.stored_rows > 0 ? writeBackAgg.matched_any_gold / writeBackAgg.stored_rows : 1,
      );
    }
    return {
      suite, harness, seam: SEAM[harness],
      gold_total: writeBackAgg.gold_total,
      gold_failed: writeBackAgg.gold_failed,
      metrics,
      fixtures: [...writeBackAgg.fixtures],
    };
  }

  if (suite === 'continuity') {
    const agg = continuityByReader.get(harness);
    if (!agg) return null;
    const rows = turnRows.filter((r) => r.harness === harness && r.suite === 'continuity');
    return {
      suite, harness, seam: SEAM[harness],
      gold_total: agg.gold_total,
      gold_failed: agg.gold_failed,
      metrics: {
        continuity_rate: round4(
          agg.gold_total > 0 ? (agg.gold_total - agg.gold_failed) / agg.gold_total : 1,
        ),
        source_isolation_violations: rows.reduce((n, r) => n + r.cross_source_slugs.length, 0),
        avg_injected_tokens: round4(avg(rows.map((r) => r.injected_tokens))),
      },
      fixtures: [...agg.fixtures],
    };
  }

  const rows = turnRows.filter((r) => r.harness === harness && r.suite === suite);
  if (rows.length === 0) return null;
  const score = suite === 'know-to-ask' ? scoreKnowToAsk(rows) : scorePush(rows);
  const fixtures = [...new Set(rows.map((r) => r.fixture_id))];
  return {
    suite, harness, seam: SEAM[harness],
    gold_total: score.gold_total,
    gold_failed: score.gold_failed,
    metrics: {
      ...Object.fromEntries(Object.entries(score.metrics).map(([k, v]) => [k, round4(v)])),
      source_isolation_violations: rows.reduce((n, r) => n + r.cross_source_slugs.length, 0),
      avg_injected_tokens: round4(avg(rows.map((r) => r.injected_tokens))),
    },
    fixtures,
  };
}

function avg(ns: number[]): number {
  return ns.length === 0 ? 0 : ns.reduce((a, b) => a + b, 0) / ns.length;
}