Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.6 KB
Raw Blame
/**
 * v0.32.3 — `gbrain eval run-all` orchestrator.
 *
 * Sweeps every requested mode × suite combination and writes per-run
 * results to `<repo>/.gbrain-evals/eval-results.jsonl` [CDX-23]. Personal
 * brain (~/.gbrain) is never touched; the repo's git history is the
 * audit trail.
 *
 * Sequential default per D9: --parallel N is opt-in. Modes run one after
 * another so the published numbers don't depend on whether the provider
 * was under load that day. --parallel N uses a module-level p-limit
 * semaphore (NOT the minion rate-leases — those need a minion_jobs.id FK
 * that a CLI eval doesn't have, per [CDX-10]).
 *
 * Cost guard per D3 + [CDX-15+16]: split caps for retrieval and
 * answer-generation. Default $5 retrieval / $20 answer. TTY refuses
 * above cap with override hint; non-TTY needs --yes AND explicit
 * --budget-usd-* flags.
 *
 * Wired suites (brainbench) run in-process; longmemeval and replay run only
 * through their own commands. A bare run sweeps the wired suites and names
 * the per-suite commands; an explicit `--suites` naming an unwired suite
 * refuses with `eval_suite_unwired` (exit 1). Any wired suite whose record
 * says `failed` makes the run exit 1, with the record already persisted.
 *
 * Per-suite implementations are documented in src/commands/eval-*.ts.
 * This file is the dispatcher + bookkeeper.
 */

import { writeFileSync, mkdirSync, appendFileSync, existsSync } from 'fs';
import { dirname, join } from 'path';
import type { BrainEngine } from '../core/engine.ts';
import { MODE_BUNDLES, SEARCH_MODES, type SearchMode } from '../core/search/mode.ts';
import { redactSecrets } from '../eval/longmemeval/run-config.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import type { Action } from '../core/agent-output.ts';
import { usageError } from '../cli/cli-error.ts';
import { intFlagValue, numberFlagValue } from '../cli/flag-values.ts';

export interface RunAllOpts {
  help: boolean;
  modes: SearchMode[];
  suites: string[];
  /** True when `--suites` named the suites; a bare run sweeps the wired ones. */
  suitesExplicit: boolean;
  limit?: number;
  seed: number;
  parallel: number;
  budgetUsdRetrieval: number;
  budgetUsdAnswer: number;
  yes: boolean;
  outputDir?: string;
  jsonOutput: boolean;
}

const VALID_SUITES = ['longmemeval', 'replay', 'brainbench'] as const;
type ValidSuite = (typeof VALID_SUITES)[number];
const WIRED_SUITES = ['brainbench'] as const;
type UnwiredSuite = Exclude<ValidSuite, (typeof WIRED_SUITES)[number]>;
/** The per-suite command for each suite run-all does not execute in-process. */
const UNWIRED_SUITE_ARGV: Record<UnwiredSuite, string[]> = {
  longmemeval: ['gbrain', 'eval', 'longmemeval', '<dataset.jsonl>', '--mode', '<mode>', '--record'],
  replay: ['gbrain', 'eval', 'replay', '--mode', '<mode>'],
};

const DEFAULT_BUDGET_USD_RETRIEVAL = 5;
const DEFAULT_BUDGET_USD_ANSWER = 20;
const DEFAULT_SEED = 42;
const DEFAULT_PARALLEL = 1;

export function parseRunAllArgs(args: string[]): RunAllOpts {
  const opts: RunAllOpts = {
    help: false,
    modes: [...SEARCH_MODES],
    suites: [...WIRED_SUITES],
    suitesExplicit: false,
    seed: DEFAULT_SEED,
    parallel: DEFAULT_PARALLEL,
    budgetUsdRetrieval: DEFAULT_BUDGET_USD_RETRIEVAL,
    budgetUsdAnswer: DEFAULT_BUDGET_USD_ANSWER,
    yes: false,
    jsonOutput: false,
  };

  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--help' || a === '-h') { opts.help = true; continue; }
    if (a === '--modes') {
      const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
      const validated: SearchMode[] = [];
      for (const m of list) {
        if (m === 'conservative' || m === 'balanced' || m === 'tokenmax') {
          validated.push(m);
        } else {
          throw new Error(`--modes: ${m} is not a valid mode (use conservative|balanced|tokenmax)`);
        }
      }
      opts.modes = validated;
      continue;
    }
    if (a === '--suite' || a === '--suites') {
      const list = (args[++i] ?? '').split(',').map(s => s.trim()).filter(Boolean);
      for (const s of list) {
        if (!(VALID_SUITES as readonly string[]).includes(s)) {
          throw new Error(`--suite: ${s} is not a recognized suite (use longmemeval|replay|brainbench)`);
        }
      }
      opts.suites = list;
      opts.suitesExplicit = true;
      continue;
    }
    // #5933 (D4): NaN/out-of-range values would bypass the cost guard; reject them (usage error, exit 2).
    if (a === '--limit') { opts.limit = intFlagValue(args[++i], '--limit', { min: 1, example: 50 }); continue; }
    if (a === '--seed') { opts.seed = intFlagValue(args[++i], '--seed', { example: 42 }); continue; }
    if (a === '--parallel') {
      const n = Number(args[++i]);
      if (!Number.isFinite(n) || n < 1) throw new Error('--parallel must be >= 1');
      opts.parallel = Math.min(n, SEARCH_MODES.length);
      continue;
    }
    if (a === '--budget-usd-retrieval') { opts.budgetUsdRetrieval = numberFlagValue(args[++i], a, { min: 0, example: 5 }); continue; }
    if (a === '--budget-usd-answer') { opts.budgetUsdAnswer = numberFlagValue(args[++i], a, { min: 0, example: 5 }); continue; }
    if (a === '--yes' || a === '-y') { opts.yes = true; continue; }
    if (a === '--output' || a === '--output-dir') { opts.outputDir = args[++i]; continue; }
    if (a === '--json') { opts.jsonOutput = true; continue; }
  }

  if (opts.modes.length === 0) throw new Error('--modes resolved to an empty list');
  if (opts.suites.length === 0) throw new Error('--suites resolved to an empty list');
  return opts;
}

function printHelp(): void {
  process.stderr.write(
    `gbrain eval run-all [flags]\n\n` +
    `Runs the eval suites wired into the orchestrator (brainbench, once per sweep)\n` +
    `and writes per-run results to <repo>/.gbrain-evals/eval-results.jsonl. Exits 1\n` +
    `when a suite fails (its record is kept). Personal brain is never touched.\n\n` +
    `Flags:\n` +
    `  --modes M1,M2,M3              Modes to evaluate (default: conservative,balanced,tokenmax).\n` +
    `  --suites S1,S2                Suites to run (default: brainbench).\n` +
    `                                run-all runs brainbench in-process. longmemeval and\n` +
    `                                replay run through their own commands, once per mode:\n` +
    `                                  ${UNWIRED_SUITE_ARGV.longmemeval.join(' ')}\n` +
    `                                  ${UNWIRED_SUITE_ARGV.replay.join(' ')}\n` +
    `                                Naming them here exits 1 (eval_suite_unwired).\n` +
    `  --limit N                     Limit each suite to N questions (default: full split).\n` +
    `  --seed N                      Random seed (default: 42).\n` +
    `  --parallel N                  Run N modes in parallel (default: 1; max ${SEARCH_MODES.length}).\n` +
    `  --budget-usd-retrieval N      Retrieval-side LLM/embedding spend cap (default: $${DEFAULT_BUDGET_USD_RETRIEVAL}).\n` +
    `  --budget-usd-answer N         Answer-gen LLM spend cap (default: $${DEFAULT_BUDGET_USD_ANSWER}).\n` +
    `  --yes                         Required (alongside --budget-usd-*) in non-TTY for over-cap runs.\n` +
    `  --output DIR                  Override .gbrain-evals/ location.\n` +
    `  --json                        Emit run summary as JSON.\n` +
    `  -h, --help                    Show this help.\n\n` +
    `Cost guard refuses non-TTY runs over the budget cap without --yes AND an\n` +
    `explicit --budget-usd-* flag (defense against agent loops + cron jobs).\n`,
  );
}

function unwiredSuiteFix(suites: UnwiredSuite[], modes: SearchMode[]): Action {
  const [suite, ...rest] = suites;
  return {
    argv: UNWIRED_SUITE_ARGV[suite],
    consent: ['paid'],
    actor: 'agent',
    why: `run-all does not run ${suite} in-process; its own command runs it, once per mode.`,
    inputs: [
      ...(suite === 'longmemeval' ? [{ name: 'dataset.jsonl', how: 'The LongMemEval dataset file (https://huggingface.co/datasets/xiaowu0162/longmemeval).' }] : []),
      { name: 'mode', how: `Each requested mode in turn: ${modes.join(', ')}.` },
    ],
    docs: 'docs/eval/SEARCH_MODE_METHODOLOGY.md#4-run-procedure',
    requires_exclusive: false,
    ...(rest.length ? { then: unwiredSuiteFix(rest, modes) } : {}),
  };
}

/** The refusal for an explicit `--suites` that names a suite run-all does not run. */
export function unwiredSuiteError(suites: string[], modes: SearchMode[]): OperationError | null {
  const unwired = suites.filter((s): s is UnwiredSuite => s in UNWIRED_SUITE_ARGV);
  if (unwired.length === 0) return null;
  const commands = unwired.map(s => UNWIRED_SUITE_ARGV[s].join(' ')).join('; ');
  return opError(
    'eval_suite_unwired',
    `gbrain eval run-all does not run ${unwired.join(' or ')}; nothing ran.`,
    `Run ${unwired.length > 1 ? 'each suite' : 'it'} with its own command, once per mode: ${commands}. Or drop ${unwired.join(', ')} from --suites.`,
    { fix: unwiredSuiteFix(unwired, modes) },
  );
}

export interface EvalRunRecord {
  /**
   * v3 (BrainBench wave, decision 16): `mode` widened to `SearchMode | 'n/a'`
   * for search-mode-independent suites — brainbench runs once per sweep and
   * records 'n/a' instead of fabricating a mode. v2 records (mode always a
   * SearchMode) parse fine under v3 readers. NOTE: eval-compare's markdown
   * renderer iterates SEARCH_MODES only, so 'n/a' rows surface in its --json
   * `records` output, not the mode table (review finding; markdown surfacing
   * is a follow-up).
   */
  schema_version: 3;
  run_id: string;
  ran_at: string;
  suite: ValidSuite;
  mode: SearchMode | 'n/a';
  commit: string;
  seed: number;
  limit?: number;
  params: Record<string, unknown>;
  status: 'completed' | 'failed' | 'skipped' | 'over_budget';
  duration_ms: number;
  error?: string;
}

function getRepoRoot(): string {
  try {
    const { execSync } = require('../core/spawn.ts') as typeof import('../core/spawn.ts');
    return execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' }).trim();
  } catch {
    return process.cwd();
  }
}

function getCommitSha(): string {
  try {
    const { execSync } = require('../core/spawn.ts') as typeof import('../core/spawn.ts');
    return execSync('git rev-parse --short HEAD', { encoding: 'utf-8' }).trim();
  } catch {
    return 'unknown';
  }
}

function evalResultsPath(repoRoot: string, outputDirOverride?: string): string {
  if (outputDirOverride) {
    return join(outputDirOverride, 'eval-results.jsonl');
  }
  return join(repoRoot, '.gbrain-evals', 'eval-results.jsonl');
}

/** Apply `redactSecrets` to every string leaf (keys untouched); JSON stays valid. */
function redactDeep<T>(value: T): T {
  if (typeof value === 'string') return redactSecrets(value) as unknown as T;
  if (Array.isArray(value)) return value.map(redactDeep) as unknown as T;
  if (value && typeof value === 'object') {
    const out: Record<string, unknown> = {};
    for (const [k, v] of Object.entries(value as Record<string, unknown>)) out[k] = redactDeep(v);
    return out as T;
  }
  return value;
}

/**
 * Secret-scrub a ledger record before it is written: `error` text and every
 * string leaf of `params` pass through `redactSecrets` (provider keys, bearer
 * tokens, DB connection strings). Lives HERE, on the one write path every
 * suite shares, so a caller that forgets to redact cannot leak a connection
 * string into the committed ledger. String leaves are redacted individually
 * (not the serialized line) so the written JSON is always valid.
 */
export function redactRunRecord(record: EvalRunRecord): EvalRunRecord {
  return {
    ...record,
    params: redactDeep(record.params ?? {}),
    ...(typeof record.error === 'string' ? { error: redactSecrets(record.error) } : {}),
  };
}

export function persistRunRecord(repoRoot: string, record: EvalRunRecord, outputDirOverride?: string): void {
  const path = evalResultsPath(repoRoot, outputDirOverride);
  mkdirSync(dirname(path), { recursive: true });
  appendFileSync(path, JSON.stringify(redactRunRecord(record)) + '\n', 'utf-8');
}

/**
 * Estimate per-run cost. Rough heuristic: per-mode estimates × #questions.
 * The methodology doc documents the assumption. Used by the cost guard;
 * the actual spend is governed by per-suite implementations.
 *
 * Returns total estimated USD across retrieval + answer-gen sides.
 */
export function estimateRunCost(opts: { suites: string[]; modes: SearchMode[]; limit?: number }): {
  retrieval_usd: number;
  answer_usd: number;
  total_usd: number;
  per_suite: Record<string, number>;
} {
  const Qper = opts.limit ?? 500;
  const perSuite: Record<string, number> = {};
  let retrieval = 0;
  let answer = 0;

  for (const suite of opts.suites) {
    for (const mode of opts.modes) {
      // Retrieval side: 1 embed + (optional Haiku) per query.
      const expansionCalls = MODE_BUNDLES[mode].expansion ? Qper : 0;
      const retCost = Qper * 0.00001 + expansionCalls * 0.001;
      // Answer side: 1 Sonnet/Opus call per query for longmemeval; 0 for replay/brainbench.
      const answerCost = suite === 'longmemeval' ? Qper * 0.015 : 0;
      retrieval += retCost;
      answer += answerCost;
      perSuite[`${suite}:${mode}`] = retCost + answerCost;
    }
  }
  return {
    retrieval_usd: retrieval,
    answer_usd: answer,
    total_usd: retrieval + answer,
    per_suite: perSuite,
  };
}

interface CostGuardResult {
  proceed: boolean;
  reason: string;
}

export function evaluateCostGuard(
  estimate: { retrieval_usd: number; answer_usd: number; total_usd: number },
  opts: { budgetUsdRetrieval: number; budgetUsdAnswer: number; yes: boolean; isTty: boolean },
): CostGuardResult {
  const retOver = estimate.retrieval_usd > opts.budgetUsdRetrieval;
  const ansOver = estimate.answer_usd > opts.budgetUsdAnswer;
  if (!retOver && !ansOver) {
    return { proceed: true, reason: 'within budget' };
  }
  if (!opts.isTty && !opts.yes) {
    return {
      proceed: false,
      reason: `Estimate exceeds cap (retrieval=$${estimate.retrieval_usd.toFixed(2)} vs cap $${opts.budgetUsdRetrieval}, answer=$${estimate.answer_usd.toFixed(2)} vs cap $${opts.budgetUsdAnswer}). Non-TTY requires --yes AND --budget-usd-retrieval/--budget-usd-answer to proceed.`,
    };
  }
  if (opts.isTty && !opts.yes) {
    return {
      proceed: false,
      reason: `Estimate ($${estimate.total_usd.toFixed(2)}) exceeds caps. Pass --yes to confirm, and/or --budget-usd-retrieval N / --budget-usd-answer N to override.`,
    };
  }
  return { proceed: true, reason: 'over cap but --yes acknowledged' };
}

/** Runs the sweep; returns the exit code (0 all wired suites completed, 1 one failed). */
export async function runEvalRunAll(_engine: BrainEngine | null, args: string[]): Promise<number> {
  let opts: RunAllOpts;
  try {
    opts = parseRunAllArgs(args);
  } catch (e) {
    throw e instanceof OperationError ? e : usageError((e as Error).message, 'Run `gbrain eval run-all --help` for the accepted flags and examples.');
  }
  if (opts.help) {
    printHelp();
    return 0;
  }
  const unwired = unwiredSuiteError(opts.suites, opts.modes);
  if (unwired) throw unwired;

  const repoRoot = getRepoRoot();
  const commit = getCommitSha();
  const estimate = estimateRunCost({ suites: opts.suites, modes: opts.modes, limit: opts.limit });
  const isTty = Boolean(process.stderr.isTTY);

  if (opts.jsonOutput) {
    process.stdout.write(JSON.stringify({
      phase: 'estimate',
      schema_version: 2,
      modes: opts.modes,
      suites: opts.suites,
      limit: opts.limit,
      seed: opts.seed,
      commit,
      estimate,
    }) + '\n');
  } else {
    process.stderr.write(`[eval run-all] commit=${commit} modes=${opts.modes.join(',')} suites=${opts.suites.join(',')}\n`);
    process.stderr.write(`[eval run-all] cost estimate: retrieval=$${estimate.retrieval_usd.toFixed(2)} answer=$${estimate.answer_usd.toFixed(2)} total=$${estimate.total_usd.toFixed(2)}\n`);
    process.stderr.write(`[eval run-all] budget caps: retrieval=$${opts.budgetUsdRetrieval} answer=$${opts.budgetUsdAnswer}\n`);
  }
  if (!opts.suitesExplicit) {
    process.stderr.write(
      `[eval run-all] running the wired suites (${WIRED_SUITES.join(', ')}). longmemeval and replay run through their own commands, once per mode:\n` +
      Object.values(UNWIRED_SUITE_ARGV).map(argv => `[eval run-all]   ${argv.join(' ')}\n`).join(''),
    );
  }

  const guard = evaluateCostGuard(estimate, {
    budgetUsdRetrieval: opts.budgetUsdRetrieval,
    budgetUsdAnswer: opts.budgetUsdAnswer,
    yes: opts.yes,
    isTty,
  });
  if (!guard.proceed) {
    process.stderr.write(`[eval run-all] REFUSED: ${guard.reason}\n`);
    process.exit(2);
  }
  if (!opts.jsonOutput) {
    process.stderr.write(`[eval run-all] ${guard.reason}, proceeding.\n`);
  }

  const failed: EvalRunRecord[] = [];
  for (const suite of opts.suites) {
    // BrainBench is search-mode-independent (decision 16): run ONCE per
    // sweep, in-process, recorded under mode 'n/a' — never multiplied by
    // modes. Unwired suites never reach this loop (unwiredSuiteError).
    if (suite === 'brainbench') {
      const startedAt = Date.now();
      const runId = `${commit}-brainbench-na-${opts.seed}`;
      const { runBrainBenchCore } = await import('./eval-brainbench.ts');
      const core = await runBrainBenchCore();
      const record: EvalRunRecord = {
        schema_version: 3,
        run_id: runId,
        ran_at: new Date().toISOString(),
        suite: 'brainbench',
        mode: 'n/a',
        commit,
        seed: opts.seed,
        limit: opts.limit,
        params: {
          budget_usd_retrieval: opts.budgetUsdRetrieval,
          budget_usd_answer: opts.budgetUsdAnswer,
          parallel: opts.parallel,
          fixtures_hash: core.fixtures_hash,
          cells: core.cells,
        },
        status: core.status,
        duration_ms: Date.now() - startedAt,
      };
      if (core.error) record.error = core.error;
      persistRunRecord(repoRoot, record, opts.outputDir);
      if (record.status === 'failed') failed.push(record);
      if (!opts.jsonOutput) {
        process.stderr.write(`[eval run-all] ${runId}: ${record.status}${record.error ? ` (${redactSecrets(record.error)})` : ''}\n`);
      }
    }
  }

  if (opts.jsonOutput) {
    process.stdout.write(JSON.stringify({
      phase: 'complete',
      commit,
      modes: opts.modes,
      suites: opts.suites,
      failed_runs: failed.map(r => r.run_id),
      output_path: evalResultsPath(repoRoot, opts.outputDir),
    }) + '\n');
  } else if (failed.length > 0) {
    process.stderr.write(`[eval run-all] FAILED: ${failed.map(r => r.run_id).join(', ')}. Records kept in ${evalResultsPath(repoRoot, opts.outputDir)}\n`);
  } else {
    process.stderr.write(`[eval run-all] complete. Audit trail: ${evalResultsPath(repoRoot, opts.outputDir)}\n`);
  }
  return failed.length > 0 ? 1 : 0;
}