Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.4 KB
Raw Blame
/**
 * v0.32.3 — eval-run-all orchestrator unit tests.
 * Pins arg parsing + cost-guard semantics + persist hook + audit trail.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { runCli } from './helpers/cli-spawn.ts';
import {
  parseRunAllArgs,
  estimateRunCost,
  evaluateCostGuard,
  persistRunRecord,
  type EvalRunRecord,
} from '../src/commands/eval-run-all.ts';

let tmp: string;

beforeAll(() => {
  tmp = mkdtempSync(join(tmpdir(), 'gbrain-eval-runall-'));
});

afterAll(() => {
  rmSync(tmp, { recursive: true, force: true });
});

describe('parseRunAllArgs', () => {
  test('defaults: all modes, the wired suites (brainbench), seed=42', () => {
    const opts = parseRunAllArgs([]);
    expect(opts.modes).toEqual(['conservative', 'balanced', 'tokenmax']);
    expect(opts.suites).toEqual(['brainbench']);
    expect(opts.suitesExplicit).toBe(false);
    expect(opts.seed).toBe(42);
    expect(opts.parallel).toBe(1);
    expect(opts.budgetUsdRetrieval).toBe(5);
    expect(opts.budgetUsdAnswer).toBe(20);
    expect(opts.yes).toBe(false);
  });

  test('--modes filters to a subset', () => {
    const opts = parseRunAllArgs(['--modes', 'conservative,tokenmax']);
    expect(opts.modes).toEqual(['conservative', 'tokenmax']);
  });

  test('--modes rejects invalid mode', () => {
    expect(() => parseRunAllArgs(['--modes', 'frontier'])).toThrow(/not a valid mode/);
  });

  test('--suites filters; rejects unknown', () => {
    const opts = parseRunAllArgs(['--suites', 'longmemeval']);
    expect(opts.suites).toEqual(['longmemeval']);
    expect(opts.suitesExplicit).toBe(true);
    expect(() => parseRunAllArgs(['--suites', 'foo'])).toThrow(/not a recognized suite/);
  });

  test('--budget-usd-retrieval + --budget-usd-answer override defaults', () => {
    const opts = parseRunAllArgs(['--budget-usd-retrieval', '15', '--budget-usd-answer', '50']);
    expect(opts.budgetUsdRetrieval).toBe(15);
    expect(opts.budgetUsdAnswer).toBe(50);
  });

  test('--parallel clamps to mode count', () => {
    const opts = parseRunAllArgs(['--parallel', '10']);
    expect(opts.parallel).toBe(3); // clamped to SEARCH_MODES.length
  });

  test('--parallel rejects 0 / negative', () => {
    expect(() => parseRunAllArgs(['--parallel', '0'])).toThrow(/must be >= 1/);
    expect(() => parseRunAllArgs(['--parallel', '-1'])).toThrow(/must be >= 1/);
  });

  test('--yes flag toggles', () => {
    expect(parseRunAllArgs(['--yes']).yes).toBe(true);
    expect(parseRunAllArgs(['-y']).yes).toBe(true);
  });

  // #5933 (absorbed in agent contract v1 D4): NaN/out-of-range numeric flags
  // are invalid_params usage errors, so the cost guard can't be bypassed.
  test('--limit requires a positive safe integer', () => {
    for (const value of ['NaN', 'Infinity', '1.5', '0', '-1', '9007199254740992']) {
      expect(() => parseRunAllArgs(['--limit', value]), `--limit ${value}`).toThrow(/--limit/);
    }
    expect(() => parseRunAllArgs(['--limit', 'NaN'])).toThrow(expect.objectContaining({ code: 'invalid_params' }));
  });

  test('--seed requires a safe integer and allows zero', () => {
    expect(parseRunAllArgs(['--seed', '0']).seed).toBe(0);
    for (const value of ['NaN', 'Infinity', '1.5', '9007199254740992']) {
      expect(() => parseRunAllArgs(['--seed', value]), `--seed ${value}`).toThrow(/--seed/);
    }
  });

  test('budget caps require finite non-negative amounts', () => {
    expect(parseRunAllArgs(['--budget-usd-answer', '0']).budgetUsdAnswer).toBe(0);
    expect(parseRunAllArgs(['--budget-usd-retrieval', '0.25']).budgetUsdRetrieval).toBe(0.25);
    for (const flag of ['--budget-usd-retrieval', '--budget-usd-answer']) {
      for (const value of ['NaN', 'Infinity', '-Infinity', '-1']) {
        expect(() => parseRunAllArgs([flag, value]), `${flag} ${value}`).toThrow(new RegExp(flag));
      }
    }
  });
});

describe('estimateRunCost', () => {
  test('returns retrieval/answer/total breakdown', () => {
    const est = estimateRunCost({ suites: ['longmemeval'], modes: ['balanced'], limit: 100 });
    expect(est.total_usd).toBe(est.retrieval_usd + est.answer_usd);
    expect(est.answer_usd).toBeGreaterThan(0); // longmemeval has answer-gen
    expect(est.retrieval_usd).toBeGreaterThan(0);
  });

  test('expansion cost follows the mode bundle: no mode expands by default since GBRA-94', () => {
    const bal = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
    const tok = estimateRunCost({ suites: ['replay'], modes: ['tokenmax'], limit: 100 });
    expect(tok.retrieval_usd).toBe(bal.retrieval_usd);
  });

  test('replay suite has zero answer cost', () => {
    const est = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
    expect(est.answer_usd).toBe(0);
  });

  test('multi-suite multi-mode estimate sums correctly', () => {
    const est = estimateRunCost({ suites: ['longmemeval', 'replay'], modes: ['conservative', 'balanced'], limit: 100 });
    expect(Object.keys(est.per_suite).length).toBe(4); // 2 × 2
  });
});

describe('evaluateCostGuard', () => {
  test('within-budget proceeds without --yes', () => {
    const g = evaluateCostGuard(
      { retrieval_usd: 1, answer_usd: 10, total_usd: 11 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
    );
    expect(g.proceed).toBe(true);
  });

  test('over-cap TTY without --yes refuses', () => {
    const g = evaluateCostGuard(
      { retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
    );
    expect(g.proceed).toBe(false);
    expect(g.reason).toContain('--yes');
  });

  test('over-cap TTY with --yes proceeds', () => {
    const g = evaluateCostGuard(
      { retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: true, isTty: true },
    );
    expect(g.proceed).toBe(true);
  });

  test('over-cap non-TTY without --yes refuses (exit 2 path)', () => {
    const g = evaluateCostGuard(
      { retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
    );
    expect(g.proceed).toBe(false);
    expect(g.reason).toContain('Non-TTY requires --yes');
  });

  test('only-retrieval-over OR only-answer-over both trigger guard', () => {
    const a = evaluateCostGuard(
      { retrieval_usd: 100, answer_usd: 1, total_usd: 101 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
    );
    expect(a.proceed).toBe(false);
    const b = evaluateCostGuard(
      { retrieval_usd: 1, answer_usd: 100, total_usd: 101 },
      { budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
    );
    expect(b.proceed).toBe(false);
  });
});

describe('persistRunRecord audit trail', () => {
  beforeEach(() => {
    rmSync(join(tmp, '.gbrain-evals'), { recursive: true, force: true });
    rmSync(join(tmp, 'eval-results.jsonl'), { force: true });
  });

  test('appends to eval-results.jsonl, creates dir if missing', () => {
    const record: EvalRunRecord = {
      schema_version: 3,
      run_id: 'abc123-longmemeval-conservative-42',
      ran_at: '2026-05-12T12:00:00Z',
      suite: 'longmemeval',
      mode: 'conservative',
      commit: 'abc123',
      seed: 42,
      params: {},
      status: 'completed',
      duration_ms: 12_345,
    };
    persistRunRecord(tmp, record, tmp);
    const path = join(tmp, 'eval-results.jsonl');
    expect(existsSync(path)).toBe(true);
    const content = readFileSync(path, 'utf-8');
    expect(content).toContain('abc123-longmemeval-conservative-42');
    const parsed = JSON.parse(content.trim());
    expect(parsed.mode).toBe('conservative');
    expect(parsed.schema_version).toBe(3);
  });

  test('appends multiple records (NDJSON)', () => {
    const base = {
      schema_version: 3 as const,
      ran_at: '2026-05-12T12:00:00Z',
      suite: 'longmemeval' as const,
      commit: 'abc',
      seed: 42,
      params: {},
      status: 'completed' as const,
      duration_ms: 1,
    };
    persistRunRecord(tmp, { ...base, run_id: 'a', mode: 'conservative' }, tmp);
    persistRunRecord(tmp, { ...base, run_id: 'b', mode: 'balanced' }, tmp);
    persistRunRecord(tmp, { ...base, run_id: 'c', mode: 'tokenmax' }, tmp);
    const content = readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8');
    const lines = content.trim().split('\n');
    expect(lines.length).toBe(3);
  });

  test('secrets in error text AND params string leaves are redacted on the write path (DB URL password, provider keys)', () => {
    const record: EvalRunRecord = {
      schema_version: 3,
      run_id: 'abc123-longmemeval-balanced-zz',
      ran_at: '2026-09-06T12:00:00Z',
      suite: 'longmemeval',
      mode: 'balanced',
      commit: 'abc123',
      seed: 0,
      params: {
        topK: 5,
        output: 'postgres://brain_user:[email protected]:5432/brain?sslmode=require',
        nested: { note: 'anthropic sk-ant-api03-ABCDEFGHIJKLMNOP failed', n: 3 },
        list: ['Authorization: Bearer abcdefghijklmnop.qrstuv', 'plain text'],
      },
      status: 'failed',
      duration_ms: 1,
      error: 'exit 1 | q1: connect failed: postgres://brain_user:[email protected]:5432/brain; openai key sk-proj-abcdefghijklmnopqrstuvwxyz0123',
    };
    // No caller-side redaction here — the write path alone must scrub.
    persistRunRecord(tmp, record, tmp);
    const line = readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8').trim();
    expect(line).not.toContain('s3cr3t-pw');
    expect(line).not.toContain('brain_user:');
    expect(line).not.toContain('sk-proj-abcdefghijklmnopqrstuvwxyz0123');
    expect(line).not.toContain('ABCDEFGHIJKLMNOP');
    expect(line).not.toContain('abcdefghijklmnop.qrstuv');
    // The line is still valid JSON with the diagnostic shape + non-string leaves intact.
    const parsed = JSON.parse(line);
    expect(parsed.error).toContain('exit 1 | q1: connect failed: postgres://<redacted>@db.example.internal:5432/brain');
    expect(parsed.error).toContain('sk-<redacted>');
    expect(parsed.params.output).toBe('postgres://<redacted>@db.example.internal:5432/brain?sslmode=require');
    expect(parsed.params.nested.note).toBe('anthropic sk-ant-<redacted> failed');
    expect(parsed.params.nested.n).toBe(3);
    expect(parsed.params.topK).toBe(5);
    expect(parsed.params.list).toEqual(['Authorization: Bearer <redacted>', 'plain text']);
    expect(parsed.status).toBe('failed');
  });

  test("v3: brainbench records carry mode 'n/a' (decision 16 — no params.mode_independent hack)", () => {
    const record: EvalRunRecord = {
      schema_version: 3,
      run_id: 'abc123-brainbench-na-42',
      ran_at: '2026-06-12T12:00:00Z',
      suite: 'brainbench',
      mode: 'n/a',
      commit: 'abc123',
      seed: 42,
      params: { fixtures_hash: 'deadbeef', cells: {} },
      status: 'completed',
      duration_ms: 1,
    };
    persistRunRecord(tmp, record, tmp);
    const parsed = JSON.parse(readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8').trim());
    expect(parsed.mode).toBe('n/a');
    expect(parsed.suite).toBe('brainbench');
    expect(parsed.params.mode_independent).toBeUndefined();
  });
});

describe('eval run-all with a suite it does not run (CLI)', () => {
  test('explicit --suites longmemeval,replay exits 1 with eval_suite_unwired, the per-suite commands as fix, and no record', async () => {
    const out = join(tmp, 'unwired');
    const r = await runCli(['eval', 'run-all', '--suites', 'longmemeval,replay', '--modes', 'balanced', '--yes', '--json', '--output', out]);
    expect(r.exitCode).toBe(1);
    const env = JSON.parse(r.stdout);
    expect(env.code).toBe('eval_suite_unwired');
    expect(env.fix.argv.slice(0, 7)).toEqual(['gbrain', 'eval', 'longmemeval', '<dataset.jsonl>', '--mode', '<mode>', '--record']);
    expect(env.fix.next).toBe('ask_user');
    expect(env.fix.then.argv.slice(0, 5)).toEqual(['gbrain', 'eval', 'replay', '--mode', '<mode>']);
    expect(r.stderr).toContain('Error [eval_suite_unwired]');
    expect(existsSync(join(out, 'eval-results.jsonl'))).toBe(false);
  });

  test('gbrain errors eval_suite_unwired explains the code', async () => {
    const r = await runCli(['errors', 'eval_suite_unwired']);
    expect(r.exitCode).toBe(0);
    expect(r.stdout).toContain('eval_suite_unwired (caller, exit 1)');
    expect(r.stdout).toContain('does not run in-process');
  });
});