Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
4.2 KB
Raw Blame
/**
 * `gbrain eval brainbench --decide ...` (System One arms) with a fixture
 * decide transport (no provider is called).
 *
 * Protects: the flags parse (and bad values are usage errors); an all-off run
 * adds no decide field to turn rows; an S6 recall_needed arm configures the
 * benchmark brain so the slot acts at the claude-code turn-context seam and
 * each replayed turn carries its `decide.recall_needed` receipt (mode,
 * effective, outcomes, latency, decide tokens); the roll-up summarizes them.
 * Serial: mutates the process-global gateway, transport and GBRAIN_DECIDE_SLOTS.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { _internal } from '../src/commands/eval-brainbench.ts';
import { DEFAULT_FIXTURES_DIR, DEFAULT_GOLD_DIR } from '../src/commands/eval-brainbench.ts';
import { loadCorpus } from '../src/eval/brainbench/fixtures.ts';
import { runBrainBench } from '../src/eval/brainbench/harness.ts';
import { newDecideEvalOptions, applyDecideEvalFlag, prepareDecideEval, summarizeDecideReceipts } from '../src/eval/decide-eval-flags.ts';
import { configureGateway, resetGateway, __setDecideTransportForTests } from '../src/core/ai/gateway.ts';
import { enableDecideEvalOverride } from '../src/core/ai/decide/config.ts';
import { __resetDecideStoreForTests } from '../src/core/ai/decide/store.ts';

const prevEnv = process.env.GBRAIN_DECIDE_SLOTS;
let calls = 0;

beforeAll(() => {
  configureGateway({ env: { TYPESAFE_API_KEY: 'sk-test-typesafe' } });
  __setDecideTransportForTests(async (_url, init) => {
    calls++;
    const body = JSON.parse(init.body as string);
    const answers = Object.fromEntries(Object.keys(body.questions).map((id) => [id, { type: 'noul', noul: 0.8 }]));
    return new Response(JSON.stringify({ model: 'jev-1.13.0', answers, usage: { input_tokens: 90, output_tokens: 0 } }));
  });
});

afterAll(() => {
  __setDecideTransportForTests(null);
  resetGateway();
  __resetDecideStoreForTests();
  if (prevEnv === undefined) delete process.env.GBRAIN_DECIDE_SLOTS;
  else process.env.GBRAIN_DECIDE_SLOTS = prevEnv;
  enableDecideEvalOverride(false);
});

describe('eval brainbench --decide', () => {
  test('flags parse; bad values are usage errors', () => {
    const ok = _internal.parseArgs(['--decide', 'recall_needed=on', '--decide-threshold', 'recall_needed=0.5', '--decide-force-on', 'recall_needed']);
    expect('usageError' in ok).toBe(false);
    expect((ok as { decide: unknown }).decide).toMatchObject({ slots: { recall_needed: 'on' }, thresholds: { recall_needed: 0.5 }, forceOn: ['recall_needed'] });
    const bad = _internal.parseArgs(['--decide', 'recall_needed=sometimes']);
    expect((bad as { usageError: string }).usageError).toContain('off|on|shadow');
  });

  test('all-off turn rows carry no decide; an S6 arm stamps per-turn receipts at the claude-code seam', async () => {
    const corpus = await loadCorpus(DEFAULT_FIXTURES_DIR, DEFAULT_GOLD_DIR);
    const base = await runBrainBench(corpus, { harnesses: ['claude-code'], suites: ['know-to-ask'], includeHoldout: false, llm: false });
    expect(base.turn_rows.length).toBeGreaterThan(0);
    expect(base.turn_rows.every((r) => r.decide === undefined)).toBe(true);
    expect(calls).toBe(0);

    const o = newDecideEvalOptions();
    for (const [f, v] of [['--decide', 'recall_needed=on'], ['--decide-threshold', 'recall_needed=0.5'], ['--decide-force-on', 'recall_needed']]) applyDecideEvalFlag(o, f!, v!);
    const decide = prepareDecideEval(o, { command: 'gbrain eval brainbench', throwaway: true })!;
    const arm = await runBrainBench(corpus, { harnesses: ['claude-code'], suites: ['know-to-ask'], includeHoldout: false, llm: false, decide });
    expect(arm.turn_rows.length).toBe(base.turn_rows.length);
    expect(arm.turn_rows.every((r) => r.decide?.recall_needed?.mode === 'on')).toBe(true);
    const acted = arm.turn_rows.filter((r) => r.decide?.recall_needed?.effective === 'on');
    expect(acted.length).toBeGreaterThan(0);
    expect(calls).toBeGreaterThan(0);
    expect(acted.some((r) => (r.decide!.recall_needed!.input_tokens ?? 0) > 0)).toBe(true);
    const summary = summarizeDecideReceipts(arm.turn_rows.map((r) => r.decide)) as Record<string, { rows: number }>;
    expect(summary.recall_needed!.rows).toBe(arm.turn_rows.length);
  }, 240_000);
});