Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.8 KB
Raw Blame
/**
 * `--decide` eval plumbing (src/eval/decide-eval-flags.ts) shared by eval
 * longmemeval / brainbench / retrieval-quality.
 *
 * Protects: flag validation; the all-off path returns null and leaves the
 * process untouched; flags win over an inherited GBRAIN_DECIDE_SLOTS and the
 * effective value is set and recorded; calibration import from `decide
 * calibrate --json` output (object, array, JSONL) and reference ids; the
 * split holdout refuses a mismatched split and a calibrate_only=false row
 * (split_mismatch); benchmark-brain config (provider, consent, private egress,
 * awaited shadow, thresholds, force_on, the Jev reranker pin for rerank=on);
 * operator-brain runs refuse write flags and name the catalogued refusal;
 * per-row receipts and the run roll-up (S2 late rate).
 * Serial: mutates process.env.GBRAIN_DECIDE_SLOTS and the eval override.
 */
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
  applyDecideEvalFlag, benchmarkBrainConfig, configureDecideBrain, decideRowReceipt, DecideEvalRefusal, extractDecideEvalFlags,
  newDecideEvalOptions, operatorBrainRefusal, parseCalibrationText, prepareDecideEval, spendDelta, summarizeDecideReceipts,
} from '../../src/eval/decide-eval-flags.ts';
import { enableDecideEvalOverride, readDecideConfig } from '../../src/core/ai/decide/config.ts';
import { splitHash, stableSplit } from '../../src/core/ai/decide/dataset.ts';
import { listCalibrations } from '../../src/core/ai/decide/store.ts';
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';

const dir = mkdtempSync(join(tmpdir(), 'gbrain-decide-eval-flags-'));
const prevEnv = process.env.GBRAIN_DECIDE_SLOTS;
const THROWAWAY = { command: 'test', throwaway: true };

/** A frozen evidence dataset over families f0..f19 (split by the production stable hash). */
const items = Array.from({ length: 20 }, (_, i) => ({
  id: `f${i}:s0`, family: `f${i}`, slot: 'evidence', split: stableSplit(`f${i}`), state: { query: 'q' }, inputs: { candidate: 'c' }, label: i % 2 === 0, rank: 0,
}));
const datasetPath = join(dir, 'evidence.jsonl');
const SPLIT = splitHash(items as never);

function calibration(over: Record<string, unknown> = {}) {
  return {
    id: 7, slot: 'evidence', call_site: 'search', provider: 'typesafe:jev-1.13.0', model_resolved: 'jev-1.13.0', threshold: 0.42,
    min_keep: 3, metric: 'f1', metric_value: 0.8, ece: 0.05, retest_sd: 0.01, repack_sd: 0.02, n: 10, dataset_hash: 'd', split_hash: SPLIT,
    calibrate_ids_hash: 'c', calibrate_only: true, pack_shape: 'evidence:v1', notes: null, precision: 0.9, recall: 0.7, f1: 0.8, reliability: [], margin: 0.05,
    ...over,
  };
}

beforeAll(() => {
  writeFileSync(datasetPath, items.map((i) => JSON.stringify(i)).join('\n') + '\n');
});

afterEach(() => {
  if (prevEnv === undefined) delete process.env.GBRAIN_DECIDE_SLOTS;
  else process.env.GBRAIN_DECIDE_SLOTS = prevEnv;
  enableDecideEvalOverride(false);
});

afterAll(() => rmSync(dir, { recursive: true, force: true }));

describe('flag parsing', () => {
  test('valid and invalid values', () => {
    const o = newDecideEvalOptions();
    expect(applyDecideEvalFlag(o, '--decide', 'rerank=on')).toBe(true);
    expect(applyDecideEvalFlag(o, '--decide-threshold', 'evidence=0.3')).toBe(true);
    expect(applyDecideEvalFlag(o, '--decide-force-on', 'evidence')).toBe(true);
    expect(applyDecideEvalFlag(o, '--limit', '3')).toBe(false);
    expect(o).toMatchObject({ slots: { rerank: 'on' }, thresholds: { evidence: 0.3 }, forceOn: ['evidence'] });
    expect(() => applyDecideEvalFlag(o, '--decide', 'rerank=maybe')).toThrow('off|on|shadow');
    expect(() => applyDecideEvalFlag(o, '--decide', 'nope=on')).toThrow('SLOT=VALUE');
    expect(() => applyDecideEvalFlag(o, '--decide-threshold', 'rerank=0.5')).toThrow('has no threshold');
    expect(() => applyDecideEvalFlag(o, '--decide-threshold', 'evidence=2')).toThrow('from 0 to 1');
    expect(() => applyDecideEvalFlag(o, '--decide-provider', 'none')).toThrow();
  });

  test('extractDecideEvalFlags strips flags and values (both spellings) from argv', () => {
    const { decide, rest } = extractDecideEvalFlags(['fixture.jsonl', '--decide', 'evidence=on', '--json', '--decide-provider=llm:openai:gpt-4o-mini']);
    expect(rest).toEqual(['fixture.jsonl', '--json']);
    expect(decide.slots).toEqual({ evidence: 'on' });
    expect(decide.provider).toBe('llm:openai:gpt-4o-mini');
    expect(() => extractDecideEvalFlags(['--decide'])).toThrow('requires a value');
  });
});

describe('prepareDecideEval', () => {
  test('all-off: null, env untouched, override not enabled', () => {
    delete process.env.GBRAIN_DECIDE_SLOTS;
    expect(prepareDecideEval(newDecideEvalOptions(), THROWAWAY)).toBeNull();
    const off = newDecideEvalOptions();
    applyDecideEvalFlag(off, '--decide', 'evidence=off');
    expect(prepareDecideEval(off, THROWAWAY)).toBeNull();
    expect(process.env.GBRAIN_DECIDE_SLOTS).toBeUndefined();
    expect(readDecideConfig({ 'decide.slots.evidence.mode': 'off' }).slots.evidence.mode).toBe('off');
  });

  test('--decide-* options without an on/shadow slot are an error', () => {
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide-threshold', 'evidence=0.3');
    expect(() => prepareDecideEval(o, THROWAWAY)).toThrow('need at least one --decide');
  });

  test('flags win over an inherited GBRAIN_DECIDE_SLOTS; the effective value is set, honored and recorded', () => {
    process.env.GBRAIN_DECIDE_SLOTS = 'rerank=on,triage=shadow';
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'rerank=off');
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    const run = prepareDecideEval(o, THROWAWAY)!;
    expect(run.slots).toEqual({ evidence: 'on', triage: 'shadow' });
    expect(process.env.GBRAIN_DECIDE_SLOTS).toBe('evidence=on,triage=shadow');
    expect(run.runConfig).toMatchObject({ gbrain_decide_slots: 'evidence=on,triage=shadow', inherited_env: 'rerank=on,triage=shadow', provider: 'typesafe:jev-1.13.0', shadow_wait: true });
    expect(readDecideConfig({}).slots.evidence.mode).toBe('on');
    expect(run.searchPins).toEqual({});
  });

  test('rerank=on pins the Jev reranker; benchmark config writes provider, consent, private egress, shadow wait, thresholds, force_on', () => {
    const o = newDecideEvalOptions();
    for (const [f, v] of [['--decide', 'rerank=on'], ['--decide', 'evidence=shadow'], ['--decide', 'recall_needed=on'], ['--decide-threshold', 'recall_needed=0.4'], ['--decide-force-on', 'recall_needed']]) applyDecideEvalFlag(o, f!, v!);
    const run = prepareDecideEval(o, THROWAWAY)!;
    expect(run.searchPins).toEqual({ 'search.reranker.enabled': 'true', 'search.reranker.model': 'typesafe:jev-1.13.0' });
    const cfg = Object.fromEntries(benchmarkBrainConfig(run, { includeSearchPins: true }));
    expect(cfg).toMatchObject({
      'decide.provider': 'typesafe:jev-1.13.0', 'decide.egress.private': 'allow', 'decide.egress.typesafe.query': 'allow',
      'decide.egress.typesafe.candidates': 'allow', 'decide.egress.typesafe.conversation': 'allow',
      'decide.slots.evidence.shadow_wait': 'on', 'decide.slots.evidence.shadow_sample': '1',
      'decide.slots.recall_needed.threshold': '0.4', 'decide.slots.recall_needed.force_on': 'true',
      'search.reranker.model': 'typesafe:jev-1.13.0',
    });
    expect(run.runConfig).toMatchObject({ force_on: ['recall_needed'], rerank_pinned: 'typesafe:jev-1.13.0' });
    expect(Object.fromEntries(benchmarkBrainConfig(run))['search.reranker.model']).toBeUndefined();
  });

  test('operator-brain runs refuse write flags', () => {
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    applyDecideEvalFlag(o, '--decide-force-on', 'evidence');
    expect(() => prepareDecideEval(o, { command: 'gbrain eval retrieval-quality', throwaway: false })).toThrow('never writes its config');
  });
});

describe('calibrations and the split holdout', () => {
  const withCal = (text: string, dataset = true) => {
    const path = join(dir, `cal-${Math.random().toString(36).slice(2)}.json`);
    writeFileSync(path, text);
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    applyDecideEvalFlag(o, '--decide-calibration', path);
    if (dataset) applyDecideEvalFlag(o, '--decide-dataset', datasetPath);
    return o;
  };

  test('parses one object, an array and JSONL', () => {
    expect(parseCalibrationText(JSON.stringify(calibration()))).toHaveLength(1);
    expect(parseCalibrationText(JSON.stringify([calibration(), calibration()]))).toHaveLength(2);
    expect(parseCalibrationText(`${JSON.stringify(calibration())}\n${JSON.stringify(calibration())}\n`)).toHaveLength(2);
  });

  test('a matching split is accepted; the eval half is exposed; ids and split_hash are recorded', () => {
    const run = prepareDecideEval(withCal(JSON.stringify(calibration(), null, 2)), THROWAWAY)!;
    expect(run.calibrations).toHaveLength(1);
    expect(run.dataset!.splitHashes.evidence).toBe(SPLIT);
    expect([...run.dataset!.evalFamilies].sort()).toEqual(items.filter((i) => i.split === 'eval').map((i) => i.family).sort());
    expect(run.runConfig).toMatchObject({ split_verified: true, calibrations: [{ slot: 'evidence', threshold: 0.42, split_hash: SPLIT }] });
  });

  test('mismatched split: refused with split_mismatch', () => {
    let err: unknown;
    try { prepareDecideEval(withCal(JSON.stringify(calibration({ split_hash: 'deadbeefdeadbeef' }))), THROWAWAY); } catch (e) { err = e; }
    expect(err).toBeInstanceOf(DecideEvalRefusal);
    expect((err as DecideEvalRefusal).reason).toBe('split_mismatch');
    expect((err as Error).message).toContain('split_mismatch');
    expect((err as Error).message).toContain('deadbeefdeadbeef');
  });

  test('calibrate_only false: refused even without a dataset', () => {
    expect(() => prepareDecideEval(withCal(JSON.stringify(calibration({ calibrate_only: false })), false), THROWAWAY)).toThrow('calibrate_only is false');
  });

  test('unknown reference ids and malformed rows are errors', () => {
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    applyDecideEvalFlag(o, '--decide-calibration', 'ref:does-not-exist');
    expect(() => prepareDecideEval(o, THROWAWAY)).toThrow('unknown reference calibration');
    expect(() => prepareDecideEval(withCal(JSON.stringify({ slot: 'evidence', threshold: 0.5 })), THROWAWAY)).toThrow('call_site is required');
  });

  test('configureDecideBrain inserts, qualifies and adopts the rows in a benchmark brain', async () => {
    const engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
    try {
      const run = prepareDecideEval(withCal(JSON.stringify(calibration({ action_precision_lb: 0.93, policy_fingerprint: 'fp' }))), THROWAWAY)!;
      await configureDecideBrain(engine, run);
      const rows = await listCalibrations(engine, { slot: 'evidence' });
      expect(rows).toHaveLength(1);
      expect(rows[0]).toMatchObject({ threshold: 0.42, split_hash: SPLIT, calibrate_only: true, action_precision_lb: 0.93, policy_fingerprint: 'fp' });
      expect(await engine.getConfig('decide.slots.evidence.calibration')).toBe(`local:${rows[0]!.id}`);
      expect(await engine.getConfig('decide.egress.private')).toBe('allow');
    } finally {
      await engine.disconnect();
    }
  });
});

describe('operator brain and receipts', () => {
  test('operatorBrainRefusal names the catalogued reason (provider, key, consent)', () => {
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    const run = prepareDecideEval(o, { command: 'rq', throwaway: false })!;
    configureGateway({ env: { TYPESAFE_API_KEY: 'sk-test-typesafe' } });
    try {
      expect(operatorBrainRefusal({}, run)).toContain('no_provider');
      expect(operatorBrainRefusal({ 'decide.provider': 'typesafe:jev-1.13.0' }, run)).toContain('egress_class_denied');
      expect(operatorBrainRefusal({ 'decide.provider': 'typesafe:jev-1.13.0', 'decide.egress.typesafe.query': 'allow', 'decide.egress.typesafe.candidates': 'allow' }, run)).toBeNull();
    } finally {
      resetGateway();
    }
  });

  test('per-row receipt from slot meta + spend delta; the roll-up counts S2 late answers', () => {
    const o = newDecideEvalOptions();
    applyDecideEvalFlag(o, '--decide', 'evidence=on');
    applyDecideEvalFlag(o, '--decide', 'intent=on');
    applyDecideEvalFlag(o, '--decide', 'triage=on');
    const run = prepareDecideEval(o, THROWAWAY)!;
    const spend = spendDelta({ evidence: { input_tokens: 100, cost_usd: 0.001, requests: 1 } }, { evidence: { input_tokens: 400, cost_usd: 0.004, requests: 2 } });
    const r1 = decideRowReceipt(run, {
      evidence: { mode: 'on', effective: 'on', threshold: 0.5, judged: 8, outcomes: { kept: 6, pruned: 2 }, latency_ms: 120, model_resolved: 'jev-1.13.0' },
      intent: { mode: 'on', effective: 'on', skipped: 'late' },
    }, spend);
    expect(r1.evidence).toEqual({ mode: 'on', effective: 'on', threshold: 0.5, outcomes: { kept: 6, pruned: 2 }, latency_ms: 120, model_resolved: 'jev-1.13.0', judged: 8, input_tokens: 300, cost_usd: 0.003 });
    expect(r1.intent).toMatchObject({ skipped: 'late', late: true });
    expect(r1.triage).toEqual({ mode: 'on', effective: null, skipped: 'not_reached' });
    const r2 = decideRowReceipt(run, { intent: { mode: 'on', effective: 'on', outcomes: { override: 1 }, latency_ms: 80 } }, {});
    const sum = summarizeDecideReceipts([r1, r2, undefined]) as Record<string, any>;
    expect(sum.intent).toMatchObject({ rows: 2, late_rate: 0.5, outcomes: { override: 1 } });
    expect(sum.evidence).toMatchObject({ rows: 2, acted: 1, input_tokens: 300, outcomes: { kept: 6, pruned: 2 } });
    expect(sum.triage.skipped).toEqual({ not_reached: 2 });
  });
});