Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.0 KB
Raw Blame
/**
 * `gbrain decide judge-agreement` end to end with a fixture decide transport
 * (no provider is called): one request per item on the background lane,
 * the kappa report, per question_type slices, Jev cost and latency, the
 * --out file with per-item rows, --dry-run (sends nothing) and the no-key
 * refusal. Serial: mutates the process-global gateway and decide transport.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { configureGateway, resetGateway, __setDecideTransportForTests } from '../../src/core/ai/gateway.ts';
import { runJudgeAgreement } from '../../src/commands/decide/judge-agreement.ts';

const dir = mkdtempSync(join(tmpdir(), 'gbrain-judge-agreement-cli-'));
const input = join(dir, 'lme-judged.jsonl');
let bodies: any[] = [];

async function capture(fn: () => Promise<number>): Promise<{ code: number; out: string; err: string }> {
  const log = console.log, error = console.error;
  let out = '', err = '';
  console.log = (...a: unknown[]) => { out += a.join(' ') + '\n'; };
  console.error = (...a: unknown[]) => { err += a.join(' ') + '\n'; };
  try { return { code: await fn(), out, err }; } finally { console.log = log; console.error = error; }
}

beforeAll(() => {
  const rows = [
    { question_id: 'q1', question_type: 'temporal-reasoning', question: 'When?', answer: 'May', hypothesis: 'In May', judge_correct: true, judge_model: 'gpt-4o' },
    { question_id: 'q2', question_type: 'temporal-reasoning', question: 'When?', answer: 'June', hypothesis: 'In July', judge_correct: false, judge_model: 'gpt-4o' },
    { question_id: 'q3', question_type: 'multi-session', question: 'How many?', answer: '3', hypothesis: 'three', judge_correct: true, judge_model: 'gpt-4o' },
    { question_id: 'q4', question_type: 'multi-session', question: 'How many?', answer: '4', hypothesis: 'two', judge_correct: false, judge_model: 'gpt-4o' },
    { question_id: 'q5', question: 'x', answer: 'y', hypothesis: 'z', judge_error: 'timeout' },
  ];
  writeFileSync(input, rows.map((r) => JSON.stringify(r)).join('\n') + '\n');
  configureGateway({ env: { TYPESAFE_API_KEY: 'sk-test-typesafe' } });
  // Jev agrees except on q4 (says yes where the LLM judge said no).
  __setDecideTransportForTests(async (_url, init) => {
    const body = JSON.parse(init.body as string);
    bodies.push(body);
    const q = body.questions['judge:0'];
    const hyp = q.instructions.hypothesis as string;
    const p = hyp === 'In May' || hyp === 'three' || hyp === 'two' ? 0.9 : 0.1;
    return new Response(JSON.stringify({ model: 'jev-1.13.0', answers: { 'judge:0': { type: 'noul', noul: p } }, usage: { input_tokens: 200, output_tokens: 1 } }));
  });
});

afterAll(() => {
  __setDecideTransportForTests(null);
  resetGateway();
  rmSync(dir, { recursive: true, force: true });
});

describe('decide judge-agreement', () => {
  test('--suite longmemeval --json: kappa, confusion, slices, cost, latency; one request per item', async () => {
    bodies = [];
    const out = join(dir, 'report.json');
    const r = await capture(() => runJudgeAgreement(['--suite', 'longmemeval', '--input', input, '--json', '--out', out]));
    expect(r.code).toBe(0);
    const report = JSON.parse(r.out);
    expect(bodies.length).toBe(4);
    expect(report).toMatchObject({ suite: 'longmemeval', n: 4, raw_agreement: 0.75, models: ['jev-1.13.0'], skipped: { judge_error: 1 } });
    expect(report.confusion).toEqual({ both_yes: 2, reference_yes_jev_no: 0, reference_no_jev_yes: 1, both_no: 1 });
    expect(report.kappa).toBe(0.5);
    expect(report.kappa_ci95).toHaveLength(2);
    expect(report.reference).toEqual({ kind: 'llm_judge', label_source: { 'llm:gpt-4o': 4 } });
    expect(Object.keys(report.slices)).toEqual(['multi-session', 'temporal-reasoning']);
    expect(report.slices['temporal-reasoning'].raw_agreement).toBe(1);
    expect(report.input_tokens).toBe(800);
    expect(report.cost_usd).toBeGreaterThan(0);
    expect(typeof report.latency_ms.p50).toBe('number');
    const written = JSON.parse(readFileSync(out, 'utf8'));
    expect(written.rows.map((x: { id: string; predicted: boolean }) => [x.id, x.predicted])).toEqual([['q1', true], ['q2', false], ['q3', true], ['q4', true]]);
  });

  test('--limit and --threshold', async () => {
    bodies = [];
    const r = await capture(() => runJudgeAgreement(['--suite', 'longmemeval', '--input', input, '--limit', '2', '--threshold', '0.95', '--json']));
    expect(r.code).toBe(0);
    const report = JSON.parse(r.out);
    expect(bodies.length).toBe(2);
    expect(report.threshold).toBe(0.95);
    expect(report.confusion).toEqual({ both_yes: 0, reference_yes_jev_no: 1, reference_no_jev_yes: 0, both_no: 1 });
  });

  test('--dry-run estimates tokens and cost and sends nothing', async () => {
    bodies = [];
    const r = await capture(() => runJudgeAgreement(['--suite', 'longmemeval', '--input', input, '--dry-run', '--json']));
    expect(r.code).toBe(0);
    const report = JSON.parse(r.out);
    expect(bodies.length).toBe(0);
    expect(report.dry_run).toBe(true);
    expect(report.items).toBe(4);
    expect(report.estimated_input_tokens).toBeGreaterThan(0);
  });

  test('usage errors and the no-key refusal', async () => {
    expect((await capture(() => runJudgeAgreement(['--suite', 'nope', '--input', input]))).code).toBe(1);
    configureGateway({ env: {} });
    const prev = { a: process.env.TYPESAFE_API_KEY, b: process.env.JEV_TYPESAFE_API_KEY };
    delete process.env.TYPESAFE_API_KEY; delete process.env.JEV_TYPESAFE_API_KEY;
    try {
      const r = await capture(() => runJudgeAgreement(['--suite', 'longmemeval', '--input', input]));
      expect(r.code).toBe(1);
      expect(r.err).toContain('no_key');
    } finally {
      if (prev.a !== undefined) process.env.TYPESAFE_API_KEY = prev.a;
      if (prev.b !== undefined) process.env.JEV_TYPESAFE_API_KEY = prev.b;
      configureGateway({ env: { TYPESAFE_API_KEY: 'sk-test-typesafe' } });
    }
  });
});