Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.40.1.0 Track D / T3+T4 — Cross-modal --batch mode tests.
*
* Hermetic via the runEval DI seam (per D5). Tests the batch loop +
* semaphore + exit precedence + receipt suppression, NOT the underlying
* runEval orchestrator (which has its own coverage in existing tests).
*
* No env mutation, no mock.module — regular *.test.ts per CLAUDE.md
* test-isolation rules.
*/
import { describe, test, expect } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { runEvalCrossModal, runWithLimit, type BatchSummary } from '../src/commands/eval-cross-modal.ts';
import { DEFAULT_SLOTS, type RunEvalOpts, type RunEvalResult } from '../src/core/cross-modal-eval/runner.ts';
import { aggregate, type AggregateResult, type SlotResult } from '../src/core/cross-modal-eval/aggregate.ts';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function writeBatchFixture(rows: object[]): string {
const tmp = mkdtempSync(join(tmpdir(), 'cm-batch-fixture-'));
const path = join(tmp, 'batch.jsonl');
writeFileSync(path, rows.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf8');
return path;
}
function makeStubRunEval(verdicts: Array<'pass' | 'fail' | 'inconclusive' | 'throw'>) {
let i = 0;
return async function stubRunEval(_opts: any): Promise<RunEvalResult> {
const idx = i++;
const v = verdicts[idx % verdicts.length];
if (v === 'throw') {
throw new Error(`stub error for question ${idx}`);
}
const aggregate: AggregateResult = {
verdict: v,
verdictMessage: `stub: ${v}`,
overall: v === 'pass' ? 8 : v === 'fail' ? 4 : 0,
perDimension: {},
successCount: 3,
modelCount: 3,
} as any;
return {
finalAggregate: aggregate,
cycles: [],
finalReceiptPath: '/tmp/fake-receipt.json',
};
};
}
// ---------------------------------------------------------------------------
// 1. runWithLimit semaphore primitive (T4)
// ---------------------------------------------------------------------------
describe('runWithLimit semaphore (v0.41.15.0 — migrated to shared helper)', () => {
// v0.41.15.0 T4 (codex #15): migrated to opts-object API from
// src/core/worker-pool.ts. Result shape gains an additive `idx` field;
// error field widened from `Error` to `unknown`. The `limit < 1`
// throw is no longer raised — the helper clamps to 1 silently,
// matching the permissive contract of runSlidingPool.
test('never exceeds the in-flight limit', async () => {
let inFlight = 0;
let maxObserved = 0;
const items = Array.from({ length: 20 }, (_, i) => i);
await runWithLimit({
items,
limit: 3,
fn: async (_item) => {
inFlight++;
if (inFlight > maxObserved) maxObserved = inFlight;
await new Promise(r => setTimeout(r, 5));
inFlight--;
return 1;
},
});
expect(maxObserved).toBeLessThanOrEqual(3);
expect(maxObserved).toBeGreaterThanOrEqual(1);
});
test('per-item errors do not abort the whole batch', async () => {
const items = [0, 1, 2, 3, 4];
const results = await runWithLimit({
items,
limit: 2,
fn: async (item) => {
if (item === 2) throw new Error(`fail-${item}`);
return item * 10;
},
});
expect(results.length).toBe(5);
expect(results[0]).toMatchObject({ ok: true, value: 0, idx: 0 });
expect(results[1]).toMatchObject({ ok: true, value: 10, idx: 1 });
expect(results[2].ok).toBe(false);
expect(results[2].idx).toBe(2);
expect(results[3]).toMatchObject({ ok: true, value: 30, idx: 3 });
expect(results[4]).toMatchObject({ ok: true, value: 40, idx: 4 });
});
test('preserves input order in results array', async () => {
const items = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
const results = await runWithLimit({
items,
limit: 4,
fn: async (item) => {
await new Promise(r => setTimeout(r, (10 - item) * 2));
return item * item;
},
});
for (let i = 0; i < items.length; i++) {
expect(results[i]).toMatchObject({ ok: true, value: i * i, idx: i });
}
});
test('limit=1 = serial', async () => {
const order: number[] = [];
const items = [0, 1, 2, 3];
await runWithLimit({
items,
limit: 1,
fn: async (item) => {
const start = Date.now();
await new Promise(r => setTimeout(r, 5));
order.push(item);
return Date.now() - start;
},
});
expect(order).toEqual([0, 1, 2, 3]);
});
test('limit > items.length still completes correctly', async () => {
const results = await runWithLimit({
items: [0, 1, 2],
limit: 100,
fn: async (x) => x + 1,
});
expect(results.length).toBe(3);
expect(results[0]).toMatchObject({ ok: true, value: 1, idx: 0 });
expect(results[1]).toMatchObject({ ok: true, value: 2, idx: 1 });
expect(results[2]).toMatchObject({ ok: true, value: 3, idx: 2 });
});
test('limit < 1 silently clamps to 1 worker (v0.41.15.0 permissive contract)', async () => {
// Prior behavior threw "limit must be >= 1". The shared helper
// clamps via Math.max(1, ...) to match runSlidingPool's permissive
// shape. Callers passing 0 get serial execution, not an exception.
const results = await runWithLimit({
items: [1, 2, 3],
limit: 0,
fn: async (x) => x,
});
expect(results.length).toBe(3);
for (let i = 0; i < 3; i++) {
expect(results[i]).toMatchObject({ ok: true, value: i + 1, idx: i });
}
});
});
// ---------------------------------------------------------------------------
// 2. End-to-end batch via stubbed runEval (T3 + D5 + D10)
// ---------------------------------------------------------------------------
describe('runEvalCrossModal --batch end-to-end (v0.40.1.0 Track D / T3, per D5+D10)', () => {
test('all-pass batch → exit 0; summary receipt has expected shape', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'what is X?', hypothesis: 'X is foo.' },
{ question_id: 'q2', question: 'what is Y?', hypothesis: 'Y is bar.' },
{ question_id: 'q3', question: 'what is Z?', hypothesis: 'Z is baz.' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '3',
'--cycles', '1', '--concurrent', '2', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'pass', 'pass']) },
);
expect(exit).toBe(0);
expect(existsSync(summaryPath)).toBe(true);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
expect(summary.kind).toBe('cross_modal_batch_summary');
expect(summary.schema_version).toBe(1);
expect(summary.verdict).toBe('pass');
expect(summary.pass_count).toBe(3);
expect(summary.fail_count).toBe(0);
expect(summary.error_count).toBe(0);
expect(summary.inconclusive_count).toBe(0);
expect(summary.per_question.length).toBe(3);
expect(summary.per_question[0].question_id).toBe('q1');
expect(summary.per_question[0].verdict).toBe('pass');
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('any FAIL → exit 1 (precedence: FAIL > INCONCLUSIVE > PASS)', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2', question: 'b', hypothesis: 'b-ans' },
{ question_id: 'q3', question: 'c', hypothesis: 'c-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '3',
'--cycles', '1', '--concurrent', '3', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'fail', 'pass']) },
);
expect(exit).toBe(1);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
expect(summary.verdict).toBe('fail');
expect(summary.fail_count).toBe(1);
expect(summary.pass_count).toBe(2);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('any per-question ERROR → exit 2 (precedence: ERROR > FAIL > INCONCLUSIVE > PASS)', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2', question: 'b', hypothesis: 'b-ans' },
{ question_id: 'q3', question: 'c', hypothesis: 'c-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '3',
'--cycles', '1', '--concurrent', '3', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'throw', 'fail']) },
);
// ERROR wins precedence over FAIL.
expect(exit).toBe(2);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
expect(summary.verdict).toBe('error');
expect(summary.error_count).toBe(1);
expect(summary.fail_count).toBe(1);
expect(summary.pass_count).toBe(1);
const q2 = summary.per_question.find(p => p.question_id === 'q2')!;
expect(q2.verdict).toBe('error');
expect(q2.error).toContain('stub error');
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('any INCONCLUSIVE (no error, no fail) → exit 2', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2', question: 'b', hypothesis: 'b-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '2',
'--cycles', '1', '--concurrent', '2', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'inconclusive']) },
);
expect(exit).toBe(2);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
expect(summary.verdict).toBe('inconclusive');
expect(summary.inconclusive_count).toBe(1);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('--batch + --task mutex → exit 1 with clear error', async () => {
const exit = await runEvalCrossModal(
['--batch', '/tmp/fake.jsonl', '--task', 'something', '--output', '/tmp/out.json'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).toBe(1);
});
test('--batch with non-existent file → exit 1', async () => {
const exit = await runEvalCrossModal(
['--batch', '/tmp/this-does-not-exist-' + Date.now() + '.jsonl', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).toBe(1);
});
test('--batch filters out by_type_summary rows (Codex #6)', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2', question: 'b', hypothesis: 'b-ans' },
// The summary row should be filtered before batch processing.
{ schema_version: 1, kind: 'by_type_summary', recall_by_type: {}, aggregate: { hit: 0, total: 0, rate: null } },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '10',
'--cycles', '1', '--concurrent', '2', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'pass']) },
);
expect(exit).toBe(0);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
// 2 rows, NOT 3 — the summary row was filtered.
expect(summary.total).toBe(2);
expect(summary.pass_count).toBe(2);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('--max-usd refusal without --yes', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
]);
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--limit', '1', '--cycles', '1', '--max-usd', '0.001'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).toBe(1);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
}
});
test('--max-usd refusal bypassed by --yes', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath,
'--limit', '1', '--cycles', '1', '--max-usd', '0.001', '--yes'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).toBe(0);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
});
// ---------------------------------------------------------------------------
// 3. Argument parser strictness (parseIntStrict / parseFloatStrict throw paths)
// ---------------------------------------------------------------------------
describe('argument parser strictness (v0.40.1.0 Track D / coverage gap)', () => {
test('--limit rejects non-integer with usage error → exit 1', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
]);
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--limit', 'not-a-number'],
{ runEval: makeStubRunEval(['pass']) },
);
// parseIntStrict throws synchronously inside parseArgs → caller catches in CLI
// dispatch and returns 1. Either path is acceptable; what matters is no PASS.
expect(exit).not.toBe(0);
} catch (err) {
// If parseArgs throws synchronously, that also counts as "rejected" — we
// assert the message rather than a specific exit code.
expect(String(err)).toMatch(/positive integer/);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
}
});
test('--max-usd rejects negative number', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
]);
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--max-usd', '-1'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).not.toBe(0);
} catch (err) {
expect(String(err)).toMatch(/non-negative number/);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
}
});
});
// ---------------------------------------------------------------------------
// 4. Codex CDX-1 + CDX-2 — upstream error rows + --limit 0 + malformed rows
// must count in the denominator and fail-loud (no silent CI bypass).
// ---------------------------------------------------------------------------
describe('codex CDX-1 + CDX-2 — denominator-bypass defenses', () => {
test('upstream-error rows (longmemeval emitted {error:...}) count as upstream_error, NOT silently dropped', async () => {
// Mimics what eval-longmemeval emits when runOneQuestion throws:
// {question_id, question, question_type, hypothesis: '', error: '...'}
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2', question: 'b', question_type: 'temporal-reasoning', hypothesis: '', error: 'upstream OOM' },
{ question_id: 'q3', question: 'c', hypothesis: 'c-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '10',
'--cycles', '1', '--concurrent', '2', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'pass']) }, // only 2 calls (q1, q3) — q2 is upstream error
);
// Exit 2 because upstream_error_count > 0 (CDX-1).
expect(exit).toBe(2);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
// Denominator is 3 (not 2!) — upstream error counted.
expect(summary.total).toBe(3);
expect(summary.upstream_error_count).toBe(1);
expect(summary.error_count).toBe(0);
expect(summary.pass_count).toBe(2);
expect(summary.verdict).toBe('error');
// q2 surfaces in per_question with verdict 'upstream_error'.
const q2 = summary.per_question.find(p => p.question_id === 'q2');
expect(q2).toBeDefined();
expect(q2!.verdict).toBe('upstream_error');
expect(q2!.error).toBe('upstream OOM');
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('malformed rows (missing question or hypothesis) count toward malformed_count + exit 2', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
{ question_id: 'q2' /* missing question and hypothesis */ },
{ question_id: 'q3', question: 'c', hypothesis: 'c-ans' },
]);
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--limit', '10',
'--cycles', '1', '--concurrent', '2', '--max-usd', '1000'],
{ runEval: makeStubRunEval(['pass', 'pass']) },
);
expect(exit).toBe(2);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
expect(summary.malformed_count).toBe(1);
expect(summary.total).toBe(3); // 2 scored + 0 upstream + 1 malformed
expect(summary.verdict).toBe('error');
} finally {
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
});
test('--limit 0 is rejected (would bypass the gate with empty result → PASS)', async () => {
const fixturePath = writeBatchFixture([
{ question_id: 'q1', question: 'a', hypothesis: 'a-ans' },
]);
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--limit', '0'],
{ runEval: makeStubRunEval(['pass']) },
);
expect(exit).toBe(1);
} finally {
rmSync(fixturePath, { recursive: true, force: true });
}
});
});
// ---------------------------------------------------------------------------
// 5. #5506: the summary and stderr name the judge panel; per-judge scores
// stay slot-indexed. Scores go through the real aggregate(), so the verdict
// and exit code are exactly what the same scores produce without the panel.
// ---------------------------------------------------------------------------
interface ScoredQuestion {
/** Per dimension, one score per slot; null: that slot returned no score for it. */
scores: Record<string, Array<number | null>>;
/** Slot positions whose judge call errored. */
erroredSlots?: number[];
}
function makeScoredRunEval(questions: ScoredQuestion[]) {
let i = 0;
return async (opts: RunEvalOpts): Promise<RunEvalResult> => {
const q = questions[i++ % questions.length]!;
const slotResults: SlotResult[] = (opts.slots ?? []).map((slot, idx) => {
if (q.erroredSlots?.includes(idx)) return { ok: false, modelId: slot.model, error: 'stub judge timeout' };
const scores: Record<string, { score: number }> = {};
for (const [dim, perSlot] of Object.entries(q.scores)) {
const score = perSlot[idx];
if (typeof score === 'number') scores[dim.toUpperCase()] = { score };
}
return { ok: true, modelId: slot.model, parsed: { scores, improvements: [] } };
});
const agg = aggregate({ slots: slotResults });
return {
finalAggregate: agg,
cycles: [{
schema_version: 1,
cycle: 1,
task: opts.task,
output_sha8: 'stub0000',
slug: opts.slug ?? 'stub',
timestamp: '2026-10-02T11:14:26Z',
dimensions: opts.dimensions ?? [],
// The receipt's id is the provider's first letter; scores must be
// read by position, never by this id.
slots: slotResults.map(s => ({
id: s.modelId.split(':')[0]!.toUpperCase().slice(0, 1),
model: s.modelId,
ok: s.ok,
error: s.ok ? undefined : s.error,
parsed: s.ok ? s.parsed : undefined,
})),
aggregate: agg,
receipt_path: '/tmp/fake-receipt.json',
}],
finalReceiptPath: '/tmp/fake-receipt.json',
};
};
}
async function runScoredBatch(slotModels: string[], questions: ScoredQuestion[]) {
const fixturePath = writeBatchFixture(questions.map((_, i) => ({
question_id: `q${i + 1}`, question: `question ${i + 1}?`, hypothesis: `answer ${i + 1}`,
})));
const summaryPath = join(mkdtempSync(join(tmpdir(), 'cm-summary-')), 'summary.json');
const stderr: string[] = [];
const originalWrite = process.stderr.write.bind(process.stderr);
process.stderr.write = ((chunk: string | Uint8Array) => {
stderr.push(String(chunk));
return true;
}) as typeof process.stderr.write;
try {
const exit = await runEvalCrossModal(
['--batch', fixturePath, '--output', summaryPath, '--cycles', '1', '--max-usd', '1000',
'--slot-a-model', slotModels[0]!, '--slot-b-model', slotModels[1]!, '--slot-c-model', slotModels[2]!],
{ runEval: makeScoredRunEval(questions) },
);
const summary = JSON.parse(readFileSync(summaryPath, 'utf8')) as BatchSummary;
return { exit, summary, stderr: stderr.join('') };
} finally {
process.stderr.write = originalWrite;
rmSync(fixturePath, { recursive: true, force: true });
rmSync(summaryPath, { force: true });
}
}
// The production night: directness 6,7,6 fails on its mean while
// correctness is 10,10,10; a second question passes.
const DIRECTNESS_FAIL: ScoredQuestion = { scores: { directness: [6, 7, 6], correctness: [10, 10, 10] } };
const ALL_PASS: ScoredQuestion = { scores: { directness: [8, 9, 8], correctness: [10, 10, 10] } };
describe('#5506: batch judge panel', () => {
const SONNET = 'anthropic:claude-sonnet-4-6';
const OPUS = 'anthropic:claude-opus-4-7';
const ALL_SCORED = { directness: [6, 7, 6], correctness: [10, 10, 10] };
const PANEL_CASES = [
{
name: 'slots A and C on one model (the production panel)',
slots: [SONNET, OPUS, SONNET],
panel: { distinct_models: 2, distinct_providers: 1, slot_scored_questions: [2, 2, 2] },
sharedLine: `[eval cross-modal batch] judge ${SONNET} holds slots A, C, so its votes count more than once.\n`,
notCrossModal: true,
},
{
name: 'three different claude-cli models',
slots: ['claude-cli:claude-opus-5-5', 'claude-cli:claude-sonnet-5', 'claude-cli:claude-haiku-4-5-20251001'],
panel: { distinct_models: 3, distinct_providers: 1, slot_scored_questions: [2, 2, 2] },
notCrossModal: true,
},
{
name: 'three different models from two providers',
slots: ['claude-cli:claude-opus-5-5', SONNET, OPUS],
panel: { distinct_models: 3, distinct_providers: 2, slot_scored_questions: [2, 2, 2] },
notCrossModal: true,
},
{
name: 'the default three-provider panel',
slots: DEFAULT_SLOTS.map(s => s.model),
panel: { distinct_models: 3, distinct_providers: 3, slot_scored_questions: [2, 2, 2] },
notCrossModal: false,
},
{
// Slot B errors on every question: two providers judged, not three.
name: 'the default panel with a slot that scored no question',
slots: DEFAULT_SLOTS.map(s => s.model),
erroredSlots: [1],
panel: { distinct_models: 2, distinct_providers: 2, slot_scored_questions: [2, 0, 2] },
silentLine: `[eval cross-modal batch] judge ${DEFAULT_SLOTS[1]!.model} in slot B scored no question, so it did not judge.\n`,
notCrossModal: true,
slotScores: { directness: [6, null, 6], correctness: [10, null, 10] },
},
{
// Slot C shares slot A's model but never scored, so no vote counts twice.
name: 'a shared model whose second slot scored no question',
slots: [SONNET, OPUS, SONNET],
erroredSlots: [2],
panel: { distinct_models: 2, distinct_providers: 1, slot_scored_questions: [2, 2, 0] },
silentLine: `[eval cross-modal batch] judge ${SONNET} in slot C scored no question, so it did not judge.\n`,
notCrossModal: true,
slotScores: { directness: [6, 7, null], correctness: [10, 10, null] },
},
];
test.each(PANEL_CASES)('$name', async ({ slots, erroredSlots, panel, sharedLine, silentLine, notCrossModal, slotScores }) => {
const { exit, summary, stderr } = await runScoredBatch(
slots,
[DIRECTNESS_FAIL, ALL_PASS].map(q => ({ ...q, erroredSlots })),
);
// Verdict and exit code are the aggregate's, whatever the panel.
expect(exit).toBe(1);
expect(summary.verdict).toBe('fail');
expect(summary.slots.map(s => s.model)).toEqual(slots);
expect(summary.panel).toEqual(panel);
expect(stderr).toContain(
`(total 2) distinct_judge_models=${panel.distinct_models} distinct_judge_providers=${panel.distinct_providers}\n`,
);
if (sharedLine) expect(stderr).toContain(sharedLine);
else expect(stderr).not.toContain('so its votes count more than once');
if (silentLine) expect(stderr).toContain(silentLine);
else expect(stderr).not.toContain('scored no question');
expect(stderr.includes('fewer than 3 providers judged, so this panel is not cross-modal.')).toBe(notCrossModal);
expect(summary.per_question[0]!.slot_scores).toEqual(slotScores ?? ALL_SCORED);
});
test('a slot that errored or gave no score holds null at its position, never shifting a vote', async () => {
const { exit, summary } = await runScoredBatch(
['anthropic:claude-sonnet-4-6', 'anthropic:claude-opus-4-7', 'anthropic:claude-sonnet-4-6'],
[
// B errored: the aggregate keeps [6, 5] for directness.
{ scores: { directness: [6, 9, 5], correctness: [10, 10, 10] }, erroredSlots: [1] },
// C answered but gave no directness score.
{ scores: { directness: [6, 6, null], correctness: [10, 10, 10] } },
],
);
expect(exit).toBe(1);
// B missed one question and still judged the other.
expect(summary.panel.slot_scored_questions).toEqual([2, 1, 2]);
const [q1, q2] = summary.per_question;
expect((q1!.final_aggregate as AggregateResult).dimensions.directness!.scores).toEqual([6, 5]);
expect(q1!.slot_scores).toEqual({ directness: [6, null, 5], correctness: [10, null, 10] });
expect(q2!.slot_scores).toEqual({ directness: [6, 6, null], correctness: [10, 10, 10] });
});
});