Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Regression: `runValidationGate` must SURFACE errors that make the whole
* gate meaningless, not swallow them as score=0.
*
* The bug the SkillOpt real-LLM eval surfaced: a Haiku run with `--max-cost`
* hit `BudgetTracker` no_pricing on the FIRST chat() of every rollout, the
* error threw before any network call, and `runWithLimit` caught it as a
* `{ok:false}` settled item which the gate turned into `median:0`. Result: the
* whole gate reported a vacuous `selScore:0` in milliseconds with zero LLM
* calls — a pricing crash masquerading as a real "0/N" measurement. The fix
* re-throws any MUST_ABORT-class error so the caller aborts loudly.
*
* #4741 widened the same rule structurally: when EVERY task in a gate fails
* (a dead provider — e.g. a logged-out claude-cli child — is a plain
* non-tagged error), or EVERY rollout carries a judge_error (dead judge
* provider), the gate throws instead of scoring the run 0.000 and letting the
* orchestrator end with a plausible-looking `no_improvement`. Partial failure
* keeps the pessimistic per-task 0 (fail-open for scoring noise).
*/
import { describe, test, expect } from 'bun:test';
import {
runValidationGate,
scoreSkillOnTasks,
SKILLOPT_ALL_ROLLOUTS_FAILED,
SKILLOPT_ALL_JUDGE_ERRORS,
} from '../../src/core/skillopt/validate-gate.ts';
import { ClaudeCliProcessError } from '../../src/core/ai/providers/claude-cli-language-model.ts';
import type { BenchmarkTask, ScoredRollout, Trajectory } from '../../src/core/skillopt/types.ts';
import type { RolloutOpts } from '../../src/core/skillopt/rollout.ts';
import { invokeAI, withAIInvocationGuard } from '../../src/core/ai/invocation-guard.ts';
const TASKS: BenchmarkTask[] = [
{ task_id: 't1', task: 'do a thing', judge: { kind: 'rule', checks: [{ op: 'contains', arg: 'x' }] } } as never,
{ task_id: 't2', task: 'do another', judge: { kind: 'rule', checks: [{ op: 'contains', arg: 'y' }] } } as never,
];
function budgetExhausted(): Error {
const e = new Error('no pricing entry for model "anthropic:claude-haiku-4-5" (kind=chat)');
(e as { tag?: string }).tag = 'BUDGET_EXHAUSTED';
return e;
}
const LOGGED_OUT = 'claude-cli reported error: Not logged in · Please run /login';
const okTrajectory = (final_text: string): Trajectory =>
({ final_text, tool_calls: [], turns: 1 }) as never;
describe('runValidationGate — must-abort errors surface', () => {
test('a BUDGET_EXHAUSTED rollout error is re-thrown, not scored 0', async () => {
const throwingRollout = (async () => {
throw budgetExhausted();
}) as never;
await expect(
runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'anthropic:claude-haiku-4-5',
runsPerTask: 1,
rolloutFn: throwingRollout,
}),
).rejects.toThrow(/no pricing entry/);
});
test('an AI spend-policy refusal on one task is re-thrown, not scored 0 beside a passing task', async () => {
const denied = new Error('spend policy refused this call');
await withAIInvocationGuard(async () => { throw denied; }, () =>
invokeAI({ operation: 'test', kind: 'chat', model: 'test:model' }, async () => 'unreached', () => null)).catch(() => {});
const mixedRollout = (async (o: RolloutOpts) => {
if ((o as { task?: BenchmarkTask }).task?.task_id === 't1') throw denied;
return okTrajectory('y');
}) as never;
await expect(
runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'anthropic:claude-haiku-4-5',
runsPerTask: 1,
rolloutFn: mixedRollout,
}),
).rejects.toBe(denied);
});
test('scoreSkillOnTasks propagates the abort too (does not return a vacuous 0)', async () => {
const throwingRollout = (async () => {
throw budgetExhausted();
}) as never;
await expect(
scoreSkillOnTasks({
engine: {} as never,
skillText: 'skill',
tasks: TASKS,
targetModel: 'anthropic:claude-haiku-4-5',
runsPerTask: 1,
rolloutFn: throwingRollout,
}),
).rejects.toThrow(/no pricing entry/);
});
test('a PARTIAL (non-abort) rollout failure still scores that task 0 — fail-open preserved', async () => {
// t1 throws a plain error (no .tag → not must-abort); t2 completes and
// passes its contains-'y' check. Pessimistic mean = (0 + 1) / 2.
const flakyRollout = (async (o: RolloutOpts) => {
if (o.task.task_id === 't1') throw new Error('transient judge hiccup');
return okTrajectory('y');
}) as never;
const gate = await runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'anthropic:claude-haiku-4-5',
runsPerTask: 1,
rolloutFn: flakyRollout,
});
expect(gate.selScore).toBe(0.5);
expect(gate.scoredRollouts).toHaveLength(1);
expect(gate.perTaskMedians.find((r) => r.task_id === 't1')?.median).toBe(0);
});
});
describe('#4741 — a 100%-failed gate is an error, not a 0.000 measurement', () => {
const loggedOutRollout = (async () => {
throw new ClaudeCliProcessError(LOGGED_OUT);
}) as never;
test('every task failing with a non-abort error throws skillopt_all_rollouts_failed carrying the first error', async () => {
await expect(
runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'claude-cli:claude-sonnet-4-6',
runsPerTask: 1,
rolloutFn: loggedOutRollout,
}),
).rejects.toThrow(new RegExp(`${SKILLOPT_ALL_ROLLOUTS_FAILED}.*Not logged in`));
});
test('scoreSkillOnTasks propagates the all-failed error too', async () => {
await expect(
scoreSkillOnTasks({
engine: {} as never,
skillText: 'skill',
tasks: TASKS,
targetModel: 'claude-cli:claude-sonnet-4-6',
runsPerTask: 1,
rolloutFn: loggedOutRollout,
}),
).rejects.toThrow(new RegExp(`${SKILLOPT_ALL_ROLLOUTS_FAILED}.*Not logged in`));
});
test('every rollout carrying a judge_error throws skillopt_all_judge_errors', async () => {
const deadJudge = (async (trajectory: Trajectory): Promise<ScoredRollout> => ({
trajectory,
score: 0,
judge_error: 'llm_call_failed: 401 Unauthorized',
})) as never;
await expect(
runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'anthropic:claude-haiku-4-5',
judgeModel: 'anthropic:claude-sonnet-4-6',
runsPerTask: 1,
rolloutFn: (async () => okTrajectory('x y')) as never,
scoreFn: deadJudge,
}),
).rejects.toThrow(new RegExp(`${SKILLOPT_ALL_JUDGE_ERRORS}.*401 Unauthorized`));
});
test('one judge_error out of two rollouts resolves (partial judge failure stays fail-open)', async () => {
const flakyJudge = (async (trajectory: Trajectory, judge: BenchmarkTask['judge']): Promise<ScoredRollout> => {
const arg = (judge as { checks: Array<{ arg: string }> }).checks[0]!.arg;
return arg === 'x'
? { trajectory, score: 0, judge_error: 'llm_call_failed: timeout' }
: { trajectory, score: 1 };
}) as never;
const gate = await runValidationGate({
engine: {} as never,
candidateSkillText: 'skill',
selSet: TASKS,
bestScore: -1,
targetModel: 'anthropic:claude-haiku-4-5',
judgeModel: 'anthropic:claude-sonnet-4-6',
runsPerTask: 1,
rolloutFn: (async () => okTrajectory('x y')) as never,
scoreFn: flakyJudge,
});
expect(gate.selScore).toBe(0.5);
expect(gate.scoredRollouts).toHaveLength(2);
});
});