Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* SkillOpt validation gate (D12 + D4).
*
* D12 — median-of-3 judge runs per sel-task + epsilon=0.05 margin.
* D4 — parallel sel-task scoring via runWithLimit cap=4.
*
* Pseudocode:
*
* For each sel-task t in selSet:
* scores[t] = median(await Promise.all([
* scoreTrajectory(rollout(candidate, t), judge), // run 1
* scoreTrajectory(rollout(candidate, t), judge), // run 2
* scoreTrajectory(rollout(candidate, t), judge), // run 3
* ]))
* sel_score = mean(scores)
* if sel_score > best_score + 0.05: ACCEPT
* else: REJECT
*
* The 3 rollout calls per sel-task share the same candidate-skill prompt
* which is cached (D11), so the effective cost ~1.3x not 3x.
*/
import { runWithLimit, type SettledItem } from '../worker-pool.ts';
import { createHash } from 'node:crypto';
import { withoutBudgetTracker } from '../ai/gateway.ts';
import { isSkilloptMustAbort, SKILLOPT_HOOK_INFRA_ERROR, SKILLOPT_RUNTIME_EXCEEDED } from './must-abort.ts';
import { runRollout, type RolloutOpts } from './rollout.ts';
import { scoreTrajectory } from './score.ts';
import type { BenchmarkTask, GateInput, GateResult, RolloutBatchHook, RolloutPhase, ScoredRollout } from './types.ts';
import { VALIDATION_EPSILON, VALIDATION_RUNS_PER_TASK } from './types.ts';
import type { BrainEngine } from '../engine.ts';
import type { OperationContext } from '../ops/contract.ts';
export interface ValidateGateOpts extends Omit<GateInput, 'selSet'> {
operationContext?: OperationContext;
selSet: BenchmarkTask[];
engine: BrainEngine;
targetModel: string;
judgeModel?: string;
/** D4: max in-flight sel-task evaluations. Default 4. */
concurrency?: number;
/**
* Runs per task for the median-of-N. Default `VALIDATION_RUNS_PER_TASK` (3)
* for sel-side acceptance gates. The orchestrator's forward pass passes 1
* because we only need a rough partition into successes/failures, not noise
* rejection. Must be >= 1.
*/
runsPerTask?: number;
abortSignal?: AbortSignal;
/**
* #4119 — wall-clock deadline (epoch ms) observed INSIDE the rollout loop:
* checked before every individual rollout, not just between orchestrator
* steps. A breach throws `skillopt_runtime_exceeded` (surfaced loudly, never
* swallowed as a score-0 task) so a long batch can't blow past
* --max-runtime by a whole gate's worth of rollouts.
*/
deadlineMs?: number;
/** Test seam — substitute rollout. */
rolloutFn?: typeof runRollout;
/** Test seam — substitute scoreTrajectory. */
scoreFn?: typeof scoreTrajectory;
/** B1.3: when set, the whole batch goes to the hook in one call; rolloutFn/scoreFn are unused. */
rolloutHook?: RolloutBatchHook;
/** Hook request labels. Default phase 'sel_gate', epoch/step 0. */
phase?: RolloutPhase;
epoch?: number;
step?: number;
}
export { SKILLOPT_RUNTIME_EXCEEDED };
interface PerTask {
task_id: string;
median: number;
runs: number[];
rollouts: ScoredRollout[];
}
/** setTimeout clamps longer delays to 1 ms; a cap beyond it relies on the post-call deadline check. */
const MAX_TIMER_MS = 2 ** 31 - 1;
/** One hook call for the whole batch; a malformed reply is an infra error, never a score. */
async function hookBatch(opts: ValidateGateOpts, hook: RolloutBatchHook, runsPerTask: number): Promise<PerTask[]> {
const deadline = opts.deadlineMs;
if (deadline !== undefined && Date.now() > deadline) throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
// The runtime cap bounds the batch: the hook gets a signal that fires at the
// deadline, and a batch still running then (or answered late) aborts the run.
const controller = new AbortController();
const onAbort = () => controller.abort(opts.abortSignal?.reason);
if (opts.abortSignal?.aborted) onAbort();
else opts.abortSignal?.addEventListener('abort', onAbort, { once: true });
let timer: ReturnType<typeof setTimeout> | undefined;
const expired = deadline === undefined || deadline - Date.now() > MAX_TIMER_MS ? null : new Promise<never>((_, reject) => {
timer = setTimeout(() => {
controller.abort(new Error(SKILLOPT_RUNTIME_EXCEEDED));
reject(new Error(SKILLOPT_RUNTIME_EXCEEDED));
}, Math.max(0, deadline - Date.now()));
});
const call = Promise.resolve().then(() => withoutBudgetTracker(() => hook({
phase: opts.phase ?? 'sel_gate',
skillText: opts.candidateSkillText,
skillSha256: createHash('sha256').update(opts.candidateSkillText).digest('hex'),
tasks: opts.selSet,
runsPerTask,
epoch: opts.epoch ?? 0,
step: opts.step ?? 0,
}, controller.signal)));
let rows: ScoredRollout[][];
try {
rows = await (expired ? Promise.race([call, expired]) : call);
} finally {
clearTimeout(timer);
opts.abortSignal?.removeEventListener('abort', onAbort);
}
if (deadline !== undefined && Date.now() > deadline) throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
const needsTrajectory = opts.phase === 'train_forward';
const badRun = (r: ScoredRollout | undefined) => typeof r?.score !== 'number' || !(r.score >= 0 && r.score <= 1)
|| (needsTrajectory && (typeof r.trajectory !== 'object' || r.trajectory === null));
if (!Array.isArray(rows) || rows.length !== opts.selSet.length || rows.some((r) => !Array.isArray(r) || r.length !== runsPerTask || r.some(badRun))) {
throw new Error(`${SKILLOPT_HOOK_INFRA_ERROR}: rollout hook returned a malformed batch for ${opts.phase ?? 'sel_gate'} (want ${opts.selSet.length} rows of ${runsPerTask} runs, each a score in [0,1]${needsTrajectory ? ' with a trajectory' : ''})`);
}
return opts.selSet.map((task, i) => {
const runs = rows[i]!.map((r) => r.score);
return { task_id: task.task_id, median: median(runs), runs, rollouts: rows[i]! };
});
}
function isRuntimeExceeded(e: unknown): boolean {
return e instanceof Error && e.message === SKILLOPT_RUNTIME_EXCEEDED;
}
/**
* #4741 — every task in a gate failed (dead target provider, e.g. a logged-out
* claude-cli child) / every rollout's judge errored (dead judge provider).
* Thrown instead of scoring the gate 0.000, so the orchestrator's catch-all
* ends the run `errored` with the first provider error as abort_detail.
*/
export const SKILLOPT_ALL_ROLLOUTS_FAILED = 'skillopt_all_rollouts_failed';
export const SKILLOPT_ALL_JUDGE_ERRORS = 'skillopt_all_judge_errors';
const errMsg = (e: unknown): string => (e instanceof Error ? e.message : String(e));
/**
* Run the validation gate against a candidate skill. Returns GateResult
* with accept/reject + per-task medians for the audit JSONL.
*/
export async function runValidationGate(opts: ValidateGateOpts): Promise<GateResult> {
const rolloutFn = opts.rolloutFn ?? runRollout;
const scoreFn = opts.scoreFn ?? scoreTrajectory;
const concurrency = opts.concurrency ?? 4;
// Forward pass uses 1; sel/baseline use VALIDATION_RUNS_PER_TASK (3). Floor
// at 1 (a runsPerTask of 0 would silently produce an empty median).
const runsPerTask = Math.max(1, opts.runsPerTask ?? VALIDATION_RUNS_PER_TASK);
// Per sel-task, run N rollouts + N judges, take median.
const settled: SettledItem<PerTask>[] = opts.rolloutHook
? (await hookBatch(opts, opts.rolloutHook, runsPerTask)).map((value, idx) => ({ ok: true, value, idx }))
: await runWithLimit({
items: opts.selSet,
limit: concurrency,
fn: async (task: BenchmarkTask): Promise<PerTask> => {
const runs: number[] = [];
const rollouts: ScoredRollout[] = [];
for (let i = 0; i < runsPerTask; i++) {
// #4119: in-rollout-loop deadline — see ValidateGateOpts.deadlineMs.
if (opts.deadlineMs !== undefined && Date.now() > opts.deadlineMs) {
throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
}
const rolloutOpts: RolloutOpts = {
operationContext: opts.operationContext,
engine: opts.engine,
skillText: opts.candidateSkillText,
task,
targetModel: opts.targetModel,
abortSignal: opts.abortSignal,
};
const trajectory = await rolloutFn(rolloutOpts);
const scored: ScoredRollout = await scoreFn(trajectory, task.judge, {
judgeModel: opts.judgeModel,
});
runs.push(scored.score);
rollouts.push(scored);
}
return { task_id: task.task_id, median: median(runs), runs, rollouts };
},
signal: opts.abortSignal,
});
// MUST-ABORT errors (budget exhaustion / no-pricing / spend-policy refusals) are NOT scoring noise —
// swallowing them as score=0 turns a pricing/cap crash into a fake "0/N" run
// (the bug the SkillOpt eval surfaced: a Haiku run with --max-cost hit
// no_pricing on every rollout and the whole gate reported a vacuous 0). Surface
// them loudly so the caller aborts instead of recording a hollow measurement.
// #4119: a deadline breach is not scoring noise either — it must surface as
// the orchestrator's runtime-exceeded abort, never a fake 0-score task.
const aborter = settled.find(
(s) => s && !s.ok && (isSkilloptMustAbort(s.error) || isRuntimeExceeded(s.error)),
);
if (aborter && !aborter.ok) throw aborter.error;
// #4741: a 100%-failed gate is not a measurement. Any plain (non-tagged)
// provider error — a logged-out claude-cli child, a dead endpoint — lands
// here as an ordinary !ok item; scoring all of them 0 let a dead provider
// finish the whole run as a plausible `no_improvement` / 0.000. Partial
// failure below keeps the pessimistic per-task 0 (scoring noise).
const failed = settled.filter((s) => s && !s.ok);
if (settled.length > 0 && failed.length === settled.length) {
const first = failed[0]!.ok ? undefined : failed[0]!.error;
throw new Error(
`${SKILLOPT_ALL_ROLLOUTS_FAILED}: every task in this gate failed (${failed.length}/${settled.length}); first error: ${errMsg(first)}`,
{ cause: first },
);
}
// SettledItem<TOut>[] — extract successful results; treat (non-abort) errors as
// score=0 (pessimistic fallback consistent with the judge fail-open posture).
// Errored tasks contribute no scoredRollouts (caller's reflect sees fewer
// trajectories rather than fabricated zero-score entries).
const perTaskMedians = settled.map((s, idx) => {
if (s && s.ok) return { task_id: s.value.task_id, median: s.value.median, runs: s.value.runs };
return { task_id: opts.selSet[idx]!.task_id, median: 0, runs: [], error: errMsg(s.error) };
});
const scoredRollouts: ScoredRollout[] = settled.flatMap((s) => (s && s.ok) ? s.value.rollouts : []);
// #4741: same rule for the judge side. scoreLlm fails open per rollout
// (score 0 + judge_error) so one flaky judge call is noise; EVERY rollout
// carrying a judge_error is a dead judge provider, not a 0.000 skill.
// Rule/qrels judges never set judge_error, so only the LLM judge trips it.
if (scoredRollouts.length > 0 && scoredRollouts.every((r) => r.judge_error)) {
throw new Error(
`${SKILLOPT_ALL_JUDGE_ERRORS}: every rollout's judge failed (${scoredRollouts.length}/${scoredRollouts.length}); first error: ${scoredRollouts[0]!.judge_error}`,
);
}
const selScore = perTaskMedians.length === 0
? 0
: perTaskMedians.reduce((acc, r) => acc + r.median, 0) / perTaskMedians.length;
// D12 accept rule: strict > best + epsilon. Ties/sub-epsilon-gains are
// rejected (paper-faithful — protects against noise-as-improvement).
const threshold = opts.bestScore + VALIDATION_EPSILON;
const accepted = selScore > threshold;
let reason: GateResult['reason'];
if (!accepted) {
if (selScore <= opts.bestScore) reason = 'below_baseline';
else reason = 'no_margin';
}
return { accepted, perTaskMedians, selScore, scoredRollouts, ...(reason ? { reason } : {}) };
}
export interface ScoreOnTasksOpts {
operationContext?: OperationContext;
engine: BrainEngine;
skillText: string;
tasks: BenchmarkTask[];
targetModel: string;
judgeModel?: string;
/** Median-of-N runs per task. Defaults to `VALIDATION_RUNS_PER_TASK` (3). */
runsPerTask?: number;
/** #4119 — in-rollout-loop deadline (epoch ms); see ValidateGateOpts. */
deadlineMs?: number;
abortSignal?: AbortSignal;
rolloutFn?: typeof runRollout;
scoreFn?: typeof scoreTrajectory;
/** B1.3: score through a harness hook (one call for the batch). */
rolloutHook?: RolloutBatchHook;
/** Hook request label. Default 'test'. */
phase?: RolloutPhase;
}
/** Per-task medians plus their mean; a task whose rollout errored has median 0, no runs and its error. */
export interface PerTaskScore {
score: number;
perTask: GateResult['perTaskMedians'];
}
/** scoreSkillOnTasks with the per-task rows (paired comparisons across arms need them). */
export async function scoreSkillPerTask(opts: ScoreOnTasksOpts): Promise<PerTaskScore> {
const gate = await runValidationGate({
operationContext: opts.operationContext,
engine: opts.engine,
candidateSkillText: opts.skillText,
selSet: opts.tasks,
bestScore: -1,
targetModel: opts.targetModel,
phase: opts.phase ?? 'test',
...(opts.judgeModel !== undefined ? { judgeModel: opts.judgeModel } : {}),
...(opts.runsPerTask !== undefined ? { runsPerTask: opts.runsPerTask } : {}),
...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}),
...(opts.abortSignal ? { abortSignal: opts.abortSignal } : {}),
...(opts.rolloutFn ? { rolloutFn: opts.rolloutFn } : {}),
...(opts.scoreFn ? { scoreFn: opts.scoreFn } : {}),
...(opts.rolloutHook ? { rolloutHook: opts.rolloutHook } : {}),
});
return { score: gate.selScore, perTask: gate.perTaskMedians };
}
/**
* Score a skill on an arbitrary task set and return the mean-of-per-task-medians
* (`selScore`). The single primitive for "how good is this skill on these tasks?"
* — used by the orchestrator's baseline + final-test eval, the held-out gate, and
* the Track B eval harnesses, so they can't drift on scoring semantics. Thin
* wrapper over `scoreSkillPerTask` (any score "accepts"; we only read the mean).
*/
export async function scoreSkillOnTasks(opts: ScoreOnTasksOpts): Promise<number> {
return (await scoreSkillPerTask(opts)).score;
}
/** Pure median for an array of numbers. Returns 0 for empty array. */
export function median(values: readonly number[]): number {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 1
? sorted[mid]!
: (sorted[mid - 1]! + sorted[mid]!) / 2;
}