Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* SkillOpt main loop — runSkillOpt.
*
* ┌─ LR cosine-decay curve (default base=4, totalSteps=10) ────────────────┐
* │ │
* │ 4 ●─● │
* │ \ │
* │ 3 ●─●─● │
* │ \ │
* │ 2 ●─●─● │
* │ \ │
* │ 1 ●─●─● │
* │ ─┼──┼──┼──┼──┼──┼──┼──┼──┼──┼─ │
* │ 1 2 3 4 5 6 7 8 9 10 │
* │ │
* │ Peaks early (most aggressive when skill is least refined), tapers │
* │ as the skill converges. Schedule lives in lr-schedule.ts. │
* └───────────────────────────────────────────────────────────────────────┘
*
* ┌─ Run state machine ────────────────────────────────────────────────────┐
* │ │
* │ start ──► models_plan ──► preflight ──► lock_acquire ──► resume │
* │ (banner; strict abort; reservation check: all before any spend) │
* │ (resume: cursor next_epoch/next_step, spec check) │
* │ │ │
* │ ▼ │
* │ ┌──── epoch_start ◄────┐ │
* │ │ │ │ │
* │ │ ▼ │ │
* │ │ forward_pass │ │
* │ │ (rollouts batch) │ │
* │ │ │ │ │
* │ │ ▼ │ │
* │ │ backward_pass │ │
* │ │ (reflect ×2 D7; each │ │
* │ │ call: edits | empty │ │
* │ │ | exactly 1 error) │ │
* │ │ │ │ │
* │ │ ▼ │ │
* │ │ rank_and_clip │ │
* │ │ (LR budget) │ │
* │ │ │ │ │
* │ │ ▼ │ │
* │ │ validation_gate ─────┘ │
* │ │ (D12 median+ε) │
* │ │ │ │
* │ │ accept ──┤── reject │
* │ │ │ │ │ │
* │ │ ▼ │ ▼ │
* │ │ commit │ rejected_buffer │
* │ │ (D8) │ │
* │ │ │ │ │
* │ │ └─►◄──┘ │
* │ │ │ │
* │ │ ▼ │
* │ │ step_end: advance cursor, │
* │ │ save cost + tally ──► 2 fully │
* │ │ │ unusable steps in a │
* │ │ │ row ──► EARLY STOP ─┐ │
* │ │ ▼ │ │
* │ │ epoch_end ──► slow_update (D6) │ │
* │ │ │ │ │
* │ └───────────┘ │ │
* │ │ all epochs done │ │
* │ ▼ │ │
* │ final_test ──► run_end ◄───────┘ │
* │ │
* │ run_end outcome (stop_reason completed | early_stop_unusable_output │
* │ | aborted): │
* │ caught abort/error ──────────────► aborted | errored (checkpoint │
* │ kept, resume_command) │
* │ any step accepted ───────────────► accepted │
* │ optimizer called, 0 usable replies ► errored │
* │ (optimizer_output_unusable, │
* │ checkpoint kept, remediation) │
* │ otherwise ───────────────────────► no_improvement │
* └────────────────────────────────────────────────────────────────────────┘
*
* ┌─ Validation gate decision tree (D12) ──────────────────────────────────┐
* │ │
* │ candidate edits applied │
* │ │ │
* │ ▼ │
* │ for each sel-task in parallel (cap=4 per D4): │
* │ median(score_run_1, score_run_2, score_run_3) ← VALIDATION_RUNS=3 │
* │ │ │
* │ ▼ │
* │ mean(per_task_medians) = sel_score │
* │ │ │
* │ ▼ │
* │ sel_score > best_score + 0.05 ? ← VALIDATION_EPSILON │
* │ │ yes │ no │
* │ ▼ ▼ │
* │ ACCEPT REJECT → rejected-buffer │
* │ commit via D8 │
* └────────────────────────────────────────────────────────────────────────┘
*/
import { assertLegacySkillWriter } from '../skillpack/writer-guard.ts';
import { createHash, randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import { BudgetExhausted, BudgetTracker, loadPricingOverrides, type NoPricingGuidance } from '../budget/budget-tracker.ts';
import type { PricingOverrides } from '../budget/reservation-cost.ts';
import { buildModelsUsed } from '../budget/models-used.ts';
import { withBudgetTracker } from '../ai/gateway.ts';
import { errorFor } from '../errors.ts';
import { applyEditBatch, getWorkingTreeStatusForFile, splitFrontmatter } from './apply-edits.ts';
import { logEvent, sha8 } from './audit.ts';
import { loadRunTasks, parseSplit } from './benchmark.ts';
import { getBundledSkillContext, shouldMutateSkillFile, assertBundledMutationHeldOut } from './bundled-skill-gate.ts';
import {
advanceCursor,
assertResumeCompatible,
bankAccounting,
deleteCheckpoint,
emptyTally,
loadCheckpoint,
priorSegments,
resumeCursor,
rewindUnusableSteps,
saveCheckpoint,
type ReflectTally,
type RunSpec,
} from './checkpoint.ts';
import { loadHeldOut, runHeldOutGate } from './held-out.ts';
import { withSkilloptLock } from './lock.ts';
import { resolveLrSchedule } from './lr-schedule.ts';
import { SKILLOPT_HOOK_BUDGET_EXHAUSTED, SKILLOPT_HOOK_INFRA_ERROR, SKILLOPT_RUNTIME_EXCEEDED } from './must-abort.ts';
import {
buildModelsPlan,
formatModelsBanner,
modelsStrictError,
resolveModelsStrict,
strictVerdict,
unknownProvenanceModels,
type ModelsPlanEntry,
type StrictVerdict,
} from './models-plan.ts';
import { resolveReflectMaxTokens, type ReflectCap } from './output-cap.ts';
import { preflight, formatPreflightReport } from './preflight.ts';
import { isRejected, loadRejectedBuffer, makeRejectedEntry, saveRejectedBuffer } from './rejected-buffer.ts';
import { runReflect, runOneShotRewrite, describeJudges, type SkillBodyTruncation } from './reflect.ts';
import { buildRemediation, errorCode } from './remediation.ts';
import {
buildResumeCommand,
recordOptimizerStep,
resolveRunOutcome,
shouldEarlyStop,
unusableClasses,
EARLY_STOP_UNUSABLE_STEPS,
} from './run-outcome.ts';
import {
acceptCandidate,
proposedPath as proposedFilePath,
revertAllPending,
skillPath,
writeProposed,
} from './version-store.ts';
import { runValidationGate } from './validate-gate.ts';
import { ROLLOUT_SUCCESS_THRESHOLD, VALIDATION_RUNS_PER_TASK } from './types.ts';
import type { SkillOptOpts, EditOp, RunReceipt, BenchmarkTask, RolloutPhase } from './types.ts';
export interface RunSkillOptResult {
outcome: 'accepted' | 'no_improvement' | 'aborted' | 'errored';
receipt: RunReceipt;
/** Final SKILL.md text (committed or proposed). */
finalText: string;
/** True when SKILL.md was actually rewritten; false for --no-mutate / bundled-skill paths. */
mutatedSkillFile: boolean;
/** When mutate was skipped, path where the proposed.md was written. */
proposedPath?: string;
sharedOptimization?: { proposal_id: string; publication?: Record<string, unknown> };
}
export async function runSkillOpt(opts: SkillOptOpts): Promise<RunSkillOptResult> {
if (opts.sharedSkill) {
const { optimizeSharedSkill } = await import('../shared-skills/optimizer.ts');
return optimizeSharedSkill(opts, runSkillOpt);
}
const { engine, skillName, skillsDir } = opts;
// ── Pre-flight gates (fail-loud BEFORE any LLM spend) ───────────────────
const skillFile = skillPath(skillsDir, skillName);
await assertLegacySkillWriter(engine, skillFile);
if (!fs.existsSync(skillFile)) {
throw errorFor({
class: 'NoSkill',
code: 'no_skill_md',
message: `Cannot find SKILL.md for '${skillName}' at ${skillFile}.`,
hint: `Create the skill first via 'gbrain skillify scaffold ${skillName}'.`,
});
}
// Working-tree gate.
if (!opts.force) {
const status = getWorkingTreeStatusForFile(skillFile);
if (status === 'dirty' && !isOwnResumeEdit(skillsDir, skillName, skillFile, opts.resumeRunId)) {
throw errorFor({
class: 'DirtyTree',
code: 'dirty_tree',
message: `${skillFile} has uncommitted changes.`,
hint: `Commit or stash changes before running skillopt, or pass --force to override.`,
});
}
}
// Bundled-skill gate (D16).
const bundledCtx = getBundledSkillContext(skillsDir, skillName);
const mutateDecision = shouldMutateSkillFile(bundledCtx, {
noMutate: opts.noMutate,
allowMutateBundled: opts.allowMutateBundled,
});
// Held-out set (F11). Loaded ONCE here so the bundled-mutation ENFORCE gate
// and the per-acceptance held-out gate share the same parsed tasks. Empty
// when --held-out is not passed.
const heldOutTasks: BenchmarkTask[] = opts.heldOutPath ? loadHeldOut(opts.heldOutPath) : [];
// D16 ENFORCE (core mutation policy — fires for EVERY caller: CLI, batch,
// fleet, cycle, background job, and the run_skillopt MCP op, since they all
// funnel through runSkillOpt). Fail-loud BEFORE the lock + any LLM spend.
assertBundledMutationHeldOut({
isBundled: bundledCtx.isBundled,
willMutate: mutateDecision.mutate,
heldOutCount: heldOutTasks.length,
skillName,
});
// Load + validate the tasks (D17 floor enforcement, D15 sentinel check, B1.2 explicit lists).
const { bench, split } = loadRunTasks(opts);
// Held-out must be INDEPENDENT of the benchmark. If a held-out file shares
// task_ids with the benchmark, an overfit candidate passes the held-out gate
// vacuously — silently voiding the gate's entire benchmark-gaming defense
// (the load-bearing safety claim for bundled mutation). Reject on overlap.
if (heldOutTasks.length > 0) {
const benchIds = new Set(bench.tasks.map((t) => t.task_id));
const overlap = heldOutTasks.filter((t) => benchIds.has(t.task_id)).map((t) => t.task_id);
if (overlap.length > 0) {
throw errorFor({
class: 'HeldOutOverlap',
code: 'held_out_overlaps_benchmark',
message: `Held-out set shares ${overlap.length} task_id(s) with the benchmark (e.g. ${overlap.slice(0, 3).join(', ')}). The held-out set must be independent.`,
hint: `Use disjoint task_ids in the held-out file — an overlapping held-out can't catch benchmark overfitting.`,
});
}
}
// ── Models plan + strict mode (#5585) — banner and verdict before any spend
const reflectCap = await resolveReflectMaxTokens(engine, opts.optimizerModel, opts.reflectMaxTokens);
const modelsPlan = await buildModelsPlan(
engine,
opts.models ?? unknownProvenanceModels(opts, 'caller-supplied'),
[...bench.tasks, ...heldOutTasks],
);
const banner = formatModelsBanner(modelsPlan, { skill: skillName, baseline: opts.modelsBannerBaseline, reflectCap });
if (banner) process.stderr.write(banner);
const strict = strictVerdict(modelsPlan, await resolveModelsStrict(engine, opts.modelsStrict));
if (strict.enabled && !strict.ok && !opts.dryRun) throw modelsStrictError(strict);
const pricingOverrides = await loadPricingOverrides(engine);
// ── Cost preflight (D3) ─────────────────────────────────────────────────
const preflightResult = preflight({
epochs: opts.epochs,
batchSize: opts.batchSize,
trainSize: split.train.length,
selSize: split.sel.length,
testSize: split.test.length,
optimizerModel: opts.optimizerModel,
targetModel: opts.targetModel,
judgeModel: opts.judgeModel,
maxCostUsd: opts.maxCostUsd,
maxCostSource: opts.maxCostSource ?? 'user',
heldOutSize: heldOutTasks.length,
interactive: process.stderr.isTTY === true,
reflectMaxTokens: reflectCap.maxTokens,
judgeModels: modelsPlan.filter((e) => e.touchpoint === 'judge' && e.active).map((e) => e.model),
optimizerMode: opts.optimizerMode,
rolloutsExternal: opts.rolloutHook !== undefined,
...(pricingOverrides ? { pricingOverrides } : {}),
});
if (opts.json !== true) {
process.stderr.write(formatPreflightReport(preflightResult.estimate, {
epochs: opts.epochs, batchSize: opts.batchSize,
trainSize: split.train.length, selSize: split.sel.length, testSize: split.test.length,
optimizerModel: opts.optimizerModel, targetModel: opts.targetModel, judgeModel: opts.judgeModel,
maxCostUsd: opts.maxCostUsd,
maxCostSource: opts.maxCostSource ?? 'user',
optimizerMode: opts.optimizerMode,
}) + '\n');
}
if (preflightResult.abort_code === 'no_pricing') {
throw errorFor({
class: 'BudgetExhausted',
code: 'no_pricing',
message: `${preflightResult.abort_reason} No model call was made; or pass --max-usd off to run uncapped.`,
});
}
if (!preflightResult.proceed) {
throw errorFor({
class: 'CostCapExceeded',
code: 'cost_cap_exceeded',
message: preflightResult.abort_reason ?? 'preflight refused to proceed',
hint: preflightResult.abort_code === 'reservation_exceeds_cap'
? `No model call was made.`
: `Raise --max-cost-usd or reduce knobs.`,
});
}
// --dry-run short-circuits BEFORE the lock + LLM calls.
if (opts.dryRun) {
const receipt: RunReceipt = {
run_id: opts.resumeRunId ?? randomUUID(),
skill: skillName,
skill_sha8: sha8(fs.readFileSync(skillFile, 'utf8')),
benchmark_sha8: bench.benchmark_sha8,
optimizer_model: opts.optimizerModel,
target_model: opts.targetModel,
judge_model: opts.judgeModel,
epochs: opts.epochs,
batch_size: opts.batchSize,
lr: opts.lr,
lr_schedule: opts.lrSchedule,
max_cost_usd: opts.maxCostUsd,
started_at: new Date().toISOString(),
outcome: 'aborted',
reflect_max_tokens: reflectCap.maxTokens,
reflect_max_tokens_source: reflectCap.source,
models_plan: modelsPlan,
models_strict: strict,
optimizer_mode: opts.optimizerMode,
cost_scope: opts.rolloutHook ? 'optimizer_only' : 'all',
};
return { outcome: 'aborted', receipt, finalText: fs.readFileSync(skillFile, 'utf8'), mutatedSkillFile: false };
}
// ── Acquire per-skill lock (D14) ────────────────────────────────────────
return await withSkilloptLock(engine, skillName, async () => {
return runOptimizationLoop(opts, bench, split, bundledCtx, mutateDecision, heldOutTasks, {
reflectCap, modelsPlan, strict, pricingOverrides,
});
});
}
/**
* Classifies a run-loop error into the abort taxonomy the receipt + audit
* trail surface. Pulled out of runOptimizationLoop's catch block so it's
* unit-testable without driving a full orchestrator run (engine, benchmark,
* lock, LLM calls) — the string-sniff bug this replaces went unexercised
* for exactly that reason: nothing could cheaply throw a real error deep
* inside the loop to catch it.
*
* `err instanceof BudgetExhausted` is the real classification signal — the
* string sniff it replaces (`msg.includes('BudgetExhausted') ||
* msg.includes('budget_exhausted')`) could never match, because every
* BudgetExhausted thrown by src/core/budget/budget-tracker.ts carries a
* human-readable message ("projected cost $X exceeds the configured cap
* $Y", "N exceeded --max-runtime Ns", ...) containing neither literal
* substring. Every cost-cap / no-pricing / (unreachable from this tracker
* today, since it's never constructed with maxRuntimeMs) runtime abort fell
* through to the generic catch-all instead, misreporting outcome='errored'
* / abortReason='error' on the receipt and audit trail.
*/
/**
* A resume may start on a SKILL.md this run itself rewrote: the file is dirty,
* but it is exactly the checkpoint's best text. Any other edit still trips the
* dirty-tree gate, so a hand edit between segments is never overwritten.
*/
function isOwnResumeEdit(skillsDir: string, skillName: string, skillFile: string, resumeRunId: string | undefined): boolean {
if (!resumeRunId) return false;
const cp = loadCheckpoint(skillsDir, skillName, resumeRunId);
if (!cp) return false;
try {
return sha8(fs.readFileSync(skillFile, 'utf8')) === sha8(cp.best_skill_text);
} catch {
return false;
}
}
export function classifyAbortError(
err: unknown,
opts: { maxRuntimeMin: number },
): {
outcome: 'aborted' | 'errored';
abortReason: 'budget_exhausted' | 'runtime_exceeded' | 'sigint' | 'error';
abortDetail: string;
/** A no_pricing abort's lookup-and-register guidance. */
pricing?: NoPricingGuidance;
} {
const msg = err instanceof Error ? err.message : String(err);
if (err instanceof BudgetExhausted) {
return {
outcome: 'aborted',
abortReason: err.reason === 'runtime' ? 'runtime_exceeded' : 'budget_exhausted',
abortDetail: msg,
...(err.pricing ? { pricing: err.pricing } : {}),
};
} else if (msg.includes(SKILLOPT_HOOK_BUDGET_EXHAUSTED)) {
return { outcome: 'aborted', abortReason: 'budget_exhausted', abortDetail: msg };
} else if (msg.includes(SKILLOPT_HOOK_INFRA_ERROR)) {
return { outcome: 'aborted', abortReason: 'error', abortDetail: msg };
} else if (msg.includes(SKILLOPT_RUNTIME_EXCEEDED)) {
return {
outcome: 'aborted',
abortReason: 'runtime_exceeded',
abortDetail: `exceeded --max-runtime-min ${opts.maxRuntimeMin}`,
};
} else if (msg.includes('SIGINT')) {
return { outcome: 'aborted', abortReason: 'sigint', abortDetail: msg };
} else {
// #3516: truthful catch-all. Pre-fix this logged reason:'sigint' for
// EVERY unrecognized error (provider failures, no_pricing hard-fails),
// which made the audit trail lie about why the run died.
return { outcome: 'errored', abortReason: 'error', abortDetail: msg };
}
}
/** Appended to a cost-cap abort: optimizer calls reserve their whole output cap up front. */
const BUDGET_ADMISSION_HINT = ' (raise --max-cost-usd; each optimizer call reserves its full output cap, so lowering --reflect-max-tokens / skillopt.reflect_max_tokens also helps)';
async function runOptimizationLoop(
opts: SkillOptOpts,
bench: ReturnType<typeof loadRunTasks>['bench'],
split: ReturnType<typeof loadRunTasks>['split'],
bundledCtx: ReturnType<typeof getBundledSkillContext>,
mutateDecision: ReturnType<typeof shouldMutateSkillFile>,
heldOutTasks: BenchmarkTask[],
resolved: {
reflectCap: ReflectCap;
modelsPlan: ModelsPlanEntry[];
strict: StrictVerdict;
pricingOverrides: PricingOverrides | undefined;
},
): Promise<RunSkillOptResult> {
const { skillName, skillsDir } = opts;
const { reflectCap, modelsPlan } = resolved;
const skillFile = skillPath(skillsDir, skillName);
// Plain-English success criteria fed to the optimizer so it targets WHAT the
// scorer rewards instead of guessing from a bare pass/fail score. Without
// this, rule-judged benchmarks (e.g. "must include a Confidence: line") are
// near-unoptimizable. Built from the train split only, with per-task
// arguments (answer keys) redacted (B1.4, defect 3): selection and test
// judges never reach the proposer. A caller's criteriaText replaces it.
const benchmarkCriteria = opts.criteriaText ?? describeJudges(split.train);
// Crash-recovery sweep (D8): revert any pending rows from a prior crashed run.
revertAllPending(skillsDir, skillName);
// Load baseline skill text.
const baselineText = fs.readFileSync(skillFile, 'utf8');
const baselineSha8 = sha8(baselineText);
const runSpec: RunSpec = {
skills_dir: skillsDir,
benchmark_path: opts.benchmarkPath ?? bench.source_path,
benchmark_sha8: bench.benchmark_sha8,
held_out_path: opts.heldOutPath ?? null,
held_out_sha8: opts.heldOutPath ? sha8(fs.readFileSync(opts.heldOutPath, 'utf8')) : null,
split: opts.tasks ? [split.train.length, split.sel.length, split.test.length] : opts.split ?? [4, 1, 5],
mutate: mutateDecision.mutate,
no_mutate: opts.noMutate,
allow_mutate_bundled: opts.allowMutateBundled,
bootstrap_reviewed: opts.bootstrapReviewed,
optimizer_model: opts.optimizerModel,
target_model: opts.targetModel,
judge_model: opts.judgeModel,
epochs: opts.epochs,
batch_size: opts.batchSize,
lr: opts.lr,
lr_schedule: opts.lrSchedule,
reflect_max_tokens: reflectCap.maxTokens,
optimizer_mode: opts.optimizerMode,
};
// Resume or init checkpoint. A resume must match what the checkpoint measured.
const runId = opts.resumeRunId ?? randomUUID();
let checkpoint = opts.resumeRunId ? loadCheckpoint(skillsDir, skillName, opts.resumeRunId) : null;
const prior = priorSegments(checkpoint);
if (checkpoint) {
assertResumeCompatible(checkpoint, runSpec);
checkpoint.run_spec = runSpec;
} else {
checkpoint = {
schema: 1,
run_id: runId,
skill: skillName,
skill_sha8: baselineSha8,
benchmark_sha8: bench.benchmark_sha8,
optimizer_model: opts.optimizerModel,
target_model: opts.targetModel,
judge_model: opts.judgeModel,
epochs: opts.epochs,
batch_size: opts.batchSize,
lr: opts.lr,
lr_schedule: opts.lrSchedule,
best_sel_score: 0,
best_skill_text: baselineText,
last_completed_epoch: 0,
last_completed_step: 0,
next_epoch: 1,
next_step: 1,
run_spec: runSpec,
tally: emptyTally(),
cumulative_cost_usd: 0,
models_used: [],
models_used_scope: 'full_run',
started_at: new Date().toISOString(),
last_updated_at: new Date().toISOString(),
};
saveCheckpoint(skillsDir, skillName, checkpoint);
}
// Initial audit row.
logEvent({
kind: 'run_start',
run_id: runId,
skill: skillName,
skill_sha8: baselineSha8,
benchmark_sha8: bench.benchmark_sha8,
target_model: opts.targetModel,
optimizer_model: opts.optimizerModel,
judge_model: opts.judgeModel,
epochs: opts.epochs,
batch_size: opts.batchSize,
lr: opts.lr,
lr_schedule: opts.lrSchedule,
max_cost_usd: opts.maxCostUsd,
reflect_mode: opts.reflectMode ?? 'both',
validation_gate_disabled: opts.disableValidationGate === true,
optimizer_mode: opts.optimizerMode,
held_out_size: heldOutTasks.length,
models_plan: modelsPlan,
} as never);
// Budget tracker for the whole run. BudgetExhausted propagates as
// MUST_ABORT through every gateway.chat call.
//
// #3516: maxCostUsd === 0 means "no cap" (--no-max-cost / --max-cost-usd 0).
// Passing undefined switches the tracker's pricing-miss behavior from a
// hard BudgetExhausted(no_pricing) abort to the legacy warn-once path, so
// unpriced model ids (openrouter:*, litellm:*) can run at the user's own risk.
const tracker = new BudgetTracker({
...(opts.maxCostUsd > 0 ? { maxCostUsd: opts.maxCostUsd, capSource: opts.maxCostSource ?? 'user' } : {}),
...(resolved.pricingOverrides ? { pricingOverrides: resolved.pricingOverrides } : {}),
label: `skillopt:${skillName}`,
});
const scheduleFn = resolveLrSchedule(opts.lrSchedule);
const totalSteps = opts.epochs * Math.max(1, Math.floor(split.train.length / opts.batchSize));
// Run the loop inside withBudgetTracker so every nested gateway call composes.
let finalText = checkpoint.champion ? baselineText : checkpoint.best_skill_text;
let totalStepsRun = 0;
// A legacy checkpoint (no tally) that already moved past the baseline text
// accepted a candidate in an earlier segment.
const tally = checkpoint.tally
?? { ...emptyTally(), accepted_steps: sha8(checkpoint.best_skill_text) !== checkpoint.skill_sha8 ? 1 : 0 };
// A resume is a fresh attempt (usually after a fix): the trailing fully
// unusable steps are re-run instead of skipped, and the early-stop streak
// restarts instead of stopping the resumed run after one step.
// The rewound cursor is written to the checkpoint together with the reset
// streak, so a resume that aborts before its first step still retries them.
const stepsPerEpoch = Math.max(1, Math.floor(split.train.length / opts.batchSize));
if (opts.resumeRunId) rewindUnusableSteps(checkpoint, tally.unusable_streak, stepsPerEpoch, opts.optimizerMode === 'one-shot-rewrite');
tally.unusable_streak = 0;
checkpoint.tally = tally;
// Tally as of the start of the in-flight step. A handled abort re-runs that
// step on --resume, so the checkpoint keeps this copy (the receipt keeps the
// live tally, which includes the calls this segment actually made).
let stepStartTally: ReflectTally | undefined;
// First error of the most recent fully unusable step (the receipt's error
// list is deduped and capped, so it cannot say what caused an early stop).
let lastUnusableError: string | undefined;
let earlyStopped = false;
let skillBodyTruncated: SkillBodyTruncation | undefined;
let caught: ReturnType<typeof classifyAbortError> | undefined;
// Hoisted for the receipt (computed inside the budget-tracker closure).
let baselineSelScore = 0;
let testScore: number | undefined;
let baselineTestScore: number | undefined;
// maxRuntimeMin enforcement (#4119): wall-clock deadline checked between
// steps AND inside every gate's rollout loop (deadlineMs is threaded into
// runValidationGate / scoreSkillOnTasks / runHeldOutGate, checked before
// each individual rollout) — a long batch can no longer blow past
// --max-runtime by a whole gate's worth of rollouts. A breach throws
// `skillopt_runtime_exceeded`, caught below → outcome 'aborted'
// (no partial commit beyond whatever already passed every gate).
const deadline = Date.now() + opts.maxRuntimeMin * 60_000;
// Scoring sites (B1.3 / B1.6): every rollout goes through runValidationGate
// (with the caller's hook when set) and is counted by phase.
const hook = opts.rolloutHook;
const rollouts: Record<RolloutPhase, number> = { baseline_sel: 0, train_forward: 0, sel_gate: 0, held_out: 0, test: 0 };
const selCandidates = new Set<string>();
const score = async (phase: RolloutPhase, skillText: string, tasks: BenchmarkTask[], o: { bestScore?: number; runsPerTask?: number; epoch?: number; step?: number } = {}) => {
const runsPerTask = o.runsPerTask ?? VALIDATION_RUNS_PER_TASK;
const result = await runValidationGate({
engine: opts.engine,
operationContext: opts.operationContext,
candidateSkillText: skillText,
selSet: tasks,
bestScore: o.bestScore ?? -1,
targetModel: opts.targetModel,
judgeModel: opts.judgeModel,
runsPerTask,
deadlineMs: deadline,
phase,
epoch: o.epoch ?? 0,
step: o.step ?? 0,
...(hook ? { rolloutHook: hook } : {}),
});
rollouts[phase] += tasks.length * runsPerTask;
if (phase === 'sel_gate' && skillText !== baselineText) selCandidates.add(createHash('sha256').update(skillText).digest('hex'));
return result;
};
// D2: with a held-out set, nothing is written until the single held-out
// check on the final champion passes; without one, each accepted step is
// written as before (so an abort keeps the best skill so far on disk).
const deferWrites = heldOutTasks.length > 0;
const writeCandidate = (text: string, selScore: number, epoch: number, step: number, edits: EditOp[]) => {
if (mutateDecision.mutate) {
acceptCandidate({ skillsDir, skillName, runId, epoch, step, edits, candidateText: text, selScore, delta: selScore - baselineSelScore });
} else {
writeProposed(skillsDir, skillName, text);
}
finalText = text;
};
const promote = (text: string, selScore: number, epoch: number, step: number, edits: EditOp[]) => {
if (deferWrites) {
const prev = checkpoint!.champion;
checkpoint!.champion = { epoch, step, steps: (prev?.steps ?? 0) + 1, edits: [...(prev?.edits ?? []), ...edits] };
} else {
writeCandidate(text, selScore, epoch, step, edits);
tally.accepted_steps += 1;
}
checkpoint!.best_sel_score = selScore;
checkpoint!.best_skill_text = text;
};
let heldOut: RunReceipt['held_out'];
try {
await withBudgetTracker(tracker, async () => {
// Baseline eval: score the baseline skill on D_sel to set best_sel_score.
// We use the FULL validation gate with median-of-3 for a stable baseline.
const baselineGate = await score('baseline_sel', baselineText, split.sel);
if (checkpoint!.best_sel_score === 0) {
// Fresh run; set baseline.
checkpoint!.best_sel_score = baselineGate.selScore;
checkpoint!.best_skill_text = baselineText;
saveCheckpoint(skillsDir, skillName, checkpoint!);
}
baselineSelScore = baselineGate.selScore;
// ── E1 one-shot rewrite (the default) ─────────────────────────────────
// The loop's first step with a whole-body rewrite instead of edits:
// forward pass on one train batch, one rewrite call, then the same D12
// selection gate a loop step faces. A rejected rewrite leaves the skill
// unchanged. Sets the cursor past the last epoch so the loop below no-ops.
if (opts.optimizerMode === 'one-shot-rewrite' && resumeCursor(checkpoint!, stepsPerEpoch).epoch <= opts.epochs) {
stepStartTally = structuredClone(tally);
totalStepsRun = 1;
const incumbent = checkpoint!.best_skill_text;
const { body: incumbentBody, bodyStart } = splitFrontmatter(incumbent);
const fwd = await score('train_forward', incumbent, split.train.slice(0, opts.batchSize), { runsPerTask: 1, epoch: 1, step: 1 });
const rewrite = await runOneShotRewrite({
skillBodyText: incumbentBody,
successes: fwd.scoredRollouts.filter((r) => r.score >= ROLLOUT_SUCCESS_THRESHOLD),
failures: fwd.scoredRollouts.filter((r) => r.score < ROLLOUT_SUCCESS_THRESHOLD),
rejected: [],
criteria: benchmarkCriteria,
optimizerModel: opts.optimizerModel,
maxTokens: reflectCap.maxTokens,
});
recordOptimizerStep(tally, {
calls: 1,
usableReplies: rewrite.error ? 0 : 1,
errors: rewrite.error ? [rewrite.error] : [],
invalidEditsDropped: 0,
});
if (rewrite.error) process.stderr.write(`[skillopt] one-shot rewrite produced no candidate (${rewrite.error})\n`);
if (rewrite.newBody) {
const candidate = incumbent.slice(0, bodyStart) + rewrite.newBody;
const gate = await score('sel_gate', candidate, split.sel, { bestScore: checkpoint!.best_sel_score, epoch: 1, step: 1 });
// Ablation (cat31 config D): disableValidationGate promotes regardless.
const accepted = opts.disableValidationGate === true || gate.accepted;
logEvent({
kind: 'step',
run_id: runId,
skill: skillName,
epoch: 1,
step: 1,
sel_score_median: gate.selScore,
sel_score_runs: gate.perTaskMedians.map((t) => t.median),
accepted,
edits_attempted: 1,
edits_applied: 1,
delta: gate.selScore - checkpoint!.best_sel_score,
...(accepted ? {} : { reason: gate.reason ?? 'rejected' }),
cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
} as never);
if (accepted) promote(candidate, gate.selScore, 1, 1, []);
}
advanceCursor(checkpoint!, opts.epochs, stepsPerEpoch, stepsPerEpoch); // no-op the epoch loop below
bankAccounting(checkpoint!, prior, tracker.snapshot());
saveCheckpoint(skillsDir, skillName, checkpoint!);
stepStartTally = undefined;
}
// Epoch loop. The cursor names the first step that has not completed.
const cursor = resumeCursor(checkpoint!, stepsPerEpoch);
epochs: for (let epoch = cursor.epoch; epoch <= opts.epochs; epoch++) {
const startStep = epoch === cursor.epoch ? cursor.step : 1;
const epochStartBest = checkpoint!.best_sel_score;
for (let step = startStep; step <= stepsPerEpoch; step++) {
if (Date.now() > deadline) throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
totalStepsRun += 1;
stepStartTally = structuredClone(tally);
const globalStep = (epoch - 1) * stepsPerEpoch + step;
const lrBudget = scheduleFn(opts.lr, globalStep, totalSteps);
// Sample a batch from D_train (round-robin to keep deterministic).
const batchStart = ((step - 1) * opts.batchSize) % split.train.length;
const batch = split.train.slice(batchStart, batchStart + opts.batchSize);
// FORWARD PASS: run rollouts on each batch task using current best.
// runsPerTask=1 — for the train batch we only need a rough partition
// into successes/failures, not the median-of-3 noise rejection the
// sel-side gate uses. ScoredRollouts come back via GateResult.
const forwardGate = await score('train_forward', checkpoint!.best_skill_text, batch, { runsPerTask: 1, epoch, step });
// Partition into successes vs failures (>= 0.5 threshold). Reflect
// gets the actual scored trajectories so failure-mode + success-mode
// analysis can ground in real agent behavior (D7).
const successes = forwardGate.scoredRollouts.filter((r) => r.score >= ROLLOUT_SUCCESS_THRESHOLD);
const failures = forwardGate.scoredRollouts.filter((r) => r.score < ROLLOUT_SUCCESS_THRESHOLD);
stepBody: {
// BACKWARD PASS: D7 two reflect calls (failures + successes).
const rejected = loadRejectedBuffer(skillsDir, skillName);
const reflectResult = await runReflect({
skillBodyText: checkpoint!.best_skill_text,
successes,
failures,
rejected,
criteria: benchmarkCriteria,
optimizerModel: opts.optimizerModel,
maxTokens: reflectCap.maxTokens,
...(opts.reflectMode ? { reflectMode: opts.reflectMode } : {}),
abortSignal: undefined,
});
recordOptimizerStep(tally, reflectResult);
if (reflectResult.calls > 0 && reflectResult.usableReplies === 0) lastUnusableError = reflectResult.errors[0];
if (reflectResult.skillBodyTruncated) skillBodyTruncated = reflectResult.skillBodyTruncated;
// Merge + rank + LR-clip.
const allEdits: EditOp[] = [...reflectResult.failureEdits, ...reflectResult.successEdits];
// Drop edits already in rejected buffer.
const fresh = allEdits.filter((e) => !isRejected(rejected, checkpoint!.best_skill_text, [e]));
// Apply under LR budget.
const applied = applyEditBatch(checkpoint!.best_skill_text, fresh, lrBudget);
// Defect 4: only edits that were actually applied are attributed to
// the gate's verdict. Edits that failed to apply are filed as
// apply_failed; edits the LR budget clipped were never tried and
// stay free to be proposed again.
const appliedEdits = applied.results.flatMap((r) => (r.outcome === 'applied' ? [r.edit] : []));
const applyFailed = applied.results.flatMap((r) =>
r.outcome === 'rejected' && r.detail !== 'lr_budget_exhausted'
? [makeRejectedEntry(checkpoint!.best_skill_text, [r.edit], 'apply_failed')]
: []);
if (fresh.length === 0) {
// #4741: the optimizer proposed NOTHING (reflect returned/parsed no
// edits, or every edit was already in the rejected buffer). This
// used to fall into the all-rejected branch below ([].every() is
// true) and log the same 'no_edits_applied' as "gated N candidates
// and rejected all" — so a whole run of zero candidates read as a
// real `no_improvement`. Say so per step; nothing to reject-buffer.
const why = reflectResult.errors[0]
? `no_edits_proposed: ${reflectResult.errors[0]}`
: 'no_edits_proposed';
process.stderr.write(`[skillopt] epoch ${epoch} step ${step}: optimizer proposed no edits (${why})\n`);
logEvent({
kind: 'step',
run_id: runId,
skill: skillName,
epoch,
step,
sel_score_median: checkpoint!.best_sel_score,
sel_score_runs: [],
accepted: false,
edits_attempted: 0,
edits_applied: 0,
delta: 0,
reason: why,
cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
...(reflectResult.errors.length > 0 ? { reflect_errors: reflectResult.errors } : {}),
...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
} as never);
break stepBody;
}
if (appliedEdits.length === 0) {
// Nothing applied; record the apply failures + skip gate.
saveRejectedBuffer(skillsDir, skillName, applyFailed);
logEvent({
kind: 'step',
run_id: runId,
skill: skillName,
epoch,
step,
sel_score_median: checkpoint!.best_sel_score,
sel_score_runs: [],
accepted: false,
edits_attempted: fresh.length,
edits_applied: 0,
delta: 0,
reason: 'no_edits_applied',
cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
} as never);
break stepBody;
}
// VALIDATION GATE (D12 median-of-3 + epsilon=0.05, D4 parallel).
const gate = await score('sel_gate', applied.newText, split.sel, { bestScore: checkpoint!.best_sel_score, epoch, step });
// Ablation (cat31 config D): disableValidationGate greedy-accepts any
// applied edit, bypassing the D12 median+epsilon check. We still ran
// the gate above to GET the score (so sel_score tracking is honest),
// but ignore its accept verdict.
const stepAccepted = opts.disableValidationGate === true ? true : gate.accepted;
const delta = gate.selScore - checkpoint!.best_sel_score;
if (stepAccepted) {
// ACCEPT: D8 commit via version-store (mutate) OR write proposed.md
// for the --no-mutate / bundled-without-allow paths; with a
// held-out set the write waits for the final held-out check (D2).
promote(applied.newText, gate.selScore, epoch, step, appliedEdits);
} else {
// REJECT: file the applied edits (and apply failures) in the rejected buffer.
saveRejectedBuffer(skillsDir, skillName, [
...appliedEdits.map((e) => makeRejectedEntry(checkpoint!.best_skill_text, [e], `validation_gate_${gate.reason ?? 'rejected'}`)),
...applyFailed,
]);
}
logEvent({
kind: 'step',
run_id: runId,
skill: skillName,
epoch,
step,
sel_score_median: gate.selScore,
sel_score_runs: gate.perTaskMedians.map((t) => t.median),
accepted: stepAccepted,
edits_attempted: fresh.length,
edits_applied: appliedEdits.length,
delta,
...(stepAccepted ? {} : { reason: gate.reason ?? 'rejected' }),
cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
} as never);
}
// Step completed (accepted, rejected or no candidate): advance the
// cursor and persist accounting so --resume continues after it.
advanceCursor(checkpoint!, epoch, step, stepsPerEpoch);
bankAccounting(checkpoint!, prior, tracker.snapshot());
saveCheckpoint(skillsDir, skillName, checkpoint!);
stepStartTally = undefined;
if (shouldEarlyStop(tally)) {
earlyStopped = true;
process.stderr.write(
`[skillopt] stopped after ${EARLY_STOP_UNUSABLE_STEPS} steps of ${unusableClasses(tally)}; remaining budget not spent\n`,
);
break epochs;
}
}
// D6 SLOW UPDATE: if no improvement this epoch, propose one meta-edit.
if (checkpoint!.best_sel_score === epochStartBest) {
logEvent({
kind: 'slow_update',
run_id: runId,
skill: skillName,
epoch,
meta_edit_proposed: false, // Simplified for v1; full meta-update is TODO.
meta_edit_accepted: false,
} as never);
}
}
// D2 HELD-OUT: one independent check of the final champion against the
// baseline, before anything is written. The proposer never sees its
// verdict, so the held-out set never becomes a second selection set.
// A refused champion is dropped: the skill stays at the baseline.
const champion = checkpoint!.champion;
if (champion) {
const ho = await runHeldOutGate({
engine: opts.engine,
operationContext: opts.operationContext,
candidateSkillText: checkpoint!.best_skill_text,
baselineSkillText: baselineText,
heldOutTasks,
deadlineMs: deadline,
targetModel: opts.targetModel,
judgeModel: opts.judgeModel,
...(hook ? { rolloutHook: hook } : {}),
});
rollouts.held_out += heldOutTasks.length * VALIDATION_RUNS_PER_TASK * 2;
heldOut = { baseline_score: ho.baselineScore, candidate_score: ho.candidateScore, passed: ho.passed };
logEvent({ kind: 'held_out', run_id: runId, skill: skillName, ...heldOut } as never);
if (ho.passed) {
writeCandidate(checkpoint!.best_skill_text, checkpoint!.best_sel_score, champion.epoch, champion.step, champion.edits);
tally.accepted_steps += champion.steps;
} else {
process.stderr.write(`[skillopt] held-out check refused the champion (${ho.candidateScore.toFixed(3)} < baseline ${ho.baselineScore.toFixed(3)}); skill unchanged\n`);
checkpoint!.best_skill_text = baselineText;
checkpoint!.best_sel_score = baselineSelScore;
}
delete checkpoint!.champion;
saveCheckpoint(skillsDir, skillName, checkpoint!);
}
// FINAL TEST: score the best skill AND the baseline on D_test so the
// receipt carries an honest generalization signal. Skipped when D_test
// is empty, and after an early stop with nothing accepted (best ===
// baseline; budget not spent).
if (split.test.length > 0 && !(earlyStopped && tally.accepted_steps === 0)) {
testScore = (await score('test', checkpoint!.best_skill_text, split.test)).selScore;
baselineTestScore = (await score('test', baselineText, split.test)).selScore;
}
});
} catch (err) {
caught = classifyAbortError(err, opts);
if (err instanceof BudgetExhausted && err.reason === 'cost') caught.abortDetail += BUDGET_ADMISSION_HINT;
logEvent({
kind: 'abort',
run_id: runId,
skill: skillName,
reason: caught.abortReason,
detail: caught.abortDetail,
} as never);
// Handled abort: bank the spend; the cursor stays on the interrupted step
// so --resume re-runs it, and the saved tally is the step-start copy so
// that re-run does not count the step's optimizer calls twice.
bankAccounting(checkpoint, prior, tracker.snapshot());
if (stepStartTally) checkpoint.tally = stepStartTally;
saveCheckpoint(skillsDir, skillName, checkpoint);
checkpoint.tally = tally;
}
const { outcome, stopReason, abortReason, abortDetail } = resolveRunOutcome({ caught, tally, earlyStopped });
if (outcome === 'errored' && !caught) {
logEvent({ kind: 'abort', run_id: runId, skill: skillName, reason: 'error', detail: abortDetail } as never);
}
// Acceptance history (not the final loop state) decides what was written:
// an abort or early stop after an accepted step still mutated SKILL.md /
// wrote proposed.md.
const acceptedAny = tally.accepted_steps > 0;
const mutatedSkillFile = mutateDecision.mutate && acceptedAny;
const proposedPath = !mutateDecision.mutate && acceptedAny ? proposedFilePath(skillsDir, skillName) : undefined;
// An early stop left work undone even when it resolves to no_improvement.
const checkpointRetained = outcome === 'aborted' || outcome === 'errored' || earlyStopped;
if (checkpointRetained && !caught) {
// The final evaluation ran after the last step save; bank its spend too.
bankAccounting(checkpoint, prior, tracker.snapshot());
saveCheckpoint(skillsDir, skillName, checkpoint);
}
const remediation = buildRemediation([...tally.reflect_errors, ...(abortDetail ? [abortDetail] : [])], abortReason);
// An early stop that still resolved accepted/no_improvement takes its
// resume remedy from the trailing unusable errors.
const failureCode = caught
? errorCode(caught.abortDetail)
: earlyStopped && lastUnusableError ? errorCode(lastUnusableError)
: abortDetail ? errorCode(abortDetail) : undefined;
const budget = tracker.snapshot();
const modelsUsed = buildModelsUsed(budget, prior.rows);
// Final receipt.
const receipt: RunReceipt = {
run_id: runId,
skill: skillName,
skill_sha8: baselineSha8,
benchmark_sha8: bench.benchmark_sha8,
optimizer_model: opts.optimizerModel,
target_model: opts.targetModel,
judge_model: opts.judgeModel,
epochs: opts.epochs,
batch_size: opts.batchSize,
lr: opts.lr,
lr_schedule: opts.lrSchedule,
max_cost_usd: opts.maxCostUsd,
started_at: checkpoint.started_at,
ended_at: new Date().toISOString(),
outcome,
// #3516: abort/error detail rides the receipt so --json consumers and the
// CLI's stderr summary both see WHY, not just that the run died.
...(abortReason !== undefined ? { abort_reason: abortReason } : {}),
...(abortDetail !== undefined ? { abort_detail: abortDetail, ...(caught?.pricing ? { no_pricing: caught.pricing } : {}) } : {}),
stop_reason: stopReason,
...(tally.reflect_errors.length > 0 ? { reflect_errors: [...tally.reflect_errors] } : {}),
...(tally.invalid_edits_dropped > 0 ? { reflect_invalid_edits_dropped: tally.invalid_edits_dropped } : {}),
...(skillBodyTruncated ? { skill_body_truncated: skillBodyTruncated } : {}),
...(remediation.length > 0 ? { remediation } : {}),
reflect_max_tokens: reflectCap.maxTokens,
reflect_max_tokens_source: reflectCap.source,
...(checkpointRetained && !opts.tasks ? { resume_command: buildResumeCommand(skillName, runId, runSpec, failureCode, {
mode: opts.mode, maxCostUsd: opts.maxCostUsd, maxRuntimeMin: opts.maxRuntimeMin,
force: opts.force,
modelsStrict: opts.modelsStrict === true,
}) } : {}),
baseline_sel_score: baselineSelScore,
best_sel_score: checkpoint.best_sel_score,
...(testScore !== undefined ? { test_score: testScore } : {}),
...(baselineTestScore !== undefined ? { baseline_test_score: baselineTestScore } : {}),
final_cost_usd: budget.cumulativeCostUsd,
prior_segments_cost_usd: prior.costUsd,
models_used: modelsUsed,
models_used_scope: prior.scope,
models_plan: modelsPlan,
models_strict: resolved.strict,
total_steps: totalStepsRun,
epochs_completed: checkpoint.last_completed_epoch,
// Ablation provenance (cat31 replayability) — only when a non-default knob set.
...(opts.reflectMode ? { reflect_mode: opts.reflectMode } : {}),
...(opts.disableValidationGate ? { validation_gate_disabled: true } : {}),
optimizer_mode: opts.optimizerMode,
rollouts,
candidates_scored_on_sel: selCandidates.size,
optimizer_calls: tally.reflect_calls,
cost_scope: hook ? 'optimizer_only' : 'all',
...(heldOut ? { held_out: heldOut } : {}),
};
logEvent({
kind: 'run_end',
run_id: runId,
skill: skillName,
outcome,
epochs_completed: checkpoint.last_completed_epoch,
total_steps: totalStepsRun,
best_sel_score: checkpoint.best_sel_score,
final_cost_usd: budget.cumulativeCostUsd,
models_used: modelsUsed,
models_used_scope: prior.scope,
stop_reason: stopReason,
...(abortDetail !== undefined ? { abort_detail: abortDetail } : {}),
} as never);
// Clean checkpoint on success (resume not needed).
if (!checkpointRetained) {
deleteCheckpoint(skillsDir, skillName, runId);
}
return {
outcome,
receipt,
finalText,
mutatedSkillFile,
...(proposedPath ? { proposedPath } : {}),
};
}
// Re-export parseSplit so the CLI can validate flags without importing
// benchmark.ts directly.
export { parseSplit };