Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
53.1 KB
Raw Blame
/**
 * SkillOpt main loop — runSkillOpt.
 *
 * ┌─ LR cosine-decay curve (default base=4, totalSteps=10) ────────────────┐
 * │                                                                       │
 * │  4 ●─●                                                                │
 * │       \                                                               │
 * │  3     ●─●─●                                                          │
 * │             \                                                         │
 * │  2           ●─●─●                                                    │
 * │                   \                                                   │
 * │  1                 ●─●─●                                              │
 * │   ─┼──┼──┼──┼──┼──┼──┼──┼──┼──┼─                                      │
 * │    1  2  3  4  5  6  7  8  9 10                                       │
 * │                                                                       │
 * │  Peaks early (most aggressive when skill is least refined), tapers    │
 * │  as the skill converges. Schedule lives in lr-schedule.ts.            │
 * └───────────────────────────────────────────────────────────────────────┘
 *
 * ┌─ Run state machine ────────────────────────────────────────────────────┐
 * │                                                                       │
 * │   start ──► models_plan ──► preflight ──► lock_acquire ──► resume     │
 * │     (banner; strict abort; reservation check: all before any spend)   │
 * │                    (resume: cursor next_epoch/next_step, spec check)   │
 * │                                                  │                     │
 * │                                                  ▼                     │
 * │                                  ┌──── epoch_start ◄────┐              │
 * │                                  │           │            │            │
 * │                                  │           ▼            │            │
 * │                                  │   forward_pass         │            │
 * │                                  │   (rollouts batch)     │            │
 * │                                  │           │            │            │
 * │                                  │           ▼            │            │
 * │                                  │   backward_pass        │            │
 * │                                  │   (reflect ×2 D7; each │            │
 * │                                  │    call: edits | empty │            │
 * │                                  │    | exactly 1 error)  │            │
 * │                                  │           │            │            │
 * │                                  │           ▼            │            │
 * │                                  │   rank_and_clip        │            │
 * │                                  │   (LR budget)          │            │
 * │                                  │           │            │            │
 * │                                  │           ▼            │            │
 * │                                  │   validation_gate ─────┘            │
 * │                                  │     (D12 median+ε)                  │
 * │                                  │           │                          │
 * │                                  │  accept ──┤── reject                │
 * │                                  │     │     │     │                    │
 * │                                  │     ▼     │     ▼                    │
 * │                                  │  commit   │  rejected_buffer        │
 * │                                  │   (D8)    │                          │
 * │                                  │     │     │                          │
 * │                                  │     └─►◄──┘                          │
 * │                                  │           │                          │
 * │                                  │           ▼                          │
 * │                                  │   step_end: advance cursor,         │
 * │                                  │   save cost + tally ──► 2 fully     │
 * │                                  │           │    unusable steps in a  │
 * │                                  │           │    row ──► EARLY STOP ─┐ │
 * │                                  │           ▼                        │ │
 * │                                  │   epoch_end ──► slow_update (D6)  │ │
 * │                                  │           │                        │ │
 * │                                  └───────────┘                        │ │
 * │                                              │ all epochs done        │ │
 * │                                              ▼                        │ │
 * │                                       final_test ──► run_end ◄───────┘ │
 * │                                                                       │
 * │  run_end outcome (stop_reason completed | early_stop_unusable_output  │
 * │  | aborted):                                                          │
 * │    caught abort/error ──────────────► aborted | errored  (checkpoint  │
 * │                                        kept, resume_command)          │
 * │    any step accepted ───────────────► accepted                        │
 * │    optimizer called, 0 usable replies ► errored                       │
 * │                                        (optimizer_output_unusable,    │
 * │                                        checkpoint kept, remediation)  │
 * │    otherwise ───────────────────────► no_improvement                  │
 * └────────────────────────────────────────────────────────────────────────┘
 *
 * ┌─ Validation gate decision tree (D12) ──────────────────────────────────┐
 * │                                                                       │
 * │  candidate edits applied                                              │
 * │             │                                                          │
 * │             ▼                                                          │
 * │  for each sel-task in parallel (cap=4 per D4):                        │
 * │    median(score_run_1, score_run_2, score_run_3)  ← VALIDATION_RUNS=3 │
 * │             │                                                          │
 * │             ▼                                                          │
 * │     mean(per_task_medians) = sel_score                                │
 * │             │                                                          │
 * │             ▼                                                          │
 * │    sel_score > best_score + 0.05  ?  ← VALIDATION_EPSILON             │
 * │      │ yes               │ no                                          │
 * │      ▼                   ▼                                             │
 * │  ACCEPT             REJECT → rejected-buffer                          │
 * │  commit via D8                                                         │
 * └────────────────────────────────────────────────────────────────────────┘
 */

import { assertLegacySkillWriter } from '../skillpack/writer-guard.ts';
import { createHash, randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import { BudgetExhausted, BudgetTracker, loadPricingOverrides, type NoPricingGuidance } from '../budget/budget-tracker.ts';
import type { PricingOverrides } from '../budget/reservation-cost.ts';
import { buildModelsUsed } from '../budget/models-used.ts';
import { withBudgetTracker } from '../ai/gateway.ts';
import { errorFor } from '../errors.ts';
import { applyEditBatch, getWorkingTreeStatusForFile, splitFrontmatter } from './apply-edits.ts';
import { logEvent, sha8 } from './audit.ts';
import { loadRunTasks, parseSplit } from './benchmark.ts';
import { getBundledSkillContext, shouldMutateSkillFile, assertBundledMutationHeldOut } from './bundled-skill-gate.ts';
import {
  advanceCursor,
  assertResumeCompatible,
  bankAccounting,
  deleteCheckpoint,
  emptyTally,
  loadCheckpoint,
  priorSegments,
  resumeCursor,
  rewindUnusableSteps,
  saveCheckpoint,
  type ReflectTally,
  type RunSpec,
} from './checkpoint.ts';
import { loadHeldOut, runHeldOutGate } from './held-out.ts';
import { withSkilloptLock } from './lock.ts';
import { resolveLrSchedule } from './lr-schedule.ts';
import { SKILLOPT_HOOK_BUDGET_EXHAUSTED, SKILLOPT_HOOK_INFRA_ERROR, SKILLOPT_RUNTIME_EXCEEDED } from './must-abort.ts';
import {
  buildModelsPlan,
  formatModelsBanner,
  modelsStrictError,
  resolveModelsStrict,
  strictVerdict,
  unknownProvenanceModels,
  type ModelsPlanEntry,
  type StrictVerdict,
} from './models-plan.ts';
import { resolveReflectMaxTokens, type ReflectCap } from './output-cap.ts';
import { preflight, formatPreflightReport } from './preflight.ts';
import { isRejected, loadRejectedBuffer, makeRejectedEntry, saveRejectedBuffer } from './rejected-buffer.ts';
import { runReflect, runOneShotRewrite, describeJudges, type SkillBodyTruncation } from './reflect.ts';
import { buildRemediation, errorCode } from './remediation.ts';
import {
  buildResumeCommand,
  recordOptimizerStep,
  resolveRunOutcome,
  shouldEarlyStop,
  unusableClasses,
  EARLY_STOP_UNUSABLE_STEPS,
} from './run-outcome.ts';
import {
  acceptCandidate,
  proposedPath as proposedFilePath,
  revertAllPending,
  skillPath,
  writeProposed,
} from './version-store.ts';
import { runValidationGate } from './validate-gate.ts';
import { ROLLOUT_SUCCESS_THRESHOLD, VALIDATION_RUNS_PER_TASK } from './types.ts';
import type { SkillOptOpts, EditOp, RunReceipt, BenchmarkTask, RolloutPhase } from './types.ts';

export interface RunSkillOptResult {
  outcome: 'accepted' | 'no_improvement' | 'aborted' | 'errored';
  receipt: RunReceipt;
  /** Final SKILL.md text (committed or proposed). */
  finalText: string;
  /** True when SKILL.md was actually rewritten; false for --no-mutate / bundled-skill paths. */
  mutatedSkillFile: boolean;
  /** When mutate was skipped, path where the proposed.md was written. */
  proposedPath?: string;
  sharedOptimization?: { proposal_id: string; publication?: Record<string, unknown> };
}

export async function runSkillOpt(opts: SkillOptOpts): Promise<RunSkillOptResult> {
  if (opts.sharedSkill) {
    const { optimizeSharedSkill } = await import('../shared-skills/optimizer.ts');
    return optimizeSharedSkill(opts, runSkillOpt);
  }
  const { engine, skillName, skillsDir } = opts;

  // ── Pre-flight gates (fail-loud BEFORE any LLM spend) ───────────────────
  const skillFile = skillPath(skillsDir, skillName);
  await assertLegacySkillWriter(engine, skillFile);
  if (!fs.existsSync(skillFile)) {
    throw errorFor({
      class: 'NoSkill',
      code: 'no_skill_md',
      message: `Cannot find SKILL.md for '${skillName}' at ${skillFile}.`,
      hint: `Create the skill first via 'gbrain skillify scaffold ${skillName}'.`,
    });
  }

  // Working-tree gate.
  if (!opts.force) {
    const status = getWorkingTreeStatusForFile(skillFile);
    if (status === 'dirty' && !isOwnResumeEdit(skillsDir, skillName, skillFile, opts.resumeRunId)) {
      throw errorFor({
        class: 'DirtyTree',
        code: 'dirty_tree',
        message: `${skillFile} has uncommitted changes.`,
        hint: `Commit or stash changes before running skillopt, or pass --force to override.`,
      });
    }
  }

  // Bundled-skill gate (D16).
  const bundledCtx = getBundledSkillContext(skillsDir, skillName);
  const mutateDecision = shouldMutateSkillFile(bundledCtx, {
    noMutate: opts.noMutate,
    allowMutateBundled: opts.allowMutateBundled,
  });

  // Held-out set (F11). Loaded ONCE here so the bundled-mutation ENFORCE gate
  // and the per-acceptance held-out gate share the same parsed tasks. Empty
  // when --held-out is not passed.
  const heldOutTasks: BenchmarkTask[] = opts.heldOutPath ? loadHeldOut(opts.heldOutPath) : [];

  // D16 ENFORCE (core mutation policy — fires for EVERY caller: CLI, batch,
  // fleet, cycle, background job, and the run_skillopt MCP op, since they all
  // funnel through runSkillOpt). Fail-loud BEFORE the lock + any LLM spend.
  assertBundledMutationHeldOut({
    isBundled: bundledCtx.isBundled,
    willMutate: mutateDecision.mutate,
    heldOutCount: heldOutTasks.length,
    skillName,
  });

  // Load + validate the tasks (D17 floor enforcement, D15 sentinel check, B1.2 explicit lists).
  const { bench, split } = loadRunTasks(opts);

  // Held-out must be INDEPENDENT of the benchmark. If a held-out file shares
  // task_ids with the benchmark, an overfit candidate passes the held-out gate
  // vacuously — silently voiding the gate's entire benchmark-gaming defense
  // (the load-bearing safety claim for bundled mutation). Reject on overlap.
  if (heldOutTasks.length > 0) {
    const benchIds = new Set(bench.tasks.map((t) => t.task_id));
    const overlap = heldOutTasks.filter((t) => benchIds.has(t.task_id)).map((t) => t.task_id);
    if (overlap.length > 0) {
      throw errorFor({
        class: 'HeldOutOverlap',
        code: 'held_out_overlaps_benchmark',
        message: `Held-out set shares ${overlap.length} task_id(s) with the benchmark (e.g. ${overlap.slice(0, 3).join(', ')}). The held-out set must be independent.`,
        hint: `Use disjoint task_ids in the held-out file — an overlapping held-out can't catch benchmark overfitting.`,
      });
    }
  }

  // ── Models plan + strict mode (#5585) — banner and verdict before any spend
  const reflectCap = await resolveReflectMaxTokens(engine, opts.optimizerModel, opts.reflectMaxTokens);
  const modelsPlan = await buildModelsPlan(
    engine,
    opts.models ?? unknownProvenanceModels(opts, 'caller-supplied'),
    [...bench.tasks, ...heldOutTasks],
  );
  const banner = formatModelsBanner(modelsPlan, { skill: skillName, baseline: opts.modelsBannerBaseline, reflectCap });
  if (banner) process.stderr.write(banner);
  const strict = strictVerdict(modelsPlan, await resolveModelsStrict(engine, opts.modelsStrict));
  if (strict.enabled && !strict.ok && !opts.dryRun) throw modelsStrictError(strict);
  const pricingOverrides = await loadPricingOverrides(engine);

  // ── Cost preflight (D3) ─────────────────────────────────────────────────
  const preflightResult = preflight({
    epochs: opts.epochs,
    batchSize: opts.batchSize,
    trainSize: split.train.length,
    selSize: split.sel.length,
    testSize: split.test.length,
    optimizerModel: opts.optimizerModel,
    targetModel: opts.targetModel,
    judgeModel: opts.judgeModel,
    maxCostUsd: opts.maxCostUsd,
    maxCostSource: opts.maxCostSource ?? 'user',
    heldOutSize: heldOutTasks.length,
    interactive: process.stderr.isTTY === true,
    reflectMaxTokens: reflectCap.maxTokens,
    judgeModels: modelsPlan.filter((e) => e.touchpoint === 'judge' && e.active).map((e) => e.model),
    optimizerMode: opts.optimizerMode,
    rolloutsExternal: opts.rolloutHook !== undefined,
    ...(pricingOverrides ? { pricingOverrides } : {}),
  });
  if (opts.json !== true) {
    process.stderr.write(formatPreflightReport(preflightResult.estimate, {
      epochs: opts.epochs, batchSize: opts.batchSize,
      trainSize: split.train.length, selSize: split.sel.length, testSize: split.test.length,
      optimizerModel: opts.optimizerModel, targetModel: opts.targetModel, judgeModel: opts.judgeModel,
      maxCostUsd: opts.maxCostUsd,
      maxCostSource: opts.maxCostSource ?? 'user',
      optimizerMode: opts.optimizerMode,
    }) + '\n');
  }
  if (preflightResult.abort_code === 'no_pricing') {
    throw errorFor({
      class: 'BudgetExhausted',
      code: 'no_pricing',
      message: `${preflightResult.abort_reason} No model call was made; or pass --max-usd off to run uncapped.`,
    });
  }
  if (!preflightResult.proceed) {
    throw errorFor({
      class: 'CostCapExceeded',
      code: 'cost_cap_exceeded',
      message: preflightResult.abort_reason ?? 'preflight refused to proceed',
      hint: preflightResult.abort_code === 'reservation_exceeds_cap'
        ? `No model call was made.`
        : `Raise --max-cost-usd or reduce knobs.`,
    });
  }

  // --dry-run short-circuits BEFORE the lock + LLM calls.
  if (opts.dryRun) {
    const receipt: RunReceipt = {
      run_id: opts.resumeRunId ?? randomUUID(),
      skill: skillName,
      skill_sha8: sha8(fs.readFileSync(skillFile, 'utf8')),
      benchmark_sha8: bench.benchmark_sha8,
      optimizer_model: opts.optimizerModel,
      target_model: opts.targetModel,
      judge_model: opts.judgeModel,
      epochs: opts.epochs,
      batch_size: opts.batchSize,
      lr: opts.lr,
      lr_schedule: opts.lrSchedule,
      max_cost_usd: opts.maxCostUsd,
      started_at: new Date().toISOString(),
      outcome: 'aborted',
      reflect_max_tokens: reflectCap.maxTokens,
      reflect_max_tokens_source: reflectCap.source,
      models_plan: modelsPlan,
      models_strict: strict,
      optimizer_mode: opts.optimizerMode,
      cost_scope: opts.rolloutHook ? 'optimizer_only' : 'all',
    };
    return { outcome: 'aborted', receipt, finalText: fs.readFileSync(skillFile, 'utf8'), mutatedSkillFile: false };
  }

  // ── Acquire per-skill lock (D14) ────────────────────────────────────────
  return await withSkilloptLock(engine, skillName, async () => {
    return runOptimizationLoop(opts, bench, split, bundledCtx, mutateDecision, heldOutTasks, {
      reflectCap, modelsPlan, strict, pricingOverrides,
    });
  });
}

/**
 * Classifies a run-loop error into the abort taxonomy the receipt + audit
 * trail surface. Pulled out of runOptimizationLoop's catch block so it's
 * unit-testable without driving a full orchestrator run (engine, benchmark,
 * lock, LLM calls) — the string-sniff bug this replaces went unexercised
 * for exactly that reason: nothing could cheaply throw a real error deep
 * inside the loop to catch it.
 *
 * `err instanceof BudgetExhausted` is the real classification signal — the
 * string sniff it replaces (`msg.includes('BudgetExhausted') ||
 * msg.includes('budget_exhausted')`) could never match, because every
 * BudgetExhausted thrown by src/core/budget/budget-tracker.ts carries a
 * human-readable message ("projected cost $X exceeds the configured cap
 * $Y", "N exceeded --max-runtime Ns", ...) containing neither literal
 * substring. Every cost-cap / no-pricing / (unreachable from this tracker
 * today, since it's never constructed with maxRuntimeMs) runtime abort fell
 * through to the generic catch-all instead, misreporting outcome='errored'
 * / abortReason='error' on the receipt and audit trail.
 */
/**
 * A resume may start on a SKILL.md this run itself rewrote: the file is dirty,
 * but it is exactly the checkpoint's best text. Any other edit still trips the
 * dirty-tree gate, so a hand edit between segments is never overwritten.
 */
function isOwnResumeEdit(skillsDir: string, skillName: string, skillFile: string, resumeRunId: string | undefined): boolean {
  if (!resumeRunId) return false;
  const cp = loadCheckpoint(skillsDir, skillName, resumeRunId);
  if (!cp) return false;
  try {
    return sha8(fs.readFileSync(skillFile, 'utf8')) === sha8(cp.best_skill_text);
  } catch {
    return false;
  }
}

export function classifyAbortError(
  err: unknown,
  opts: { maxRuntimeMin: number },
): {
  outcome: 'aborted' | 'errored';
  abortReason: 'budget_exhausted' | 'runtime_exceeded' | 'sigint' | 'error';
  abortDetail: string;
  /** A no_pricing abort's lookup-and-register guidance. */
  pricing?: NoPricingGuidance;
} {
  const msg = err instanceof Error ? err.message : String(err);
  if (err instanceof BudgetExhausted) {
    return {
      outcome: 'aborted',
      abortReason: err.reason === 'runtime' ? 'runtime_exceeded' : 'budget_exhausted',
      abortDetail: msg,
      ...(err.pricing ? { pricing: err.pricing } : {}),
    };
  } else if (msg.includes(SKILLOPT_HOOK_BUDGET_EXHAUSTED)) {
    return { outcome: 'aborted', abortReason: 'budget_exhausted', abortDetail: msg };
  } else if (msg.includes(SKILLOPT_HOOK_INFRA_ERROR)) {
    return { outcome: 'aborted', abortReason: 'error', abortDetail: msg };
  } else if (msg.includes(SKILLOPT_RUNTIME_EXCEEDED)) {
    return {
      outcome: 'aborted',
      abortReason: 'runtime_exceeded',
      abortDetail: `exceeded --max-runtime-min ${opts.maxRuntimeMin}`,
    };
  } else if (msg.includes('SIGINT')) {
    return { outcome: 'aborted', abortReason: 'sigint', abortDetail: msg };
  } else {
    // #3516: truthful catch-all. Pre-fix this logged reason:'sigint' for
    // EVERY unrecognized error (provider failures, no_pricing hard-fails),
    // which made the audit trail lie about why the run died.
    return { outcome: 'errored', abortReason: 'error', abortDetail: msg };
  }
}

/** Appended to a cost-cap abort: optimizer calls reserve their whole output cap up front. */
const BUDGET_ADMISSION_HINT = ' (raise --max-cost-usd; each optimizer call reserves its full output cap, so lowering --reflect-max-tokens / skillopt.reflect_max_tokens also helps)';

async function runOptimizationLoop(
  opts: SkillOptOpts,
  bench: ReturnType<typeof loadRunTasks>['bench'],
  split: ReturnType<typeof loadRunTasks>['split'],
  bundledCtx: ReturnType<typeof getBundledSkillContext>,
  mutateDecision: ReturnType<typeof shouldMutateSkillFile>,
  heldOutTasks: BenchmarkTask[],
  resolved: {
    reflectCap: ReflectCap;
    modelsPlan: ModelsPlanEntry[];
    strict: StrictVerdict;
    pricingOverrides: PricingOverrides | undefined;
  },
): Promise<RunSkillOptResult> {
  const { skillName, skillsDir } = opts;
  const { reflectCap, modelsPlan } = resolved;
  const skillFile = skillPath(skillsDir, skillName);

  // Plain-English success criteria fed to the optimizer so it targets WHAT the
  // scorer rewards instead of guessing from a bare pass/fail score. Without
  // this, rule-judged benchmarks (e.g. "must include a Confidence: line") are
  // near-unoptimizable. Built from the train split only, with per-task
  // arguments (answer keys) redacted (B1.4, defect 3): selection and test
  // judges never reach the proposer. A caller's criteriaText replaces it.
  const benchmarkCriteria = opts.criteriaText ?? describeJudges(split.train);

  // Crash-recovery sweep (D8): revert any pending rows from a prior crashed run.
  revertAllPending(skillsDir, skillName);

  // Load baseline skill text.
  const baselineText = fs.readFileSync(skillFile, 'utf8');
  const baselineSha8 = sha8(baselineText);

  const runSpec: RunSpec = {
    skills_dir: skillsDir,
    benchmark_path: opts.benchmarkPath ?? bench.source_path,
    benchmark_sha8: bench.benchmark_sha8,
    held_out_path: opts.heldOutPath ?? null,
    held_out_sha8: opts.heldOutPath ? sha8(fs.readFileSync(opts.heldOutPath, 'utf8')) : null,
    split: opts.tasks ? [split.train.length, split.sel.length, split.test.length] : opts.split ?? [4, 1, 5],
    mutate: mutateDecision.mutate,
    no_mutate: opts.noMutate,
    allow_mutate_bundled: opts.allowMutateBundled,
    bootstrap_reviewed: opts.bootstrapReviewed,
    optimizer_model: opts.optimizerModel,
    target_model: opts.targetModel,
    judge_model: opts.judgeModel,
    epochs: opts.epochs,
    batch_size: opts.batchSize,
    lr: opts.lr,
    lr_schedule: opts.lrSchedule,
    reflect_max_tokens: reflectCap.maxTokens,
    optimizer_mode: opts.optimizerMode,
  };

  // Resume or init checkpoint. A resume must match what the checkpoint measured.
  const runId = opts.resumeRunId ?? randomUUID();
  let checkpoint = opts.resumeRunId ? loadCheckpoint(skillsDir, skillName, opts.resumeRunId) : null;
  const prior = priorSegments(checkpoint);
  if (checkpoint) {
    assertResumeCompatible(checkpoint, runSpec);
    checkpoint.run_spec = runSpec;
  } else {
    checkpoint = {
      schema: 1,
      run_id: runId,
      skill: skillName,
      skill_sha8: baselineSha8,
      benchmark_sha8: bench.benchmark_sha8,
      optimizer_model: opts.optimizerModel,
      target_model: opts.targetModel,
      judge_model: opts.judgeModel,
      epochs: opts.epochs,
      batch_size: opts.batchSize,
      lr: opts.lr,
      lr_schedule: opts.lrSchedule,
      best_sel_score: 0,
      best_skill_text: baselineText,
      last_completed_epoch: 0,
      last_completed_step: 0,
      next_epoch: 1,
      next_step: 1,
      run_spec: runSpec,
      tally: emptyTally(),
      cumulative_cost_usd: 0,
      models_used: [],
      models_used_scope: 'full_run',
      started_at: new Date().toISOString(),
      last_updated_at: new Date().toISOString(),
    };
    saveCheckpoint(skillsDir, skillName, checkpoint);
  }

  // Initial audit row.
  logEvent({
    kind: 'run_start',
    run_id: runId,
    skill: skillName,
    skill_sha8: baselineSha8,
    benchmark_sha8: bench.benchmark_sha8,
    target_model: opts.targetModel,
    optimizer_model: opts.optimizerModel,
    judge_model: opts.judgeModel,
    epochs: opts.epochs,
    batch_size: opts.batchSize,
    lr: opts.lr,
    lr_schedule: opts.lrSchedule,
    max_cost_usd: opts.maxCostUsd,
    reflect_mode: opts.reflectMode ?? 'both',
    validation_gate_disabled: opts.disableValidationGate === true,
    optimizer_mode: opts.optimizerMode,
    held_out_size: heldOutTasks.length,
    models_plan: modelsPlan,
  } as never);

  // Budget tracker for the whole run. BudgetExhausted propagates as
  // MUST_ABORT through every gateway.chat call.
  //
  // #3516: maxCostUsd === 0 means "no cap" (--no-max-cost / --max-cost-usd 0).
  // Passing undefined switches the tracker's pricing-miss behavior from a
  // hard BudgetExhausted(no_pricing) abort to the legacy warn-once path, so
  // unpriced model ids (openrouter:*, litellm:*) can run at the user's own risk.
  const tracker = new BudgetTracker({
    ...(opts.maxCostUsd > 0 ? { maxCostUsd: opts.maxCostUsd, capSource: opts.maxCostSource ?? 'user' } : {}),
    ...(resolved.pricingOverrides ? { pricingOverrides: resolved.pricingOverrides } : {}),
    label: `skillopt:${skillName}`,
  });

  const scheduleFn = resolveLrSchedule(opts.lrSchedule);
  const totalSteps = opts.epochs * Math.max(1, Math.floor(split.train.length / opts.batchSize));

  // Run the loop inside withBudgetTracker so every nested gateway call composes.
  let finalText = checkpoint.champion ? baselineText : checkpoint.best_skill_text;
  let totalStepsRun = 0;
  // A legacy checkpoint (no tally) that already moved past the baseline text
  // accepted a candidate in an earlier segment.
  const tally = checkpoint.tally
    ?? { ...emptyTally(), accepted_steps: sha8(checkpoint.best_skill_text) !== checkpoint.skill_sha8 ? 1 : 0 };
  // A resume is a fresh attempt (usually after a fix): the trailing fully
  // unusable steps are re-run instead of skipped, and the early-stop streak
  // restarts instead of stopping the resumed run after one step.
  // The rewound cursor is written to the checkpoint together with the reset
  // streak, so a resume that aborts before its first step still retries them.
  const stepsPerEpoch = Math.max(1, Math.floor(split.train.length / opts.batchSize));
  if (opts.resumeRunId) rewindUnusableSteps(checkpoint, tally.unusable_streak, stepsPerEpoch, opts.optimizerMode === 'one-shot-rewrite');
  tally.unusable_streak = 0;
  checkpoint.tally = tally;
  // Tally as of the start of the in-flight step. A handled abort re-runs that
  // step on --resume, so the checkpoint keeps this copy (the receipt keeps the
  // live tally, which includes the calls this segment actually made).
  let stepStartTally: ReflectTally | undefined;
  // First error of the most recent fully unusable step (the receipt's error
  // list is deduped and capped, so it cannot say what caused an early stop).
  let lastUnusableError: string | undefined;
  let earlyStopped = false;
  let skillBodyTruncated: SkillBodyTruncation | undefined;
  let caught: ReturnType<typeof classifyAbortError> | undefined;
  // Hoisted for the receipt (computed inside the budget-tracker closure).
  let baselineSelScore = 0;
  let testScore: number | undefined;
  let baselineTestScore: number | undefined;
  // maxRuntimeMin enforcement (#4119): wall-clock deadline checked between
  // steps AND inside every gate's rollout loop (deadlineMs is threaded into
  // runValidationGate / scoreSkillOnTasks / runHeldOutGate, checked before
  // each individual rollout) — a long batch can no longer blow past
  // --max-runtime by a whole gate's worth of rollouts. A breach throws
  // `skillopt_runtime_exceeded`, caught below → outcome 'aborted'
  // (no partial commit beyond whatever already passed every gate).
  const deadline = Date.now() + opts.maxRuntimeMin * 60_000;

  // Scoring sites (B1.3 / B1.6): every rollout goes through runValidationGate
  // (with the caller's hook when set) and is counted by phase.
  const hook = opts.rolloutHook;
  const rollouts: Record<RolloutPhase, number> = { baseline_sel: 0, train_forward: 0, sel_gate: 0, held_out: 0, test: 0 };
  const selCandidates = new Set<string>();
  const score = async (phase: RolloutPhase, skillText: string, tasks: BenchmarkTask[], o: { bestScore?: number; runsPerTask?: number; epoch?: number; step?: number } = {}) => {
    const runsPerTask = o.runsPerTask ?? VALIDATION_RUNS_PER_TASK;
    const result = await runValidationGate({
      engine: opts.engine,
      operationContext: opts.operationContext,
      candidateSkillText: skillText,
      selSet: tasks,
      bestScore: o.bestScore ?? -1,
      targetModel: opts.targetModel,
      judgeModel: opts.judgeModel,
      runsPerTask,
      deadlineMs: deadline,
      phase,
      epoch: o.epoch ?? 0,
      step: o.step ?? 0,
      ...(hook ? { rolloutHook: hook } : {}),
    });
    rollouts[phase] += tasks.length * runsPerTask;
    if (phase === 'sel_gate' && skillText !== baselineText) selCandidates.add(createHash('sha256').update(skillText).digest('hex'));
    return result;
  };
  // D2: with a held-out set, nothing is written until the single held-out
  // check on the final champion passes; without one, each accepted step is
  // written as before (so an abort keeps the best skill so far on disk).
  const deferWrites = heldOutTasks.length > 0;
  const writeCandidate = (text: string, selScore: number, epoch: number, step: number, edits: EditOp[]) => {
    if (mutateDecision.mutate) {
      acceptCandidate({ skillsDir, skillName, runId, epoch, step, edits, candidateText: text, selScore, delta: selScore - baselineSelScore });
    } else {
      writeProposed(skillsDir, skillName, text);
    }
    finalText = text;
  };
  const promote = (text: string, selScore: number, epoch: number, step: number, edits: EditOp[]) => {
    if (deferWrites) {
      const prev = checkpoint!.champion;
      checkpoint!.champion = { epoch, step, steps: (prev?.steps ?? 0) + 1, edits: [...(prev?.edits ?? []), ...edits] };
    } else {
      writeCandidate(text, selScore, epoch, step, edits);
      tally.accepted_steps += 1;
    }
    checkpoint!.best_sel_score = selScore;
    checkpoint!.best_skill_text = text;
  };
  let heldOut: RunReceipt['held_out'];

  try {
    await withBudgetTracker(tracker, async () => {
      // Baseline eval: score the baseline skill on D_sel to set best_sel_score.
      // We use the FULL validation gate with median-of-3 for a stable baseline.
      const baselineGate = await score('baseline_sel', baselineText, split.sel);
      if (checkpoint!.best_sel_score === 0) {
        // Fresh run; set baseline.
        checkpoint!.best_sel_score = baselineGate.selScore;
        checkpoint!.best_skill_text = baselineText;
        saveCheckpoint(skillsDir, skillName, checkpoint!);
      }
      baselineSelScore = baselineGate.selScore;

      // ── E1 one-shot rewrite (the default) ─────────────────────────────────
      // The loop's first step with a whole-body rewrite instead of edits:
      // forward pass on one train batch, one rewrite call, then the same D12
      // selection gate a loop step faces. A rejected rewrite leaves the skill
      // unchanged. Sets the cursor past the last epoch so the loop below no-ops.
      if (opts.optimizerMode === 'one-shot-rewrite' && resumeCursor(checkpoint!, stepsPerEpoch).epoch <= opts.epochs) {
        stepStartTally = structuredClone(tally);
        totalStepsRun = 1;
        const incumbent = checkpoint!.best_skill_text;
        const { body: incumbentBody, bodyStart } = splitFrontmatter(incumbent);
        const fwd = await score('train_forward', incumbent, split.train.slice(0, opts.batchSize), { runsPerTask: 1, epoch: 1, step: 1 });
        const rewrite = await runOneShotRewrite({
          skillBodyText: incumbentBody,
          successes: fwd.scoredRollouts.filter((r) => r.score >= ROLLOUT_SUCCESS_THRESHOLD),
          failures: fwd.scoredRollouts.filter((r) => r.score < ROLLOUT_SUCCESS_THRESHOLD),
          rejected: [],
          criteria: benchmarkCriteria,
          optimizerModel: opts.optimizerModel,
          maxTokens: reflectCap.maxTokens,
        });
        recordOptimizerStep(tally, {
          calls: 1,
          usableReplies: rewrite.error ? 0 : 1,
          errors: rewrite.error ? [rewrite.error] : [],
          invalidEditsDropped: 0,
        });
        if (rewrite.error) process.stderr.write(`[skillopt] one-shot rewrite produced no candidate (${rewrite.error})\n`);
        if (rewrite.newBody) {
          const candidate = incumbent.slice(0, bodyStart) + rewrite.newBody;
          const gate = await score('sel_gate', candidate, split.sel, { bestScore: checkpoint!.best_sel_score, epoch: 1, step: 1 });
          // Ablation (cat31 config D): disableValidationGate promotes regardless.
          const accepted = opts.disableValidationGate === true || gate.accepted;
          logEvent({
            kind: 'step',
            run_id: runId,
            skill: skillName,
            epoch: 1,
            step: 1,
            sel_score_median: gate.selScore,
            sel_score_runs: gate.perTaskMedians.map((t) => t.median),
            accepted,
            edits_attempted: 1,
            edits_applied: 1,
            delta: gate.selScore - checkpoint!.best_sel_score,
            ...(accepted ? {} : { reason: gate.reason ?? 'rejected' }),
            cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
          } as never);
          if (accepted) promote(candidate, gate.selScore, 1, 1, []);
        }
        advanceCursor(checkpoint!, opts.epochs, stepsPerEpoch, stepsPerEpoch); // no-op the epoch loop below
        bankAccounting(checkpoint!, prior, tracker.snapshot());
        saveCheckpoint(skillsDir, skillName, checkpoint!);
        stepStartTally = undefined;
      }

      // Epoch loop. The cursor names the first step that has not completed.
      const cursor = resumeCursor(checkpoint!, stepsPerEpoch);
      epochs: for (let epoch = cursor.epoch; epoch <= opts.epochs; epoch++) {
        const startStep = epoch === cursor.epoch ? cursor.step : 1;
        const epochStartBest = checkpoint!.best_sel_score;

        for (let step = startStep; step <= stepsPerEpoch; step++) {
          if (Date.now() > deadline) throw new Error(SKILLOPT_RUNTIME_EXCEEDED);
          totalStepsRun += 1;
          stepStartTally = structuredClone(tally);
          const globalStep = (epoch - 1) * stepsPerEpoch + step;
          const lrBudget = scheduleFn(opts.lr, globalStep, totalSteps);

          // Sample a batch from D_train (round-robin to keep deterministic).
          const batchStart = ((step - 1) * opts.batchSize) % split.train.length;
          const batch = split.train.slice(batchStart, batchStart + opts.batchSize);

          // FORWARD PASS: run rollouts on each batch task using current best.
          // runsPerTask=1 — for the train batch we only need a rough partition
          // into successes/failures, not the median-of-3 noise rejection the
          // sel-side gate uses. ScoredRollouts come back via GateResult.
          const forwardGate = await score('train_forward', checkpoint!.best_skill_text, batch, { runsPerTask: 1, epoch, step });
          // Partition into successes vs failures (>= 0.5 threshold). Reflect
          // gets the actual scored trajectories so failure-mode + success-mode
          // analysis can ground in real agent behavior (D7).
          const successes = forwardGate.scoredRollouts.filter((r) => r.score >= ROLLOUT_SUCCESS_THRESHOLD);
          const failures = forwardGate.scoredRollouts.filter((r) => r.score < ROLLOUT_SUCCESS_THRESHOLD);

          stepBody: {
            // BACKWARD PASS: D7 two reflect calls (failures + successes).
            const rejected = loadRejectedBuffer(skillsDir, skillName);
            const reflectResult = await runReflect({
              skillBodyText: checkpoint!.best_skill_text,
              successes,
              failures,
              rejected,
              criteria: benchmarkCriteria,
              optimizerModel: opts.optimizerModel,
              maxTokens: reflectCap.maxTokens,
              ...(opts.reflectMode ? { reflectMode: opts.reflectMode } : {}),
              abortSignal: undefined,
            });
            recordOptimizerStep(tally, reflectResult);
            if (reflectResult.calls > 0 && reflectResult.usableReplies === 0) lastUnusableError = reflectResult.errors[0];
            if (reflectResult.skillBodyTruncated) skillBodyTruncated = reflectResult.skillBodyTruncated;

            // Merge + rank + LR-clip.
            const allEdits: EditOp[] = [...reflectResult.failureEdits, ...reflectResult.successEdits];
            // Drop edits already in rejected buffer.
            const fresh = allEdits.filter((e) => !isRejected(rejected, checkpoint!.best_skill_text, [e]));
            // Apply under LR budget.
            const applied = applyEditBatch(checkpoint!.best_skill_text, fresh, lrBudget);
            // Defect 4: only edits that were actually applied are attributed to
            // the gate's verdict. Edits that failed to apply are filed as
            // apply_failed; edits the LR budget clipped were never tried and
            // stay free to be proposed again.
            const appliedEdits = applied.results.flatMap((r) => (r.outcome === 'applied' ? [r.edit] : []));
            const applyFailed = applied.results.flatMap((r) =>
              r.outcome === 'rejected' && r.detail !== 'lr_budget_exhausted'
                ? [makeRejectedEntry(checkpoint!.best_skill_text, [r.edit], 'apply_failed')]
                : []);

            if (fresh.length === 0) {
              // #4741: the optimizer proposed NOTHING (reflect returned/parsed no
              // edits, or every edit was already in the rejected buffer). This
              // used to fall into the all-rejected branch below ([].every() is
              // true) and log the same 'no_edits_applied' as "gated N candidates
              // and rejected all" — so a whole run of zero candidates read as a
              // real `no_improvement`. Say so per step; nothing to reject-buffer.
              const why = reflectResult.errors[0]
                ? `no_edits_proposed: ${reflectResult.errors[0]}`
                : 'no_edits_proposed';
              process.stderr.write(`[skillopt] epoch ${epoch} step ${step}: optimizer proposed no edits (${why})\n`);
              logEvent({
                kind: 'step',
                run_id: runId,
                skill: skillName,
                epoch,
                step,
                sel_score_median: checkpoint!.best_sel_score,
                sel_score_runs: [],
                accepted: false,
                edits_attempted: 0,
                edits_applied: 0,
                delta: 0,
                reason: why,
                cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
                ...(reflectResult.errors.length > 0 ? { reflect_errors: reflectResult.errors } : {}),
                ...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
              } as never);
              break stepBody;
            }

            if (appliedEdits.length === 0) {
              // Nothing applied; record the apply failures + skip gate.
              saveRejectedBuffer(skillsDir, skillName, applyFailed);
              logEvent({
                kind: 'step',
                run_id: runId,
                skill: skillName,
                epoch,
                step,
                sel_score_median: checkpoint!.best_sel_score,
                sel_score_runs: [],
                accepted: false,
                edits_attempted: fresh.length,
                edits_applied: 0,
                delta: 0,
                reason: 'no_edits_applied',
                cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
                ...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
              } as never);
              break stepBody;
            }

            // VALIDATION GATE (D12 median-of-3 + epsilon=0.05, D4 parallel).
            const gate = await score('sel_gate', applied.newText, split.sel, { bestScore: checkpoint!.best_sel_score, epoch, step });

            // Ablation (cat31 config D): disableValidationGate greedy-accepts any
            // applied edit, bypassing the D12 median+epsilon check. We still ran
            // the gate above to GET the score (so sel_score tracking is honest),
            // but ignore its accept verdict.
            const stepAccepted = opts.disableValidationGate === true ? true : gate.accepted;
            const delta = gate.selScore - checkpoint!.best_sel_score;
            if (stepAccepted) {
              // ACCEPT: D8 commit via version-store (mutate) OR write proposed.md
              // for the --no-mutate / bundled-without-allow paths; with a
              // held-out set the write waits for the final held-out check (D2).
              promote(applied.newText, gate.selScore, epoch, step, appliedEdits);
            } else {
              // REJECT: file the applied edits (and apply failures) in the rejected buffer.
              saveRejectedBuffer(skillsDir, skillName, [
                ...appliedEdits.map((e) => makeRejectedEntry(checkpoint!.best_skill_text, [e], `validation_gate_${gate.reason ?? 'rejected'}`)),
                ...applyFailed,
              ]);
            }
            logEvent({
              kind: 'step',
              run_id: runId,
              skill: skillName,
              epoch,
              step,
              sel_score_median: gate.selScore,
              sel_score_runs: gate.perTaskMedians.map((t) => t.median),
              accepted: stepAccepted,
              edits_attempted: fresh.length,
              edits_applied: appliedEdits.length,
              delta,
              ...(stepAccepted ? {} : { reason: gate.reason ?? 'rejected' }),
              cumulative_cost_usd: tracker.snapshot().cumulativeCostUsd,
              ...(reflectResult.invalidEditsDropped > 0 ? { invalid_edits_dropped: reflectResult.invalidEditsDropped } : {}),
            } as never);
          }

          // Step completed (accepted, rejected or no candidate): advance the
          // cursor and persist accounting so --resume continues after it.
          advanceCursor(checkpoint!, epoch, step, stepsPerEpoch);
          bankAccounting(checkpoint!, prior, tracker.snapshot());
          saveCheckpoint(skillsDir, skillName, checkpoint!);
          stepStartTally = undefined;
          if (shouldEarlyStop(tally)) {
            earlyStopped = true;
            process.stderr.write(
              `[skillopt] stopped after ${EARLY_STOP_UNUSABLE_STEPS} steps of ${unusableClasses(tally)}; remaining budget not spent\n`,
            );
            break epochs;
          }
        }

        // D6 SLOW UPDATE: if no improvement this epoch, propose one meta-edit.
        if (checkpoint!.best_sel_score === epochStartBest) {
          logEvent({
            kind: 'slow_update',
            run_id: runId,
            skill: skillName,
            epoch,
            meta_edit_proposed: false, // Simplified for v1; full meta-update is TODO.
            meta_edit_accepted: false,
          } as never);
        }

      }

      // D2 HELD-OUT: one independent check of the final champion against the
      // baseline, before anything is written. The proposer never sees its
      // verdict, so the held-out set never becomes a second selection set.
      // A refused champion is dropped: the skill stays at the baseline.
      const champion = checkpoint!.champion;
      if (champion) {
        const ho = await runHeldOutGate({
          engine: opts.engine,
          operationContext: opts.operationContext,
          candidateSkillText: checkpoint!.best_skill_text,
          baselineSkillText: baselineText,
          heldOutTasks,
          deadlineMs: deadline,
          targetModel: opts.targetModel,
          judgeModel: opts.judgeModel,
          ...(hook ? { rolloutHook: hook } : {}),
        });
        rollouts.held_out += heldOutTasks.length * VALIDATION_RUNS_PER_TASK * 2;
        heldOut = { baseline_score: ho.baselineScore, candidate_score: ho.candidateScore, passed: ho.passed };
        logEvent({ kind: 'held_out', run_id: runId, skill: skillName, ...heldOut } as never);
        if (ho.passed) {
          writeCandidate(checkpoint!.best_skill_text, checkpoint!.best_sel_score, champion.epoch, champion.step, champion.edits);
          tally.accepted_steps += champion.steps;
        } else {
          process.stderr.write(`[skillopt] held-out check refused the champion (${ho.candidateScore.toFixed(3)} < baseline ${ho.baselineScore.toFixed(3)}); skill unchanged\n`);
          checkpoint!.best_skill_text = baselineText;
          checkpoint!.best_sel_score = baselineSelScore;
        }
        delete checkpoint!.champion;
        saveCheckpoint(skillsDir, skillName, checkpoint!);
      }

      // FINAL TEST: score the best skill AND the baseline on D_test so the
      // receipt carries an honest generalization signal. Skipped when D_test
      // is empty, and after an early stop with nothing accepted (best ===
      // baseline; budget not spent).
      if (split.test.length > 0 && !(earlyStopped && tally.accepted_steps === 0)) {
        testScore = (await score('test', checkpoint!.best_skill_text, split.test)).selScore;
        baselineTestScore = (await score('test', baselineText, split.test)).selScore;
      }
    });
  } catch (err) {
    caught = classifyAbortError(err, opts);
    if (err instanceof BudgetExhausted && err.reason === 'cost') caught.abortDetail += BUDGET_ADMISSION_HINT;
    logEvent({
      kind: 'abort',
      run_id: runId,
      skill: skillName,
      reason: caught.abortReason,
      detail: caught.abortDetail,
    } as never);
    // Handled abort: bank the spend; the cursor stays on the interrupted step
    // so --resume re-runs it, and the saved tally is the step-start copy so
    // that re-run does not count the step's optimizer calls twice.
    bankAccounting(checkpoint, prior, tracker.snapshot());
    if (stepStartTally) checkpoint.tally = stepStartTally;
    saveCheckpoint(skillsDir, skillName, checkpoint);
    checkpoint.tally = tally;
  }

  const { outcome, stopReason, abortReason, abortDetail } = resolveRunOutcome({ caught, tally, earlyStopped });
  if (outcome === 'errored' && !caught) {
    logEvent({ kind: 'abort', run_id: runId, skill: skillName, reason: 'error', detail: abortDetail } as never);
  }

  // Acceptance history (not the final loop state) decides what was written:
  // an abort or early stop after an accepted step still mutated SKILL.md /
  // wrote proposed.md.
  const acceptedAny = tally.accepted_steps > 0;
  const mutatedSkillFile = mutateDecision.mutate && acceptedAny;
  const proposedPath = !mutateDecision.mutate && acceptedAny ? proposedFilePath(skillsDir, skillName) : undefined;
  // An early stop left work undone even when it resolves to no_improvement.
  const checkpointRetained = outcome === 'aborted' || outcome === 'errored' || earlyStopped;
  if (checkpointRetained && !caught) {
    // The final evaluation ran after the last step save; bank its spend too.
    bankAccounting(checkpoint, prior, tracker.snapshot());
    saveCheckpoint(skillsDir, skillName, checkpoint);
  }
  const remediation = buildRemediation([...tally.reflect_errors, ...(abortDetail ? [abortDetail] : [])], abortReason);
  // An early stop that still resolved accepted/no_improvement takes its
  // resume remedy from the trailing unusable errors.
  const failureCode = caught
    ? errorCode(caught.abortDetail)
    : earlyStopped && lastUnusableError ? errorCode(lastUnusableError)
      : abortDetail ? errorCode(abortDetail) : undefined;
  const budget = tracker.snapshot();
  const modelsUsed = buildModelsUsed(budget, prior.rows);

  // Final receipt.
  const receipt: RunReceipt = {
    run_id: runId,
    skill: skillName,
    skill_sha8: baselineSha8,
    benchmark_sha8: bench.benchmark_sha8,
    optimizer_model: opts.optimizerModel,
    target_model: opts.targetModel,
    judge_model: opts.judgeModel,
    epochs: opts.epochs,
    batch_size: opts.batchSize,
    lr: opts.lr,
    lr_schedule: opts.lrSchedule,
    max_cost_usd: opts.maxCostUsd,
    started_at: checkpoint.started_at,
    ended_at: new Date().toISOString(),
    outcome,
    // #3516: abort/error detail rides the receipt so --json consumers and the
    // CLI's stderr summary both see WHY, not just that the run died.
    ...(abortReason !== undefined ? { abort_reason: abortReason } : {}),
    ...(abortDetail !== undefined ? { abort_detail: abortDetail, ...(caught?.pricing ? { no_pricing: caught.pricing } : {}) } : {}),
    stop_reason: stopReason,
    ...(tally.reflect_errors.length > 0 ? { reflect_errors: [...tally.reflect_errors] } : {}),
    ...(tally.invalid_edits_dropped > 0 ? { reflect_invalid_edits_dropped: tally.invalid_edits_dropped } : {}),
    ...(skillBodyTruncated ? { skill_body_truncated: skillBodyTruncated } : {}),
    ...(remediation.length > 0 ? { remediation } : {}),
    reflect_max_tokens: reflectCap.maxTokens,
    reflect_max_tokens_source: reflectCap.source,
    ...(checkpointRetained && !opts.tasks ? { resume_command: buildResumeCommand(skillName, runId, runSpec, failureCode, {
      mode: opts.mode, maxCostUsd: opts.maxCostUsd, maxRuntimeMin: opts.maxRuntimeMin,
      force: opts.force,
      modelsStrict: opts.modelsStrict === true,
    }) } : {}),
    baseline_sel_score: baselineSelScore,
    best_sel_score: checkpoint.best_sel_score,
    ...(testScore !== undefined ? { test_score: testScore } : {}),
    ...(baselineTestScore !== undefined ? { baseline_test_score: baselineTestScore } : {}),
    final_cost_usd: budget.cumulativeCostUsd,
    prior_segments_cost_usd: prior.costUsd,
    models_used: modelsUsed,
    models_used_scope: prior.scope,
    models_plan: modelsPlan,
    models_strict: resolved.strict,
    total_steps: totalStepsRun,
    epochs_completed: checkpoint.last_completed_epoch,
    // Ablation provenance (cat31 replayability) — only when a non-default knob set.
    ...(opts.reflectMode ? { reflect_mode: opts.reflectMode } : {}),
    ...(opts.disableValidationGate ? { validation_gate_disabled: true } : {}),
    optimizer_mode: opts.optimizerMode,
    rollouts,
    candidates_scored_on_sel: selCandidates.size,
    optimizer_calls: tally.reflect_calls,
    cost_scope: hook ? 'optimizer_only' : 'all',
    ...(heldOut ? { held_out: heldOut } : {}),
  };

  logEvent({
    kind: 'run_end',
    run_id: runId,
    skill: skillName,
    outcome,
    epochs_completed: checkpoint.last_completed_epoch,
    total_steps: totalStepsRun,
    best_sel_score: checkpoint.best_sel_score,
    final_cost_usd: budget.cumulativeCostUsd,
    models_used: modelsUsed,
    models_used_scope: prior.scope,
    stop_reason: stopReason,
    ...(abortDetail !== undefined ? { abort_detail: abortDetail } : {}),
  } as never);

  // Clean checkpoint on success (resume not needed).
  if (!checkpointRetained) {
    deleteCheckpoint(skillsDir, skillName, runId);
  }

  return {
    outcome,
    receipt,
    finalText,
    mutatedSkillFile,
    ...(proposedPath ? { proposedPath } : {}),
  };
}

// Re-export parseSplit so the CLI can validate flags without importing
// benchmark.ts directly.
export { parseSplit };