Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.8 KB
Raw Blame
/**
 * SkillOpt cost preflight (D3).
 *
 * Estimates the total USD cost of a run BEFORE any LLM call fires.
 * Refuses to start when estimate > --max-cost-usd. In TTY, prompts the
 * user with a 10-second Ctrl-C grace window (mirrors the progressive-batch
 * cost-prompt UX).
 *
 * Cost model (rough but consistent):
 *
 *   Per step:
 *     - batch_size rollouts × target-model price
 *     - 2 reflect calls (D7) × optimizer-model price
 *     - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × target-model price
 *     - sel_size sel-tasks × VALIDATION_RUNS_PER_TASK × judge-model price
 *
 *   Total:
 *     - 1× baseline eval on D_sel
 *     - epochs × steps_per_epoch × per-step cost
 *     - epochs × 1 slow-update reflect call (if no improvement that epoch)
 *     - 1× final test eval on D_test
 *
 * Expected cost is a heuristic. Separately, every run role's largest single
 * call is priced exactly as `BudgetTracker.reserve()` will price it (shared
 * `reservationCostUsd`, effective output caps: the reflect cap, the target's
 * default output cap, `skilloptOutputCap` for each active judge, same
 * pricing overrides, input counted as 0 so the check never over-refuses).
 * When one call's reservation alone exceeds the cap the run can never make
 * that call, so preflight refuses before any spend and names the role and
 * the control that fixes it (`reservation_exceeds_cap`).
 *
 * Prices come from the shared resolver (`priceFor`: operator
 * `pricing.overrides`, the claude-cli → Anthropic sibling, canonical rows),
 * the same chain the run's BudgetTracker uses. A model nothing prices is
 * reported as unpriced (`est_cost_usd: null`, `unpriced_models`), never at an
 * invented rate (#5563 / A-N4). Under a cap the user set, an unpriced model
 * refuses before any spend (`no_pricing`, the shared guidance); under the
 * default cap it runs with a warning, as the tracker does.
 */

import { defaultMaxOutputTokens } from '../ai/gateway.ts';
import { peakRateNote, priceFor, reservationCostUsd, type PricingOverrides } from '../budget/reservation-cost.ts';
import { noPricingGuidance, noPricingMessage, type NoPricingGuidance } from '../budget/no-pricing.ts';
import { skilloptOutputCap } from './output-cap.ts';
import { JUDGE_SITE_MAX_TOKENS } from './score.ts';
import { VALIDATION_RUNS_PER_TASK, type OptimizerMode } from './types.ts';

/** Conservative per-rollout token estimates (input + output). */
const ROLLOUT_INPUT_TOKENS = 3000; // skill + task prompt + tool defs
const ROLLOUT_OUTPUT_TOKENS = 800;
const REFLECT_INPUT_TOKENS = 8000; // skill + trajectories + rejected buffer
const REFLECT_OUTPUT_TOKENS = 1500;
const JUDGE_INPUT_TOKENS = 2000; // rubric + agent output
const JUDGE_OUTPUT_TOKENS = 200;

export interface PreflightOpts {
  epochs: number;
  batchSize: number;
  trainSize: number;
  selSize: number;
  testSize: number;
  optimizerModel: string;
  targetModel: string;
  judgeModel: string;
  maxCostUsd: number;
  /** Where a positive `maxCostUsd` came from: `user` (a flag; the default when unset) refuses an unpriced model, `default` warns. */
  maxCostSource?: 'user' | 'default';
  /**
   * Held-out task count (F11). When > 0, the held-out gate scores baseline +
   * candidate on the held-out set once, on the final champion (D2). Priced
   * priced as if EVERY step accepts (upper bound) so the cap is honest.
   * 0 / omitted = no held-out gate.
   */
  heldOutSize?: number;
  /** When true, print the prompt to stderr + use Ctrl-C grace. Default false (non-TTY). */
  interactive?: boolean;
  /** Effective optimizer output cap; default `defaultMaxOutputTokens(optimizerModel)`. */
  reflectMaxTokens?: number;
  /**
   * Every judge model the run can call (default judge when active + task
   * `judge.model` overrides). Empty = no LLM judge (rule/qrels benchmark).
   * Default `[judgeModel]`.
   */
  judgeModels?: string[];
  /** Operator `pricing.overrides`, the same map the run's BudgetTracker uses. */
  pricingOverrides?: PricingOverrides;
  /** E1: one-shot is one step with one optimizer call; default (absent) prices the loop. */
  optimizerMode?: OptimizerMode;
  /** B1.3: a rollout hook runs (and pays for) every rollout, so only optimizer calls are priced here. */
  rolloutsExternal?: boolean;
}

export type ReservationRole = 'optimizer' | 'target' | 'judge';

/** What one call of a role reserves up front (input counted as 0). */
export interface CallReservation {
  role: ReservationRole;
  model: string;
  max_output_tokens: number;
  /** null = unpriced (left to the tracker's no_pricing contract at run time). */
  reservation_usd: number | null;
}

const RESERVATION_CONTROL: Record<ReservationRole, string> = {
  optimizer: 'lower --reflect-max-tokens / skillopt.reflect_max_tokens, or raise --max-cost-usd',
  target: 'use a cheaper --target-model (models.tier.subagent), or raise --max-cost-usd',
  judge: 'use a cheaper --judge-model (models.tier.reasoning) or task judge.model, or raise --max-cost-usd',
};

export function singleCallReservations(opts: PreflightOpts): CallReservation[] {
  const reserve = (role: ReservationRole, model: string, maxOut: number): CallReservation =>
    ({ role, model, max_output_tokens: maxOut, reservation_usd: reservationCostUsd(model, 'chat', 0, maxOut, opts.pricingOverrides) });
  const optimizer = reserve('optimizer', opts.optimizerModel, opts.reflectMaxTokens ?? defaultMaxOutputTokens(opts.optimizerModel));
  if (opts.rolloutsExternal) return [optimizer];
  return [
    optimizer,
    reserve('target', opts.targetModel, defaultMaxOutputTokens(opts.targetModel)),
    ...(opts.judgeModels ?? [opts.judgeModel]).map((m) => reserve('judge', m, skilloptOutputCap(m, JUDGE_SITE_MAX_TOKENS))),
  ];
}

function largestReservation(opts: PreflightOpts): CallReservation | undefined {
  let best: CallReservation | undefined;
  for (const r of singleCallReservations(opts)) {
    if (r.reservation_usd !== null && (!best || r.reservation_usd > best.reservation_usd!)) best = r;
  }
  return best;
}

export interface PreflightEstimate {
  steps_per_epoch: number;
  total_steps: number;
  rollout_calls: number;
  reflect_calls: number;
  judge_calls: number;
  est_input_tokens: number;
  est_output_tokens: number;
  /** null when a model the run calls has no price (see `unpriced_models`). */
  est_cost_usd: number | null;
  /** Per-model breakdown for the audit; null for an unpriced model. */
  per_model_cost_usd: Record<string, number | null>;
  /** Models the run can call that nothing prices. */
  unpriced_models: string[];
  /** True when est_cost_usd > maxCostUsd (caller should refuse or prompt). */
  exceeds_cap: boolean;
  /** The priciest single call any role will reserve (separate from expected cost). */
  largest_reservation?: CallReservation;
}

export interface PreflightResult {
  estimate: PreflightEstimate;
  /** When false, caller should abort. When true, run may proceed. */
  proceed: boolean;
  /** Reason for abort, if proceed=false. */
  abort_reason?: string;
  /** Machine code for the abort: one call can never fit the cap, the expected total exceeds it, or a user cap meets an unpriced model. */
  abort_code?: 'reservation_exceeds_cap' | 'cost_cap_exceeded' | 'no_pricing';
  /** Set with abort_code 'no_pricing': the lookup-and-register guidance. */
  no_pricing?: NoPricingGuidance;
}

export function estimateCost(opts: PreflightOpts): PreflightEstimate {
  const oneShot = opts.optimizerMode === 'one-shot-rewrite';
  const stepsPerEpoch = oneShot ? 1 : Math.max(1, Math.floor(opts.trainSize / opts.batchSize));
  const totalSteps = oneShot ? 1 : opts.epochs * stepsPerEpoch;

  // Per-step counts.
  const rolloutsPerStep = oneShot ? Math.min(opts.batchSize, opts.trainSize) : opts.batchSize;
  const reflectsPerStep = oneShot ? 1 : 2; // D7: two reflect calls; one-shot: one rewrite
  const sel_runs_per_step = opts.selSize * VALIDATION_RUNS_PER_TASK;

  // D2 held-out: baseline + candidate scored once, on the final champion.
  const heldOutSize = opts.heldOutSize ?? 0;
  const heldOutRollouts = heldOutSize > 0 ? heldOutSize * VALIDATION_RUNS_PER_TASK * 2 : 0;

  // Cumulative counts across the whole run.
  const rollout_calls = totalSteps * rolloutsPerStep
    + opts.selSize * VALIDATION_RUNS_PER_TASK // baseline sel eval
    + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step sel validation
    + opts.testSize * 2 // final test eval (best + baseline)
    + heldOutRollouts; // D2 held-out check (baseline+candidate, once)
  const reflect_calls = totalSteps * reflectsPerStep
    + (oneShot ? 0 : opts.epochs); // slow-update meta calls
  const judgeModels = opts.judgeModels ?? [opts.judgeModel];
  const judgeCallsRaw = opts.selSize // baseline (1 per task; median-of-3 is in the rollout count already? — no, judge runs per rollout)
    + opts.selSize * VALIDATION_RUNS_PER_TASK * totalSteps // per-step validation
    + opts.testSize * 2 // final test judges (best + baseline)
    + heldOutRollouts; // F11 held-out judges (1 per held-out rollout)
  const judge_calls = judgeModels.length > 0 && !opts.rolloutsExternal ? judgeCallsRaw : 0;

  // Cost per call type. Judge calls price at the priciest judge the run can reach.
  const lookup = (model: string) => priceFor(model, 'chat', opts.pricingOverrides)?.pricing ?? null;
  const unpriced = [...new Set([...(opts.rolloutsExternal ? [] : [opts.targetModel]), opts.optimizerModel, ...(judge_calls > 0 ? judgeModels : [])])]
    .filter((m) => lookup(m) === null);
  const zero = { input: 0, output: 0 };
  const targetPrice = lookup(opts.targetModel) ?? zero;
  const optimizerPrice = lookup(opts.optimizerModel) ?? zero;
  const judgePrice = judgeModels.map((m) => lookup(m) ?? zero).reduce(
    (a, b) => (b.input + b.output > a.input + a.output ? b : a),
    zero,
  );
  const unpricedJudge = judge_calls > 0 && judgeModels.some((m) => unpriced.includes(m));

  const rolloutCost = opts.rolloutsExternal ? 0 : rollout_calls * (
    (ROLLOUT_INPUT_TOKENS * targetPrice.input) / 1_000_000
    + (ROLLOUT_OUTPUT_TOKENS * targetPrice.output) / 1_000_000
  );
  const reflectCost = reflect_calls * (
    (REFLECT_INPUT_TOKENS * optimizerPrice.input) / 1_000_000
    + (REFLECT_OUTPUT_TOKENS * optimizerPrice.output) / 1_000_000
  );
  const judgeCost = judge_calls * (
    (JUDGE_INPUT_TOKENS * judgePrice.input) / 1_000_000
    + (JUDGE_OUTPUT_TOKENS * judgePrice.output) / 1_000_000
  );

  // D11 prompt caching gives ~50% discount on stable layers. Apply
  // conservatively (assume 50% of optimizer + judge tokens are cached).
  const cachedReflectCost = reflectCost * 0.6;
  const cachedJudgeCost = judgeCost * 0.6;

  const total = unpriced.length > 0 ? null : rolloutCost + cachedReflectCost + cachedJudgeCost;
  void sel_runs_per_step;
  const largest = largestReservation(opts);

  return {
    steps_per_epoch: stepsPerEpoch,
    total_steps: totalSteps,
    rollout_calls,
    reflect_calls,
    judge_calls,
    est_input_tokens: rollout_calls * ROLLOUT_INPUT_TOKENS + reflect_calls * REFLECT_INPUT_TOKENS + judge_calls * JUDGE_INPUT_TOKENS,
    est_output_tokens: rollout_calls * ROLLOUT_OUTPUT_TOKENS + reflect_calls * REFLECT_OUTPUT_TOKENS + judge_calls * JUDGE_OUTPUT_TOKENS,
    est_cost_usd: total,
    per_model_cost_usd: {
      [opts.targetModel]: unpriced.includes(opts.targetModel) ? null : rolloutCost,
      [opts.optimizerModel]: unpriced.includes(opts.optimizerModel) ? null : cachedReflectCost,
      [opts.judgeModel]: unpricedJudge ? null : cachedJudgeCost,
    },
    unpriced_models: unpriced,
    // #3516: maxCostUsd === 0 means uncapped (--no-max-cost) — never refuse.
    exceeds_cap: opts.maxCostUsd > 0 && total !== null && total > opts.maxCostUsd,
    ...(largest ? { largest_reservation: largest } : {}),
  };
}

/**
 * Render a human-readable preflight summary to stderr. Caller may follow
 * with a Ctrl-C grace prompt in TTY mode (see runPreflightPrompt).
 */
export function formatPreflightReport(est: PreflightEstimate, opts: PreflightOpts): string {
  return [
    opts.optimizerMode === 'one-shot-rewrite'
      ? `[skillopt] Cost estimate for one rewrite (one-shot; --loop runs the iterative optimizer):`
      : `[skillopt] Cost estimate for ${opts.epochs} epochs × ${est.steps_per_epoch} steps × ${opts.batchSize} rollouts:`,
    `  Rollouts:   ${est.rollout_calls.toLocaleString()} calls`,
    `  Reflects:   ${est.reflect_calls.toLocaleString()} calls`,
    `  Judges:     ${est.judge_calls.toLocaleString()} calls`,
    `  Tokens:     ~${(est.est_input_tokens / 1000).toFixed(0)}K in / ~${(est.est_output_tokens / 1000).toFixed(0)}K out`,
    `  Est. cost:  ${est.est_cost_usd === null ? `unpriced (no price for ${est.unpriced_models.join(', ')}; register a rate with gbrain pricing set)` : `$${est.est_cost_usd.toFixed(2)}${peakRateNote([opts.targetModel, opts.optimizerModel, ...(opts.judgeModels ?? [opts.judgeModel])])}`} (cap: ${opts.maxCostUsd > 0 ? `$${opts.maxCostUsd.toFixed(2)}${opts.maxCostSource === 'default' ? ' default' : ''}` : 'uncapped'})`,
    est.largest_reservation
      ? `  Per call:   largest single call reserves $${est.largest_reservation.reservation_usd!.toFixed(2)} (${est.largest_reservation.role} ${est.largest_reservation.model}, ${est.largest_reservation.max_output_tokens} output tokens)`
      : '',
    est.exceeds_cap ? `  WARNING:    estimate exceeds --max-cost-usd cap.` : '',
  ].filter(Boolean).join('\n');
}

/**
 * Decision wrapper. Returns {proceed: false, abort_reason} when the
 * estimate over-shoots the cap (caller exits 2). Returns {proceed: true}
 * otherwise. Interactive=true callers should print formatPreflightReport
 * + a Ctrl-C grace window separately.
 */
export function preflight(opts: PreflightOpts): PreflightResult {
  const estimate = estimateCost(opts);
  const unpriced = estimate.unpriced_models[0];
  if (opts.maxCostUsd > 0 && (opts.maxCostSource ?? 'user') === 'user' && unpriced) {
    const guidance = noPricingGuidance(unpriced, 'chat');
    return {
      estimate,
      proceed: false,
      abort_code: 'no_pricing',
      abort_reason: noPricingMessage(guidance, { label: 'skillopt', capUsd: opts.maxCostUsd }),
      no_pricing: guidance,
    };
  }
  const big = estimate.largest_reservation;
  if (opts.maxCostUsd > 0 && big && big.reservation_usd! > opts.maxCostUsd) {
    return {
      estimate,
      proceed: false,
      abort_code: 'reservation_exceeds_cap',
      abort_reason: `reservation_exceeds_cap: a single ${big.role} call reserves $${big.reservation_usd!.toFixed(2)} (${big.model}, ${big.max_output_tokens} output tokens), which exceeds the $${opts.maxCostUsd.toFixed(2)} cap on its own, so it can never run. Fix: ${RESERVATION_CONTROL[big.role]}.`,
    };
  }
  if (estimate.exceeds_cap) {
    return {
      estimate,
      proceed: false,
      abort_code: 'cost_cap_exceeded',
      abort_reason: `estimated cost $${estimate.est_cost_usd!.toFixed(2)} exceeds --max-cost-usd $${opts.maxCostUsd.toFixed(2)}. Raise the cap with --max-usd ${Math.ceil(estimate.est_cost_usd!)} or reduce --epochs/--batch-size.`,
    };
  }
  return { estimate, proceed: true };
}