Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.5 KB
Raw Blame
/**
 * takes-quality-eval/runner — orchestrator for one `eval takes-quality run`.
 *
 * Three-model panel scored over a sample of takes. Each cycle dispatches
 * gateway.chat() to all 3 models in parallel via Promise.allSettled (thinking
 * off, #5331), parses the JSON via the shared eval-shared/json-repair, re-asks
 * each malformed slot once with the same model and sample (#5325; a correction
 * replaces the first attempt only when it validates), drops models with
 * incomplete dim scores (codex review #5), aggregates, and stops early on
 * PASS or INCONCLUSIVE.
 *
 * Budget enforcement: if --budget-usd is set, the runner aborts BEFORE the
 * next call's projected cost would exceed the cap. Pricing comes from
 * pricing.ts (registered overrides, then the canonical table). An unpriced
 * model refuses up front under a cap (`no_pricing`, fix: register the rate)
 * and runs with a warning without one.
 *
 * NB: this module is engine-aware (samples takes from DB) but the runner
 * itself doesn't write the receipt — that's `runEval()`'s caller's job
 * (the CLI wires receipt-write after the runner returns).
 */
import type { BrainEngine } from '../engine.ts';
import { chat, thinkingOffOutputCap } from '../ai/gateway.ts';
import { parseModelJSON } from '../eval-shared/json-repair.ts';
import { aggregate, slotFormatFailure, type SlotResult, type AggregateResult } from './aggregate.ts';
import {
  RUBRIC_VERSION,
  rubricSha8,
  renderJudgePrompt,
} from './rubric.ts';
import {
  corpusSha8,
  modelSetSha8,
} from './receipt-name.ts';
import type { TakesQualityCorrection, TakesQualityReceipt } from './receipt.ts';
import { estimateCost, getPricing, unpricedUnderCapError, unpricedWarning } from './pricing.ts';
import type { PricingOverrides } from '../budget/reservation-cost.ts';
import { loadPricingOverrides } from '../budget/budget-tracker.ts';
import { DEFAULT_CYCLES_NONTTY } from '../eval/cycle-default.ts';

/**
 * Three distinct providers (uncorrelated judge blind spots). Every entry MUST
 * be listed in its recipe's chat touchpoint AND priced for takes-quality
 * (pricing.ts) — pinned by test/default-model-panels.test.ts.
 * google:gemini-1.5-pro (retired by Google) and openai:gpt-4o (dropped from
 * the OpenAI recipe's chat list) sat here dead until #3510; gemini-2.0-flash
 * replaced the former and was itself retired before it was ever swept.
 */
/** Receipt `protocol_version`: thinking-off judges plus one correction per malformed slot. */
export const TAKES_QUALITY_PROTOCOL_VERSION = 2;

function correctionPrompt(prompt: string, formatFailure: string): string {
  return `${prompt}\n\nYour previous response failed validation: ${formatFailure}\n` +
    'Return a complete replacement in the requested JSON shape, with a score for every dimension. ' +
    'Do not invent scores when evidence is insufficient.';
}

/** Each judge call's requested output cap; `judgeCallCostUsd` prices the cap `chat()` actually sends. */
export const JUDGE_MAX_TOKENS = 2000;
/** Assumed judge prompt size for the pre-call projection (real usage is counted after each call). */
const PROJECTED_INPUT_TOKENS = 5000;

/** Worst-case price of one judge call (or correction) to `modelId`, or null when unpriced. */
export function judgeCallCostUsd(modelId: string, overrides?: PricingOverrides): number | null {
  return estimateCost(modelId, PROJECTED_INPUT_TOKENS, thinkingOffOutputCap(modelId, JUDGE_MAX_TOKENS), overrides);
}

export const DEFAULT_MODEL_PANEL = [
  'openai:gpt-5.2',
  'anthropic:claude-opus-4-7',
  'google:gemini-2.5-flash',
] as const;

export interface RunOpts {
  /** Sample size from the takes table. Default 100. */
  limit?: number;
  /** Optional deterministic seed; same seed + same corpus = same sample. */
  seed?: number;
  /** Budget cap in USD; runner aborts before next call would exceed. null = no cap. */
  budgetUsd?: number | null;
  /** 'db' samples from takes table; 'fs' walks markdown (not yet wired). */
  source?: 'db' | 'fs';
  /** Filter to slugs starting with this prefix (DB source). */
  slugPrefix?: string | null;
  /** Cycles to run per panel. Default 3 in TTY, 1 in non-TTY. */
  cycles?: number;
  /** Override the default 3-model panel. */
  models?: readonly string[];
  /** Abort signal from the CLI (Ctrl-C, etc.). */
  abortSignal?: AbortSignal;
}

export interface RunResult {
  receipt: TakesQualityReceipt;
  /** True when the budget cap aborted the run before all cycles ran. */
  budgetAborted: boolean;
}

/**
 * Sample N takes from the DB, render them as text the judge model sees.
 * Random sampling via tablesample (Postgres) or ORDER BY random() (PGLite,
 * acceptable on 100K rows for an eval). For deterministic re-runs we'd
 * need a seed; v1 ships non-seeded random sampling.
 */
async function sampleTakesAsText(
  engine: BrainEngine,
  opts: { limit: number; slugPrefix: string | null },
): Promise<{ takesText: string; nTakes: number }> {
  const params: any[] = [opts.limit];
  let where = '';
  if (opts.slugPrefix) {
    params.push(opts.slugPrefix + '%');
    where = `JOIN pages p ON p.id = t.page_id WHERE p.slug LIKE $${params.length}`;
  }
  const rows = await engine.executeRaw<{
    claim: string;
    kind: string;
    holder: string;
    weight: number;
    since_date: string | null;
    source: string | null;
    page_slug: string | null;
  }>(
    `SELECT t.claim, t.kind, t.holder, t.weight, t.since_date, t.source,
            ${opts.slugPrefix ? 'p.slug' : 'NULL'} AS page_slug
       FROM takes t ${where || 'JOIN pages p ON p.id = t.page_id'}
       ${where ? '' : ''}
       ORDER BY random()
       LIMIT $1`,
    params,
  );
  const lines = rows.map(r => {
    const since = r.since_date ?? '—';
    const src = r.source ?? '—';
    const slug = r.page_slug ? ` [page=${r.page_slug}]` : '';
    return `- ${r.kind} | holder=${r.holder} | weight=${r.weight} | since=${since} | src=${src}${slug}\n  ${r.claim}`;
  });
  return { takesText: lines.join('\n'), nTakes: rows.length };
}

async function callOneModel(
  modelId: string,
  systemPrompt: string,
  abortSignal?: AbortSignal,
): Promise<SlotResult & { _usage?: { input_tokens: number; output_tokens: number } }> {
  try {
    const result = await chat({
      model: modelId,
      system: 'You are an evaluation judge. Return strict JSON in the requested shape. Do not include markdown fences in your final response.',
      messages: [{ role: 'user', content: systemPrompt }],
      maxTokens: JUDGE_MAX_TOKENS,
      abortSignal,
      allowFallback: false,
      thinking: 'off',
    });
    try {
      const parsed = parseModelJSON(result.text);
      return {
        ok: true,
        modelId,
        parsed,
        _usage: { input_tokens: result.usage.input_tokens, output_tokens: result.usage.output_tokens },
      } as SlotResult & { _usage: { input_tokens: number; output_tokens: number } };
    } catch (parseErr) {
      return {
        ok: false,
        modelId,
        error: `parse_failed: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
        _usage: { input_tokens: result.usage.input_tokens, output_tokens: result.usage.output_tokens },
      } as SlotResult & { _usage: { input_tokens: number; output_tokens: number } };
    }
  } catch (err) {
    return {
      ok: false,
      modelId,
      error: `provider_error: ${err instanceof Error ? err.message : String(err)}`,
    };
  }
}

export async function runEval(engine: BrainEngine, opts: RunOpts = {}): Promise<RunResult> {
  const limit = opts.limit ?? 100;
  // Library core stays TTY-agnostic (#1784): default to the cost-conservative
  // value. The CLI layer (eval-takes-quality.ts) owns the TTY=3 upgrade + the
  // banner annotation; it always passes an explicit `cycles` down.
  const cycles = opts.cycles ?? DEFAULT_CYCLES_NONTTY;
  const models = opts.models ?? DEFAULT_MODEL_PANEL;
  const budgetUsd = opts.budgetUsd ?? null;
  const source = opts.source ?? 'db';

  if (source === 'fs') {
    throw new Error('fs source not yet wired in v0.32; use --source db');
  }

  // Pre-flight pricing: a cap is enforceable only when every model is priced.
  const overrides = await loadPricingOverrides(engine);
  for (const m of models) {
    if (getPricing(m, overrides)) continue;
    if (budgetUsd !== null) throw unpricedUnderCapError(m, budgetUsd);
    process.stderr.write(`${unpricedWarning(m)}\n`);
  }
  for (const m of models) {
    const cap = thinkingOffOutputCap(m, JUDGE_MAX_TOKENS);
    if (cap === JUDGE_MAX_TOKENS) continue;
    process.stderr.write(
      `[eval takes-quality] note: ${m} cannot turn thinking off; its calls send and are priced at a ${cap}-token output cap\n`,
    );
  }

  // Sample the corpus.
  const { takesText, nTakes } = await sampleTakesAsText(engine, { limit, slugPrefix: opts.slugPrefix ?? null });
  if (nTakes === 0) {
    throw new Error('no takes to evaluate (empty corpus). Run `gbrain extract takes` first or check --slug-prefix.');
  }

  const corpus_sha8 = corpusSha8(takesText);
  const { prompt, sha8: prompt_sha8 } = renderJudgePrompt(takesText);
  const models_sha8 = modelSetSha8(models);
  const rubric_sha8 = rubricSha8();

  const ts = new Date().toISOString();
  const successes_per_cycle: number[] = [];
  let cumulativeCost = 0;
  let budgetAborted = false;
  let lastAggregate: AggregateResult | null = null;
  const corrections: TakesQualityCorrection[] = [];

  for (let cycle = 0; cycle < cycles; cycle++) {
    if (opts.abortSignal?.aborted) {
      process.stderr.write('[eval takes-quality] aborted by signal\n');
      break;
    }

    // Project the worst-case spend for this cycle if budget is set: ~5k
    // input tokens plus the output cap each call sends. The cap fires before
    // the call if the projection would exceed remaining budget. (Real usage
    // is captured post-call from result.usage.)
    if (budgetUsd !== null) {
      let projected = 0;
      for (const m of models) {
        projected += judgeCallCostUsd(m, overrides) ?? 0;
      }
      if (cumulativeCost + projected > budgetUsd) {
        process.stderr.write(
          `[eval takes-quality] budget cap hit: cumulative=$${cumulativeCost.toFixed(2)} ` +
          `projected_next=$${projected.toFixed(2)} cap=$${budgetUsd.toFixed(2)}; aborting before cycle ${cycle + 1}\n`,
        );
        budgetAborted = true;
        break;
      }
    }

    const settled = await Promise.allSettled(
      models.map(m => callOneModel(m, prompt, opts.abortSignal)),
    );
    const slots: SlotResult[] = [];
    for (let i = 0; i < settled.length; i++) {
      const s = settled[i];
      const m = models[i];
      if (s.status === 'fulfilled') {
        const r = s.value as SlotResult & { _usage?: { input_tokens: number; output_tokens: number } };
        if (r._usage) {
          cumulativeCost += estimateCost(m, r._usage.input_tokens, r._usage.output_tokens, overrides) ?? 0;
        }
        slots.push(r);
      } else {
        slots.push({ ok: false, modelId: m, error: `allSettled_rejected: ${String(s.reason)}` });
      }
    }
    // #5325: one same-model correction per malformed slot, priced against the
    // cap before it is sent. Provider errors and valid low scores never re-run.
    for (let i = 0; i < slots.length; i++) {
      const firstError = slotFormatFailure(slots[i]!);
      if (!firstError) continue;
      const m = models[i]!;
      const skipped = opts.abortSignal?.aborted ? 'aborted'
        : budgetUsd !== null && cumulativeCost + (judgeCallCostUsd(m, overrides) ?? 0) > budgetUsd ? 'budget'
        : null;
      if (skipped) {
        corrections.push({ cycle, modelId: m, first_error: firstError, corrected: null, skipped_reason: skipped });
        continue;
      }
      const retry = await callOneModel(m, correctionPrompt(prompt, firstError), opts.abortSignal);
      if (retry._usage) cumulativeCost += estimateCost(m, retry._usage.input_tokens, retry._usage.output_tokens, overrides) ?? 0;
      const retryError = retry.ok ? slotFormatFailure(retry) : retry.error;
      corrections.push({
        cycle, modelId: m, first_error: firstError,
        corrected: retryError === null ? 'valid' : 'invalid',
        ...(retryError !== null ? { corrected_error: retryError } : {}),
      });
      if (retryError === null) slots[i] = retry;
    }
    const agg = aggregate({ slots });
    lastAggregate = agg;
    successes_per_cycle.push(agg.successes);
    if (agg.verdict === 'pass' || agg.verdict === 'inconclusive') break;
  }

  if (!lastAggregate) {
    // Budget aborted before any cycle ran.
    lastAggregate = {
      verdict: 'inconclusive',
      successes: 0,
      failures: 0,
      dimensions: {},
      overall: undefined,
      topImprovements: [],
      errors: [],
      verdictMessage: 'INCONCLUSIVE: budget cap exceeded before any cycle completed.',
    };
  }

  const receipt: TakesQualityReceipt = {
    schema_version: 1,
    ts,
    rubric_version: RUBRIC_VERSION,
    rubric_sha8,
    corpus: { source, n_takes: nTakes, slug_prefix: opts.slugPrefix ?? null, corpus_sha8 },
    prompt_sha8,
    models_sha8,
    models: [...models],
    cycles_run: successes_per_cycle.length,
    successes_per_cycle,
    verdict: lastAggregate.verdict,
    scores: lastAggregate.dimensions,
    overall_score: lastAggregate.overall ?? null,
    cost_usd: roundCost(cumulativeCost),
    improvements: lastAggregate.topImprovements,
    errors: lastAggregate.errors,
    verdictMessage: lastAggregate.verdictMessage,
    protocol_version: TAKES_QUALITY_PROTOCOL_VERSION,
    correction_selection_rule: 'corrected_if_valid',
    ...(corrections.length > 0 ? { corrections } : {}),
  };

  return { receipt, budgetAborted };
}

function roundCost(n: number): number {
  return Math.round(n * 10000) / 10000;
}