Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
24.1 KB
Raw Blame
/**
 * `--decide` eval plumbing shared by `gbrain eval longmemeval`, `eval
 * brainbench` and `eval retrieval-quality` (System One matched runs: arms
 * differ only in the slot under test).
 *
 *   --decide <slot>=<off|on|shadow>   repeatable; merged over an inherited
 *                                     GBRAIN_DECIDE_SLOTS (flags win), then
 *                                     set for the process with the eval
 *                                     override enabled
 *   --decide-provider <id>            default typesafe:jev-1.13.0
 *   --decide-calibration <file|ref:<id>>   `decide calibrate --json` /
 *                                     `decide calibrations list --json` output
 *                                     (object, array or JSONL) or a reference id
 *   --decide-threshold <slot>=<x>     operator override (the gate still applies)
 *   --decide-force-on <slot>          bypass the action-precision gate (loud)
 *   --decide-dataset <jsonl>          the frozen labelled split: calibrations
 *                                     must match its split_hash and be
 *                                     calibrate-only (else split_mismatch)
 *
 * Throwaway benchmark brains (LongMemEval, BrainBench) are configured so the
 * slot can act: provider, consent for the slot's egress classes,
 * decide.egress.private=allow (public benchmark data only), awaited shadow,
 * thresholds, force_on, and the supplied calibrations inserted and adopted.
 * retrieval-quality runs on the operator's brain: nothing is written there;
 * a missing provider, key or consent refuses with the catalogued line.
 *
 * All-off contract: with no non-off slot (flags and inherited env combined),
 * prepareDecideEval returns null and the command's output is byte-identical.
 */
import { readFileSync } from 'node:fs';
import { basename } from 'node:path';
import type { BrainEngine } from '../core/engine.ts';
import {
  DEFAULT_TYPESAFE_PROVIDER, enableDecideEvalOverride, isValidProvider, parseEvalSlots, readDecideConfig,
} from '../core/ai/decide/config.ts';
import { parseDatasetJsonl, splitHash, type DatasetItem } from '../core/ai/decide/dataset.ts';
import { hasTypesafeKey } from '../core/ai/decide/index.ts';
import { percentile } from '../core/ai/decide/judge-agreement.ts';
import { refusalLine } from '../core/ai/decide/outcomes.ts';
import { REFERENCE_CALIBRATIONS } from '../core/ai/decide/reference-calibrations.ts';
import { SLOT_SPECS } from '../core/ai/decide/slots.ts';
import { flushDecideWrites, insertCalibration, storeQualification, type NewCalibration } from '../core/ai/decide/store.ts';
import { DECIDE_SLOTS, type DecideMode, type DecideSlot } from '../core/ai/decide/types.ts';
import { usageCostUsd } from '../core/budget/reservation-cost.ts';
import { resetDecideSearchCache, type DecideSearchMeta, type DecideSlotMeta } from '../core/search/decide-stage.ts';

export interface DecideEvalOptions {
  slots: Partial<Record<DecideSlot, DecideMode>>;
  provider?: string;
  calibrations: string[];
  thresholds: Partial<Record<DecideSlot, number>>;
  forceOn: DecideSlot[];
  datasetPath?: string;
}

export function newDecideEvalOptions(): DecideEvalOptions {
  return { slots: {}, calibrations: [], thresholds: {}, forceOn: [] };
}

/** Flag table (name, value placeholder, help lines) each command renders in its own help. */
export const DECIDE_EVAL_FLAGS: ReadonlyArray<{ name: string; arg: string; help: string[] }> = [
  { name: '--decide', arg: 'SLOT=MODE', help: [
    'System One arm: set a decide slot to off|on|shadow for this run (repeatable;',
    'flags win over an inherited GBRAIN_DECIDE_SLOTS). Slots: ' + DECIDE_SLOTS.join(', ') + '.'] },
  { name: '--decide-provider', arg: 'ID', help: [`Decide provider for the run (default ${DEFAULT_TYPESAFE_PROVIDER}).`] },
  { name: '--decide-calibration', arg: 'FILE|ref:ID', help: [
    'Calibration(s) for on slots: `gbrain decide calibrate --slot X --json` output',
    '(object, array or JSONL) or a reference id (repeatable).'] },
  { name: '--decide-threshold', arg: 'SLOT=X', help: ['Operator threshold override (the action-precision gate still applies).'] },
  { name: '--decide-force-on', arg: 'SLOT', help: ['Bypass the action-precision gate for SLOT (recorded in run_config).'] },
  { name: '--decide-dataset', arg: 'JSONL', help: [
    'Frozen labelled dataset (gbrain decide dataset): calibrations must match its',
    'split_hash and be calibrate-only (else split_mismatch, exit 1).'] },
];

const isSlot = (s: string): s is DecideSlot => (DECIDE_SLOTS as readonly string[]).includes(s);

function slotEq(flag: string, value: string): [DecideSlot, string] {
  const eq = value.indexOf('=');
  const slot = eq > 0 ? value.slice(0, eq).trim() : '';
  if (!isSlot(slot)) throw new Error(`${flag} takes SLOT=VALUE with SLOT one of ${DECIDE_SLOTS.join(', ')} (got: ${value})`);
  return [slot, value.slice(eq + 1).trim()];
}

/** Apply one decide flag; returns false when `flag` is not a decide flag. Throws on an invalid value. */
export function applyDecideEvalFlag(o: DecideEvalOptions, flag: string, value: string): boolean {
  switch (flag) {
    case '--decide': {
      const [slot, mode] = slotEq(flag, value);
      if (mode !== 'off' && mode !== 'on' && mode !== 'shadow') throw new Error(`--decide ${slot} mode must be off|on|shadow (got: ${mode})`);
      o.slots[slot] = mode;
      return true;
    }
    case '--decide-provider':
      if (!isValidProvider(value) || value === 'none') throw new Error(`--decide-provider must be typesafe:<model> or llm:<provider:model> (got: ${value})`);
      o.provider = value;
      return true;
    case '--decide-calibration':
      o.calibrations.push(value);
      return true;
    case '--decide-threshold': {
      const [slot, raw] = slotEq(flag, value);
      const x = Number(raw);
      if (!raw || !Number.isFinite(x) || x < 0 || x > 1) throw new Error(`--decide-threshold ${slot} must be a number from 0 to 1 (got: ${raw})`);
      if (!SLOT_SPECS[slot].thresholded) throw new Error(`--decide-threshold: ${slot} has no threshold`);
      o.thresholds[slot] = x;
      return true;
    }
    case '--decide-force-on':
      if (!isSlot(value)) throw new Error(`--decide-force-on takes a slot: ${DECIDE_SLOTS.join(', ')} (got: ${value})`);
      if (!o.forceOn.includes(value)) o.forceOn.push(value);
      return true;
    case '--decide-dataset':
      o.datasetPath = value;
      return true;
    default:
      return false;
  }
}

/** For hand-rolled parsers: split decide flags (and their values) out of argv. */
export function extractDecideEvalFlags(argv: readonly string[]): { decide: DecideEvalOptions; rest: string[] } {
  const decide = newDecideEvalOptions();
  const rest: string[] = [];
  const names = new Set(DECIDE_EVAL_FLAGS.map((f) => f.name));
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i]!;
    const eq = a.indexOf('=');
    const name = a.startsWith('--') && eq > 0 ? a.slice(0, eq) : a;
    if (!names.has(name)) { rest.push(a); continue; }
    const value = name !== a ? a.slice(eq + 1) : argv[++i];
    if (value === undefined || value === '' || (name === a && value.startsWith('--'))) throw new Error(`${name} requires a value`);
    applyDecideEvalFlag(decide, name, value);
  }
  return { decide, rest };
}

// ---------------------------------------------------------------------------
// Run preparation (validation, env, calibrations, split holdout)
// ---------------------------------------------------------------------------

/** A refused run: print `message` and exit non-zero. */
export class DecideEvalRefusal extends Error {
  constructor(readonly reason: string, message: string) {
    super(message);
    this.name = 'DecideEvalRefusal';
  }
}

export interface EvalCalibration {
  /** Where it came from: `ref:<id>` or `<file basename>#<n>`. */
  id: string;
  ref?: string;
  row?: NewCalibration & { action_precision_lb?: number | null; qualification?: string | null; policy_fingerprint?: string | null };
  slot: DecideSlot;
  call_site: string;
  provider: string;
  model_resolved: string;
  threshold: number;
  split_hash: string | null;
  calibrate_only: boolean;
}

export interface DecideEvalRun {
  /** Non-off slots and their modes (flags over inherited env). */
  slots: Partial<Record<DecideSlot, DecideMode>>;
  /** The GBRAIN_DECIDE_SLOTS value set for this process. */
  envValue: string;
  provider: string;
  calibrations: EvalCalibration[];
  thresholds: Partial<Record<DecideSlot, number>>;
  forceOn: DecideSlot[];
  /** Frozen dataset: its eval-half families and per-slot split hashes. */
  dataset: { name: string; evalFamilies: Set<string>; splitHashes: Partial<Record<DecideSlot, string>> } | null;
  /** search.* pins the run adds (`--decide rerank=on` pins the Jev reranker). */
  searchPins: Record<string, string>;
  /** The run-metadata block (run_config.decide / result.decide). */
  runConfig: Record<string, unknown>;
}

function str(v: unknown): string | null {
  return typeof v === 'string' && v.length > 0 ? v : null;
}

function numOrNull(v: unknown): number | null {
  return typeof v === 'number' && Number.isFinite(v) ? v : null;
}

/** Parse calibration JSON: one object, an array, or JSONL. */
export function parseCalibrationText(text: string): Record<string, unknown>[] {
  const t = text.trim();
  if (!t) return [];
  try {
    const whole = JSON.parse(t) as unknown;
    return (Array.isArray(whole) ? whole : [whole]) as Record<string, unknown>[];
  } catch {
    return t.split('\n').filter((l) => l.trim()).map((l, n) => {
      try { return JSON.parse(l) as Record<string, unknown>; } catch { throw new Error(`calibration line ${n + 1}: not JSON`); }
    });
  }
}

function calibrationFromJson(raw: Record<string, unknown>, id: string): EvalCalibration {
  const slot = str(raw.slot);
  const threshold = numOrNull(raw.threshold);
  const need = { call_site: str(raw.call_site), provider: str(raw.provider), model_resolved: str(raw.model_resolved), pack_shape: str(raw.pack_shape) };
  if (!slot || !isSlot(slot)) throw new Error(`calibration ${id}: slot must be one of ${DECIDE_SLOTS.join(', ')}`);
  for (const [k, v] of Object.entries(need)) if (!v) throw new Error(`calibration ${id}: ${k} is required (use gbrain decide calibrate --json output)`);
  if (threshold === null || threshold < 0 || threshold > 1) throw new Error(`calibration ${id}: threshold must be a number from 0 to 1`);
  const row: EvalCalibration['row'] = {
    slot, call_site: need.call_site!, provider: need.provider!, model_resolved: need.model_resolved!, threshold,
    min_keep: numOrNull(raw.min_keep), metric: str(raw.metric) ?? 'f1', metric_value: numOrNull(raw.metric_value), ece: numOrNull(raw.ece),
    retest_sd: numOrNull(raw.retest_sd), repack_sd: numOrNull(raw.repack_sd), n: numOrNull(raw.n) ?? 0,
    dataset_hash: str(raw.dataset_hash), split_hash: str(raw.split_hash), calibrate_ids_hash: str(raw.calibrate_ids_hash),
    calibrate_only: raw.calibrate_only === true, pack_shape: need.pack_shape!, notes: `eval import ${id}`,
    action_precision_lb: numOrNull(raw.action_precision_lb), qualification: str(raw.qualification), policy_fingerprint: str(raw.policy_fingerprint),
  };
  return { id, row, slot, call_site: row.call_site, provider: row.provider, model_resolved: row.model_resolved, threshold, split_hash: row.split_hash, calibrate_only: row.calibrate_only };
}

function loadCalibrations(specs: readonly string[]): EvalCalibration[] {
  return specs.flatMap((spec) => {
    if (spec.startsWith('ref:')) {
      const r = REFERENCE_CALIBRATIONS.find((x) => x.id === spec.slice(4));
      if (!r) throw new Error(`--decide-calibration ${spec}: unknown reference calibration (this build ships ${REFERENCE_CALIBRATIONS.length})`);
      return [{ id: spec, ref: spec, slot: r.slot, call_site: r.call_site, provider: r.provider, model_resolved: r.model_resolved, threshold: r.threshold, split_hash: r.split_hash, calibrate_only: true }];
    }
    const rows = parseCalibrationText(readFileSync(spec, 'utf8'));
    if (rows.length === 0) throw new Error(`--decide-calibration ${spec}: no calibration rows`);
    return rows.map((raw, n) => calibrationFromJson(raw, `${basename(spec)}#${n + 1}`));
  });
}

/**
 * Validate the flags, merge them over an inherited GBRAIN_DECIDE_SLOTS, set
 * the env and opt this process in, load calibrations and enforce the split
 * holdout. Null when no slot is on or shadow (the all-off path). Throws Error
 * on bad flags and DecideEvalRefusal on split_mismatch.
 */
export function prepareDecideEval(o: DecideEvalOptions, opts: { command: string; throwaway: boolean }): DecideEvalRun | null {
  const inherited = process.env.GBRAIN_DECIDE_SLOTS || null;
  const merged = { ...parseEvalSlots(inherited ?? undefined), ...o.slots };
  const slots = Object.fromEntries(DECIDE_SLOTS.filter((s) => merged[s] && merged[s] !== 'off').map((s) => [s, merged[s]!])) as Partial<Record<DecideSlot, DecideMode>>;
  const active = Object.keys(slots) as DecideSlot[];
  const extras = o.calibrations.length > 0 || Object.keys(o.thresholds).length > 0 || o.forceOn.length > 0 || o.datasetPath !== undefined || o.provider !== undefined;
  if (active.length === 0) {
    if (extras) throw new Error(`--decide-* options need at least one --decide <slot>=on|shadow`);
    return null;
  }
  if (!opts.throwaway && (o.calibrations.length > 0 || Object.keys(o.thresholds).length > 0 || o.forceOn.length > 0)) {
    throw new Error(`${opts.command} runs on your connected brain and never writes its config: --decide-calibration, --decide-threshold and --decide-force-on are for benchmark brains (calibrate/adopt with gbrain decide instead)`);
  }
  const provider = o.provider ?? DEFAULT_TYPESAFE_PROVIDER;
  const calibrations = loadCalibrations(o.calibrations);
  let dataset: DecideEvalRun['dataset'] = null;
  if (o.datasetPath) {
    const items: DatasetItem[] = parseDatasetJsonl(readFileSync(o.datasetPath, 'utf8'));
    const splitHashes = Object.fromEntries(DECIDE_SLOTS.filter((s) => items.some((i) => i.slot === s)).map((s) => [s, splitHash(items.filter((i) => i.slot === s))])) as Partial<Record<DecideSlot, string>>;
    dataset = { name: basename(o.datasetPath), evalFamilies: new Set(items.filter((i) => i.split === 'eval').map((i) => i.family)), splitHashes };
  }
  for (const c of calibrations) {
    const expected = dataset?.splitHashes[c.slot];
    if (!c.calibrate_only || (dataset && c.split_hash !== expected)) {
      const why = !c.calibrate_only ? 'calibrate_only is false' : `split_hash ${c.split_hash ?? 'none'} differs from ${dataset!.name} (${expected ?? `no ${c.slot} items`})`;
      throw new DecideEvalRefusal('split_mismatch', `${opts.command}: calibration ${c.id} refused: ${why}.\n${refusalLine('split_mismatch', c.slot)}`);
    }
    if (c.provider !== provider) process.stderr.write(`[decide] WARN calibration ${c.id} is for ${c.provider}, the run uses ${provider}: it will not resolve a threshold\n`);
  }
  if (calibrations.length > 0 && !dataset) process.stderr.write(`[decide] WARN split holdout not verified: pass --decide-dataset <jsonl> so calibrations are checked against the eval split\n`);
  for (const s of o.forceOn) process.stderr.write(`[decide] WARN force_on ${s}: the action-precision gate is BYPASSED for this eval run (recorded in run_config)\n`);
  const envValue = active.map((s) => `${s}=${slots[s]}`).join(',');
  process.env.GBRAIN_DECIDE_SLOTS = envValue;
  enableDecideEvalOverride();
  const rerankModel = provider.startsWith('typesafe:') ? provider : DEFAULT_TYPESAFE_PROVIDER;
  const searchPins: Record<string, string> = slots.rerank === 'on' ? { 'search.reranker.enabled': 'true', 'search.reranker.model': rerankModel } : {};
  const runConfig: Record<string, unknown> = {
    slots, gbrain_decide_slots: envValue, inherited_env: inherited, provider,
    calibrations: calibrations.map((c) => ({ id: c.id, slot: c.slot, call_site: c.call_site, provider: c.provider, model_resolved: c.model_resolved, threshold: c.threshold, split_hash: c.split_hash })),
    thresholds: o.thresholds, force_on: o.forceOn,
    dataset: dataset ? { name: dataset.name, split_hash: dataset.splitHashes, eval_families: dataset.evalFamilies.size } : null,
    split_verified: calibrations.length === 0 ? null : dataset !== null,
    ...(slots.rerank === 'on' ? { rerank_pinned: rerankModel } : {}),
    ...(opts.throwaway ? { shadow_wait: active.some((s) => slots[s] === 'shadow') } : {}),
  };
  return { slots, envValue, provider, calibrations, thresholds: o.thresholds, forceOn: o.forceOn, dataset, searchPins, runConfig };
}

/** Config keys a throwaway benchmark brain needs so every active slot can act. */
export function benchmarkBrainConfig(run: DecideEvalRun, opts: { includeSearchPins?: boolean } = {}): Array<[string, string]> {
  const writes: Array<[string, string]> = [['decide.provider', run.provider], ['decide.egress.private', 'allow']];
  for (const slot of Object.keys(run.slots) as DecideSlot[]) {
    for (const c of SLOT_SPECS[slot].egressClasses) writes.push([`decide.egress.typesafe.${c}`, 'allow']);
    if (run.slots[slot] === 'shadow') writes.push([`decide.slots.${slot}.shadow_wait`, 'on'], [`decide.slots.${slot}.shadow_sample`, '1']);
  }
  for (const [slot, x] of Object.entries(run.thresholds)) writes.push([`decide.slots.${slot}.threshold`, String(x)]);
  for (const slot of run.forceOn) writes.push([`decide.slots.${slot}.force_on`, 'true']);
  if (opts.includeSearchPins) writes.push(...Object.entries(run.searchPins));
  return [...new Map(writes).entries()];
}

/** Configure one throwaway benchmark brain: config keys, then insert and adopt the calibrations. */
export async function configureDecideBrain(engine: BrainEngine, run: DecideEvalRun, opts: { includeSearchPins?: boolean } = {}): Promise<void> {
  for (const [k, v] of benchmarkBrainConfig(run, opts)) await engine.setConfig(k, v);
  for (const c of run.calibrations) {
    if (c.ref) { await engine.setConfig(`decide.slots.${c.slot}.calibration`, c.ref); continue; }
    const { action_precision_lb, qualification, policy_fingerprint, ...row } = c.row!;
    const id = await insertCalibration(engine, row);
    if (action_precision_lb !== null && action_precision_lb !== undefined) {
      await storeQualification(engine, id, { action_precision_lb, qualification: qualification ?? '{}', policy_fingerprint: policy_fingerprint ?? '' });
    }
    await engine.setConfig(`decide.slots.${c.slot}.calibration`, `local:${id}`);
  }
  resetDecideSearchCache();
}

/**
 * retrieval-quality on the operator's brain: the catalogued refusal line when
 * an active slot has no provider, key or consent (config is never written).
 */
export function operatorBrainRefusal(snapshot: Record<string, string>, run: DecideEvalRun): string | null {
  const cfg = readDecideConfig(snapshot, { evalSlots: run.envValue });
  for (const slot of Object.keys(run.slots) as DecideSlot[]) {
    const provider = slot === 'rerank' ? cfg.provider : cfg.slots[slot].provider;
    if (provider === 'none') return refusalLine('no_provider', slot);
    if ((provider.startsWith('typesafe:') || slot === 'rerank') && !hasTypesafeKey()) return refusalLine('no_key', slot);
    if (slot === 'rerank' || !provider.startsWith('typesafe:')) continue;
    if (SLOT_SPECS[slot].egressClasses.some((c) => !cfg.consent[c])) return refusalLine('egress_class_denied', slot);
    if (SLOT_SPECS[slot].egressClasses.includes('conversation') && cfg.egressPrivate !== 'allow' && cfg.egressFallback === 'none') return refusalLine('egress_private_denied', slot);
  }
  return null;
}

// ---------------------------------------------------------------------------
// Receipts: per row / per turn / per query, and the run-level roll-up
// ---------------------------------------------------------------------------

export type DecideSpend = Partial<Record<DecideSlot, { input_tokens: number; cost_usd: number; requests: number }>>;

/** decide_spend totals by slot in `engine` (flushes buffered rows first). */
export async function decideSpendTotals(engine: BrainEngine): Promise<DecideSpend> {
  await flushDecideWrites();
  const rows = await engine.executeRaw<{ slot: DecideSlot; tokens: number | string; cost: number | string; n: number | string }>(
    'SELECT slot, COALESCE(SUM(input_tokens), 0) AS tokens, COALESCE(SUM(cost_usd), 0) AS cost, COUNT(*) AS n FROM decide_spend GROUP BY slot',
  ).catch(() => []);
  return Object.fromEntries(rows.map((r) => [r.slot, { input_tokens: Number(r.tokens), cost_usd: Number(r.cost), requests: Number(r.n) }]));
}

export function spendDelta(before: DecideSpend, after: DecideSpend): DecideSpend {
  const out: DecideSpend = {};
  for (const slot of Object.keys(after) as DecideSlot[]) {
    const a = after[slot]!, b = before[slot] ?? { input_tokens: 0, cost_usd: 0, requests: 0 };
    if (a.requests > b.requests) out[slot] = { input_tokens: a.input_tokens - b.input_tokens, cost_usd: a.cost_usd - b.cost_usd, requests: a.requests - b.requests };
  }
  return out;
}

export interface DecideSlotReceipt {
  mode: DecideMode;
  /** What the slot did on this row; null when its call site never ran (`skipped: not_reached`). */
  effective: DecideMode | null;
  skipped?: string;
  threshold?: number | null;
  outcomes?: Record<string, number>;
  latency_ms?: number;
  model_resolved?: string;
  judged?: number;
  answer?: string;
  input_tokens?: number;
  cost_usd?: number;
  /** S2: the answer arrived after decide.slots.intent.wait_ms (the regex label stood). */
  late?: boolean;
}

/** One receipt per active slot from the visible slot meta plus this row's decide_spend delta. */
export function decideRowReceipt(run: DecideEvalRun, meta: DecideSearchMeta | undefined, spend: DecideSpend): Partial<Record<DecideSlot, DecideSlotReceipt>> {
  const out: Partial<Record<DecideSlot, DecideSlotReceipt>> = {};
  for (const slot of Object.keys(run.slots) as DecideSlot[]) {
    const m: DecideSlotMeta | undefined = meta?.[slot];
    const s = spend[slot];
    const rerankTokens = slot === 'rerank' && m?.input_tokens !== undefined ? m.input_tokens : undefined;
    const tokens = s ? s.input_tokens + (rerankTokens ?? 0) : rerankTokens;
    const cost = (s?.cost_usd ?? 0) + (rerankTokens ? usageCostUsd(m?.provider ?? run.provider, rerankTokens, 0, 'rerank') ?? 0 : 0);
    out[slot] = {
      mode: run.slots[slot]!,
      effective: m?.effective ?? null,
      ...(m ? (m.skipped ? { skipped: m.skipped } : {}) : { skipped: 'not_reached' }),
      ...(m?.threshold !== undefined ? { threshold: m.threshold } : {}),
      ...(m?.outcomes ? { outcomes: m.outcomes } : {}),
      ...(m?.latency_ms !== undefined ? { latency_ms: m.latency_ms } : {}),
      ...(m?.model_resolved ? { model_resolved: m.model_resolved } : {}),
      ...(m?.judged !== undefined ? { judged: m.judged } : {}),
      ...(m?.answer ? { answer: m.answer } : {}),
      ...(tokens !== undefined ? { input_tokens: tokens, cost_usd: Number(cost.toFixed(8)) } : {}),
      ...(slot === 'intent' && m ? { late: m.skipped === 'late' } : {}),
    };
  }
  return out;
}

/** Run-level roll-up of per-row receipts: per slot rows reached, outcome tally, latency p50/p95, tokens, cost, S2 late rate. */
export function summarizeDecideReceipts(receipts: ReadonlyArray<Partial<Record<DecideSlot, DecideSlotReceipt>> | undefined>): Record<string, unknown> {
  const out: Record<string, unknown> = {};
  for (const slot of DECIDE_SLOTS) {
    const rows = receipts.map((r) => r?.[slot]).filter((r): r is DecideSlotReceipt => r !== undefined);
    if (rows.length === 0) continue;
    const outcomes: Record<string, number> = {};
    const skipped: Record<string, number> = {};
    for (const r of rows) {
      for (const [k, v] of Object.entries(r.outcomes ?? {})) outcomes[k] = (outcomes[k] ?? 0) + v;
      if (r.skipped) skipped[r.skipped] = (skipped[r.skipped] ?? 0) + 1;
    }
    const lat = rows.map((r) => r.latency_ms).filter((x): x is number => typeof x === 'number');
    const intentRows = rows.filter((r) => r.late !== undefined);
    out[slot] = {
      rows: rows.length, acted: rows.filter((r) => r.effective === 'on').length, outcomes, skipped,
      latency_ms: { p50: percentile(lat, 0.5), p95: percentile(lat, 0.95) },
      input_tokens: rows.reduce((n, r) => n + (r.input_tokens ?? 0), 0),
      cost_usd: Number(rows.reduce((n, r) => n + (r.cost_usd ?? 0), 0).toFixed(8)),
      ...(intentRows.length > 0 ? { late_rate: Number((intentRows.filter((r) => r.late).length / intentRows.length).toFixed(4)) } : {}),
    };
  }
  return out;
}