Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.2 KB
Raw Blame
/**
 * `gbrain decide calibrate | qualify | calibrations | dataset`.
 *
 * calibrate: runs the slot's production-shaped requests over the dataset's
 * calibrate half, picks the threshold, measures reliability/ECE, retest_sd
 * (items re-asked 3 times) and repack_sd (re-asked with resampled co-packed
 * neighbours, 3 draws), and stores a decide_calibrations row.
 * qualify: evaluates the newest calibration on the eval half through the
 * production action reducer, family-level Wilson bound, stores
 * action_precision_lb and per-slice results bound to the policy fingerprint.
 */
import { accessSync, chmodSync, constants, readFileSync, statSync, writeFileSync } from 'node:fs';
import { dirname } from 'node:path';
import type { BrainEngine } from '../../core/engine.ts';
import { meanItemSd, qualifyActions, reliability, requiredN, searchThreshold, type CalibrationTarget, type HarmfulAction, type LabelledValue } from '../../core/ai/decide/calibrate.ts';
import { datasetAdapter, datasetBuilder, datasetHash, datasetSources, families, idsHash, itemForCallSite, parseDatasetJsonl, splitHash, toJsonl, type DatasetItem } from '../../core/ai/decide/dataset.ts';
import { runDecide } from '../../core/ai/decide/index.ts';
import { runDecideUnpacked } from '../../core/ai/decide/unpacked.ts';
import { estimateContextTokens, planBatches } from '../../core/ai/decide/pack.ts';
import { lookupModel, marginFor, policyFingerprint } from '../../core/ai/decide/policy.ts';
import { refusalLine } from '../../core/ai/decide/outcomes.ts';
import { REFERENCE_CALIBRATIONS } from '../../core/ai/decide/reference-calibrations.ts';
import { toWireQuestion } from '../../core/ai/decide/providers/typesafe.ts';
import { SLOT_SPECS } from '../../core/ai/decide/slots.ts';
import { getCalibration, insertCalibration, listCalibrations, setCalibrationRetired, storeQualification } from '../../core/ai/decide/store.ts';
import { DECIDE_SLOTS, DecideError, thresholdValue, type DecideAnswer, type DecideSlot } from '../../core/ai/decide/types.ts';
import { usageCostUsd } from '../../core/budget/reservation-cost.ts';
import { evidenceCoPackedSlots, resetDecideSearchCache } from '../../core/search/decide-stage.ts';
import { injectionQuestion } from '../../core/ai/decide/injection.ts';
import { flagValue, loadDecideState, slotPackShape } from '../decide.ts';
import '../../core/ai/decide/recall-needed.ts';

const REPEATS = 3;

function slotArg(args: string[]): DecideSlot | null {
  const slot = flagValue(args, '--slot');
  return slot && (DECIDE_SLOTS as readonly string[]).includes(slot) ? slot as DecideSlot : null;
}

function isPositive(item: DatasetItem): boolean {
  return item.label === true || (typeof item.label === 'string' && item.label !== 'false');
}

/**
 * The request production sends for a family. S3 carries S5's questions when
 * injection rides the S3 request, so the shape asked is the shape recorded
 * (pack_shape evidence+injection); the co-packed answers are not scored.
 */
export function productionRequest(slot: DecideSlot, fam: readonly DatasetItem[], cfg: Awaited<ReturnType<typeof loadDecideState>>['cfg']): ReturnType<NonNullable<ReturnType<typeof datasetAdapter>>['request']> {
  const req = datasetAdapter(slot)!.request(fam);
  if (slot !== 'evidence' || !evidenceCoPackedSlots(cfg).includes('injection')) return req;
  const coPacked = req.questions.map((q, i) => injectionQuestion(`injection:${i}`, q.rank ?? i, q.inputs!.candidate!, q.protected === true));
  return { ...req, questions: [...req.questions, ...coPacked] };
}

/** Ask every family once (production shape); returns value per item id and the resolved model. */
async function askFamilies(engine: BrainEngine, slot: DecideSlot, provider: string, fams: DatasetItem[][], cfgOverride: Awaited<ReturnType<typeof loadDecideState>>['cfg']): Promise<{ values: Record<string, number | null>; answers: Record<string, DecideAnswer | undefined>; model: string }> {
  const adapter = datasetAdapter(slot)!;
  const values: Record<string, number | null> = {};
  const answers: Record<string, DecideAnswer | undefined> = {};
  const models = new Set<string>();
  for (const fam of fams) {
    const req = productionRequest(slot, fam, cfgOverride);
    const base = { slot, callSite: adapter.callSite, state: req.state, questions: req.questions, provider, lane: 'background' as const };
    const r = adapter.unpacked
      ? await runDecideUnpacked(base, { engine, config: cfgOverride }, { deadlineAt: Date.now() + 60_000, concurrency: cfgOverride.backgroundConcurrency })
      : await runDecide({ ...base, deadlineMs: 60_000 }, { engine, config: cfgOverride });
    if (r.model_resolved) models.add(r.model_resolved);
    const perItem = new Map<string, Array<number | null>>();
    for (const q of req.questions) {
      const item = req.itemFor[q.id];
      if (!item) continue;
      const a = r.answers[q.id];
      perItem.set(item.id, [...(perItem.get(item.id) ?? []), a ? (adapter.calibrationValue ? adapter.calibrationValue(a) : thresholdValue(a)) : null]);
      answers[item.id] = a;
    }
    for (const [id, list] of perItem) {
      values[id] = adapter.aggregate === 'max'
        ? (list.some((v) => v === null) ? null : Math.max(...(list as number[])))
        : list[list.length - 1] ?? null;
    }
  }
  if (models.size > 1) throw new DecideError('mixed_model', `calibration answered by ${models.size} different models`);
  return { values, answers, model: [...models][0] ?? '' };
}

function sample<T>(items: readonly T[], n: number, seed: string): T[] {
  let h = 0;
  for (const c of seed) h = (h * 31 + c.charCodeAt(0)) >>> 0;
  const out = [...items];
  for (let i = out.length - 1; i > 0; i--) {
    h = (h * 1103515245 + 12345) >>> 0;
    const j = h % (i + 1);
    [out[i], out[j]] = [out[j]!, out[i]!];
  }
  return out.slice(0, n);
}

async function cmdCalibrate(engine: BrainEngine, args: string[]): Promise<number> {
  const json = args.includes('--json');
  const slot = slotArg(args);
  const path = flagValue(args, '--dataset');
  if (!slot || !path) { console.error('Usage: gbrain decide calibrate --slot <slot> --dataset <jsonl> [--target precision|recall|f1] [--min <x>] [--call-site <site>] [--dry-run]'); return 1; }
  const adapter = datasetAdapter(slot);
  if (!adapter || !SLOT_SPECS[slot].thresholded) { console.error(refusalLine('slot_unavailable', slot)); return 1; }
  const text = readFileSync(path, 'utf8');
  const callSite = flagValue(args, '--call-site') ?? adapter.callSite;
  const all = parseDatasetJsonl(text).filter((i) => i.slot === slot && itemForCallSite(i, callSite));
  const calibrate = all.filter((i) => i.split === 'calibrate');
  if (calibrate.length === 0) { console.error('dataset has no calibrate-half items for this slot (build it with gbrain decide dataset)'); return 1; }
  const state = await loadDecideState(engine);
  const provider = state.cfg.slots[slot].provider;
  if (provider === 'none') { console.error(refusalLine('no_provider', slot)); return 1; }
  const fams = [...families(calibrate).values()];
  const retestN = Math.min(state.cfg.retestN, fams.length);
  const estimateTokens = fams.reduce((n, fam) => {
    const req = productionRequest(slot, fam, state.cfg);
    return n + planBatches(estimateContextTokens(Object.fromEntries(Object.entries(req.state).map(([k, v]) => [k, v.text]))), req.questions.map((q) => estimateContextTokens(toWireQuestion(q)))).reduce((m, b) => m + b.estimatedInputTokens, 0);
  }, 0);
  const totalTokens = estimateTokens * (1 + (retestN / Math.max(1, fams.length)) * (REPEATS - 1) * 2);
  const estUsd = usageCostUsd(provider, totalTokens, 0, 'decide');
  if (args.includes('--dry-run')) {
    const out = { slot, provider, families: fams.length, items: calibrate.length, retest_families: retestN, estimated_input_tokens: Math.round(totalTokens), estimated_usd: estUsd };
    console.log(json ? JSON.stringify(out, null, 2) : `Dry run: ${calibrate.length} items in ${fams.length} families, ~${Math.round(totalTokens)} input tokens, ~$${(estUsd ?? 0).toFixed(4)} with ${provider}.`);
    return 0;
  }
  // Calibration is an explicit local operation on a labelled dataset: consent for its classes is implied.
  const cfg = { ...state.cfg, consent: { query: true, candidates: true, facts: true, conversation: true }, egressPrivate: 'allow' as const };
  try {
    const first = await askFamilies(engine, slot, provider, fams, cfg);
    const labelled: LabelledValue[] = calibrate.filter((i) => first.values[i.id] !== null && first.values[i.id] !== undefined).map((i) => ({ value: first.values[i.id]!, label: adapter.positive ? adapter.positive(i, first.answers[i.id]) : isPositive(i) }));
    const target = (flagValue(args, '--target') ?? 'f1') as CalibrationTarget;
    const minRaw = flagValue(args, '--min');
    const choice = searchThreshold(labelled, target, minRaw === undefined ? undefined : Number(minRaw));
    if (!choice) { console.error(`no threshold meets --min ${minRaw} for ${target}`); return 1; }
    const rel = reliability(labelled);
    const extra = adapter.calibrateExtra?.(calibrate, first.answers, choice.threshold) ?? null;
    const retestFams = sample(fams, retestN, `${slot}:retest`);
    const retest = new Map<string, number[]>();
    const repack = new Map<string, number[]>();
    for (const fam of retestFams) for (const it of fam) { retest.set(it.id, [first.values[it.id] ?? 0]); repack.set(it.id, [first.values[it.id] ?? 0]); }
    for (let rep = 1; rep < REPEATS; rep++) {
      const again = await askFamilies(engine, slot, provider, retestFams, cfg);
      for (const [id, v] of Object.entries(again.values)) if (v !== null) retest.get(id)?.push(v);
      const pool = calibrate.filter((i) => !retestFams.some((f) => f.includes(i)));
      const repacked = retestFams.map((fam, fi) => [...fam, ...sample(pool, Math.min(3, pool.length), `${slot}:repack:${rep}:${fi}`).map((n, k) => ({ ...n, id: `${n.id}#neighbour${rep}.${k}`, family: fam[0]!.family, rank: 1000 + k }))]);
      const packed = await askFamilies(engine, slot, provider, repacked, cfg);
      for (const [id, v] of Object.entries(packed.values)) if (v !== null && repack.has(id)) repack.get(id)!.push(v);
    }
    const row = {
      slot, call_site: callSite, provider, model_resolved: first.model, threshold: choice.threshold,
      min_keep: SLOT_SPECS[slot].defaultMinKeep ?? null, metric: target, metric_value: choice.metric_value, ece: rel.ece,
      retest_sd: meanItemSd(retest), repack_sd: meanItemSd(repack), n: labelled.length, dataset_hash: datasetHash(text),
      split_hash: splitHash(all), calibrate_ids_hash: idsHash(calibrate), calibrate_only: true, pack_shape: slotPackShape(slot, state.cfg), notes: extra ? JSON.stringify(extra) : null,
    };
    const id = await insertCalibration(engine, row);
    resetDecideSearchCache();
    const out = { id, ...row, precision: choice.precision, recall: choice.recall, f1: choice.f1, reliability: rel.table, margin: marginFor(state.cfg.marginFloor, { retest_sd: row.retest_sd, repack_sd: row.repack_sd }) };
    if (json) { console.log(JSON.stringify(out, null, 2)); return 0; }
    console.log(`Calibration local:${id} for ${slot} (${callSite}) on ${provider} → ${first.model}: threshold ${choice.threshold.toFixed(3)} (${target} ${choice.metric_value.toFixed(3)}; precision ${choice.precision.toFixed(3)}, recall ${choice.recall.toFixed(3)}), n=${labelled.length}`);
    console.log(`ECE ${rel.ece.toFixed(3)}; retest_sd ${row.retest_sd.toFixed(3)}; repack_sd ${row.repack_sd.toFixed(3)}; margin ${out.margin.toFixed(3)}`);
    console.log('reliability (probability bin: n, mean probability, observed rate):');
    for (const b of rel.table) if (b.n) console.log(`  ${b.lo.toFixed(1)}-${b.hi.toFixed(1)}: ${b.n}, ${b.mean_p.toFixed(3)}, ${b.observed.toFixed(3)}`);
    console.log(`Next: gbrain decide qualify --slot ${slot} --dataset ${path}`);
    return 0;
  } catch (err) {
    console.error(`calibrate failed: ${refusalLine(err instanceof DecideError ? err.reason : 'provider_error')}${err instanceof DecideError ? '' : ` (${err instanceof Error ? err.message : err})`}`);
    return 1;
  }
}

async function cmdQualify(engine: BrainEngine, args: string[]): Promise<number> {
  const json = args.includes('--json');
  const slot = slotArg(args);
  const path = flagValue(args, '--dataset');
  const itemsOut = flagValue(args, '--items-out');
  if (!slot || !path) { console.error('Usage: gbrain decide qualify --slot <slot> --dataset <jsonl> [--call-site <site>] [--items-out <path>]'); return 1; }
  const adapter = datasetAdapter(slot);
  if (!adapter?.harmfulActions) { console.error(refusalLine('slot_unavailable', slot)); return 1; }
  const unwritable = itemsOut ? itemsOutProblem(itemsOut) : null;
  if (unwritable) { console.error(`qualify: --items-out ${itemsOut} is not writable (${unwritable}); nothing was sent`); return 1; }
  const text = readFileSync(path, 'utf8');
  const callSite = flagValue(args, '--call-site') ?? adapter.callSite;
  const all = parseDatasetJsonl(text).filter((i) => i.slot === slot && itemForCallSite(i, callSite));
  const state = await loadDecideState(engine);
  const provider = state.cfg.slots[slot].provider;
  const model = lookupModel(provider, state.lastResolved);
  const rows = (await listCalibrations(engine, { slot })).filter((c) => c.call_site === callSite && c.provider === provider && (!model || c.model_resolved === model));
  const cal = rows[0];
  if (!cal) { console.error(refusalLine('no_calibration', slot)); return 1; }
  if (cal.split_hash !== splitHash(all) || !cal.calibrate_only) { console.error(refusalLine('split_mismatch', slot)); return 1; }
  const evalItems = all.filter((i) => i.split === 'eval');
  const cfg = { ...state.cfg, consent: { query: true, candidates: true, facts: true, conversation: true }, egressPrivate: 'allow' as const };
  const minKeep = state.cfg.slots[slot].minKeep ?? cal.min_keep ?? SLOT_SPECS[slot].defaultMinKeep ?? 0;
  const suppressBelow = state.cfg.slots[slot].suppressBelow;
  const policy = { threshold: state.cfg.slots[slot].threshold ?? cal.threshold, margin: marginFor(state.cfg.marginFloor, { retest_sd: cal.retest_sd ?? 0, repack_sd: cal.repack_sd ?? 0 }), minKeep, ...(suppressBelow !== undefined ? { suppressBelow } : {}) };
  try {
    const fams = [...families(evalItems).values()];
    const answered = await askFamilies(engine, slot, provider, fams, cfg);
    if (answered.model !== cal.model_resolved) { console.error(refusalLine('model_drift', slot)); return 1; }
    const judged = fams.flatMap((fam) => adapter.harmfulActions!(fam, answered.values, policy));
    const actions: HarmfulAction[] = judged.map((a) => ({ family: a.item.family, correct: a.correct, slice: a.item.slice }));
    const minPrecision = state.cfg.slots[slot].minActionPrecision;
    const q = qualifyActions(actions, minPrecision);
    const fingerprint = policyFingerprint({ slot, callSite, threshold: policy.threshold, marginFloor: state.cfg.marginFloor, minKeep, packShape: cal.pack_shape, suppressBelow });
    await storeQualification(engine, cal.id, { action_precision_lb: q.action_precision_lb, qualification: JSON.stringify({ ...q, min_action_precision: minPrecision }), policy_fingerprint: fingerprint });
    resetDecideSearchCache();
    if (itemsOut) {
      const header = { calibration: `local:${cal.id}`, slot, call_site: callSite, provider, model_resolved: answered.model, split_hash: cal.split_hash, policy_fingerprint: fingerprint, min_action_precision: minPrecision, status: q.status };
      const items = judged.map((a) => ({ id: a.item.id, family: a.item.family, split: a.item.split, slice: a.item.slice ?? null, protected: a.item.protected === true, value: answered.values[a.item.id] ?? null, correct: a.correct }));
      try {
        writeFileSync(itemsOut, JSON.stringify({ header, items }, null, 2) + '\n', { mode: 0o600 });
        chmodSync(itemsOut, 0o600);
      } catch (err) {
        console.error(`qualify: the qualification is stored, but writing --items-out ${itemsOut} failed (${err instanceof Error ? err.message : err})`);
        return 1;
      }
    }
    const out = { calibration: `local:${cal.id}`, ...q, min_action_precision: minPrecision, policy_fingerprint: fingerprint, next: q.status === 'qualified' ? `gbrain decide enable ${slot}` : null };
    if (json) { console.log(JSON.stringify(out, null, 2)); return q.status === 'qualified' ? 0 : 1; }
    console.log(`Qualification of local:${cal.id} (${slot}, ${callSite}): ${q.correct_families}/${q.families} families correct over ${q.actions} harmful actions; action_precision_lb ${q.action_precision_lb === null ? 'n/a' : q.action_precision_lb.toFixed(3)} (gate ${minPrecision})`);
    for (const [slice, s] of Object.entries(q.slices)) console.log(`  slice ${slice}: ${s.correct}/${s.families}, lb ${s.lb.toFixed(3)} (advisory)`);
    if (q.status === 'qualified') { console.log(`Qualified. Next: gbrain decide enable ${slot}`); return 0; }
    if (q.status === 'insufficient_n') console.error(`${refusalLine('insufficient_n', slot)} (need at least ${requiredN(minPrecision)} families with a harmful action; have ${q.families})`);
    else console.error(refusalLine('action_precision_low', slot));
    return 1;
  } catch (err) {
    console.error(`qualify failed: ${refusalLine(err instanceof DecideError ? err.reason : 'provider_error')}`);
    return 1;
  }
}

/** Why `--items-out <path>` cannot be written, checked before any provider call; null when it can. */
function itemsOutProblem(path: string): string | null {
  try {
    if (statSync(path, { throwIfNoEntry: false })?.isDirectory()) return 'it is a directory';
    accessSync(statSync(path, { throwIfNoEntry: false }) ? path : dirname(path), constants.W_OK);
    return null;
  } catch (err) {
    return err instanceof Error ? err.message : String(err);
  }
}

async function cmdCalibrations(engine: BrainEngine, args: string[]): Promise<number> {
  const action = args[0] ?? 'list';
  if (action === 'list') {
    const slot = slotArg(args) ?? undefined;
    const rows = await listCalibrations(engine, { slot, includeRetired: true });
    if (args.includes('--json')) { console.log(JSON.stringify(rows, null, 2)); return 0; }
    if (rows.length === 0) { console.log('No local calibrations. Build one: gbrain decide calibrate --slot <slot> --dataset <jsonl>'); return 0; }
    for (const r of rows) {
      console.log(`local:${r.id}  ${r.slot}/${r.call_site}  ${r.provider} → ${r.model_resolved}  threshold ${r.threshold.toFixed(3)}  n=${r.n}  lb=${r.action_precision_lb === null ? 'unqualified' : r.action_precision_lb.toFixed(3)}${r.retired_at ? '  (retired)' : ''}`);
    }
    return 0;
  }
  const raw = String(args[1] ?? '');
  if (action === 'adopt' && raw.startsWith('ref:')) {
    const ref = REFERENCE_CALIBRATIONS.find((r) => `ref:${r.id}` === raw);
    if (!ref) { console.error(`reference calibration ${raw} not found in this build (gbrain decide calibrations list shows local ones; shipped references appear in gbrain decide status)`); return 1; }
    await engine.setConfig(`decide.slots.${ref.slot}.calibration`, raw);
    resetDecideSearchCache();
    console.log(`${ref.slot} (${ref.call_site}) now uses ${raw} (threshold ${ref.threshold.toFixed(3)} for ${ref.model_resolved}).`);
    return 0;
  }
  const id = Number(raw.replace(/^local:/, ''));
  if (!Number.isInteger(id) || id <= 0) { console.error(`Usage: gbrain decide calibrations ${action} <id>  (adopt also takes ref:<id>)`); return 1; }
  const row = await getCalibration(engine, id);
  if (!row) { console.error(`calibration local:${id} not found`); return 1; }
  if (action === 'adopt') {
    await engine.setConfig(`decide.slots.${row.slot}.calibration`, `local:${id}`);
    await setCalibrationRetired(engine, id, false);
    resetDecideSearchCache();
    console.log(`${row.slot} now uses local:${id} (threshold ${row.threshold.toFixed(3)} for ${row.model_resolved}).`);
    return 0;
  }
  if (action === 'retire' || action === 'restore') {
    await setCalibrationRetired(engine, id, action === 'retire');
    resetDecideSearchCache();
    console.log(`local:${id} ${action === 'retire' ? 'retired' : 'restored'}.`);
    return 0;
  }
  console.error('Usage: gbrain decide calibrations list|adopt|retire|restore');
  return 1;
}

async function cmdDataset(args: string[]): Promise<number> {
  const slot = slotArg(args);
  const source = flagValue(args, '--from');
  const path = source ? args[args.indexOf('--from') + 2] : undefined;
  if (!slot || !source || !path) { console.error(`Usage: gbrain decide dataset --slot <slot> --from <source> <path> [--out <file>]. Sources in this build: ${datasetSources().join(', ')}`); return 1; }
  const builder = datasetBuilder(source);
  if (!builder) { console.error(`dataset source '${source}' is not available in this build. Sources: ${datasetSources().join(', ')}`); return 1; }
  const items = await builder(path, { slot });
  const out = flagValue(args, '--out');
  const body = toJsonl(items);
  if (out) writeFileSync(out, body);
  else process.stdout.write(body);
  const fams = families(items);
  console.error(`${items.length} items in ${fams.size} families (calibrate ${items.filter((i) => i.split === 'calibrate').length}, eval ${items.filter((i) => i.split === 'eval').length}); split_hash ${splitHash(items)}`);
  return 0;
}

export async function runCalibrationSubcommand(engine: BrainEngine, sub: string, args: string[]): Promise<number> {
  if (sub === 'calibrate') return cmdCalibrate(engine, args);
  if (sub === 'qualify') return cmdQualify(engine, args);
  if (sub === 'calibrations') return cmdCalibrations(engine, args);
  return cmdDataset(args);
}