Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.8 KB
Raw Blame
/**
 * Reference calibrations shipped in the binary: produced by the maintainer's
 * eval runs (docs/eval/system-one/), keyed by slot, call site, provider, model
 * and pack_shape, with dataset and split hashes. Used when a brain has no
 * local row; a local row always wins. An enabled slot records the calibration
 * it adopted (`decide.slots.<slot>.calibration`), so a newer binary never
 * silently switches it: `decide status` shows "newer reference available".
 *
 * Rows exist only for slots whose recorded verdict is a win and, for
 * harmful-direction slots, whose action_precision_lb passes the 0.90 gate
 * (docs/eval/system-one/README.md).
 *
 * `recommendedSlots` is the one place that turns these rows into a slot set:
 * `gbrain decide enable --recommended` turns it on, and with a TypeSafe key
 * present it is the key-aware default-on set (readDecideConfig), so the two
 * can never disagree.
 */
import { SLOT_SPECS } from './slots.ts';
import type { DecideSlot } from './types.ts';

export interface ReferenceCalibration {
  /** Stable id, referenced as `ref:<id>`. */
  id: string;
  slot: DecideSlot;
  call_site: string;
  provider: string;
  model_resolved: string;
  threshold: number;
  min_keep: number | null;
  retest_sd: number;
  repack_sd: number;
  action_precision_lb: number | null;
  policy_fingerprint: string | null;
  pack_shape: string;
  dataset_hash: string;
  split_hash: string;
  /** S9 only: proposal floor calibrated on supersede labels. */
  proposal_floor?: number;
  /** Recorded eval verdict for the slot on this model. `enable --recommended` needs `win`. */
  verdict: 'win' | 'no_change' | 'regression' | 'not_measured';
  /** Binary version that shipped the row. */
  shipped_in: string;
}

export const REFERENCE_CALIBRATIONS: readonly ReferenceCalibration[] = [
  {
    // S7: Cat 35 + synthetic routine/buried-signal transcripts (docs/eval/system-one/datasets/s7-triage.jsonl),
    // calibrated for recall 1.0 on the calibrate half; qualified 39/39 correct rejections on the eval half.
    id: 'triage-jev-1.13.0-2026-09-30', slot: 'triage', call_site: 'dream', provider: 'typesafe:jev-1.13.0', model_resolved: 'jev-1.13.0',
    threshold: 0.77, min_keep: null, retest_sd: 0.0138, repack_sd: 0.0127, action_precision_lb: 0.9103, policy_fingerprint: '4d1429790e67d15d',
    pack_shape: 'v1:order=rank-id:max=1:slots=triage', dataset_hash: '91bb3dec2951e309', split_hash: 'de2b13e9820b5ffb',
    verdict: 'win', shipped_in: '0.60.17.0',
  },
  {
    // S9: labelled fact pairs (docs/eval/system-one/datasets/s9-conflict.jsonl); duplicate threshold by F1 and the
    // proposal floor calibrated on supersede labels. Not a harmful-direction slot (proposals only).
    id: 'conflict-jev-1.13.0-2026-09-30', slot: 'conflict', call_site: 'sweep', provider: 'typesafe:jev-1.13.0', model_resolved: 'jev-1.13.0',
    threshold: 0.52, min_keep: null, retest_sd: 0.0013, repack_sd: 0.0046, action_precision_lb: null, policy_fingerprint: null,
    pack_shape: 'v1:order=rank-id:max=budget:slots=conflict', dataset_hash: '37fe4083e9722308', split_hash: '2ec631956c4b9e33', proposal_floor: 0.65,
    verdict: 'win', shipped_in: '0.60.17.0',
  },
  {
    // P8 withdrawal review (conflict slot, review_withdraw call site): held-out qualification by the custodian,
    // gbrain-evals docs/benchmarks/2026-10-05-heldout-verdicts/p8-withdraw-heldout-2026-10-05.json. 240 families
    // (66 custodian-checked model-written paraphrase pairs per the 2026-10-05 amendment); threshold calibrated on the
    // 116-family calibrate half (precision and recall 1.0); 124/124 eval families correct over 248 withdraw actions.
    // The record reports no retest/repack spread (dev seed A measured 0.000) and no dataset hashes: the ids name it.
    id: 'conflict-review-withdraw-jev-1.13.0-2026-10-05', slot: 'conflict', call_site: 'review_withdraw', provider: 'typesafe:jev-1.13.0', model_resolved: 'jev-1.13.0',
    threshold: 0.62, min_keep: null, retest_sd: 0, repack_sd: 0, action_precision_lb: 0.970, policy_fingerprint: null,
    pack_shape: 'v1:order=rank-id:max=budget:slots=conflict', dataset_hash: 'heldout:p8-withdraw-2026-10-05', split_hash: 'heldout:p8-withdraw-2026-10-05',
    verdict: 'win', shipped_in: 'P8',
  },
];

/**
 * Wired slots with a `win` reference row for `provider` whose harmful-direction
 * gate passes (action_precision_lb at or above the slot's min_action_precision).
 */
export function recommendedSlots(
  provider: string,
  minActionPrecision: (slot: DecideSlot) => number = () => 0.9,
  references: readonly ReferenceCalibration[] = REFERENCE_CALIBRATIONS,
): DecideSlot[] {
  const slots = new Set<DecideSlot>();
  for (const r of references) {
    if (r.provider !== provider || r.verdict !== 'win' || !SLOT_SPECS[r.slot].wired) continue;
    if (SLOT_SPECS[r.slot].harmful && (r.action_precision_lb ?? 0) < minActionPrecision(r.slot)) continue;
    slots.add(r.slot);
  }
  return [...slots];
}

/**
 * #6497: the newest shipped reference for the calibration a slot policy uses,
 * as `ref:<id>`, or null when none applies. A row counts only when it matches
 * the slot, call site, provider and resolved model `selectCalibration` would
 * match on (a `conflict` row for `review_withdraw` is never newer for the
 * `sweep` call site) and differs from the calibration in use.
 */
export function newerReference(
  policy: { slot: DecideSlot; callSite: string; provider: string; model: string | null; calibration?: { ref: string } },
  references: readonly ReferenceCalibration[] = REFERENCE_CALIBRATIONS,
): string | null {
  if (!policy.calibration || !policy.model) return null;
  const newest = references
    .filter((r) => r.slot === policy.slot && r.call_site === policy.callSite && r.provider === policy.provider && r.model_resolved === policy.model)
    .at(-1);
  if (!newest || `ref:${newest.id}` === policy.calibration.ref) return null;
  return `ref:${newest.id}`;
}