Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.5 KB
Raw Blame
/**
 * BrainBench — cross-harness memory conformance suite (Cathedral 2).
 *
 * Type layer for the fixture corpus, harness adapters, and result documents.
 * The fixture + result shapes are PUBLISHED interchange formats (mirrored as
 * JSON Schemas in evals/brainbench/schema/) so foreign runners — notably the
 * sibling gbrain-evals repo — can drive `gbrain eval brainbench --fixtures DIR
 * --gold DIR --json --out FILE` against their own corpora. Breaking changes
 * bump FIXTURE_SCHEMA_VERSION / RESULT_SCHEMA_VERSION; additive-only within a
 * version.
 *
 * Sealed-gold discipline (gbrain-evals convention): fixture files carry ONLY
 * what an adapter may see (turns, seed content). Gold annotations live in a
 * separate gold dir, joined by the loader, and the harness hands adapters a
 * sanitized PublicTurn. A `gold` key inside a fixture turn is a VALIDATION
 * ERROR, not a convenience.
 */

import type { ReflexPointer } from '../../core/context/retrieval-reflex.ts';
import type { BrainEngine } from '../../core/engine.ts';

export const FIXTURE_SCHEMA_VERSION = 1;
export const RESULT_SCHEMA_VERSION = 1;

// ---------------------------------------------------------------------------
// Suites + harnesses
// ---------------------------------------------------------------------------

export const ALL_SUITES = [
  'know-to-ask', 'push', 'write-back', 'continuity',
  'trust', 'state-resolution', 'poisoning', 'deletion',
] as const;
export type BrainBenchSuite = (typeof ALL_SUITES)[number];

/**
 * Memory-trust suites (#5575). Their fixtures drive real tiered write paths
 * (`trust_steps`) on a persistence-enabled brain instead of the seeder, and
 * are scored from the sealed `trust` gold block. One suite per fixture.
 */
export const TRUST_SUITES = ['trust', 'state-resolution', 'poisoning', 'deletion'] as const;
export type TrustSuite = (typeof TRUST_SUITES)[number];
export function isTrustSuite(s: string): s is TrustSuite {
  return (TRUST_SUITES as readonly string[]).includes(s);
}

export const ALL_HARNESSES = ['openclaw', 'claude-code', 'codex'] as const;
export type HarnessName = (typeof ALL_HARNESSES)[number];

/**
 * 'production' — exercises a shipped integration seam byte-for-byte.
 * 'contract'  — grades gbrain primitives through a harness-shaped injection
 *               contract a later PR will wire to the real harness. Printed on
 *               every scoreboard row; see docs/eval/BRAINBENCH.md.
 */
export type SeamKind = 'production' | 'contract';

// ---------------------------------------------------------------------------
// Fixture (adapter-visible) shapes
// ---------------------------------------------------------------------------

export interface SeedPage {
  slug: string;
  /** Full markdown content incl. frontmatter. Imported with noEmbed. */
  content: string;
  /** Which source this page seeds into. Default 'default'. */
  source_id?: string;
}

export interface SeedFact {
  fact: string;
  entity_slug?: string | null;
  /** Provenance string; default 'bench:seed'. */
  source?: string;
  source_session?: string | null;
  source_id?: string;
}

export interface FixtureTurn {
  turn_id: number;
  role: 'user' | 'assistant';
  text: string;
  /**
   * ISO timestamp. Required for write-back fixtures (the conversation page
   * rendering + segment splitting need real times); optional elsewhere.
   */
  ts?: string;
}

export interface BrainBenchFixture {
  schema_version: number;
  fixture_id: string;
  /** Which metric suites consume this fixture. */
  suites: BrainBenchSuite[];
  /** Generator category (kta-pos, kta-neg, push, write-back, continuity, multi-source, adversarial). */
  category?: string;
  /**
   * Excluded from the CI gate; scored only in published runs (--include-holdout).
   * Gaming resistance per decision 22.
   */
  holdout?: boolean;
  /**
   * Extra source ids to create beyond 'default' (multi-source fixtures,
   * decision 14). Seed pages/facts route via their own source_id.
   */
  sources?: string[];
  /** The source the conversation happens in. Default 'default'. */
  active_source?: string;
  seed_pages?: SeedPage[];
  seed_facts?: SeedFact[];
  turns: FixtureTurn[];
  /** Present on continuity fixtures only (pairing metadata, not gold). */
  continuity?: {
    pair_id: string;
    pair_role: 'writer' | 'reader';
  };
  /**
   * Memory-trust suites only: ordered writes through real channels (owner
   * sync, local and remote agents, a connector capture, owner actions). They
   * run before the turns, which then replay as a later session.
   */
  trust_steps?: TrustStep[];
}

/**
 * Who performs a trust step. `owner` writes and syncs files in the
 * fixture's own git-backed source (operator_curated) and runs owner actions
 * on a confirmed terminal; `local_agent` / `remote_agent` call operations as
 * the local CLI / an MCP connection with read+write scopes; `connector` is a
 * webhook capture (external); `raw` is a process with a database connection
 * attempting a direct tier raise.
 */
export const TRUST_ACTORS = ['owner', 'local_agent', 'remote_agent', 'connector', 'raw'] as const;
export type TrustActor = (typeof TRUST_ACTORS)[number];
export const TRUST_OPS = ['write_file', 'sync', 'put_page', 'capture', 'remember', 'forget', 'confirm', 'purge', 'raise_tier'] as const;
export type TrustOp = (typeof TRUST_OPS)[number];
/** Which actors may perform which op (validated by the loader). */
export const TRUST_ACTOR_OPS: Readonly<Record<TrustActor, readonly TrustOp[]>> = {
  owner: ['write_file', 'sync', 'confirm', 'purge'],
  local_agent: ['put_page', 'remember', 'confirm'],
  remote_agent: ['put_page', 'capture', 'remember', 'forget', 'confirm', 'purge'],
  connector: ['capture'],
  raw: ['raise_tier'],
};

export interface TrustStep {
  step_id: string;
  actor: TrustActor;
  op: TrustOp;
  /** Page slug (write_file, put_page, capture). */
  slug?: string;
  /** Page markdown (write_file, put_page) or capture body. */
  content?: string;
  /** remember: the claim. */
  fact?: string;
  entity?: string;
  kind?: 'fact' | 'preference' | 'commitment' | 'event' | 'belief';
  content_origin?: 'user_said' | 'tool_output' | 'inferred';
  /** confirm / purge / forget / raise_tier / remember.replaces: the step whose row is acted on. */
  target?: string;
  /** Picks one fact row of a page step (an owner file's facts fence) by its claim text. */
  target_claim?: string;
}
/**
 * What an adapter is allowed to see of a turn. Structurally sealed: built by
 * `toPublicTurn`, which picks exactly these fields — anything else (incl. a
 * smuggled `gold`) is dropped.
 */
export interface PublicTurn {
  turn_id: number;
  role: 'user' | 'assistant';
  text: string;
  ts?: string;
}

/**
 * The canonical 4-decimal rounding (decision 10 — baseline diff-stability).
 * ONE implementation; scoreboard + harness import it (review DRY finding).
 */
export function round4(n: number): number {
  return Math.round(n * 10000) / 10000;
}

export function toPublicTurn(turn: FixtureTurn): PublicTurn {
  const out: PublicTurn = { turn_id: turn.turn_id, role: turn.role, text: turn.text };
  if (turn.ts !== undefined) out.ts = turn.ts;
  return out;
}

// ---------------------------------------------------------------------------
// Gold (sealed) shapes — evals/brainbench/gold/<fixture_id>.gold.json
// ---------------------------------------------------------------------------

export interface GoldFactSpec {
  /** Human label for the fact ("pricing concern"). */
  gist: string;
  /** The exact fact text the gold extractor emits into the production pipeline. */
  fact: string;
  entity_slug: string | null;
  /** Keyword probe: every keyword must appear (case-insensitive) in the stored fact. */
  match_keywords: string[];
  kind?: 'event' | 'preference' | 'commitment' | 'belief' | 'fact';
}

export interface TurnGold {
  should_retrieve: boolean;
  /** Slugs that SHOULD be injected (recall denominator). */
  gold_slugs?: string[];
  /** Additionally-acceptable slugs (count for precision, not required for recall). */
  acceptable_slugs?: string[];
  /** Write-back gold: facts this turn contributes (consumed via the gold extractor). */
  gold_facts?: GoldFactSpec[];
}

export interface ContinuityDecisionGold {
  decision_id: string;
  /** Reader-side success: any of these slugs injected on the probe turn... */
  expected_slugs: string[];
  /** ...or a stored fact matching all keywords is recallable. */
  match_keywords: string[];
}

export interface FixtureGold {
  fixture_id: string;
  /** Keyed by String(turn_id). Turns without an entry have no gold (assistant turns, filler). */
  turns: Record<string, TurnGold>;
  continuity?: {
    pair_id: string;
    decisions: ContinuityDecisionGold[];
  };
  /** Memory-trust suites: what each step should have produced. */
  trust?: { items: TrustGoldItem[] };
}

/**
 * One scored expectation of a memory-trust fixture.
 *   tier       the step's row is stored and read back (get_page, search, recall) at exactly `tier` (trust)
 *   max_tier   the step's row is stored at `tier` or lower: no laundering upward (trust)
 *   refused    the step was refused and its target's tier did not rise (trust: self-promotion)
 *   current    the step's fact is the active one recall returns (state-resolution)
 *   stale      the step's fact is not returned as current (state-resolution)
 *   history    the step's fact row is kept (expired or superseded, never deleted) (state-resolution)
 *   guarded    the step's fact stays active after a lower-tier write tried to replace or forget it (state-resolution)
 *   poison     an instruction-like payload carrying `canary` (poisoning)
 *   benign     ordinary content carrying `canary` (poisoning)
 *   purged     the purge step removed `probe` from every live store, its receipt names them, nothing resurrects (deletion)
 */
export const TRUST_CHECKS = ['tier', 'max_tier', 'refused', 'current', 'stale', 'history', 'guarded', 'poison', 'benign', 'purged'] as const;
export type TrustCheck = (typeof TRUST_CHECKS)[number];
export interface TrustGoldItem {
  item_id: string;
  check: TrustCheck;
  step: string;
  /** Fact selector within a page step. */
  claim?: string;
  /** Exact tier (`tier`) or ceiling (`max_tier`). */
  tier?: import('../../core/trust/tier.ts').TrustTier;
  /** poisoning arm. */
  arm?: 'external' | 'agent_relayed';
  /** poisoning: unique token inside the payload or benign text. */
  canary?: string;
  /** poisoning: the payload is a standing preference or commitment fact. */
  preference?: boolean;
  /** deletion: the exact text the purge must remove. */
  probe?: string;
}

/** Loader output: fixture joined with its gold. Internal to the harness — never crosses to adapters. */
export interface LoadedFixture {
  fixture: BrainBenchFixture;
  gold: FixtureGold;
  /** Absolute path the fixture was loaded from (error reporting). */
  path: string;
}

export interface LoadedCorpus {
  fixtures: LoadedFixture[];
  /** sha256 over sorted relative-path + content of every fixture AND gold file. */
  fixtures_hash: string;
  fixture_dir: string;
  gold_dir: string;
}

// ---------------------------------------------------------------------------
// Adapter contract
// ---------------------------------------------------------------------------

export interface HarnessTurnResult {
  /** The text the harness would inject this turn (null = stayed silent). */
  injectedText: string | null;
  /** Normalized slugs referenced by the injection — what metrics score. */
  injectedSlugs: string[];
  pointers: ReflexPointer[];
  /** Estimated tokens of injectedText (chars/4 heuristic; intrusion diagnostics). */
  injectedTokens: number;
  latencyMs: number;
  /** System One (turn-context seams): the per-turn decide meta, present only when a decide slot is not off. */
  decide?: import('../../core/search/decide-stage.ts').DecideSearchMeta;
}

export interface HarnessAdapter {
  readonly name: HarnessName;
  readonly seam: SeamKind;
  /**
   * v0.46.15 — optional RUN-scoped lifecycle (outside-voice F4/R2-5). The
   * harness constructs ONE adapter per harness per run and calls setupRun
   * before the first fixture / teardownRun after the last, so a production
   * seam can own long-lived infrastructure (the claude-code adapter's
   * resolve-IPC server + temp home) instead of paying setup per fixture.
   */
  setupRun?(): Promise<void>;
  teardownRun?(): Promise<void>;
  /** Called once per (fixture, adapter) before any turn. */
  beginConversation(engine: BrainEngine, fixture: AdapterFixtureView): Promise<void>;
  /**
   * Replay one turn. `priorContextText` is the joined text of PRIOR turns +
   * prior injections — adapters whose seam has no conversation memory (e.g.
   * the claude-code hook contract) ignore it by config, and that delta is
   * part of what the bench measures.
   */
  replayTurn(turn: PublicTurn, priorContextText: string): Promise<HarnessTurnResult>;
  endConversation(): Promise<void>;
}

/** The slice of a fixture an adapter may see (no gold, no category metadata). */
export interface AdapterFixtureView {
  fixture_id: string;
  active_source: string;
  turns: PublicTurn[];
}

// ---------------------------------------------------------------------------
// Per-turn evaluation rows + metric outputs
// ---------------------------------------------------------------------------

export interface TurnRow {
  fixture_id: string;
  turn_id: number;
  harness: HarnessName;
  suite: BrainBenchSuite;
  injected_slugs: string[];
  injected_tokens: number;
  gold: TurnGold | null;
  /** Slugs injected from a source other than the fixture's active source (decision 14). */
  cross_source_slugs: string[];
  latency_ms: number;
  /** System One arm (`--decide`): per-slot receipt for this turn (S6 recall_needed meta + decide spend). Absent when every slot is off. */
  decide?: Partial<Record<import('../../core/ai/decide/types.ts').DecideSlot, import('../decide-eval-flags.ts').DecideSlotReceipt>>;
}

/** One harness × suite cell of the scoreboard. Counts first; rates derived. */
export interface SuiteMetrics {
  suite: BrainBenchSuite;
  harness: HarnessName;
  seam: SeamKind;
  /** Gold items evaluated / failed — the count-aware gate operates on these. */
  gold_total: number;
  gold_failed: number;
  /** Named metric values (registered in metric-glossary.ts). */
  metrics: Record<string, number>;
  /** Fixture ids that contributed (excludes holdout in gate mode). */
  fixtures: string[];
}

export interface BrainBenchReceipt {
  result_schema_version: number;
  fixtures_hash: string;
  harness_sha: string;
  ts: string;
  cmd_args: string[];
  seed: number;
  include_holdout: boolean;
  llm: boolean;
}

export interface BrainBenchResult {
  receipt: BrainBenchReceipt;
  cells: SuiteMetrics[];
  turn_rows: TurnRow[];
  /** Fixtures that failed to seed (decision 12) — run exits 2 when non-empty. */
  seed_failures: Array<{ fixture_id: string; error: string }>;
  _meta?: { metric_glossary: Record<string, unknown> };
  /** System One arm (`--decide`): flags, provider, calibrations, split and the per-slot roll-up. Absent when every slot is off. */
  decide?: Record<string, unknown>;
}

// ---------------------------------------------------------------------------
// Canonical committed baseline (decision 10) — diff-stable, receipts excluded
// ---------------------------------------------------------------------------

export interface BrainBenchBaseline {
  schema_version: number;
  fixtures_hash: string;
  /**
   * Run configuration the numbers were produced under (red-team finding:
   * fixtures_hash covers files only — a holdout-inclusive or --llm baseline
   * is byte-plausible under the same hash but incomparable). compareBaselines
   * returns inconclusive on mismatch.
   */
  config: {
    include_holdout: boolean;
    llm: boolean;
    harnesses: string[];
    suites: string[];
  };
  /**
   * Required when a regression vs the prior baseline is being blessed
   * (decision 4) — visible in the PR diff, review-enforced.
   */
  justification?: string;
  /** `${harness}/${suite}` → metric name → value rounded to 4 decimals, keys sorted. */
  cells: Record<string, Record<string, number>>;
  /** `${harness}/${suite}` → { gold_total, gold_failed } for the count-aware gate. */
  counts: Record<string, { gold_total: number; gold_failed: number }>;
}

/** Verdict of a compare run. Maps to exit codes 0 / 1 / 2. */
export type CompareVerdict = 'pass' | 'regression' | 'inconclusive';

export interface CompareOutcome {
  verdict: CompareVerdict;
  mode: 'same-hash' | 'corpus-bless';
  breaches: Array<{
    cell: string;
    metric: string;
    baseline: number;
    current: number;
    detail: string;
  }>;
  notes: string[];
}