Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.4 KB
Raw Blame
/**
 * autocut-replay.ts — pure replay of the autocut floor sweep over captured
 * rerank pools (plan Phase C / D24).
 *
 * INVARIANT: the replay uses the SAME `applyAutocut` the live search path
 * uses, over the SAME pre-autocut reranked pool, with the SAME preserve
 * predicate — so `validateLive` can demand byte-for-byte agreement with the
 * recorded live decision before any other floor cell is trusted. Slicing to
 * k happens AFTER autocut, exactly like hybrid.ts (autocut runs on the full
 * post-rerank pool, then `slice(offset, offset + limit)`); a replay over only
 * the returned k rows would find different cliffs.
 *
 * Metric semantics mirror the harness (plan 0d): `distinct` = distinct
 * session ids among the first k CHUNK rows of the kept pool;
 * `recall_all` = gold ⊆ distinct; `recall_any` = gold ∩ distinct ≠ ∅. The
 * benefit metric (why autocut exists) is the mean kept result count and the
 * mean kept `est_tokens` over that returned window. No I/O here — the script
 * `scripts/replay-autocut-floor.ts` owns argv, file reading and printing.
 */

import { applyAutocut, DEFAULT_AUTOCUT, type AutocutDecision } from '../../core/search/autocut.ts';
import { mulberry32 } from './bootstrap.ts';

/**
 * One captured row of the EXACT pre-autocut `returnPool` (harness
 * `--capture-pool`): scored rows carry a finite `rerank_score`; alias-hop /
 * exact-lookup injections arrive post-rerank and are UNSCORED
 * (`rerank_score` absent or null) but flagged `alias_hit` / `exact_lookup`;
 * relational pins are flagged `relational_pinned` (scored low by construction,
 * so hybrid.ts keeps them OUT of the cliff math and preserves them). All kinds
 * are carried — dropping a row would change what `applyAutocut` sees (and
 * `decision.total`) versus the live call.
 */
export interface PoolRow {
  slug: string;
  chunk_id?: string | number | null;
  session_id: string;
  rrf_rank?: number;
  rerank_score?: number | null;
  alias_hit?: boolean;
  exact_lookup?: boolean;
  relational_pinned?: boolean;
  est_tokens?: number;
}

/** Live decision as recorded by the harness (`search_meta.autocut`). */
export interface LiveAutocut {
  applied: boolean;
  kept: number;
  total?: number;
  cut?: number;
  gapRatio?: number;
  signal?: string;
}

export interface ReplayRow {
  question_id: string;
  question_type?: string;
  rerank_pool: PoolRow[];
  /** The live decision for the capture arm (recorded at floor 0.35). */
  autocut?: LiveAutocut | null;
  /** Optional exact kept set the harness recorded (`slug#chunk_id` keys). */
  autocut_kept_keys?: string[];
  answer_session_ids: string[];
}

/** A floor is the `minTopScore` weak-top floor; 'off' disables autocut. */
export type Floor = number | 'off';

export interface ReplayKnobs {
  jumpRatio: number;
  minKeep: number;
}

export const DEFAULT_REPLAY_KNOBS: ReplayKnobs = Object.freeze({
  jumpRatio: DEFAULT_AUTOCUT.jumpRatio,
  minKeep: DEFAULT_AUTOCUT.minKeep,
});

export function poolKey(r: PoolRow): string {
  return `${r.slug}#${r.chunk_id ?? ''}`;
}

/**
 * Same score accessor hybrid.ts passes to applyAutocut: a relational-pinned
 * row is UNSCORED for the cliff math (live: `x.relational_pinned ? undefined :
 * x.rerank_score`) — its low-by-construction score must not manufacture or
 * mask a cliff. Every other row scores by `rerank_score` (null = unscored).
 */
export function scoreOf(r: PoolRow): number | null | undefined {
  return r.relational_pinned === true ? undefined : r.rerank_score;
}

/**
 * Same preserve predicate hybrid.ts passes to applyAutocut (alias hop / exact
 * lookup / relational pin survive the cut). Live it reads `x.alias_hit === true
 * || x.exact_lookup !== undefined || x.relational_pinned === true`; the capture
 * writes `exact_lookup: true` exactly when the live row had `exact_lookup !==
 * undefined` (and `relational_pinned: true` iff live), so on the captured shape
 * this is the identical predicate.
 */
export function preservePredicate(r: PoolRow): boolean {
  return r.alias_hit === true || r.exact_lookup === true || r.relational_pinned === true;
}

/**
 * Normalize one captured pool row. Throws on a row that cannot be replayed
 * (no slug / session_id): a silently dropped row would shift `total` and every
 * cliff. `rerank_score` is carried as a finite number or null (unscored);
 * booleans are carried only when literally `true` (the capture's own shape).
 */
export function normalizePoolRow(raw: unknown, where: string): PoolRow {
  if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) throw new Error(`${where}: pool row is not an object`);
  const o = raw as Record<string, unknown>;
  if (typeof o.slug !== 'string' || o.slug.length === 0) throw new Error(`${where}: pool row has no slug`);
  if (typeof o.session_id !== 'string' || o.session_id.length === 0) throw new Error(`${where}: pool row ${o.slug} has no session_id`);
  const row: PoolRow = { slug: o.slug, session_id: o.session_id };
  if (typeof o.chunk_id === 'string' || typeof o.chunk_id === 'number') row.chunk_id = o.chunk_id;
  else if (o.chunk_id === null) row.chunk_id = null;
  if (typeof o.rrf_rank === 'number' && Number.isFinite(o.rrf_rank)) row.rrf_rank = o.rrf_rank;
  row.rerank_score = typeof o.rerank_score === 'number' && Number.isFinite(o.rerank_score) ? o.rerank_score : null;
  if (o.alias_hit === true) row.alias_hit = true;
  if (o.exact_lookup === true) row.exact_lookup = true;
  if (o.relational_pinned === true) row.relational_pinned = true;
  if (typeof o.est_tokens === 'number' && Number.isFinite(o.est_tokens)) row.est_tokens = o.est_tokens;
  return row;
}

export function parseFloors(spec: string): Floor[] {
  const out: Floor[] = [];
  for (const raw of spec.split(',')) {
    const t = raw.trim();
    if (t.length === 0) continue;
    if (t.toLowerCase() === 'off') {
      out.push('off');
      continue;
    }
    const n = Number(t);
    if (!Number.isFinite(n) || n < 0 || n > 1) throw new Error(`invalid autocut floor '${t}' (expected 'off' or a number in [0, 1])`);
    out.push(n);
  }
  if (out.length === 0) throw new Error('no floors given');
  return out;
}

export function floorLabel(f: Floor): string {
  return f === 'off' ? 'off' : f.toFixed(2);
}

export interface RowReplay {
  question_id: string;
  question_type: string;
  floor: Floor;
  decision: AutocutDecision;
  /** Kept pool (pre-slice), in pool order. */
  kept: PoolRow[];
  /** The returned window: first k kept chunk rows. */
  returned: PoolRow[];
  distinct_sessions: string[];
  recall_all_hit: boolean;
  recall_any_hit: boolean;
  returned_count: number;
  returned_est_tokens: number;
}

/** Replay one row at one floor. Pure. */
export function replayRow(row: ReplayRow, floor: Floor, k: number, knobs: ReplayKnobs = DEFAULT_REPLAY_KNOBS): RowReplay {
  const pool = row.rerank_pool;
  let kept: PoolRow[];
  let decision: AutocutDecision;
  if (floor === 'off') {
    kept = pool;
    decision = { applied: false, signal: 'none', cut: pool.length, kept: pool.length, total: pool.length, gapRatio: 0 };
  } else {
    const r = applyAutocut(pool, scoreOf, { enabled: true, jumpRatio: knobs.jumpRatio, minKeep: knobs.minKeep, minTopScore: floor }, preservePredicate);
    kept = r.kept;
    decision = r.decision;
  }
  const returned = kept.slice(0, k);
  const distinct: string[] = [];
  for (const r of returned) if (!distinct.includes(r.session_id)) distinct.push(r.session_id);
  const gold = new Set(row.answer_session_ids ?? []);
  const distinctSet = new Set(distinct);
  const recall_all_hit = gold.size > 0 && [...gold].every((g) => distinctSet.has(g));
  const recall_any_hit = [...gold].some((g) => distinctSet.has(g));
  return {
    question_id: row.question_id,
    question_type: row.question_type ?? 'unknown',
    floor,
    decision,
    kept,
    returned,
    distinct_sessions: distinct,
    recall_all_hit,
    recall_any_hit,
    returned_count: returned.length,
    returned_est_tokens: returned.reduce((s, r) => s + (Number.isFinite(r.est_tokens) ? (r.est_tokens as number) : 0), 0),
  };
}

export interface TypeSummary {
  n: number;
  all_hit: number;
  any_hit: number;
}

export interface FloorSummary {
  floor: string;
  n: number;
  recall_all_hit: number;
  recall_all_rate: number;
  recall_any_hit: number;
  recall_any_rate: number;
  autocut_applied: number;
  mean_returned_results: number;
  mean_returned_est_tokens: number;
  mean_kept_pool: number;
  by_type: Record<string, TypeSummary>;
}

export function summarizeFloor(replays: RowReplay[], floor: Floor): FloorSummary {
  const n = replays.length;
  const by_type: Record<string, TypeSummary> = {};
  let all = 0;
  let any = 0;
  let applied = 0;
  let results = 0;
  let tokens = 0;
  let keptPool = 0;
  for (const r of replays) {
    const t = (by_type[r.question_type] ??= { n: 0, all_hit: 0, any_hit: 0 });
    t.n++;
    if (r.recall_all_hit) {
      all++;
      t.all_hit++;
    }
    if (r.recall_any_hit) {
      any++;
      t.any_hit++;
    }
    if (r.decision.applied) applied++;
    results += r.returned_count;
    tokens += r.returned_est_tokens;
    keptPool += r.kept.length;
  }
  const mean = (x: number) => (n === 0 ? 0 : x / n);
  return {
    floor: floorLabel(floor),
    n,
    recall_all_hit: all,
    recall_all_rate: mean(all),
    recall_any_hit: any,
    recall_any_rate: mean(any),
    autocut_applied: applied,
    mean_returned_results: mean(results),
    mean_returned_est_tokens: mean(tokens),
    mean_kept_pool: mean(keptPool),
    by_type,
  };
}

export interface PairedDelta {
  floor: string;
  baseline_floor: string;
  wins: number;
  losses: number;
  net: number;
  /** Per-type net (all_hit at floor − all_hit at baseline). */
  by_type_net: Record<string, number>;
  lost_question_ids: string[];
}

/** Paired recall_all comparison of `floor` against `baseline`, row by row. */
export function pairedDelta(atFloor: RowReplay[], atBaseline: RowReplay[]): PairedDelta {
  const base = new Map(atBaseline.map((r) => [r.question_id, r]));
  let wins = 0;
  let losses = 0;
  const by_type_net: Record<string, number> = {};
  const lost: string[] = [];
  for (const r of atFloor) {
    const b = base.get(r.question_id);
    if (!b) continue;
    const d = Number(r.recall_all_hit) - Number(b.recall_all_hit);
    if (d > 0) wins++;
    if (d < 0) {
      losses++;
      lost.push(r.question_id);
    }
    by_type_net[r.question_type] = (by_type_net[r.question_type] ?? 0) + d;
  }
  return {
    floor: atFloor[0] ? floorLabel(atFloor[0].floor) : '?',
    baseline_floor: atBaseline[0] ? floorLabel(atBaseline[0].floor) : '?',
    wins,
    losses,
    net: wins - losses,
    by_type_net,
    lost_question_ids: lost,
  };
}

export interface LiveMismatch {
  question_id: string;
  reason: string;
  live: LiveAutocut | null | undefined;
  replayed: AutocutDecision;
}

/**
 * Assert the replay at `floor` reproduces every row's recorded live decision:
 * `applied` and `kept` must agree (and `total` / `gapRatio` when recorded);
 * when the harness recorded `autocut_kept_keys`, the kept SET must match too.
 * Rows without a recorded decision are mismatches (nothing to validate against).
 */
export function validateLive(rows: ReplayRow[], floor: Floor, knobs: ReplayKnobs = DEFAULT_REPLAY_KNOBS): LiveMismatch[] {
  const out: LiveMismatch[] = [];
  for (const row of rows) {
    const rep = replayRow(row, floor, Number.MAX_SAFE_INTEGER, knobs);
    const live = row.autocut;
    if (!live) {
      out.push({ question_id: row.question_id, reason: 'no live autocut decision recorded', live, replayed: rep.decision });
      continue;
    }
    const reasons: string[] = [];
    // Pool-size disagreement first: it explains every downstream difference
    // (a missing unscored alias row shifts total, kept and the cliff), so name
    // it directly instead of letting it surface as an opaque kept mismatch.
    if (typeof live.total === 'number' && live.total !== rep.decision.total) {
      const diff = live.total - rep.decision.total;
      reasons.push(
        diff > 0
          ? `live pool held ${diff} row(s) the capture omitted (live total=${live.total}, captured pool=${rep.decision.total})`
          : `captured pool has ${-diff} row(s) the live pool lacked (live total=${live.total}, captured pool=${rep.decision.total})`,
      );
    }
    if (live.applied !== rep.decision.applied) reasons.push(`applied live=${live.applied} replay=${rep.decision.applied}`);
    if (live.kept !== rep.decision.kept) reasons.push(`kept live=${live.kept} replay=${rep.decision.kept}`);
    if (typeof live.gapRatio === 'number' && Math.abs(live.gapRatio - rep.decision.gapRatio) > 1e-9) {
      reasons.push(`gapRatio live=${live.gapRatio} replay=${rep.decision.gapRatio}`);
    }
    if (Array.isArray(row.autocut_kept_keys)) {
      const liveKeys = [...row.autocut_kept_keys].sort();
      const repKeys = rep.kept.map(poolKey).sort();
      if (liveKeys.length !== repKeys.length || liveKeys.some((k, i) => k !== repKeys[i])) reasons.push('kept set differs from autocut_kept_keys');
    }
    if (reasons.length > 0) out.push({ question_id: row.question_id, reason: reasons.join('; '), live, replayed: rep.decision });
  }
  return out;
}

/** 32-bit FNV-1a over a string → uint32 (deterministic seed derivation). */
function fnv1a32(s: string): number {
  let h = 0x811c9dc5;
  for (let i = 0; i < s.length; i++) {
    h ^= s.charCodeAt(i);
    h = Math.imul(h, 0x01000193);
  }
  return h >>> 0;
}

/**
 * Seeded 50/50 split by question_id (sorted first so input order is
 * irrelevant). Half A gets the ceil when n is odd. Deterministic for a seed.
 */
export function splitHalf<T extends { question_id: string }>(rows: T[], seed: string): { a: T[]; b: T[] } {
  const sorted = [...rows].sort((x, y) => (x.question_id < y.question_id ? -1 : x.question_id > y.question_id ? 1 : 0));
  const rnd = mulberry32(fnv1a32(seed));
  for (let i = sorted.length - 1; i > 0; i--) {
    const j = Math.floor(rnd() * (i + 1));
    [sorted[i], sorted[j]] = [sorted[j], sorted[i]];
  }
  const mid = Math.ceil(sorted.length / 2);
  return { a: sorted.slice(0, mid), b: sorted.slice(mid) };
}

/**
 * Histogram of each row's TOP rerank score (the weak-top floor's input) — over
 * the same `scoreOf` view autocut sees, so pinned rows never supply the top.
 */
export function topScoreHistogram(rows: ReplayRow[], binWidth = 0.1): Array<{ bin_start: number; bin_end: number; count: number }> {
  const bins = Math.max(1, Math.round(1 / binWidth));
  const counts = new Array<number>(bins).fill(0);
  let above = 0;
  for (const row of rows) {
    const scores = row.rerank_pool.map(scoreOf).filter((s): s is number => typeof s === 'number' && Number.isFinite(s));
    if (scores.length === 0) continue;
    const top = Math.max(...scores);
    if (top >= 1) {
      above++;
      continue;
    }
    // +1e-9 guards the binary-float edge (0.3 / 0.1 = 2.9999…) so 0.3 lands in [0.3, 0.4).
    counts[Math.min(bins - 1, Math.max(0, Math.floor(top / binWidth + 1e-9)))]++;
  }
  const out = counts.map((count, i) => ({ bin_start: +(i * binWidth).toFixed(6), bin_end: +((i + 1) * binWidth).toFixed(6), count }));
  if (above > 0) out[out.length - 1].count += above;
  return out;
}

export interface SweepResult {
  k: number;
  knobs: ReplayKnobs;
  floors: string[];
  summaries: FloorSummary[];
  /** Paired deltas of every floor against the FIRST floor in the list. */
  paired: PairedDelta[];
  top_score_histogram: ReturnType<typeof topScoreHistogram>;
  split?: { seed: string; a: FloorSummary[]; b: FloorSummary[] };
}

/** Run the whole sweep. Pure. */
export function sweepFloors(rows: ReplayRow[], floors: Floor[], k: number, opts: { knobs?: ReplayKnobs; splitSeed?: string } = {}): SweepResult {
  const knobs = opts.knobs ?? DEFAULT_REPLAY_KNOBS;
  const perFloor = floors.map((f) => rows.map((row) => replayRow(row, f, k, knobs)));
  const summaries = perFloor.map((reps, i) => summarizeFloor(reps, floors[i]));
  const paired = perFloor.map((reps) => pairedDelta(reps, perFloor[0]));
  const result: SweepResult = {
    k,
    knobs,
    floors: floors.map(floorLabel),
    summaries,
    paired,
    top_score_histogram: topScoreHistogram(rows),
  };
  if (opts.splitSeed !== undefined) {
    const { a, b } = splitHalf(rows, opts.splitSeed);
    result.split = {
      seed: opts.splitSeed,
      a: floors.map((f) => summarizeFloor(a.map((row) => replayRow(row, f, k, knobs)), f)),
      b: floors.map((f) => summarizeFloor(b.map((row) => replayRow(row, f, k, knobs)), f)),
    };
  }
  return result;
}

export interface ParsedNdjson {
  rows: ReplayRow[];
  skipped_error_rows: number;
  skipped_non_question_rows: number;
}

/**
 * Parse harness ndjson text into replay rows. Question rows carry
 * `question_id`; rows with an `error` field are skipped (counted); a question
 * row WITHOUT `rerank_pool` aborts, naming the row (plan error registry:
 * "autocut replay missing pool"). Summary/meta rows are skipped. Every pool
 * row is normalized through `normalizePoolRow` — unscored alias / exact-lookup
 * rows and relational pins are carried, malformed rows abort naming row + index.
 */
export function parseReplayNdjson(text: string): ParsedNdjson {
  const rows: ReplayRow[] = [];
  let skipped_error_rows = 0;
  let skipped_non_question_rows = 0;
  const lines = text.split('\n');
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i].trim();
    if (line.length === 0) continue;
    let obj: Record<string, unknown>;
    try {
      obj = JSON.parse(line) as Record<string, unknown>;
    } catch {
      throw new Error(`line ${i + 1}: not valid JSON`);
    }
    if (typeof obj.question_id !== 'string') {
      skipped_non_question_rows++;
      continue;
    }
    if (obj.error !== undefined && obj.error !== null) {
      skipped_error_rows++;
      continue;
    }
    if (!Array.isArray(obj.rerank_pool)) {
      throw new Error(`row ${obj.question_id} (line ${i + 1}) has no rerank_pool — capture the arm with --capture-pool`);
    }
    rows.push({
      question_id: obj.question_id,
      question_type: typeof obj.question_type === 'string' ? obj.question_type : undefined,
      rerank_pool: (obj.rerank_pool as unknown[]).map((r, j) => normalizePoolRow(r, `row ${obj.question_id} (line ${i + 1}) rerank_pool[${j}]`)),
      autocut: (obj.autocut ?? (obj.search_meta as { autocut?: LiveAutocut } | undefined)?.autocut ?? null) as LiveAutocut | null,
      autocut_kept_keys: Array.isArray(obj.autocut_kept_keys) ? (obj.autocut_kept_keys as string[]) : undefined,
      answer_session_ids: Array.isArray(obj.answer_session_ids) ? (obj.answer_session_ids as string[]) : [],
    });
  }
  return { rows, skipped_error_rows, skipped_non_question_rows };
}