Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.7 KB
Raw Blame
/**
 * scorer.ts — pure scoring for the takes-bootstrap classifier eval (H1 /
 * TODOS TODO-E). No I/O, no gateway imports: unit-testable keyless.
 *
 * Contract (test-gap plan, CEO Finding 6): a fixture whose model output was
 * malformed/unparseable counts as a FAILURE (`malformed`), never a skip — a
 * skip would silently shrink the denominator.
 *
 * Graduation (SCORER_VERSION 1): per-kind precision >= 0.80 AND per-kind
 * recall >= 0.70 (kinds with zero expected instances are exempt from recall),
 * zero malformed cases, forbid-violations = 0. Autopilot tier for
 * takes-bootstrap stays manual_only until a LIVE run passes this bar
 * (TODOS.md TODO-E); loosening the bar is a reviewer-visible edit here.
 *
 * Besides the per-kind verdict, the report breaks results down per variant
 * (one row per case) and per archetype (its variants rolled up). A case
 * passes when its output parsed, every expected claim matched, every
 * prediction was precise and no forbid pattern fired; these rows explain the
 * verdict and never change it.
 */

export const SCORER_VERSION = 1;
export const GRADUATION = { minPrecision: 0.8, minRecall: 0.7 } as const;

export type TakeKindLabel = 'fact' | 'take' | 'bet' | 'hunch';

export interface CorpusCase {
  id: string;
  /** The hand-authored archetype this case is a label-invariant variant of. */
  archetype: string;
  category: string;
  page: { slug: string; type: string; title: string; body: string };
  expected: Array<{ claim_re: string; kind: TakeKindLabel; weight_min: number; weight_max: number }>;
  forbid: string[];
  notes: string;
}

export interface CasePrediction {
  id: string;
  /** null = the classifier produced no parseable output for this case. */
  claims: Array<{ claim: string; kind: string; weight: number }> | null;
}

export interface KindScore {
  kind: TakeKindLabel;
  expected: number;
  matched: number;      // expected instances matched by >=1 prediction (recall numerator)
  predicted: number;
  precise: number;      // predictions that satisfy >=1 expected (precision numerator)
  precision: number;    // 1 when predicted === 0
  recall: number;       // 1 when expected === 0
}

export interface CaseTally {
  expected: number;
  matched: number;
  predicted: number;
  precise: number;
  forbid_violations: number;
}

export interface VariantScore extends CaseTally {
  id: string;
  archetype: string;
  category: string;
  malformed: boolean;
  pass: boolean;
}

export interface ArchetypeScore extends CaseTally {
  archetype: string;
  category: string;
  variants: number;
  variants_passed: number;
  malformed: number;
}

export interface ScoreReport {
  scorer_version: number;
  cases: number;
  malformed: string[];          // case ids with claims === null — FAILURES
  forbid_violations: Array<{ id: string; forbid_re: string; claim: string }>;
  by_kind: KindScore[];
  by_archetype: ArchetypeScore[];
  by_variant: VariantScore[];
  overall: { precision: number; recall: number; f1: number };
  graduated: boolean;
  failures: string[];           // human-readable reasons graduation failed
}

const KINDS: TakeKindLabel[] = ['fact', 'take', 'bet', 'hunch'];

export function scoreCorpus(corpus: CorpusCase[], predictions: CasePrediction[]): ScoreReport {
  const predById = new Map(predictions.map(p => [p.id, p]));
  const malformed: string[] = [];
  const forbidViolations: ScoreReport['forbid_violations'] = [];
  const tally = new Map<TakeKindLabel, { expected: number; matched: number; predicted: number; precise: number }>(
    KINDS.map(k => [k, { expected: 0, matched: 0, predicted: 0, precise: 0 }]),
  );

  const byVariant: VariantScore[] = [];

  for (const c of corpus) {
    const pred = predById.get(c.id);
    const v: VariantScore = {
      id: c.id, archetype: c.archetype, category: c.category, malformed: false, pass: false,
      expected: c.expected.length, matched: 0, predicted: 0, precise: 0, forbid_violations: 0,
    };
    byVariant.push(v);
    // Missing prediction row OR null claims = malformed = FAILURE, never skip.
    if (!pred || pred.claims === null) {
      malformed.push(c.id);
      v.malformed = true;
      for (const e of c.expected) tally.get(e.kind)!.expected += 1; // still owed
      continue;
    }
    const claims = pred.claims;
    const violationsBefore = forbidViolations.length;

    for (const forbidRe of c.forbid) {
      // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- the pattern comes from the COMMITTED corpus (evals/takes-bootstrap/corpus.jsonl, reviewed in PRs and validated by test/eval-takes-bootstrap.test.ts's regex-compiles check), never from runtime user input; the eval harness is an offline instrument
      const re = new RegExp(forbidRe, 'i');
      for (const cl of claims) {
        if (re.test(cl.claim)) forbidViolations.push({ id: c.id, forbid_re: forbidRe, claim: cl.claim });
      }
    }

    for (const e of c.expected) {
      const t = tally.get(e.kind)!;
      t.expected += 1;
      // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- corpus-committed pattern, same rationale as the forbid loop above
      const re = new RegExp(e.claim_re, 'i');
      const hit = claims.some(cl =>
        cl.kind === e.kind && re.test(cl.claim) && cl.weight >= e.weight_min && cl.weight <= e.weight_max,
      );
      if (hit) {
        t.matched += 1;
        v.matched += 1;
      }
    }

    for (const cl of claims) {
      const kind = (KINDS as string[]).includes(cl.kind) ? (cl.kind as TakeKindLabel) : null;
      if (!kind) continue; // parseClaimsJson already filters; unknown kinds count nowhere
      const t = tally.get(kind)!;
      t.predicted += 1;
      v.predicted += 1;
      // Precision credit: matches any expected of the same kind by regex
      // (weight range NOT required for precision — a right claim with an
      // off-range weight is a recall miss, not a hallucination).
      // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- corpus-committed pattern, same rationale as the forbid loop above
      const ok = c.expected.some(e => e.kind === kind && new RegExp(e.claim_re, 'i').test(cl.claim));
      if (ok) {
        t.precise += 1;
        v.precise += 1;
      }
    }
    v.forbid_violations = forbidViolations.length - violationsBefore;
    v.pass = v.matched === v.expected && v.precise === v.predicted && v.forbid_violations === 0;
  }

  const archetypes = new Map<string, ArchetypeScore>();
  for (const v of byVariant) {
    const a = archetypes.get(v.archetype) ?? {
      archetype: v.archetype, category: v.category, variants: 0, variants_passed: 0, malformed: 0,
      expected: 0, matched: 0, predicted: 0, precise: 0, forbid_violations: 0,
    };
    a.variants += 1;
    if (v.pass) a.variants_passed += 1;
    if (v.malformed) a.malformed += 1;
    a.expected += v.expected;
    a.matched += v.matched;
    a.predicted += v.predicted;
    a.precise += v.precise;
    a.forbid_violations += v.forbid_violations;
    archetypes.set(v.archetype, a);
  }

  const byKind: KindScore[] = KINDS.map(kind => {
    const t = tally.get(kind)!;
    return {
      kind, ...t,
      precision: t.predicted === 0 ? 1 : t.precise / t.predicted,
      recall: t.expected === 0 ? 1 : t.matched / t.expected,
    };
  });

  const totals = byKind.reduce(
    (a, k) => ({ expected: a.expected + k.expected, matched: a.matched + k.matched, predicted: a.predicted + k.predicted, precise: a.precise + k.precise }),
    { expected: 0, matched: 0, predicted: 0, precise: 0 },
  );
  const precision = totals.predicted === 0 ? 1 : totals.precise / totals.predicted;
  const recall = totals.expected === 0 ? 1 : totals.matched / totals.expected;
  const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);

  const failures: string[] = [];
  if (malformed.length > 0) failures.push(`${malformed.length} malformed case(s): ${malformed.slice(0, 5).join(', ')}${malformed.length > 5 ? ', …' : ''}`);
  if (forbidViolations.length > 0) failures.push(`${forbidViolations.length} forbid violation(s) (attribution/injection leaks)`);
  for (const k of byKind) {
    if (k.predicted > 0 && k.precision < GRADUATION.minPrecision) failures.push(`${k.kind} precision ${k.precision.toFixed(3)} < ${GRADUATION.minPrecision}`);
    if (k.expected > 0 && k.recall < GRADUATION.minRecall) failures.push(`${k.kind} recall ${k.recall.toFixed(3)} < ${GRADUATION.minRecall}`);
  }

  return {
    scorer_version: SCORER_VERSION,
    cases: corpus.length,
    malformed,
    forbid_violations: forbidViolations,
    by_kind: byKind,
    by_archetype: [...archetypes.values()],
    by_variant: byVariant,
    overall: { precision, recall, f1 },
    graduated: failures.length === 0,
    failures,
  };
}