Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.3 KB
Raw Blame
/**
 * F-Eval — hermetic $0 write-path mini-eval (eval fix wave, CEO-review E1).
 *
 * A frozen 3-transcript mini-corpus with expected salient units drives the
 * REAL runPhaseSynthesize end to end on PGLite: real triage parse + gate
 * (incl. the F2 rescue), real fan-out + oneshot drain, real quote
 * verify/repair, real provenance stamp + reverse-write + telemetry. The ONLY
 * stub is the gateway chat transport (__setChatTransportForTests — the
 * test/e2e/dream-synthesize-pglite.test.ts seam), serving a scripted triage
 * judge and a scripted oneshot child.
 *
 * HONESTY (outside-voice amendment 6): a scripted child cannot measure
 * whether PROMPT changes improve model output — that is the paid Cat 35
 * benchmark's job. This harness is the $0 regression pin for the MECHANICAL
 * write path: gate/rescue admission, page emission, the repair ladder acting
 * on real written pages, file dual-write, and telemetry shape. The
 * salient-unit presence score at the bottom is the canary: any mechanical
 * regression (emission, chunk slug rewrite, repair over-deletion) drops it
 * below 100%.
 *
 * Corpus discipline: fixtures are frozen HERE, distinct from the Cat 35
 * corpus (no tuning coupling), placeholder names only.
 */
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { withEnv } from './helpers/with-env.ts';
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { runPhaseSynthesize } from '../src/core/cycle/synthesize.ts';
import { importFromContent } from '../src/core/import-file.ts';
import { normForGrounding } from '../src/core/cycle/synthesize-verify.ts';
import { __setChatTransportForTests, resetGateway, type ChatOpts, type ChatResult } from '../src/core/ai/gateway.ts';

// ── The frozen mini-corpus ────────────────────────────────────────────────

interface MiniFixture {
  basename: string;
  band: 'high' | 'buried' | 'routine';
  content: string;
  /** Verbatim salient anchors that MUST survive into a written page. */
  expect_units: string[];
  /** Judge segments (verbatim slices) served by the scripted triage. */
  segments: string[];
}

const PAD = 'User: also check the build status.\nAssistant: Green.\n'.repeat(30);

const FIXTURES: MiniFixture[] = [
  {
    basename: '2026-08-30-high-thesis.txt',
    band: 'high',
    expect_units: [
      'memory portability is the wedge because nobody wants memories locked to an agent vendor',
      'we charge for durability, not storage',
    ],
    segments: [
      'memory portability is the wedge because nobody wants memories locked to an agent vendor',
      'we charge for durability, not storage',
    ],
    content: `User: Two things crystallized today. First: memory portability is the wedge because nobody wants memories locked to an agent vendor. Second: we charge for durability, not storage — storage is a race to zero.\nAssistant: Both worth pages.\n${PAD}`,
  },
  {
    basename: '2026-08-30-buried-nugget.txt',
    band: 'buried',
    expect_units: [
      'our churn spikes line up with the annual renewal emails, not with outages',
      'rewrite renewal emails as value receipts sixty days before the invoice',
    ],
    segments: [
      'our churn spikes line up with the annual renewal emails, not with outages',
      'rewrite renewal emails as value receipts sixty days before the invoice',
    ],
    content: `${PAD}User: Oh, while I remember — I finally see it: our churn spikes line up with the annual renewal emails, not with outages. So: rewrite renewal emails as value receipts sixty days before the invoice.\nAssistant: Noted.\n${PAD}`,
  },
  {
    basename: '2026-08-30-routine-control.txt',
    band: 'routine',
    expect_units: [],
    segments: [],
    content: `User: Move my 9am and file the parking receipt.\nAssistant: Done and filed.\n${PAD}`,
  },
];

// The scripted child writes ONE reflections page per transcript. Its body
// exercises the repair ladder deliberately:
//   - an EXACT quote (survives untouched),
//   - a normalized-drift quote (straight apostrophe + hyphen where the
//     transcript has none → actually: whitespace drift) → repaired verbatim,
//   - a FABRICATED quote → its unit is quarantined (out of the body, kept in
//     frontmatter `unverified_claims`),
// plus every expected salient unit as plain prose (the presence canary).
const FABRICATED = 'we should rewrite the entire product in a weekend, obviously';

function scriptedTransport(brainScaffoldSlug: string) {
  return async (opts: ChatOpts): Promise<ChatResult> => {
    const system = opts.system ?? '';
    const userMsg = String(opts.messages?.[0]?.content ?? '');
    let text: string;
    if (system.startsWith('You triage a conversation transcript')) {
      // The triage user message carries the extension-less basename; match on
      // the stem so the scripted judge maps to the right fixture.
      const f = FIXTURES.find(x => userMsg.includes(x.basename.replace(/\.txt$/, '')));
      const band = f?.band ?? 'routine';
      const score = band === 'high' ? 0.85 : band === 'buried' ? 0.4 : 0.1;
      text = JSON.stringify({
        score,
        content_type: band === 'high' ? 'reflection' : band === 'buried' ? 'mixed' : 'routine',
        segments: (f?.segments ?? []).map(q => ({ quote: q, note: 'scripted' })),
        entities: [],
        reasons: [band],
      });
    } else if (system.startsWith('You are a knowledge-synthesis engine')) {
      const f = FIXTURES.find(x => userMsg.includes(x.content.slice(0, 60)))
        ?? FIXTURES.find(x => (x.segments[0] ? userMsg.includes(x.segments[0]) : false));
      const hash = /hash suffix \(USE THIS in slugs\): ([a-z0-9-]+)/i.exec(userMsg)?.[1] ?? 'nohash';
      const units = f?.expect_units ?? [];
      const exact = f?.segments[0] ?? 'no segment';
      // Normalized drift: collapse the exact quote's spaces oddly + uppercase
      // one word — repairable to the verbatim transcript slice (rung 2).
      const drifted = (f?.segments[1] ?? 'no second segment').replace(/^rewrite|^we/, m => m.toUpperCase()).replace(/ /g, '  ');
      const body = [
        `A working session distilled. See [[${brainScaffoldSlug}]] for context.`,
        '',
        `Key units: ${units.join('; ')}.`,
        '',
        `The user said: "${exact}"`,
        '',
        `They also said: "${drifted}"`,
        '',
        `And allegedly: "${FABRICATED}"`,
      ].join('\n');
      text = JSON.stringify({
        pages: [{
          slug: `wiki/personal/reflections/2026-08-30-mini-eval-${hash}`,
          title: 'Mini-eval reflection',
          type: 'note',
          body,
        }],
        skipped: false,
        skip_reason: null,
      });
    } else {
      text = '{}';
    }
    return {
      text,
      blocks: [{ type: 'text', text } as never],
      stopReason: 'end',
      usage: { input_tokens: 1000, output_tokens: 200, cache_read_tokens: 0, cache_creation_tokens: 0 },
      model: opts.model ?? 'anthropic:scripted',
      providerId: 'anthropic',
    };
  };
}

describe('F-Eval: hermetic write-path mini-eval (real phase, scripted transport)', () => {
  let engine: PGLiteEngine;
  let brainDir: string;
  let corpusDir: string;
  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({ engine: 'pglite' } as never);
    await engine.initSchema();
    brainDir = mkdtempSync(join(tmpdir(), 'gbrain-minieval-brain-'));
    corpusDir = mkdtempSync(join(tmpdir(), 'gbrain-minieval-corpus-'));
  });
  afterAll(async () => {
    __setChatTransportForTests(null);
    resetGateway();
    try { await engine.disconnect(); } catch { /* best-effort */ }
    try { rmSync(brainDir, { recursive: true, force: true }); } catch { /* */ }
    try { rmSync(corpusDir, { recursive: true, force: true }); } catch { /* */ }
  });

  test('gate+rescue admit the right transcripts; repair ladder fixes real pages; units survive to disk', async () => {
    {
      // Scaffold page so the wikilink mandate is satisfiable (Cat 35 pattern).
      const scaffold = 'people/alice-example';
      await importFromContent(engine, scaffold, '---\ntype: person\n---\nAlice Example, a founder.', { noEmbed: true, remote: false, sourceId: 'default' });

      await engine.setConfig('dream.synthesize.enabled', 'true');
      await engine.setConfig('dream.synthesize.session_corpus_dir', corpusDir);
      await engine.setConfig('dream.synthesize.cooldown_hours', '0');
      // The frozen fixtures are deliberately small; the 2000-char discovery
      // floor is not what this harness pins.
      await engine.setConfig('dream.synthesize.min_chars', '200');
      for (const f of FIXTURES) writeFileSync(join(corpusDir, f.basename), f.content);

      __setChatTransportForTests(scriptedTransport(scaffold));
      // Fake key via withEnv (R1): hasAnthropicKey() must see one so the
      // judge client exists — the transport stub intercepts every real call.
      const result = await withEnv({ ANTHROPIC_API_KEY: 'sk-test-mini-eval' }, () =>
        runPhaseSynthesize(engine, { brainDir, dryRun: false }));
      expect(result.status).toBe('ok');
      const details = result.details as {
        pages_written: number;
        written_slugs: string[];
        verdicts: Array<{ filePath: string; worth: boolean; rescued?: boolean; score: number | null }>;
        triage: { rescue_fired: number; rescue_checked: number; below_threshold: number; tokens_in: number };
        synthesis: {
          quote_verify: { pages_checked: number; quotes_total: number; exact: number; normalized_fixed: number; near_fixed: number; quarantined_claims: number; quote_not_in_source: number; pages_repaired: number; errors: number } | null;
          spend: { cost_basis: string; children: { tokens_in: number }; triage: { tokens_in: number } };
          children_zero_pages: number;
        };
      };

      // ── Admission: high passes plainly, buried passes VIA RESCUE, routine never fires.
      const byPath = new Map(details.verdicts.map(v => [v.filePath, v]));
      const v = (name: string) => byPath.get(join(corpusDir, name))!;
      expect(v('2026-08-30-high-thesis.txt').worth).toBe(true);
      expect(v('2026-08-30-high-thesis.txt').rescued).toBeUndefined();
      expect(v('2026-08-30-buried-nugget.txt').worth).toBe(true);
      expect(v('2026-08-30-buried-nugget.txt').rescued).toBe(true);
      expect(v('2026-08-30-routine-control.txt').worth).toBe(false);
      expect(details.triage.rescue_fired).toBe(1);
      expect(details.triage.below_threshold).toBe(1); // routine only — rescued is NOT below-threshold
      expect(details.pages_written).toBe(2);

      // ── Repair ladder on the REAL written pages (per page: 1 exact, 1 repaired, 1 quarantined).
      const qv = details.synthesis.quote_verify!;
      expect(qv.pages_checked).toBe(2);
      expect(qv.errors).toBe(0);
      expect(qv.exact).toBe(2);
      expect(qv.normalized_fixed).toBe(2);
      expect(qv.quote_not_in_source).toBe(2);
      expect(qv.quarantined_claims).toBe(2);
      expect(qv.pages_repaired).toBe(2);

      // ── Salient-unit presence canary: every expected unit, normalized, in a written page body.
      let present = 0;
      let total = 0;
      const bodies: string[] = [];
      for (const slug of details.written_slugs) {
        const page = await engine.getPage(slug, { sourceId: 'default' });
        bodies.push(normForGrounding(`${page?.compiled_truth ?? ''}\n${page?.timeline ?? ''}`));
      }
      for (const f of FIXTURES) {
        for (const u of f.expect_units) {
          total++;
          if (bodies.some(b => b.includes(normForGrounding(u)))) present++;
        }
      }
      expect(total).toBe(4);
      expect(present).toBe(total); // 100% — the mechanical canary

      // ── Dual-write: the reverse-written .md carries the REPAIRED body.
      const mdPaths = details.written_slugs.map(s => join(brainDir, `${s}.md`));
      for (const p of mdPaths) expect(existsSync(p)).toBe(true);
      const mdAll = mdPaths.map(p => readFileSync(p, 'utf8')).join('\n');
      // The fabricated quote is out of every page body (no chunks, no
      // search) and preserved for review in frontmatter.
      for (const slug of details.written_slugs) {
        const page = await engine.getPage(slug, { sourceId: 'default' });
        expect(normForGrounding(`${page!.compiled_truth}\n${page!.timeline}`)).not.toContain(normForGrounding(FABRICATED));
        const records = page!.frontmatter.unverified_claims as Array<{ text: string; reason: string }>;
        expect(records.map(r => r.reason)).toEqual(['quote_not_in_source']);
        expect(records[0].text).toContain(FABRICATED);
      }
      expect(mdAll).toContain('unverified_claims:');
      // The drifted quote was repaired to a verbatim transcript slice.
      expect(normForGrounding(mdAll)).toContain(normForGrounding('rewrite renewal emails as value receipts sixty days before the invoice'));

      // ── Telemetry shape: spend block + rule-D counter present and honest.
      expect(details.synthesis.spend.cost_basis).toBe('in+out+cache_read');
      expect(details.synthesis.spend.triage.tokens_in).toBeGreaterThan(0);
      expect(details.synthesis.children_zero_pages).toBe(0);
    }
  }, 120_000);

  test('kill switch: quote_verify=false leaves fabricated quotes intact and reports quote_verify null; a declining child completes and counts in children_zero_pages (#5590)', async () => {
    const brainDir2 = mkdtempSync(join(tmpdir(), 'gbrain-minieval-brain2-'));
    const corpusDir2 = mkdtempSync(join(tmpdir(), 'gbrain-minieval-corpus2-'));
    try {
      const scaffold = 'people/alice-example';
      await importFromContent(engine, scaffold, '---\ntype: person\n---\nAlice Example, a founder.', { noEmbed: true, remote: false, sourceId: 'default' });
      await engine.setConfig('dream.synthesize.session_corpus_dir', corpusDir2);
      await engine.setConfig('dream.synthesize.quote_verify', 'false');   // the incident escape hatch
      await engine.setConfig('dream.synthesize.last_completion_ts', '');
      // One signal transcript (child writes, quotes unrepaired) + one that the
      // child DECLINES (task D) despite passing the gate — the rule-D case.
      // Distinct content from test 1 (different content hash → fresh
      // idempotency keys, fresh slugs — no cross-test page reuse).
      const high = { ...FIXTURES[0], basename: '2026-08-31-killswitch-writer.txt', content: `KILLSWITCH RUN.\n${FIXTURES[0].content}` };
      writeFileSync(join(corpusDir2, high.basename), high.content);
      const decliner = { basename: '2026-08-31-decliner.txt', content: `DECLINER RUN.\n${FIXTURES[0].content}` };
      writeFileSync(join(corpusDir2, decliner.basename), decliner.content);

      __setChatTransportForTests(async (opts) => {
        const system = opts.system ?? '';
        const userMsg = String(opts.messages?.[0]?.content ?? '');
        const isDecliner = userMsg.includes('decliner');
        let text: string;
        if (system.startsWith('You triage a conversation transcript')) {
          text = JSON.stringify({ score: 0.85, content_type: 'reflection', segments: [{ quote: high.segments[0], note: 's' }], entities: [], reasons: ['high'] });
        } else if (isDecliner) {
          text = JSON.stringify({ pages: [], skipped: true, skip_reason: 'still routine' });
        } else {
          const hash = /hash suffix \(USE THIS in slugs\): ([a-z0-9-]+)/i.exec(userMsg)?.[1] ?? 'nohash';
          text = JSON.stringify({ pages: [{ slug: `wiki/personal/reflections/2026-08-30-killswitch-${hash}`, title: 'T', type: 'note', body: `Body. See [[${scaffold}]].\n\nAlleged: "${FABRICATED}"` }], skipped: false, skip_reason: null });
        }
        return { text, blocks: [{ type: 'text', text } as never], stopReason: 'end', usage: { input_tokens: 100, output_tokens: 20, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'anthropic:scripted', providerId: 'anthropic' };
      });

      const result = await withEnv({ ANTHROPIC_API_KEY: 'sk-test-mini-eval' }, () =>
        runPhaseSynthesize(engine, { brainDir: brainDir2, dryRun: false }));
      expect(result.status).toBe('ok');
      const d = result.details as { written_slugs: string[]; child_outcomes: Array<{ status: string }>; synthesis: { quote_verify: unknown; children_zero_pages: number; non_completed_jobs: number } };
      expect(d.synthesis.quote_verify).toBeNull();               // pass skipped entirely
      expect(d.synthesis.children_zero_pages).toBe(1);           // rule-D decliner counted
      expect(d.synthesis.non_completed_jobs).toBe(0);
      expect(d.child_outcomes.every(child => child.status === 'completed')).toBe(true);
      expect(await engine.getConfig('dream.synthesize.last_completion_ts')).not.toBe('');
      const killSlug = d.written_slugs.find(s => s.includes('killswitch'));
      expect(killSlug).toBeDefined();
      const page = await engine.getPage(killSlug!, { sourceId: 'default' });
      expect(`${page?.compiled_truth ?? ''}`).toContain(`"${FABRICATED}"`); // unrepaired, marks intact
    } finally {
      await engine.setConfig('dream.synthesize.quote_verify', 'true');
      try { rmSync(brainDir2, { recursive: true, force: true }); } catch { /* */ }
      try { rmSync(corpusDir2, { recursive: true, force: true }); } catch { /* */ }
    }
  }, 120_000);
});