Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.1 KB
Raw Blame
/**
 * test/eval-canary.test.ts — hermetic retrieval canary (deterministic
 * embedder + runner script).
 *
 * Pins:
 *   1. basisEmbedding determinism + dims (src/eval/deterministic-embed.ts).
 *   2. buildQrelsQueryEmbedFn maps fixture queries to their basis dims and
 *      falls back deterministically (FNV-1a-derived dim) for unknown texts.
 *   3. scripts/run-eval-canary.ts end-to-end in check mode: real CLI
 *      subprocess, exit 0, metrics at/above the qrels default floors.
 *   4. Determinism: two in-process runs of the correctness gate through the
 *      queryEmbedFn seam produce identical metrics.
 *   5. Check mode writes nothing to tracked files (git status unchanged).
 *
 * Fully hermetic: no API keys, no network, no DATABASE_URL. The PGLite
 * engine is scoped to the one describe that needs it (the queryEmbedFn-seam
 * determinism test) so the pure-function and subprocess tests don't pay
 * engine init/reset.
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join, resolve } from 'node:path';
import { execSync, spawnSync } from 'node:child_process';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { basisEmbedding, buildQrelsQueryEmbedFn, fnv1a } from '../src/eval/deterministic-embed.ts';
import { parseLegacyQrels, seedCanaryCorpus } from '../scripts/run-eval-canary.ts';
import { runCorrectnessGate, type CorrectnessGateOpts } from '../src/core/bench/correctness-gate.ts';
import { parseQrelsFile, DEFAULT_QRELS_THRESHOLDS } from '../src/core/bench/qrels-file.ts';
import { hybridSearch } from '../src/core/search/hybrid.ts';

const ROOT = resolve(import.meta.dir, '..');
const QRELS_PATH = join(ROOT, 'test', 'fixtures', 'eval-baselines', 'qrels-search.json');

// ---------------------------------------------------------------------------
// 1. basisEmbedding
// ---------------------------------------------------------------------------

describe('basisEmbedding', () => {
  test('default dim 1536, 1.0 at idx, 0.0 elsewhere', () => {
    const e = basisEmbedding(5);
    expect(e.length).toBe(1536);
    expect(e[5]).toBe(1.0);
    expect(e[0]).toBe(0.0);
    expect(e.reduce((s, v) => s + v, 0)).toBe(1.0);
  });

  test('custom dim + idx wraparound', () => {
    const e = basisEmbedding(103, 100);
    expect(e.length).toBe(100);
    expect(e[3]).toBe(1.0); // 103 % 100
    expect(e.reduce((s, v) => s + v, 0)).toBe(1.0);
  });

  test('deterministic across calls', () => {
    expect([...basisEmbedding(42)]).toEqual([...basisEmbedding(42)]);
  });
});

// ---------------------------------------------------------------------------
// 2. buildQrelsQueryEmbedFn
// ---------------------------------------------------------------------------

describe('buildQrelsQueryEmbedFn', () => {
  const raw = readFileSync(QRELS_PATH, 'utf-8');

  test('maps every fixture query to its embedding_dim basis vector', () => {
    const fn = buildQrelsQueryEmbedFn(raw);
    const fixture = parseLegacyQrels(raw);
    expect(fixture.length).toBeGreaterThanOrEqual(10);
    for (const q of fixture) {
      expect([...fn(q.query)]).toEqual([...basisEmbedding(q.embedding_dim)]);
    }
  });

  test('unknown text falls back to a deterministic FNV-1a-derived dim in [100, 1099]', () => {
    const fn = buildQrelsQueryEmbedFn(raw);
    const unknown = 'a query text that is definitely not in the fixture';
    const a = fn(unknown);
    const b = fn(unknown);
    expect([...a]).toEqual([...b]);
    const idx = a.findIndex(v => v === 1.0);
    expect(idx).toBe(100 + (Math.abs(fnv1a(unknown)) % 1000));
    expect(idx).toBeGreaterThanOrEqual(100);
    expect(idx).toBeLessThanOrEqual(1099);
  });

  test('throws on malformed input', () => {
    expect(() => buildQrelsQueryEmbedFn('not json')).toThrow();
    expect(() => buildQrelsQueryEmbedFn('{"no_queries": true}')).toThrow(/queries/);
  });
});

// ---------------------------------------------------------------------------
// 4. In-process determinism through the queryEmbedFn seam
// ---------------------------------------------------------------------------

describe('correctness gate through the queryEmbedFn seam', () => {
  // Canonical PGLite block (CLAUDE.md R3+R4), scoped to this describe: the
  // seam test is the ONLY engine consumer in the file — the pure-function
  // and subprocess describes must not pay PGLite init/reset.
  let engine: PGLiteEngine;

  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
  });

  afterAll(async () => {
    await engine.disconnect();
  });

  beforeEach(async () => {
    await resetPgliteState(engine);
  });

  test('two in-process runs produce identical metrics, at/above the default floors', async () => {
    const raw = readFileSync(QRELS_PATH, 'utf-8');
    await seedCanaryCorpus(engine, parseLegacyQrels(raw));
    const queryEmbedFn = buildQrelsQueryEmbedFn(raw);
    const qrels = parseQrelsFile(raw);
    const searchFn: NonNullable<CorrectnessGateOpts['searchFn']> = async (e, q, o) => {
      const results = await hybridSearch(e, q, { limit: o.limit, queryEmbedFn });
      return results.map(r => ({ source_id: r.source_id, slug: r.slug }));
    };

    const a = await runCorrectnessGate(engine, qrels, { searchFn });
    const b = await runCorrectnessGate(engine, qrels, { searchFn });

    expect(a.summary).toEqual(b.summary);
    expect(a.per_query).toEqual(b.per_query);

    expect(a.summary.queries_errored).toBe(0);
    expect(a.summary.mean_recall_at_k).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.recall_at_k);
    expect(a.summary.first_relevant_hit_rate).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.first_relevant_hit);
    expect(a.summary.expected_top1_denominator).toBeGreaterThan(0);
    expect(a.summary.expected_top1_hit_rate).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.expected_top1);
  }, 60_000);
});

// ---------------------------------------------------------------------------
// 3 + 5. Runner end-to-end (check mode) + tracked-file invariance
// ---------------------------------------------------------------------------

describe('run-eval-canary.ts (check mode)', () => {
  // Captured by the end-to-end test; asserted separately below so a
  // tracked-file write shows up as its own named failure.
  let statusBefore: string | null = null;
  let statusAfter: string | null = null;
  let runExit: number | null = null;

  test('spawns the real CLI gate hermetically and passes the floors', () => {
    // Status scoped to the paths check mode could plausibly touch — a
    // whole-tree porcelain diff flakes when a concurrent shard sibling (or a
    // developer save) creates an unrelated file during the ~30s window.
    const STATUS_SCOPE = 'git status --porcelain -- .gbrain-evals test/fixtures docs/eval';
    statusBefore = execSync(STATUS_SCOPE, { cwd: ROOT, encoding: 'utf-8' });
    const child = spawnSync(
      process.execPath,
      [join(ROOT, 'scripts', 'run-eval-canary.ts')],
      // Outer budget strictly above the runner's inner CLI-child timeout so
      // the runner's own diagnostics always win the race.
      { cwd: ROOT, encoding: 'utf-8', timeout: 118_000 },
    );
    statusAfter = execSync(STATUS_SCOPE, { cwd: ROOT, encoding: 'utf-8' });
    runExit = child.status;

    const combined = (child.stdout ?? '') + (child.stderr ?? '');
    expect(child.status).toBe(0);

    const m = combined.match(
      /mean_recall_at_k=([\d.]+) first_relevant_hit_rate=([\d.]+) expected_top1_hit_rate=([\d.]+)/,
    );
    expect(m).not.toBeNull();
    expect(Number(m![1])).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.recall_at_k);
    expect(Number(m![2])).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.first_relevant_hit);
    expect(Number(m![3])).toBeGreaterThanOrEqual(DEFAULT_QRELS_THRESHOLDS.expected_top1);
  }, 120_000);

  test('check mode writes nothing to tracked files (git status unchanged)', () => {
    // Guard: the end-to-end test above must have actually run + passed spawn.
    expect(runExit).toBe(0);
    expect(statusBefore).not.toBeNull();
    expect(statusAfter).toBe(statusBefore!);
  });
});