Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
4.6 KB
Raw Blame
/**
 * evals/longmemeval/splits-seed42.json + dev-slice-seed42.txt — integrity of
 * the committed, seeded LongMemEval question-id splits (ids only; the file
 * never carries question text or session content).
 *
 * Every pre-registered decision rule reads these lists, so a drifted split
 * (an id in both halves, a dev id leaking into the decision set, an
 * abstention `_abs` id, or a dev-slice txt that no longer equals dev40) would
 * silently corrupt the receipts. Pure file pins — no engine, no network.
 */

import { describe, test, expect } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';

const ROOT = join(import.meta.dir, '..');
const SPLITS_PATH = join(ROOT, 'evals', 'longmemeval', 'splits-seed42.json');
const DEV_SLICE_PATH = join(ROOT, 'evals', 'longmemeval', 'dev-slice-seed42.txt');

interface Splits {
  schema_version: number;
  dataset_sha256: string;
  scored: number;
  abstention_excluded: number;
  seeds: { dev: number; halves470: number; halves430: number };
  type_counts: Record<string, number>;
  dev40_type_counts: Record<string, number>;
  dev40: string[];
  decision430: string[];
  halfA470: string[];
  halfB470: string[];
  halfA430: string[];
  halfB430: string[];
}

const splits = JSON.parse(readFileSync(SPLITS_PATH, 'utf-8')) as Splits;
const LISTS = ['dev40', 'decision430', 'halfA470', 'halfB470', 'halfA430', 'halfB430'] as const;

function isPartition(a: string[], b: string[], whole: Set<string>): boolean {
  const union = new Set([...a, ...b]);
  if (union.size !== whole.size || a.length + b.length !== whole.size) return false;
  for (const id of union) if (!whole.has(id)) return false;
  return true;
}

describe('evals/longmemeval/splits-seed42.json', () => {
  test('schema + provenance fields', () => {
    expect(splits.schema_version).toBe(1);
    expect(splits.dataset_sha256).toMatch(/^[0-9a-f]{64}$/);
    expect(splits.scored).toBe(470);
    expect(splits.abstention_excluded).toBe(30);
    expect(splits.seeds).toEqual({ dev: 42, halves470: 4242, halves430: 4243 });
  });

  test('every list is unique string ids, no abstention (_abs) id anywhere', () => {
    for (const name of LISTS) {
      const list = splits[name];
      expect(Array.isArray(list), name).toBe(true);
      expect(new Set(list).size, `${name} has duplicates`).toBe(list.length);
      for (const id of list) {
        expect(typeof id, name).toBe('string');
        expect(id.length, name).toBeGreaterThan(0);
        expect(id.endsWith('_abs'), `${name}: ${id}`).toBe(false);
      }
    }
  });

  test('dev40 ∩ decision430 = ∅ and dev40 ∪ decision430 = the 470 scored ids', () => {
    expect(splits.dev40.length).toBe(40);
    expect(splits.decision430.length).toBe(430);
    const dev = new Set(splits.dev40);
    for (const id of splits.decision430) expect(dev.has(id), `dev id leaked into decision430: ${id}`).toBe(false);
    const all = new Set([...splits.dev40, ...splits.decision430]);
    expect(all.size).toBe(splits.scored);
  });

  test('halfA470 / halfB470 partition the 470; halfA430 / halfB430 partition decision430', () => {
    const all470 = new Set([...splits.dev40, ...splits.decision430]);
    expect(splits.halfA470.length).toBe(235);
    expect(splits.halfB470.length).toBe(235);
    expect(isPartition(splits.halfA470, splits.halfB470, all470)).toBe(true);
    const decision = new Set(splits.decision430);
    expect(splits.halfA430.length).toBe(215);
    expect(splits.halfB430.length).toBe(215);
    expect(isPartition(splits.halfA430, splits.halfB430, decision)).toBe(true);
    // The 430-halves never touch the dev slice.
    const dev = new Set(splits.dev40);
    for (const id of [...splits.halfA430, ...splits.halfB430]) expect(dev.has(id), id).toBe(false);
  });

  test('type_counts sum to 470 and dev40_type_counts sum to 40 over the same type set', () => {
    const sum = (o: Record<string, number>) => Object.values(o).reduce((a, b) => a + b, 0);
    expect(sum(splits.type_counts)).toBe(470);
    expect(sum(splits.dev40_type_counts)).toBe(40);
    expect(Object.keys(splits.dev40_type_counts).sort()).toEqual(Object.keys(splits.type_counts).sort());
    for (const [t, n] of Object.entries(splits.dev40_type_counts)) expect(n, t).toBeLessThanOrEqual(splits.type_counts[t]);
  });

  test('dev-slice-seed42.txt equals dev40 as a set (one id per line, for --question-ids)', () => {
    const lines = readFileSync(DEV_SLICE_PATH, 'utf-8')
      .split('\n')
      .map((l) => l.trim())
      .filter((l) => l.length > 0);
    expect(lines.length).toBe(40);
    expect(new Set(lines).size).toBe(40);
    expect([...new Set(lines)].sort()).toEqual([...splits.dev40].sort());
  });
});