Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
3.1 KB
Raw Blame
/**
 * Regenerates docs/eval/system-one/datasets/HASHES.md from the committed
 * datasets, with the same hash functions `gbrain decide calibrate/qualify`
 * record (datasetHash = sha256 of the file text, first 16 hex; splitHash over
 * the id:split assignment).
 *
 *   bun docs/eval/system-one/generators/hashes.ts
 */
import { readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { datasetHash, families, parseDatasetJsonl, splitHash } from '../../../../src/core/ai/decide/dataset.ts';

const dir = join(import.meta.dir, '..', 'datasets');
const FILES = [
  's6-recall-needed.brainbench.jsonl',
  's6-recall-needed.extra.jsonl',
  's6-recall-needed.combined.jsonl',
  's9-conflict.jsonl',
  's9-conflict.sweep-eligible.jsonl',
  's8-grounding.jsonl',
  's7-triage.jsonl',
  's2-intent.jsonl',
  's5-injection.jsonl',
];

/** Datasets whose lines predate the per-line label_source field (their builders do not emit it). */
const PROVENANCE: Record<string, (text: string) => string> = {
  's7-triage.jsonl': (text) => {
    const ids = text.split('\n').filter((l) => l.trim()).map((l) => (JSON.parse(l) as { id: string }).id);
    const syn = ids.filter((id) => id.startsWith('syn-')).length;
    return `synthetic-construction ${syn} (s7-triage/gold/*.json), upstream-gold ${ids.length - syn} (Cat 35 expected_triage)`;
  },
  's2-intent.jsonl': (text) => `upstream-gold ${text.split('\n').filter((l) => l.trim()).length} (LongMemEval question_type, BrainBench relational fixtures)`,
  's5-injection.jsonl': (text) => `upstream-gold ${text.split('\n').filter((l) => l.trim()).length} (test/fixtures/decide/injection-cases.jsonl)`,
};

const rows = FILES.map((f) => {
  const text = readFileSync(join(dir, f), 'utf8');
  const items = parseDatasetJsonl(text);
  const mix = new Map<string, number>();
  for (const line of text.split('\n').filter((l) => l.trim())) {
    const raw = JSON.parse(line) as { label_source?: string; origin?: string };
    const src = `${raw.label_source ?? 'unmarked'}${raw.origin ? ` (${raw.origin})` : ''}`;
    mix.set(src, (mix.get(src) ?? 0) + 1);
  }
  const cal = items.filter((i) => i.split === 'calibrate').length;
  const provenance = PROVENANCE[f]?.(text) ?? [...mix].sort().map(([k, v]) => `${k} ${v}`).join(', ');
  return `| \`${f}\` | \`${datasetHash(text)}\` | \`${splitHash(items)}\` | ${items.length} | ${families(items).size} | ${cal} / ${items.length - cal} | ${provenance} |`;
});

writeFileSync(join(dir, 'HASHES.md'), `# System One dataset hashes

Generated by \`bun docs/eval/system-one/generators/hashes.ts\`. \`sha256\` is the dataset hash
\`gbrain decide calibrate\` stores (\`datasetHash\`: sha256 of the file text, first 16 hex characters);
\`split_hash\` is the hash of the frozen id:split assignment that calibrations and evals must share (the
same value \`gbrain decide dataset\` prints). Items, families and the calibrate/eval split are counted with
the repository's own parser.

| Dataset file | sha256 (16) | split_hash | Items | Families | Calibrate / eval | Label provenance |
|---|---|---|---|---|---|---|
${rows.join('\n')}
`);
console.error(rows.join('\n'));