Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* S7 triage eval: synthetic transcripts that extend the Cat 35 corpus with
* enough routine negatives (and buried-signal positives) for qualification.
*
* bun docs/eval/system-one/generators/s7-synthetic-transcripts.ts [--out DIR] [--dry-run]
*
* Labels are BY CONSTRUCTION (label_source "synthetic-construction"): each
* spec tells the generator model what the transcript must and must not
* contain; nobody hand-labelled these. Specs are deterministic (seeded
* PRNG over fixed theme/signal/name pools); the text comes from
* openai:gpt-5.6-luna (reasoning none, temperature 1), cached per id so a
* rerun only fills gaps. Output is the cat35 builder layout:
* gold/<id>.json {transcript_id, scenario, expected_triage, label_source, generator, spec}
* transcripts-txt/<date>-<id>.txt
* Spend is appended to docs/eval/system-one/ledger.jsonl.
*/
import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync } from 'node:fs';
import { join } from 'node:path';
const MODEL = 'gpt-5.6-luna';
const PRICE = { input: 0.2, output: 1.2 };
const HARD_STOP_USD = 2.5;
const SEED = 7_350_001;
const args = process.argv.slice(2);
const OUT = args.includes('--out') ? args[args.indexOf('--out') + 1]! : 'docs/eval/system-one/datasets/s7-triage';
const DRY = args.includes('--dry-run');
const LEDGER = 'docs/eval/system-one/ledger.jsonl';
function mulberry32(seed: number) {
let a = seed >>> 0;
return () => {
a = (a + 0x6d2b79f5) >>> 0;
let t = a;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
const rand = mulberry32(SEED);
const pick = <T>(xs: readonly T[]): T => xs[Math.floor(rand() * xs.length)]!;
const ROUTINE_THEMES = [
'planning the week\'s dinners around leftovers and writing a grocery list',
'tracking a delayed package and drafting a polite support message',
'converting recipe units and oven temperatures for a scaled recipe',
'a shell session freeing disk space with pasted `du` output and deletions',
'fixing a spreadsheet formula that returns #REF after a column was moved',
'working out a call time across three timezones with daylight-saving edge cases',
'reformatting a CSV export so a budgeting app will import it',
'troubleshooting home wifi dropping in one room: router restart, channel change',
'renaming a folder of vacation photos by date with a small script',
'resolving a git merge conflict in a lockfile step by step, pure mechanics',
'debugging a failing unit test caused by a typo in a fixture path, no reflection',
'choosing between two identical-looking phone chargers by wattage specs',
'setting up a printer on a laptop and clearing a stuck print queue',
'asking for synonyms and grammar fixes on a dull office notice about parking',
'comparing grocery unit prices for olive oil and paper towels',
'looking up how to reset a forgotten router admin password',
'formatting a bibliography in a citation style for a school handout',
'figuring out a bus and train route across town for a dentist appointment',
'a regex to strip trailing whitespace from log lines, with test strings',
'converting a short list of temperatures and distances between metric and imperial',
'cleaning up an overflowing email inbox with filter rules for newsletters',
'picking a houseplant watering schedule from care-label instructions',
'sorting out a laptop that will not connect to bluetooth headphones',
'trivia chat about why the sky is blue and how rainbows form',
'building a packing checklist for a two-night trip with laundry logistics',
'a linter complaining about import order, fixed by running the formatter',
'calculating tip and splitting a restaurant bill among four people',
'installing a Python package that fails because of a missing system library',
'drafting a generic out-of-office auto-reply with dates filled in',
'deciding how many AA batteries a set of toys needs and where to buy them',
] as const;
const SIGNALS = [
{ kind: 'decision', brief: 'the user makes a consequential decision about their own project (for example committing to shut down a side product, pick a co-founder role, or change pricing), with the reason' },
{ kind: 'commitment', brief: 'the user commits to a specific future action with a named person and a date (for example promising to send a named collaborator a term sheet by Friday)' },
{ kind: 'person-fact', brief: 'the user reveals a new durable fact about a named person they work with (their new role, a health situation they are handling, a move, a preference that matters for working together)' },
{ kind: 'project-fact', brief: 'the user states a new durable fact about their project or company (a metric milestone, a lost customer and why, a key hire, a legal constraint)' },
{ kind: 'idea', brief: 'the user articulates an original idea or thesis (a product concept, a mental model about their market) in a few sentences with some reasoning' },
{ kind: 'reflection', brief: 'the user reflects on themselves: names a recurring pattern in how they work or feel, and what they want to change' },
] as const;
const FIRST = ['Marisol', 'Tobin', 'Anneke', 'Dario', 'Priyanka', 'Otto', 'Leilani', 'Bram', 'Yusra', 'Cormac', 'Ingrid', 'Teo', 'Wren', 'Hollis', 'Saoirse', 'Ravi'];
const LAST = ['Veen', 'Arkwright', 'Solberg', 'Quintero', 'Hale-Ober', 'Lindqvist', 'Moradi', 'Okonkwo-Bell', 'Fairweather', 'Castellan', 'Drummond', 'Ishikawa-Roe'];
const PROJECTS = ['fernpost', 'lumenkit', 'harbor-ledger', 'quillmesh', 'saltbox', 'northgate-labs', 'tidewell', 'brambleworks'];
export interface Spec {
id: string;
scenario: 'synthetic-routine' | 'synthetic-buried-signal';
expected_triage: 'low' | 'high';
theme: string;
turns: number;
signal?: { kind: string; brief: string; person: string; project: string; position: string };
}
export function buildSpecs(): Spec[] {
const specs: Spec[] = [];
for (let i = 0; i < 110; i++) {
const theme = ROUTINE_THEMES[i % ROUTINE_THEMES.length]!;
specs.push({ id: `syn-routine-${String(i + 1).padStart(3, '0')}`, scenario: 'synthetic-routine', expected_triage: 'low', theme, turns: 10 + Math.floor(rand() * 14) });
}
for (let i = 0; i < 50; i++) {
const sig = SIGNALS[i % SIGNALS.length]!;
specs.push({
id: `syn-buried-${String(i + 1).padStart(3, '0')}`, scenario: 'synthetic-buried-signal', expected_triage: 'high',
theme: pick(ROUTINE_THEMES), turns: 12 + Math.floor(rand() * 14),
signal: { kind: sig.kind, brief: sig.brief, person: `${pick(FIRST)} ${pick(LAST)}`, project: pick(PROJECTS), position: pick(['in the middle', 'about two thirds of the way through', 'near the end, before the wrap-up']) },
});
}
// Second batch (appended after the first so the first batch's PRNG draws are unchanged):
// more routine negatives so the eval half can reach the qualification minimum.
for (let i = 110; i < 180; i++) {
const theme = ROUTINE_THEMES[(i * 7) % ROUTINE_THEMES.length]!;
specs.push({ id: `syn-routine-${String(i + 1).padStart(3, '0')}`, scenario: 'synthetic-routine', expected_triage: 'low', theme, turns: 10 + Math.floor(rand() * 14) });
}
return specs;
}
function prompt(s: Spec): string {
const base = `Write a realistic chat transcript between a user and an AI assistant, ${s.turns} turns total, alternating, starting with the user.
Format exactly: a line "[user]" or "[assistant]", then the message, then a blank line. No title, no commentary, no markdown fences.
Length: about ${s.turns * 450} characters overall; assistant replies are practical and specific; include pasted command output or lists where natural.
Topic: ${s.theme}.`;
if (s.scenario === 'synthetic-routine') {
return `${base}
This transcript must be ENTIRELY ROUTINE: nothing a personal knowledge base should remember. Hard rules: no decisions of consequence, no commitments to other people, no facts about named people or companies, no original ideas or theses, no self-reflection or emotions beyond mild annoyance or relief. Do not name any real or fictional person. Mundane details (quantities, times, file names) are fine.`;
}
const g = s.signal!;
return `${base}
The transcript is mostly routine, but ${g.position} the user briefly goes off-topic for one or two messages in which ${g.brief}. Use the person name "${g.person}" and/or the project name "${g.project}" where a name fits. Keep that passage natural and concrete (4-8 sentences from the user), then the conversation returns to the routine topic. Everything else follows these rules: no other people, decisions or reflections.`;
}
async function generate(s: Spec): Promise<{ text: string; usage: { in: number; out: number } }> {
const res = await fetch('https://api.openai.com/v1/chat/completions', {
method: 'POST',
headers: { Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, 'Content-Type': 'application/json' },
body: JSON.stringify({ model: MODEL, reasoning_effort: 'none', temperature: 1, max_completion_tokens: 6000, messages: [{ role: 'user', content: prompt(s) }] }),
});
if (!res.ok) throw new Error(`openai ${res.status}: ${(await res.text()).slice(0, 200)}`);
const d = await res.json() as { choices: Array<{ message: { content: string } }>; usage: { prompt_tokens: number; completion_tokens: number } };
return { text: d.choices[0]!.message.content.trim() + '\n', usage: { in: d.usage.prompt_tokens, out: d.usage.completion_tokens } };
}
async function main(): Promise<void> {
const specs = buildSpecs();
mkdirSync(join(OUT, 'gold'), { recursive: true });
mkdirSync(join(OUT, 'transcripts-txt'), { recursive: true });
const todo = specs.filter((s) => !existsSync(join(OUT, 'gold', `${s.id}.json`)));
console.log(`${specs.length} specs, ${todo.length} to generate`);
if (DRY) return;
let usd = 0, tin = 0, tout = 0;
let next = 0;
const worker = async () => {
while (next < todo.length) {
const s = todo[next++]!;
if (usd > HARD_STOP_USD) throw new Error(`hard stop at $${usd.toFixed(3)}`);
const { text, usage } = await generate(s);
tin += usage.in; tout += usage.out;
usd += (usage.in * PRICE.input + usage.out * PRICE.output) / 1e6;
if (!/^\[user\]/m.test(text) || !/^\[assistant\]/m.test(text)) { console.error(`${s.id}: malformed, skipped`); continue; }
writeFileSync(join(OUT, 'transcripts-txt', `2026-09-30-${s.id}.txt`), text);
writeFileSync(join(OUT, 'gold', `${s.id}.json`), JSON.stringify({
schema_version: 1, transcript_id: s.id, scenario: s.scenario, expected_triage: s.expected_triage,
label_source: 'synthetic-construction', generator: { model: `openai:${MODEL}`, reasoning: 'none', temperature: 1, seed: SEED }, spec: s,
}, null, 2) + '\n');
}
};
await Promise.all(Array.from({ length: 8 }, worker));
appendFileSync(LEDGER, JSON.stringify({ ts: new Date().toISOString(), purpose: 'S7 synthetic transcripts', provider: 'openai', model: MODEL, input_tokens: tin, output_tokens: tout, usd: Number(usd.toFixed(4)) }) + '\n');
console.log(`done: ${todo.length} generated, $${usd.toFixed(4)} (${tin} in / ${tout} out)`);
}
if (import.meta.main) await main();