Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.8 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Guard mutants for the TLA+ specs (spec/tla/README.md). Each mutant in spec/tla/<spec>/mutants.json removes one guard:
 *
 * - its `spec` edit is applied to a copy of the model, and TLC must report the named property (or nothing, for a
 *   mutant marked `survives`: a guard that is defense in depth at these bounds);
 * - its `code` edits, when present, are applied to the implementation in a scratch copy of the working tree, and each
 *   named test must fail on its own assertions under them and pass without them. That ties each guard the model relies
 *   on to the lines that implement it (every engine branch) and the tests that pin it.
 *
 *   bun scripts/spec/mutants.ts [<spec>] [--only <id>] [--spec-only | --code-only] [--require-postgres]
 *
 * Failure codes: MUTANT_USAGE (unknown or contradictory flags), MUTANT_NOTHING_CHECKED (every selected check was skipped), MUTANT_STALE (an edit's `before` text is not found exactly
 * once: the spec or code moved; update the mutant), MUTANT_SURVIVED (TLC found no violation, or the test passed under the
 * code edit: a property or a test no longer catches the guard), MUTANT_WRONG_PROPERTY, MUTANT_INCOMPLETE (TLC did not
 * finish, or a test failed without the edit, matched no test, or did not run to a clean assertion failure with it: a
 * signal, timeout, load error or non-assertion failure proves nothing). Each invocation writes to its own directory under TLC_OUT
 * (default .context/tlc), printed first: the scratch tree, every child's full output and each test's JUnit report.
 */
import { spawnSync } from 'node:child_process';
import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import { join, resolve } from 'node:path';

interface Edit { file: string; before: string; after: string }
interface CodeTest { test: string; filter: string; postgres?: boolean }
interface CodeEdit { edits: Edit[]; tests: CodeTest[] }
interface Mutant { id: string; kind: 'guard' | 'assumption' | 'survivor'; title: string; expect: string; reason?: string; config?: string; spec: Edit; code?: CodeEdit }
interface Catalog { module: string; config: string; mutants: Mutant[] }

const ROOT = resolve(import.meta.dir, '../..');
const args = process.argv.slice(2);
const flag = (name: string) => args.includes(name);
const FLAGS = new Set(['--only', '--spec-only', '--code-only', '--require-postgres']);
const unknown = args.filter(a => a.startsWith('--') && !FLAGS.has(a));
if (unknown.length) { console.error(`MUTANT_USAGE: unknown flag ${unknown.join(', ')}. Usage: bun scripts/spec/mutants.ts [<spec>] [--only <id>] [--spec-only | --code-only] [--require-postgres]`); process.exit(2); }
if (flag('--spec-only') && flag('--code-only')) { console.error('MUTANT_USAGE: --spec-only and --code-only together would check nothing; pass at most one.'); process.exit(2); }
const only = args.includes('--only') ? args[args.indexOf('--only') + 1] : undefined;
const specName = args.find((a, i) => !a.startsWith('--') && args[i - 1] !== '--only') ?? 'drain-lease';
const specDir = join(ROOT, 'spec/tla', specName);
const catalog = JSON.parse(readFileSync(join(specDir, 'mutants.json'), 'utf8')) as Catalog;
const out = join(process.env.TLC_OUT ?? join(ROOT, '.context/tlc'), `mutants-${specName}-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`);
const timeoutS = Number(process.env.TLC_TIMEOUT_S ?? 300);
const failures: string[] = [];
let skipped = 0, checked = 0;
const fail = (code: string, id: string, message: string) => { failures.push(`${code} ${id}: ${message}`); console.error(`FAIL ${code} ${id}: ${message}`); };

function applyOnce(text: string, edit: Edit): string | null {
  const at = text.indexOf(edit.before);
  if (at < 0 || text.indexOf(edit.before, at + 1) >= 0) return null;
  return text.slice(0, at) + edit.after + text.slice(at + edit.before.length);
}

function jar(): string {
  const r = spawnSync('bash', [join(ROOT, 'scripts/spec/tla-jar.sh')], { encoding: 'utf8' });
  if (r.status !== 0) { process.stderr.write(r.stderr); process.exit(2); }
  return r.stdout.trim();
}

function runSpec(m: Mutant, tla2tools: string): void {
  const work = join(out, m.id);
  rmSync(work, { recursive: true, force: true });
  mkdirSync(join(work, 'tmp'), { recursive: true });
  const config = m.config ?? catalog.config;
  for (const f of [`${catalog.module}.tla`, config]) cpSync(join(specDir, f), join(work, f));
  const target = join(work, m.spec.file);
  const mutated = applyOnce(readFileSync(target, 'utf8'), m.spec);
  if (mutated === null) return fail('MUTANT_STALE', m.id, `spec text not found exactly once in ${m.spec.file}: ${JSON.stringify(m.spec.before)}`);
  writeFileSync(target, mutated);
  const tlc = (cfg: string) => spawnSync('java', ['-XX:+UseParallelGC', `-Djava.io.tmpdir=${join(work, 'tmp')}`, '-cp', tla2tools, 'tlc2.TLC', '-workers', 'auto',
    '-config', cfg, '-metadir', join(work, `states-${cfg}`), `${catalog.module}.tla`], { cwd: work, encoding: 'utf8', timeout: timeoutS * 1000 });
  const r = tlc(config);
  const log = `${r.stdout ?? ''}${r.stderr ?? ''}`;
  writeFileSync(join(work, 'tlc.log'), log);
  const spawnCode = (r.error as NodeJS.ErrnoException | undefined)?.code;
  if (spawnCode === 'ENOENT') return fail('MUTANT_INCOMPLETE', m.id, `cannot run java (ENOENT); TLC did not start. Install Java 11 or later on PATH and rerun: bun scripts/spec/mutants.ts --only ${m.id} --spec-only`);
  if (r.error || r.signal) return fail('MUTANT_INCOMPLETE', m.id, `TLC did not finish within TLC_TIMEOUT_S (${timeoutS} s) or was killed (${spawnCode ?? r.signal}). If the spec or config changed, find the action that multiplies states; otherwise rerun once. Log: ${work}/tlc.log`);
  checked++;
  let violated = log.match(/(?:Invariant|Action property|Property) (\w+) is violated/)?.[1];
  // TLC names no property when a temporal one fails: check the config's properties one at a time to name it.
  if (!violated && log.includes('Temporal properties were violated.')) {
    const text = readFileSync(join(work, config), 'utf8');
    for (const property of [...text.matchAll(/^PROPERTY (\w+)$/gm)].map(match => match[1]!)) {
      const single = `${property}-${config}`;
      writeFileSync(join(work, single), text.replace(/^PROPERTY \w+\n/gm, '') + `PROPERTY ${property}\n`);
      const each = tlc(single);
      if (`${each.stdout ?? ''}${each.stderr ?? ''}`.includes('Temporal properties were violated.')) { violated = property; break; }
    }
  }
  const clean = log.includes('Model checking completed. No error has been found.');
  if (m.expect === 'survives') {
    if (clean) return void console.log(`ok   ${m.id} (survivor): survives as expected (defense in depth)`);
    if (violated) return fail('MUTANT_WRONG_PROPERTY', m.id, `expected to survive, but ${violated} is violated: the guard is now load-bearing; give it an expect and a code test. Log: ${work}/tlc.log`);
    return fail('MUTANT_INCOMPLETE', m.id, `TLC did not finish (exit ${r.status ?? r.signal}). Log: ${work}/tlc.log`);
  }
  if (violated === m.expect) return void console.log(`ok   ${m.id} (${m.kind}): caught by ${violated}`);
  if (violated) return fail('MUTANT_WRONG_PROPERTY', m.id, `expected ${m.expect}, but ${violated} caught it. Log: ${work}/tlc.log`);
  if (clean) return fail('MUTANT_SURVIVED', m.id, `no property caught "${m.title}"; ${m.expect} no longer depends on this guard. Log: ${work}/tlc.log`);
  fail('MUTANT_INCOMPLETE', m.id, `TLC did not finish (exit ${r.status ?? r.signal}). Log: ${work}/tlc.log`);
}

/** The code checks run in a scratch copy of the working tree, so an interrupted run never leaves the checkout edited. */
function scratchCopy(): string {
  const dir = join(out, 'tree');
  mkdirSync(dir, { recursive: true });
  const files = spawnSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], { cwd: ROOT, encoding: 'utf8', maxBuffer: 1 << 28 }).stdout.split('\0').filter(Boolean);
  for (const f of files) if (existsSync(join(ROOT, f))) cpSync(join(ROOT, f), join(dir, f), { recursive: true });
  symlinkSync(join(ROOT, 'node_modules'), join(dir, 'node_modules'));
  return dir;
}

/**
 * One run of one test file, its full output kept in `log` and its JUnit report beside it. `outcome` is `passed` (exit 0,
 * every selected test passed), `assertion` (exit 1, and every failed test failed on an expect(): an AssertionError in the
 * JUnit report) or `broken`: a signal, a timeout, a spawn error, a load or bootstrap error, no test selected, or a
 * failure of any other type, such as an Error thrown after a passing expect(). Only `assertion` counts as a caught
 * guard; `broken` proves nothing either way.
 */
interface TestRun { outcome: 'passed' | 'assertion' | 'broken'; why: string; log: string }
function bunTest(cwd: string, t: CodeTest, log: string): TestRun {
  const junit = log.replace(/[.]log$/, '.xml');
  rmSync(junit, { force: true });
  const argv = ['test', '--timeout=300000', '--reporter=junit', `--reporter-outfile=${junit}`, t.test, '-t', t.filter];
  const r = spawnSync('bun', argv, { cwd, encoding: 'utf8', env: process.env, timeout: 600_000 });
  const text = `${r.stdout ?? ''}${r.stderr ?? ''}`;
  writeFileSync(log, `$ bun ${argv.map(a => JSON.stringify(a)).join(' ')}\nexit ${r.status} signal ${r.signal} error ${r.error?.message ?? 'none'}\n\n${text}`);
  const broken = (why: string): TestRun => ({ outcome: 'broken', why, log });
  if (r.error) return broken(`bun did not run (${(r.error as NodeJS.ErrnoException).code ?? r.error.message})`);
  if (r.signal) return broken(`bun was terminated by ${r.signal}`);
  if (['Unhandled error between tests', 'SyntaxError', 'Cannot find module'].some(s => text.includes(s))) return broken('the file did not load, or an error escaped between tests');
  if (!existsSync(junit)) return broken('bun wrote no JUnit report');
  const cases = readFileSync(junit, 'utf8').split('<testcase ').slice(1).map(c => c.split('</testcase>')[0]!).filter(c => !c.includes('<skipped'));
  const failures = cases.map(c => c.includes('<failure') ? (c.split('<failure type="')[1]?.split('"')[0] ?? 'unknown') : null).filter((f): f is string => f !== null);
  if (cases.length === 0) return broken(`-t "${t.filter}" matched no test`);
  if (r.status === 0 && failures.length === 0) return { outcome: 'passed', why: `${cases.length} passed`, log };
  const other = failures.filter(type => type !== 'AssertionError');
  if (r.status === 1 && failures.length > 0 && other.length === 0) return { outcome: 'assertion', why: `${failures.length} failed on an assertion`, log };
  return broken(`exit ${r.status}, ${failures.length} failed${other.length ? ` (${other.join(', ')}, not an assertion)` : ''}`);
}

function runCode(m: Mutant, tree: string): void {
  const code = m.code!;
  const tests = code.tests.filter(t => {
    if (!t.postgres || process.env.DATABASE_URL) return true;
    if (flag('--require-postgres')) fail('MUTANT_INCOMPLETE', m.id, `${t.test} needs DATABASE_URL (a test Postgres)`);
    else { skipped++; console.log(`skip ${m.id} (code): ${t.test} needs DATABASE_URL`); }
    return false;
  });
  if (tests.length === 0) return;
  const originals = new Map<string, string>();
  let mutated = new Map<string, string>();
  for (const edit of code.edits) {
    const text = mutated.get(edit.file) ?? readFileSync(join(tree, edit.file), 'utf8');
    originals.set(edit.file, originals.get(edit.file) ?? text);
    const next = applyOnce(text, edit);
    if (next === null) return fail('MUTANT_STALE', m.id, `code text not found exactly once in ${edit.file}: ${JSON.stringify(edit.before)}`);
    mutated.set(edit.file, next);
  }
  const logs = join(out, m.id);
  mkdirSync(logs, { recursive: true });
  for (const [i, t] of tests.entries()) {
    checked++;
    const base = bunTest(tree, t, join(logs, `test-${i}-without-edit.log`));
    if (base.outcome !== 'passed') return fail('MUTANT_INCOMPLETE', m.id, `${t.test} -t "${t.filter}" must pass without the edit, but ${base.why}. Fix the test first. Log: ${base.log}`);
    try {
      for (const [file, text] of mutated) writeFileSync(join(tree, file), text);
      const under = bunTest(tree, t, join(logs, `test-${i}-with-edit.log`));
      if (under.outcome === 'broken') return fail('MUTANT_INCOMPLETE', m.id, `with the edit, ${under.why}, so the run proves nothing. Rerun once: bun scripts/spec/mutants.ts --only ${m.id} --code-only. Log: ${under.log}`);
      if (under.outcome === 'passed') return fail('MUTANT_SURVIVED', m.id, `${t.test} -t "${t.filter}" passes with the guard removed from ${[...mutated.keys()].join(', ')}; the test no longer pins it. Log: ${under.log}`);
      console.log(`ok   ${m.id} (code): ${t.test} fails on an assertion with the guard removed from ${[...mutated.keys()].join(', ')}`);
    } finally {
      for (const [file, text] of originals) writeFileSync(join(tree, file), text);
    }
  }
}

const selected = catalog.mutants.filter(m => !only || m.id === only);
if (selected.length === 0) { console.error(`no mutant ${only} in ${specName}`); process.exit(2); }
console.log(`logs: ${out}`);
if (!flag('--code-only')) { const tla2tools = jar(); for (const m of selected) runSpec(m, tla2tools); }
if (!flag('--spec-only') && selected.some(m => m.code)) {
  const tree = scratchCopy();
  for (const m of selected) if (m.code) runCode(m, tree);
  rmSync(tree, { recursive: true, force: true });
}
if (failures.length === 0 && checked === 0) { console.error(`MUTANT_NOTHING_CHECKED: every check the selection asked for was skipped (${skipped} Postgres code check(s) without DATABASE_URL); nothing was verified.`); process.exit(1); }
if (failures.length) { console.error(`\n${failures.length} mutant check(s) failed; spec/tla/README.md says what each code means.`); process.exit(1); }
console.log(`\nall ${selected.length} mutant(s) behave as catalogued${skipped ? `; partial: ${skipped} Postgres code check(s) skipped (set DATABASE_URL to a test database, or pass --require-postgres to fail instead)` : ''}`);