Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
feat(loop-drill): fire drills that prove guardrails actually fire (#600)
THRISHAL DOMA336c3296bcd5
diff --git a/package.json b/package.jsonindex b7108a9..4f0cd3e 100644--- a/package.json+++ b/package.json@@ -20,13 +20,14 @@"test:goal-init": "cd tools/goal-init && npm test","test:readiness-core": "cd tools/readiness-core && npm test","test:goal-audit": "cd tools/goal-audit && npm test",- "test:tools": "npm run test:readiness-core && npm run test:loop-audit && npm run test:loop-init && npm run test:loop-cost && npm run test:loop-sync && npm run test:loop-context && npm run test:loop-gate && npm run test:mcp-server && npm run test:loop-worktree && npm run test:loop-sandbox && npm run test:loop-swarm && npm run test:loop && npm run test:loop-metrics && npm run test:goal-audit && npm run test:goal-init",- "build:tools": "cd tools/readiness-core && npm run build && cd ../loop-audit && npm run build && cd ../loop-init && npm run build && cd ../loop-cost && npm run build && cd ../loop-sync && npm run build && cd ../loop-context && npm run build && cd ../loop-gate && npm run build && cd ../mcp-server && npm run build && cd ../loop-worktree && npm run build && cd ../loop-sandbox && npm run build && cd ../loop-swarm && npm run build && cd ../loop && npm run build && cd ../loop-metrics && npm run build && cd ../goal-audit && npm run build && cd ../goal-init && npm run build",+ "test:tools": "npm run test:readiness-core && npm run test:loop-audit && npm run test:loop-init && npm run test:loop-cost && npm run test:loop-sync && npm run test:loop-context && npm run test:loop-gate && npm run test:mcp-server && npm run test:loop-worktree && npm run test:loop-sandbox && npm run test:loop-swarm && npm run test:loop && npm run test:loop-metrics && npm run test:loop-drill && npm run test:goal-audit && npm run test:goal-init",+ "build:tools": "cd tools/readiness-core && npm run build && cd ../loop-audit && npm run build && cd ../loop-init && npm run build && cd ../loop-cost && npm run build && cd ../loop-sync && npm run build && cd ../loop-context && npm run build && cd ../loop-gate && npm run build && cd ../mcp-server && npm run build && cd ../loop-worktree && npm run build && cd ../loop-sandbox && npm run build && cd ../loop-swarm && npm run build && cd ../loop && npm run build && cd ../loop-metrics && npm run build && cd ../loop-drill && npm run build && cd ../goal-audit && npm run build && cd ../goal-init && npm run build","contributors:generate": "node scripts/generate-contributors.mjs",- "contributors:check": "node scripts/generate-contributors.mjs --check"+ "contributors:check": "node scripts/generate-contributors.mjs --check",+ "test:loop-drill": "cd tools/loop-drill && npm test"},"devDependencies": {"ajv": "^8.17.1","yaml": "^2.8.0"}-}\ No newline at end of file+}diff --git a/scripts/ci-validate-gates.sh b/scripts/ci-validate-gates.shindex 65cd29f..15f35fa 100755--- a/scripts/ci-validate-gates.sh+++ b/scripts/ci-validate-gates.sh@@ -106,4 +106,25 @@ cd ../mcp-servernpm cinpm test+echo "Building and testing loop-drill…"+cd ../loop-drill+npm ci+npm test++# Dogfood: prove this repo's own gate.yaml and circuit breaker actually fire.+# Exit 1 means a drill was skipped (an unconfigured guardrail) and is tolerated;+# exit 2 means a guardrail failed to fire and must fail CI.+#+# Written as an explicit if rather than `cmd || [ $? -eq 1 ]`: under `set -e`+# a failing command on the right of `||` does not exit the shell, so that+# shorter form silently passes an exit-2 drill.+echo "Running loop-drill against the reference repo…"+cd ../..+drill_status=0+node tools/loop-drill/dist/cli.js . || drill_status=$?+if [ "$drill_status" -ge 2 ]; then+ echo "loop-drill: a guardrail failed to fire (exit $drill_status)" >&2+ exit "$drill_status"+fi+echo "validate gates passed ✓"diff --git a/tools/loop-drill/README.md b/tools/loop-drill/README.mdnew file mode 100644index 0000000..92adc64--- /dev/null+++ b/tools/loop-drill/README.md@@ -0,0 +1,127 @@+# loop-drill++**Fire drills for loop guardrails.** Injects a known fault and asserts the guardrail actually fires.++`loop-audit` scores whether your guardrails are **present**. `loop-drill` tests whether they **work**.++## Why++[`docs/failure-modes.md`](../../docs/failure-modes.md) names ten ways loops fail. Several have mechanical counterparts — [`loop-gate`](../loop-gate) for path scope, [`loop-context`](../loop-context)'s circuit breaker for runaway retries. `loop-audit` awards points for having them.++Nothing checked whether they fire.++The sharpest case is the verifier. `loop-audit` gives its joint-largest signal (14 points) for a verifier and gates L3 on it, by checking whether a file whose *name* contains `verifier` exists:++```ts+if (base.includes('verifier') || base === 'loop-verifier') // auditor.ts:181+```++An empty file passes. On a scratch repo, `touch .claude/agents/verifier.md` moves the score from **34 (L0)** to **55 (L1)**. `docs/failure-modes.md` calls the resulting failure **Verifier Theater** and rates it S2; `docs/primitives.md` calls maker/checker *"the single most important structural pattern for reliable loops"*.++loop-drill closes that gap by running the real verifier against real seeded defects.++## Usage++```bash+# Offline drills — no agent, no tokens, no network+npx @cobusgreyling/loop-drill .++# The verifier canary+npx @cobusgreyling/loop-drill . --only verifier \+ --verifier-cmd "npm test" --setup "npm ci"+```++### Options++| Flag | Meaning |+|------|---------|+| `--only <drills>` | `gate`, `breaker`, `verifier` (default: `gate,breaker`) |+| `--verifier-cmd <cmd>` | Verifier command. Non-zero exit = rejected. Required for the canary |+| `--setup <cmd>` | Run in each worktree before verifying (e.g. `npm ci`) |+| `--mutants <n>` | Seeded defects (default: 3) |+| `--scope <path>` | Restrict mutation to a repo-relative path |+| `--timeout <ms>` | Per-run timeout (default: 120000) |+| `--gate-file <path>` | Policy file (default: `gate.yaml`) |+| `--benign-path <path>` | Path the specificity drill treats as ordinary |+| `--token-budget <n>` | Token budget for the breaker drill |+| `--json` | Machine-readable output |++Exit codes match `loop-gate` and `loop-context` so control scripts chain all three: **0** all passed, **1** some skipped, **2** a guardrail failed to fire.++## The drills++Each drill is tagged with the `docs/failure-modes.md` entry it exercises, so the report reads as coverage of that document.++| Drill | Failure mode | Injected fault | Passes if |+|-------|--------------|----------------|-----------|+| `gate.denylist[glob]` | Over-Reach | A path matching each denylist entry | Blocked by the denylist |+| `gate.file-count` | Over-Reach | `maxFiles + 1` benign paths | Blocked by file-count |+| `gate.auto-merge` | Over-Reach | Non-allowlisted path, `--action auto-merge` | Blocked |+| `gate.benign` | Over-Reach | An ordinary doc change | **Allowed** |+| `breaker.stagnation` | Infinite Fix Loop | N identical failures | Escalates |+| `breaker.no-progress` | Infinite Fix Loop | N consecutive failures | Escalates |+| `breaker.token-budget` | Token Burn | Attempt over budget | Escalates |+| `breaker.healthy` | Infinite Fix Loop | A healthy run | **Does not escalate** |+| `verifier.control` | Verifier Theater | Nothing (clean tree) | **Accepts** |+| `verifier.mutant[op]` | Verifier Theater | One seeded defect | Rejects |++### Both directions, always++Every guardrail is drilled twice, because only one direction is easy:++- **sensitivity** — the fault is caught. A guardrail that never fires is theater.+- **specificity** — benign input is *not* caught. A guardrail that blocks everything passes every sensitivity drill while being useless.++A `denylist: ["**"]` catches every seeded fault and still fails, because it blocks an ordinary docs change too.++## How the verifier canary works++Borrowed from mutation testing. Rather than asking a model to invent a defect — non-deterministic and token-hungry — it applies a small mechanical edit to real source and checks whether the verifier notices:++```+control (repo) no defect -> must ACCEPT (exit 0)+control (worktree) no defect -> must ACCEPT (exit 0)+mutant × N one defect -> must REJECT (non-zero)+```++**Mutation score** = mutants rejected / mutants run.++Three properties make the result trustworthy:++1. **Isolation.** Each run happens in an ephemeral `git worktree`, removed even on throw, so a verifier that writes or builds cannot touch your checkout.+2. **The worktree control.** A fresh worktree has no `node_modules`. Without this second control, `npm test` would fail there for reasons unrelated to code quality and *every mutant would score as caught* — a broken setup reporting a perfect score. When it fails, mutants are **skipped, not scored**, and you're pointed at `--setup`.+3. **Bail on a failed control.** A verifier that rejects a clean tree catches every mutant while being worthless. The mutants never run, so it can't report 100%.++Operators are restricted to edits that change behaviour in any C-family language and that a reviewer would call a bug: `===`↔`!==`, `<=`→`<`, `>=`→`>`, `&&`→`||`, `return true`↔`return false`. Ambiguous edits are deliberately excluded — `<`→`>` breaks TS generics and JSX, `+`→`-` mangles string concatenation, and either would make a *correct* verifier look broken. Matches inside comments are skipped, and test files, `dist/`, and `node_modules/` are never mutated.++## Example++A rubber-stamp verifier, caught:++```+Verifier Theater — ❌ NOT PROVEN+ ✅ verifier accepts an unmodified tree+ ✅ verifier accepts a clean worktree+ ❌ verifier rejects strict-equality-flip in src/gate.ts:34+ expected: non-zero exit (rejected)+ actual: exit 0 (approved a seeded defect)+ flip === to !== (typeof v === 'string' -> typeof v !== 'string')++Mutation score: 0% of seeded defects rejected+```++## Limitations++- **Mutation is regex-based, not AST-based.** It cannot construct a semantically interesting bug, only a mechanically detectable one. A verifier that catches every mutant is not proven to catch subtle logic errors — this establishes a floor, not a ceiling.+- **The canary costs whatever your verifier costs**, once per mutant plus two controls. Start with `--mutants 1` and `--scope`.+- **`Escalation Failure` and `Notification Fatigue`** have no drills yet — they need a notification sink to observe.++## Development++```bash+npm ci && npm test+```++## License++MITdiff --git a/tools/loop-drill/dist/canary.d.ts b/tools/loop-drill/dist/canary.d.tsnew file mode 100644index 0000000..59460f2--- /dev/null+++ b/tools/loop-drill/dist/canary.d.ts@@ -0,0 +1,63 @@+/**+ * The verifier canary — the drill that gave this package its reason to exist.+ *+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and+ * gates L3 on it, by checking whether a file whose name contains "verifier"+ * exists. An empty file passes. docs/failure-modes.md calls the resulting+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls+ * maker/checker "the single most important structural pattern for reliable+ * loops". Nothing tested whether the verifier verifies.+ *+ * This runs the real verifier command against real seeded defects in an+ * isolated worktree, and scores it on what it caught:+ *+ * control — clean tree -> the verifier must ACCEPT (exit 0)+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)+ *+ * Both directions are required. A verifier that rejects everything catches+ * every mutant while being worthless, and the control run is what exposes it.+ */+import { type Mutant } from './mutations.js';+import { type DrillResult } from './drill.js';+export interface CanaryOptions {+ /** Repo root to drill. */+ root: string;+ /** The verifier command. Non-zero exit is read as "rejected". */+ command: string;+ /** How many mutants to seed. More is slower but a sharper score. */+ count: number;+ /** Per-run timeout in ms. A verifier that hangs is a failed verifier. */+ timeoutMs: number;+ /** Restrict mutation to files under this repo-relative path. */+ scope?: string;+ /** Command run inside each worktree before verifying (e.g. "npm ci"). */+ setup?: string;+}+export interface VerifierRun {+ accepted: boolean;+ exitCode: number | null;+ timedOut: boolean;+ output: string;+}+/**+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything+ * else — including a timeout — means "rejected". A verifier that cannot be+ * launched at all is a distinct failure, surfaced by the caller.+ */+export declare function runVerifier(command: string, cwd: string, timeoutMs: number): Promise<VerifierRun>;+/** Files git tracks, filtered to plausible mutation targets. */+export declare function candidateFiles(root: string, scope?: string): Promise<string[]>;+/**+ * Build up to `count` mutants from distinct files. Distinct files matter:+ * seeding three mutants into one module mostly measures whether the verifier+ * reads that module.+ */+export declare function collectMutants(root: string, files: string[], count: number): Promise<Mutant[]>;+export interface CanaryReport {+ results: DrillResult[];+ /** Mutants rejected / mutants run. Null when no mutant could be built. */+ mutationScore: number | null;+ caught: number;+ escaped: number;+}+export declare function runCanary(options: CanaryOptions): Promise<CanaryReport>;diff --git a/tools/loop-drill/dist/canary.js b/tools/loop-drill/dist/canary.jsnew file mode 100644index 0000000..afd0f8e--- /dev/null+++ b/tools/loop-drill/dist/canary.js@@ -0,0 +1,235 @@+/**+ * The verifier canary — the drill that gave this package its reason to exist.+ *+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and+ * gates L3 on it, by checking whether a file whose name contains "verifier"+ * exists. An empty file passes. docs/failure-modes.md calls the resulting+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls+ * maker/checker "the single most important structural pattern for reliable+ * loops". Nothing tested whether the verifier verifies.+ *+ * This runs the real verifier command against real seeded defects in an+ * isolated worktree, and scores it on what it caught:+ *+ * control — clean tree -> the verifier must ACCEPT (exit 0)+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)+ *+ * Both directions are required. A verifier that rejects everything catches+ * every mutant while being worthless, and the control run is what exposes it.+ */+import { exec as execShellCb, execFile as execFileCb } from 'node:child_process';+import { readFile, writeFile, rm } from 'node:fs/promises';+import path from 'node:path';+import { promisify } from 'node:util';+import { isMutationCandidate, mutate } from './mutations.js';+import { skip } from './drill.js';+/**+ * Two spawn helpers, kept honest about their signatures.+ *+ * `execShell` runs a caller-supplied command string (--verifier-cmd, --setup)+ * through a shell, which is what those flags mean. `execFile` runs git with an+ * argv array and no shell, so repo paths never reach a shell for re-parsing.+ *+ * Earlier this file called promisify(execFile) with an options object in the+ * `args` position behind an `as never` cast -- it happened to work because Node+ * detects a non-array second argument, but it hid the real signature from tsc+ * and made timeout/killed handling harder to reason about.+ */+const execShell = promisify(execShellCb);+const exec = promisify(execFileCb);+/**+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything+ * else — including a timeout — means "rejected". A verifier that cannot be+ * launched at all is a distinct failure, surfaced by the caller.+ */+export async function runVerifier(command, cwd, timeoutMs) {+ try {+ const { stdout, stderr } = await execShell(command, {+ cwd,+ timeout: timeoutMs,+ maxBuffer: 8 * 1024 * 1024,+ });+ return { accepted: true, exitCode: 0, timedOut: false, output: `${stdout}${stderr}`.trim() };+ }+ catch (err) {+ const e = err;+ const timedOut = e.killed === true || e.code === 'ETIMEDOUT';+ return {+ accepted: false,+ exitCode: typeof e.code === 'number' ? e.code : null,+ timedOut,+ output: `${e.stdout ?? ''}${e.stderr ?? ''}`.trim(),+ };+ }+}+/** Files git tracks, filtered to plausible mutation targets. */+export async function candidateFiles(root, scope) {+ const args = ['-C', root, 'ls-files'];+ if (scope)+ args.push('--', scope);+ const { stdout } = await exec('git', args, { maxBuffer: 32 * 1024 * 1024 });+ return stdout+ .split('\n')+ .map((l) => l.trim())+ .filter((l) => l.length > 0 && isMutationCandidate(l));+}+/**+ * Build up to `count` mutants from distinct files. Distinct files matter:+ * seeding three mutants into one module mostly measures whether the verifier+ * reads that module.+ */+export async function collectMutants(root, files, count) {+ const mutants = [];+ for (const file of files) {+ if (mutants.length >= count)+ break;+ let content;+ try {+ content = await readFile(path.join(root, file), 'utf8');+ }+ catch {+ continue;+ }+ const mutant = mutate(content, file);+ if (mutant)+ mutants.push(mutant);+ }+ return mutants;+}+/**+ * Run the verifier inside an ephemeral git worktree, so a verifier that writes,+ * builds, or fixes cannot touch the real checkout. `mutant` is null for the+ * worktree control run. The worktree is always removed, including on throw.+ */+async function runInWorktree(root, mutant, command, timeoutMs, setup) {+ const worktree = path.join(root, `.loop-drill-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);+ await exec('git', ['-C', root, 'worktree', 'add', '--detach', '--quiet', worktree], {+ maxBuffer: 8 * 1024 * 1024,+ });+ try {+ if (setup) {+ // Setup failure is not a verifier verdict — surface it as such.+ const setupRun = await runVerifier(setup, worktree, timeoutMs);+ if (!setupRun.accepted) {+ return { ...setupRun, output: `[setup failed] ${setupRun.output}` };+ }+ }+ if (mutant)+ await writeFile(path.join(worktree, mutant.file), mutant.mutated, 'utf8');+ return await runVerifier(command, worktree, timeoutMs);+ }+ finally {+ // --force because the verifier may have left build output behind.+ await exec('git', ['-C', root, 'worktree', 'remove', '--force', worktree]).catch(() => { });+ await rm(worktree, { recursive: true, force: true }).catch(() => { });+ }+}+export async function runCanary(options) {+ const { root, command, count, timeoutMs, scope, setup } = options;+ const results = [];+ const mode = 'Verifier Theater';+ // Control first, and bail if it fails: every mutant result is meaningless+ // when the verifier rejects a clean tree, and running them would report a+ // reject-everything verifier as a perfect score.+ const control = await runVerifier(command, root, timeoutMs);+ if (control.accepted) {+ results.push({+ id: 'verifier.control',+ name: 'verifier accepts an unmodified tree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'passed',+ expected: 'exit 0 on clean tree',+ actual: 'exit 0',+ });+ }+ else {+ results.push({+ id: 'verifier.control',+ name: 'verifier accepts an unmodified tree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'failed',+ expected: 'exit 0 on clean tree',+ actual: control.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${control.exitCode}`,+ detail: 'The verifier rejects code with no seeded defect, so it cannot distinguish good from bad. Mutant drills were skipped — their results would be meaningless. Fix the verifier command (or the tree it runs against) first.',+ });+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }+ // Second control, in a clean worktree. A fresh checkout has no node_modules+ // and no build output, so a verifier like "npm test" fails there for reasons+ // that have nothing to do with code quality. Without this run, every mutant+ // would be scored as "caught" and a completely broken setup would report a+ // perfect mutation score — the precise false confidence this tool exists to+ // prevent.+ const worktreeControl = await runInWorktree(root, null, command, timeoutMs, setup);+ if (!worktreeControl.accepted) {+ results.push({+ id: 'verifier.worktree-control',+ name: 'verifier accepts a clean worktree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'skipped',+ expected: 'exit 0 in a clean worktree',+ actual: worktreeControl.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${worktreeControl.exitCode}`,+ detail: 'The verifier passes in your checkout but fails in a fresh worktree, so mutant results would measure the environment rather than the verifier. Pass --setup to prepare the worktree first (e.g. --setup "npm ci"). ' ++ (worktreeControl.output ? `Output: ${truncate(worktreeControl.output)}` : ''),+ });+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }+ results.push({+ id: 'verifier.worktree-control',+ name: 'verifier accepts a clean worktree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'passed',+ expected: 'exit 0 in a clean worktree',+ actual: 'exit 0',+ });+ const files = await candidateFiles(root, scope);+ const mutants = await collectMutants(root, files, count);+ if (mutants.length === 0) {+ results.push(skip('verifier.mutants', 'verifier rejects seeded defects', mode, 'sensitivity', `No mutable source found${scope ? ` under ${scope}` : ''}. loop-drill mutates tracked, non-test, non-generated source in common languages — pass --scope to point it at yours.`));+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }+ let caught = 0;+ for (const mutant of mutants) {+ const run = await runInWorktree(root, mutant, command, timeoutMs, setup);+ const where = `${mutant.file}:${mutant.line}`;+ const edit = `${mutant.operator.description} (${mutant.before} -> ${mutant.after})`;+ if (!run.accepted) {+ caught++;+ results.push({+ id: `verifier.mutant[${mutant.operator.id}]`,+ name: `verifier rejects ${mutant.operator.id} in ${where}`,+ failureMode: mode,+ direction: 'sensitivity',+ outcome: 'passed',+ expected: 'non-zero exit (rejected)',+ actual: run.timedOut ? 'timed out (treated as rejected)' : `exit ${run.exitCode}`,+ detail: edit,+ });+ }+ else {+ results.push({+ id: `verifier.mutant[${mutant.operator.id}]`,+ name: `verifier rejects ${mutant.operator.id} in ${where}`,+ failureMode: mode,+ direction: 'sensitivity',+ outcome: 'failed',+ expected: 'non-zero exit (rejected)',+ actual: 'exit 0 (approved a seeded defect)',+ detail: `${edit}. The verifier approved this change. That is Verifier Theater — see docs/failure-modes.md.`,+ });+ }+ }+ return {+ results,+ mutationScore: caught / mutants.length,+ caught,+ escaped: mutants.length - caught,+ };+}+function truncate(text, max = 400) {+ return text.length <= max ? text : `${text.slice(0, max)}…`;+}diff --git a/tools/loop-drill/dist/cli.d.ts b/tools/loop-drill/dist/cli.d.tsnew file mode 100644index 0000000..4f55271--- /dev/null+++ b/tools/loop-drill/dist/cli.d.ts@@ -0,0 +1,8 @@+#!/usr/bin/env node+/**+ * loop-drill CLI.+ *+ * Exit codes follow loop-gate / loop-context so control scripts can chain all+ * three: 0 proceed, 1 warnings, 2 escalate.+ */+export {};diff --git a/tools/loop-drill/dist/cli.js b/tools/loop-drill/dist/cli.jsnew file mode 100644index 0000000..857392c--- /dev/null+++ b/tools/loop-drill/dist/cli.js@@ -0,0 +1,174 @@+#!/usr/bin/env node+/**+ * loop-drill CLI.+ *+ * Exit codes follow loop-gate / loop-context so control scripts can chain all+ * three: 0 proceed, 1 warnings, 2 escalate.+ */+import path from 'node:path';+import { loadGateConfig } from '@cobusgreyling/loop-gate';+import { DEFAULT_BREAKER } from '@cobusgreyling/loop-context';+import { buildReport, exitCodeFor, runBreakerDrills, runGateDrills, skip, } from './drill.js';+import { runCanary } from './canary.js';+import { formatReport } from './report.js';+const HELP = `loop-drill — fire drills for loop guardrails++Injects a known fault and asserts the guardrail actually fires. Turns L3+readiness from "the files exist" into "the guardrails demonstrably work".++Usage: loop-drill [path] [options]++Options:+ --only <drills> Comma-separated: gate, breaker, verifier (default: gate,breaker)+ --verifier-cmd <cmd> Verifier command to drill. Non-zero exit = rejected.+ Required to run the verifier canary.+ --mutants <n> Seeded defects for the canary (default: 3)+ --scope <path> Restrict mutation to this repo-relative path+ --setup <cmd> Command run in each worktree before verifying+ (e.g. "npm ci") — a fresh worktree has no node_modules+ --timeout <ms> Per-verifier-run timeout (default: 120000)+ --gate-file <path> Policy file (default: gate.yaml)+ --benign-path <path> Path the gate specificity drill treats as ordinary+ (default: docs/README.md)+ --token-budget <n> Token budget for the breaker drill+ --json Machine-readable output+ --help, -h Show this message++Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.++Examples:+ loop-drill .+ loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"+ loop-drill . --only gate,breaker --json+`;+function parseArgs(argv) {+ const flags = { root: '.', json: false, help: false, mutants: 3, timeoutMs: 120_000 };+ const positional = [];+ for (let i = 0; i < argv.length; i++) {+ const arg = argv[i];+ const next = () => {+ const v = argv[++i];+ if (v === undefined)+ throw new Error(`${arg} requires a value`);+ return v;+ };+ switch (arg) {+ case '--help':+ case '-h':+ flags.help = true;+ break;+ case '--json':+ flags.json = true;+ break;+ case '--only':+ flags.only = next();+ break;+ case '--verifier-cmd':+ flags.verifierCmd = next();+ break;+ case '--mutants':+ flags.mutants = parsePositiveInt(next(), '--mutants');+ break;+ case '--scope':+ flags.scope = next();+ break;+ case '--setup':+ flags.setup = next();+ break;+ case '--timeout':+ flags.timeoutMs = parsePositiveInt(next(), '--timeout');+ break;+ case '--gate-file':+ flags.gateFile = next();+ break;+ case '--benign-path':+ flags.benignPath = next();+ break;+ case '--token-budget':+ flags.tokenBudget = parsePositiveInt(next(), '--token-budget');+ break;+ default:+ if (arg.startsWith('-'))+ throw new Error(`Unknown option: ${arg}`);+ positional.push(arg);+ }+ }+ if (positional.length > 0)+ flags.root = positional[0];+ return flags;+}+function parsePositiveInt(value, flag) {+ const n = Number(value);+ if (!Number.isInteger(n) || n <= 0)+ throw new Error(`${flag} expects a positive integer, got: ${value}`);+ return n;+}+async function main() {+ let flags;+ try {+ flags = parseArgs(process.argv.slice(2));+ }+ catch (err) {+ console.error(`${err.message}\n`);+ console.error(HELP);+ process.exit(2);+ }+ if (flags.help) {+ console.log(HELP);+ return;+ }+ const root = path.resolve(flags.root);+ const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));+ const results = [];+ let mutationScore = null;+ if (selected.has('gate')) {+ const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');+ try {+ const config = await loadGateConfig(gateFile);+ results.push(...runGateDrills({ config, benignPath: flags.benignPath }));+ }+ catch (err) {+ results.push(skip('gate', 'gate drills', 'Over-Reach (Wrong Scope)', 'sensitivity', `${err.message} Nothing mechanically stops a loop from editing a sensitive path.`));+ }+ }+ if (selected.has('breaker')) {+ const config = flags.tokenBudget === undefined+ ? DEFAULT_BREAKER+ : { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };+ results.push(...runBreakerDrills(config));+ }+ if (selected.has('verifier')) {+ if (!flags.verifierCmd) {+ results.push(skip('verifier', 'verifier canary', 'Verifier Theater', 'sensitivity', 'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.'));+ }+ else {+ const canary = await runCanary({+ root,+ command: flags.verifierCmd,+ count: flags.mutants,+ timeoutMs: flags.timeoutMs,+ scope: flags.scope,+ setup: flags.setup,+ });+ results.push(...canary.results);+ mutationScore = canary.mutationScore;+ }+ }+ if (results.length === 0) {+ console.error(`No drills selected. --only accepts: gate, breaker, verifier\n`);+ process.exit(2);+ }+ const report = buildReport(results);+ const code = exitCodeFor(report);+ if (flags.json) {+ console.log(JSON.stringify({ ...report, mutationScore, exitCode: code }, null, 2));+ }+ else {+ console.log(formatReport(report, mutationScore));+ }+ process.exit(code);+}+main().catch((err) => {+ console.error(`loop-drill failed: ${err.message}`);+ process.exit(2);+});diff --git a/tools/loop-drill/dist/drill.d.ts b/tools/loop-drill/dist/drill.d.tsnew file mode 100644index 0000000..09020c3--- /dev/null+++ b/tools/loop-drill/dist/drill.d.ts@@ -0,0 +1,89 @@+/**+ * loop-drill — fire drills for loop guardrails.+ *+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores+ * whether those counterparts are *present*. Nothing checked whether they+ * actually fire.+ *+ * A drill injects a known fault and asserts the guardrail responds. Every+ * drill carries the failure mode it exercises, so the report reads as coverage+ * of docs/failure-modes.md rather than a list of anonymous assertions.+ *+ * Each guardrail gets both directions, because only one of them is easy:+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)+ * - specificity: benign input is NOT caught (a guardrail that blocks+ * everything passes every sensitivity drill while being useless)+ * A guardrail is only credited when it passes both.+ */+import { type GateConfig } from '@cobusgreyling/loop-gate';+import { type CircuitBreakerConfig } from '@cobusgreyling/loop-context';+export type DrillOutcome = 'passed' | 'failed' | 'skipped';+/** Named failure modes from docs/failure-modes.md. */+export type FailureMode = 'Infinite Fix Loop' | 'Token Burn' | 'Over-Reach (Wrong Scope)' | 'Verifier Theater' | 'Escalation Failure';+export interface DrillResult {+ /** Stable id, e.g. 'gate.denylist'. */+ id: string;+ name: string;+ /** The docs/failure-modes.md entry this drill exercises. */+ failureMode: FailureMode;+ /** 'sensitivity' = fault must be caught; 'specificity' = benign must pass. */+ direction: 'sensitivity' | 'specificity';+ outcome: DrillOutcome;+ expected: string;+ actual: string;+ /** Why a drill was skipped, or extra context on a failure. */+ detail?: string;+}+export interface DrillReport {+ results: DrillResult[];+ passed: number;+ failed: number;+ skipped: number;+ /** Failure modes with at least one passing drill and no failing drill. */+ covered: FailureMode[];+ /** Failure modes with no passing drill (unexercised or broken). */+ uncovered: FailureMode[];+ timestamp: string;+}+export declare function skip(id: string, name: string, failureMode: FailureMode, direction: DrillResult['direction'], detail: string): DrillResult;+/**+ * Turn a denylist glob into a concrete path that should match it, so the drill+ * exercises the real matcher rather than re-implementing glob semantics.+ *+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a+ * literal, and the caller verifies the synthesized path actually matches+ * before asserting anything about it (see runGateDrills) — a glob we+ * synthesize badly is reported as skipped, never as a guardrail failure.+ */+export declare function samplePathFor(glob: string): string;+export interface GateDrillOptions {+ config: GateConfig;+ /** Path assumed benign for the specificity drill. */+ benignPath?: string;+}+/**+ * Exercise loop-gate against its own gate.yaml: every denylist entry must+ * block, the file-count cap must block, auto-merge must reject a+ * non-allowlisted path, and an ordinary doc change must still be allowed.+ */+export declare function runGateDrills(options: GateDrillOptions): DrillResult[];+/**+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated+ * identical failures must trip stagnation, a long run of unrelated failures+ * must trip no-progress, blowing the token budget must trip, and a healthy run+ * must not.+ *+ * Each drill asserts the *specific* trigger, not merely that the breaker+ * escalated. Escalating for another reason means the rule under test is still+ * unproven — the same standard the gate drills apply to `trigger !==+ * 'denylist'`.+ */+export declare function runBreakerDrills(config?: CircuitBreakerConfig): DrillResult[];+export declare function buildReport(results: DrillResult[]): DrillReport;+/**+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's+ * "2 means escalate" convention so control scripts can chain all three.+ */+export declare function exitCodeFor(report: DrillReport): 0 | 1 | 2;diff --git a/tools/loop-drill/dist/drill.js b/tools/loop-drill/dist/drill.jsnew file mode 100644index 0000000..3b3a384--- /dev/null+++ b/tools/loop-drill/dist/drill.js@@ -0,0 +1,271 @@+/**+ * loop-drill — fire drills for loop guardrails.+ *+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores+ * whether those counterparts are *present*. Nothing checked whether they+ * actually fire.+ *+ * A drill injects a known fault and asserts the guardrail responds. Every+ * drill carries the failure mode it exercises, so the report reads as coverage+ * of docs/failure-modes.md rather than a list of anonymous assertions.+ *+ * Each guardrail gets both directions, because only one of them is easy:+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)+ * - specificity: benign input is NOT caught (a guardrail that blocks+ * everything passes every sensitivity drill while being useless)+ * A guardrail is only credited when it passes both.+ */+import { checkGate } from '@cobusgreyling/loop-gate';+import { checkCircuitBreaker, DEFAULT_BREAKER, } from '@cobusgreyling/loop-context';+function pass(id, name, failureMode, direction, expected, actual) {+ return { id, name, failureMode, direction, outcome: 'passed', expected, actual };+}+function fail(id, name, failureMode, direction, expected, actual, detail) {+ return { id, name, failureMode, direction, outcome: 'failed', expected, actual, detail };+}+export function skip(id, name, failureMode, direction, detail) {+ return {+ id,+ name,+ failureMode,+ direction,+ outcome: 'skipped',+ expected: '—',+ actual: '—',+ detail,+ };+}+// ── Glob → concrete sample path ────────────────────────────────────+/**+ * Turn a denylist glob into a concrete path that should match it, so the drill+ * exercises the real matcher rather than re-implementing glob semantics.+ *+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a+ * literal, and the caller verifies the synthesized path actually matches+ * before asserting anything about it (see runGateDrills) — a glob we+ * synthesize badly is reported as skipped, never as a guardrail failure.+ */+export function samplePathFor(glob) {+ const concrete = glob+ .split('/')+ .map((segment) => {+ if (segment === '**')+ return 'src';+ if (segment === '*')+ return 'sample';+ // '*_key*' -> 'drill_key_sample'; '.env.*' -> '.env.sample'+ return segment.replace(/\*/g, (_m, offset) => (offset === 0 ? 'drill' : 'sample'));+ })+ .join('/');+ // A trailing '**' segment became 'src', which is a directory, not a file.+ return /(^|\/)src$/.test(concrete) ? `${concrete}/drill-file.txt` : concrete;+}+/**+ * Exercise loop-gate against its own gate.yaml: every denylist entry must+ * block, the file-count cap must block, auto-merge must reject a+ * non-allowlisted path, and an ordinary doc change must still be allowed.+ */+export function runGateDrills(options) {+ const { config, benignPath = 'docs/README.md' } = options;+ const results = [];+ const mode = 'Over-Reach (Wrong Scope)';+ // Sensitivity: each denylist glob blocks a path that matches it.+ for (const glob of config.denylist) {+ const id = `gate.denylist[${glob}]`;+ const name = `denylist blocks ${glob}`;+ const sample = samplePathFor(glob);+ const decision = checkGate({ config, action: 'commit', paths: [sample] });+ if (decision.allowed) {+ results.push(fail(id, name, mode, 'sensitivity', `commit touching ${sample} is blocked`, `allowed (${decision.trigger})`, `Denylist entry "${glob}" did not match the sample path this drill synthesized for it. Either the glob is unreachable in practice, or the sample is unrepresentative — check it by hand with: loop-gate check --action commit --paths ${sample}`));+ continue;+ }+ if (decision.trigger !== 'denylist') {+ // Blocked, but by file-count or the allowlist — the denylist itself is+ // unproven, so this is not a pass.+ results.push(fail(id, name, mode, 'sensitivity', `blocked by the denylist`, `blocked by ${decision.trigger}`, 'The path was stopped, but not by the rule under test.'));+ continue;+ }+ results.push(pass(id, name, mode, 'sensitivity', 'blocked by denylist', decision.reason));+ }+ // Sensitivity: the file-count cap fires one file past the threshold.+ if (config.maxFiles !== undefined) {+ const id = 'gate.file-count';+ const name = `file-count cap fires above ${config.maxFiles} files`;+ const paths = Array.from({ length: config.maxFiles + 1 }, (_, i) => `docs/drill-${i}.md`);+ const decision = checkGate({ config, action: 'commit', paths });+ results.push(decision.trigger === 'file-count'+ ? pass(id, name, mode, 'sensitivity', 'blocked by file-count', decision.reason)+ : fail(id, name, mode, 'sensitivity', `${paths.length} files is blocked by file-count`, decision.allowed ? 'allowed' : `blocked by ${decision.trigger}`));+ }+ else {+ results.push(skip('gate.file-count', 'file-count cap', mode, 'sensitivity', 'gate.yaml sets no maxFiles — a loop may propose an arbitrarily large diff.'));+ }+ // Sensitivity: auto-merge rejects a path outside the allowlist.+ if (config.autoMergeAllowlist && config.autoMergeAllowlist.length > 0) {+ const id = 'gate.auto-merge';+ const name = 'auto-merge rejects a non-allowlisted path';+ const outside = 'src/drill-not-allowlisted.bin';+ const decision = checkGate({ config, action: 'auto-merge', paths: [outside] });+ results.push(!decision.allowed+ ? pass(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, decision.reason)+ : fail(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, 'allowed'));+ }+ // Specificity: an ordinary change is still allowed. A gate that blocks+ // everything would pass every drill above while being useless in practice.+ const benign = checkGate({ config, action: 'commit', paths: [benignPath] });+ results.push(benign.allowed+ ? pass('gate.benign', 'ordinary change is still allowed', mode, 'specificity', `commit touching ${benignPath} is allowed`, benign.reason)+ : fail('gate.benign', 'ordinary change is still allowed', mode, 'specificity', `commit touching ${benignPath} is allowed`, `blocked by ${benign.trigger}`, 'The gate blocks a benign path. A guardrail that stops everything will pass every sensitivity drill while making the loop unusable — pass --benign-path if this path is genuinely sensitive in your project.'));+ return results;+}+// ── Circuit breaker drills (Infinite Fix Loop / Token Burn) ────────+function ledgerOf(attempts) {+ return { goal: 'loop-drill synthetic run', attempts };+}+/**+ * Errors with no shared structure, used for the no-progress drill.+ *+ * They must stay dissimilar *after* errorSignature() normalization, which+ * collapses every number to '#'. Errors that differ only by a number ("module 1+ * not found", "module 2 not found") normalize to the same signature and trip+ * stagnation instead — the drill would pass while leaving no-progress+ * unexercised.+ */+const DISTINCT_ERRORS = [+ 'TypeError: undefined is not a function',+ 'ECONNREFUSED connecting to the database',+ 'SyntaxError: unexpected token in JSON',+ 'AssertionError: expected ok but got null',+ 'Permission denied while writing the artifact',+ 'OutOfMemoryError during the bundle step',+ 'Segmentation fault in the native addon',+];+/**+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated+ * identical failures must trip stagnation, a long run of unrelated failures+ * must trip no-progress, blowing the token budget must trip, and a healthy run+ * must not.+ *+ * Each drill asserts the *specific* trigger, not merely that the breaker+ * escalated. Escalating for another reason means the rule under test is still+ * unproven — the same standard the gate drills apply to `trigger !==+ * 'denylist'`.+ */+export function runBreakerDrills(config = DEFAULT_BREAKER) {+ const results = [];+ // Sensitivity: the same error repeated trips stagnation.+ {+ const id = 'breaker.stagnation';+ const name = `${config.stagnationThreshold} identical failures trip stagnation`;+ const mode = 'Infinite Fix Loop';+ const attempts = Array.from({ length: config.stagnationThreshold }, (_, i) => ({+ iteration: i + 1,+ action: 'retry the failing test',+ outcome: 'failure',+ error: "TypeError: Cannot read properties of undefined (reading 'id')",+ }));+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(decision.trigger === 'stagnation'+ ? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)+ : fail(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate+ ? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.'));+ }+ // Sensitivity: a long run of unrelated failures trips no-progress.+ {+ const id = 'breaker.no-progress';+ const name = `${config.noProgressThreshold} unrelated failures trip no-progress`;+ const mode = 'Infinite Fix Loop';+ const attempts = Array.from({ length: config.noProgressThreshold }, (_, i) => ({+ iteration: i + 1,+ action: `attempt strategy ${i + 1}`,+ outcome: 'failure',+ error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],+ }));+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(decision.trigger === 'no-progress'+ ? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)+ : fail(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate+ ? 'The breaker stopped the loop, but not by the rule under test — no-progress stays unproven.'+ : undefined));+ }+ // Sensitivity: exceeding the token budget trips.+ if (config.tokenBudget !== undefined) {+ const id = 'breaker.token-budget';+ const name = 'exceeding the token budget trips the breaker';+ const mode = 'Token Burn';+ const attempts = [+ {+ iteration: 1,+ action: 'a very expensive sub-agent chain',+ outcome: 'success',+ tokensUsed: config.tokenBudget + 1,+ },+ ];+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(decision.trigger === 'token-budget'+ ? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)+ : fail(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.escalate ? `escalated via ${decision.trigger}` : 'continued'));+ }+ else {+ results.push(skip('breaker.token-budget', 'token budget cap', 'Token Burn', 'sensitivity', 'No tokenBudget configured — nothing caps spend mid-run. See loop-budget.md.'));+ }+ // Specificity: a healthy run is NOT escalated.+ {+ const id = 'breaker.healthy';+ const name = 'a healthy run is not escalated';+ const mode = 'Infinite Fix Loop';+ const attempts = [+ { iteration: 1, action: 'read the failing test', outcome: 'success' },+ { iteration: 2, action: 'apply a minimal fix', outcome: 'success' },+ ];+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(!decision.escalate+ ? pass(id, name, mode, 'specificity', 'continue', 'continued')+ : fail(id, name, mode, 'specificity', 'continue', `escalated (${decision.trigger})`, 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.'));+ }+ return results;+}+// ── Report assembly ────────────────────────────────────────────────+const ALL_MODES = [+ 'Infinite Fix Loop',+ 'Token Burn',+ 'Over-Reach (Wrong Scope)',+ 'Verifier Theater',+ 'Escalation Failure',+];+export function buildReport(results) {+ const covered = [];+ const uncovered = [];+ for (const mode of ALL_MODES) {+ const forMode = results.filter((r) => r.failureMode === mode);+ const anyPassed = forMode.some((r) => r.outcome === 'passed');+ const anyFailed = forMode.some((r) => r.outcome === 'failed');+ if (anyPassed && !anyFailed)+ covered.push(mode);+ else+ uncovered.push(mode);+ }+ return {+ results,+ passed: results.filter((r) => r.outcome === 'passed').length,+ failed: results.filter((r) => r.outcome === 'failed').length,+ skipped: results.filter((r) => r.outcome === 'skipped').length,+ covered,+ uncovered,+ timestamp: new Date().toISOString(),+ };+}+/**+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's+ * "2 means escalate" convention so control scripts can chain all three.+ */+export function exitCodeFor(report) {+ if (report.failed > 0)+ return 2;+ if (report.skipped > 0)+ return 1;+ return 0;+}diff --git a/tools/loop-drill/dist/mutations.d.ts b/tools/loop-drill/dist/mutations.d.tsnew file mode 100644index 0000000..513caeb--- /dev/null+++ b/tools/loop-drill/dist/mutations.d.ts@@ -0,0 +1,49 @@+/**+ * Mutation operators for the verifier canary.+ *+ * The canary needs a defect a competent verifier should reject. Rather than+ * asking a model to invent one (which makes the drill non-deterministic and+ * costs tokens), we borrow mutation testing: apply a small, mechanical,+ * behaviour-changing edit to real source and see whether the verifier notices.+ *+ * Operators are deliberately restricted to edits that change runtime behaviour+ * in every C-family language, and that a reviewer would call a bug rather than+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:+ * a false mutant would report a verifier as broken when it was right to+ * approve.+ */+export interface MutationOperator {+ id: string;+ description: string;+ /** Matches the token to mutate. Must be global. */+ pattern: RegExp;+ replace: string;+}+/**+ * Order matters: the first operator with a match in a file wins, so the+ * least ambiguous edits come first.+ */+export declare const OPERATORS: MutationOperator[];+export interface Mutant {+ operator: MutationOperator;+ /** Repo-relative path of the mutated file. */+ file: string;+ mutated: string;+ /** 1-based line number of the edit, for the report. */+ line: number;+ /** The line before and after, for the report. */+ before: string;+ after: string;+}+/**+ * Apply the first applicable operator to `content`, mutating only the first+ * occurrence. One edit per mutant keeps the signal clean: if the verifier+ * rejects, it rejected *this* defect.+ *+ * Skips occurrences inside line comments — a verifier is right not to care+ * about a flipped operator in a comment, and crediting it for catching one+ * would overstate the result.+ */+export declare function mutate(content: string, file: string, operators?: MutationOperator[]): Mutant | null;+export declare function isMutationCandidate(relativePath: string): boolean;diff --git a/tools/loop-drill/dist/mutations.js b/tools/loop-drill/dist/mutations.jsnew file mode 100644index 0000000..0ffc3bc--- /dev/null+++ b/tools/loop-drill/dist/mutations.js@@ -0,0 +1,120 @@+/**+ * Mutation operators for the verifier canary.+ *+ * The canary needs a defect a competent verifier should reject. Rather than+ * asking a model to invent one (which makes the drill non-deterministic and+ * costs tokens), we borrow mutation testing: apply a small, mechanical,+ * behaviour-changing edit to real source and see whether the verifier notices.+ *+ * Operators are deliberately restricted to edits that change runtime behaviour+ * in every C-family language, and that a reviewer would call a bug rather than+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:+ * a false mutant would report a verifier as broken when it was right to+ * approve.+ */+/**+ * Order matters: the first operator with a match in a file wins, so the+ * least ambiguous edits come first.+ */+export const OPERATORS = [+ {+ id: 'strict-equality-flip',+ description: 'flip === to !==',+ pattern: /===/g,+ replace: '!==',+ },+ {+ id: 'strict-inequality-flip',+ description: 'flip !== to ===',+ pattern: /!==/g,+ replace: '===',+ },+ {+ id: 'boundary-off-by-one',+ description: 'weaken <= to <',+ pattern: /<=/g,+ replace: '<',+ },+ {+ id: 'boundary-off-by-one-gte',+ description: 'weaken >= to >',+ pattern: />=/g,+ replace: '>',+ },+ {+ id: 'logical-and-to-or',+ description: 'flip && to ||',+ pattern: /&&/g,+ replace: '||',+ },+ {+ id: 'return-true-to-false',+ description: 'flip `return true` to `return false`',+ pattern: /\breturn true\b/g,+ replace: 'return false',+ },+ {+ id: 'return-false-to-true',+ description: 'flip `return false` to `return true`',+ pattern: /\breturn false\b/g,+ replace: 'return true',+ },+];+/**+ * Apply the first applicable operator to `content`, mutating only the first+ * occurrence. One edit per mutant keeps the signal clean: if the verifier+ * rejects, it rejected *this* defect.+ *+ * Skips occurrences inside line comments — a verifier is right not to care+ * about a flipped operator in a comment, and crediting it for catching one+ * would overstate the result.+ */+export function mutate(content, file, operators = OPERATORS) {+ for (const operator of operators) {+ const pattern = new RegExp(operator.pattern.source, 'g');+ let match;+ while ((match = pattern.exec(content)) !== null) {+ const index = match.index;+ if (isInLineComment(content, index))+ continue;+ const mutated = content.slice(0, index) + operator.replace + content.slice(index + match[0].length);+ const line = content.slice(0, index).split('\n').length;+ return {+ operator,+ file,+ mutated,+ line,+ before: lineAt(content, line).trim(),+ after: lineAt(mutated, line).trim(),+ };+ }+ }+ return null;+}+function lineAt(content, line) {+ return content.split('\n')[line - 1] ?? '';+}+/** True when `index` sits after a `//` or `#` on its own line. */+function isInLineComment(content, index) {+ const lineStart = content.lastIndexOf('\n', index - 1) + 1;+ const prefix = content.slice(lineStart, index);+ return prefix.includes('//') || /(^|\s)#/.test(prefix);+}+/**+ * Files worth mutating: real source, not build output, vendored code, tests,+ * or generated artifacts. Mutating a test file would make the canary trivially+ * easy (the suite fails loudly); mutating dist/ would make it meaningless+ * (nothing reads it).+ */+const SKIP_DIRECTORIES = /(^|\/)(node_modules|dist|build|out|coverage|vendor|\.git|__snapshots__)(\/|$)/;+const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|java|go|rs|c|cc|cpp|cs|kt|swift|php|rb)$/;+const TEST_FILE = /(^|\/)(test|tests|spec|__tests__)(\/|$)|\.(test|spec)\.[a-z]+$/;+export function isMutationCandidate(relativePath) {+ const normalized = relativePath.replace(/\\/g, '/');+ if (SKIP_DIRECTORIES.test(normalized))+ return false;+ if (TEST_FILE.test(normalized))+ return false;+ return SOURCE_EXTENSIONS.test(normalized);+}diff --git a/tools/loop-drill/dist/report.d.ts b/tools/loop-drill/dist/report.d.tsnew file mode 100644index 0000000..8d038f1--- /dev/null+++ b/tools/loop-drill/dist/report.d.ts@@ -0,0 +1,2 @@+import type { DrillReport } from './drill.js';+export declare function formatReport(report: DrillReport, mutationScore: number | null): string;diff --git a/tools/loop-drill/dist/report.js b/tools/loop-drill/dist/report.jsnew file mode 100644index 0000000..3b12478--- /dev/null+++ b/tools/loop-drill/dist/report.js@@ -0,0 +1,46 @@+const MARK = {+ passed: '✅',+ failed: '❌',+ skipped: '⚠️',+};+export function formatReport(report, mutationScore) {+ const lines = [];+ lines.push('Loop Drill — guardrail fire drill');+ lines.push('═'.repeat(50));+ lines.push('');+ // Group by failure mode so the report reads as coverage of+ // docs/failure-modes.md rather than a flat list of assertions.+ const modes = [...new Set(report.results.map((r) => r.failureMode))];+ for (const mode of modes) {+ const forMode = report.results.filter((r) => r.failureMode === mode);+ const failed = forMode.filter((r) => r.outcome === 'failed').length;+ const status = failed > 0 ? '❌ NOT PROVEN' : forMode.some((r) => r.outcome === 'skipped') ? '⚠️ PARTIAL' : '✅ PROVEN';+ lines.push(`${mode} — ${status}`);+ for (const r of forMode) {+ lines.push(` ${MARK[r.outcome]} ${r.name}`);+ if (r.outcome !== 'passed') {+ lines.push(` expected: ${r.expected}`);+ lines.push(` actual: ${r.actual}`);+ if (r.detail)+ lines.push(` ${r.detail}`);+ }+ }+ lines.push('');+ }+ if (mutationScore !== null) {+ const pct = Math.round(mutationScore * 100);+ lines.push(`Mutation score: ${pct}% of seeded defects rejected`);+ if (pct < 100) {+ lines.push('A verifier that approves seeded defects is Verifier Theater (docs/failure-modes.md).');+ }+ lines.push('');+ }+ lines.push(`Passed: ${report.passed} Failed: ${report.failed} Skipped: ${report.skipped}`);+ if (report.uncovered.length > 0) {+ lines.push('');+ lines.push('Failure modes not proven by this run:');+ for (const mode of report.uncovered)+ lines.push(` - ${mode}`);+ }+ return lines.join('\n');+}diff --git a/tools/loop-drill/package-lock.json b/tools/loop-drill/package-lock.jsonnew file mode 100644index 0000000..196d435--- /dev/null+++ b/tools/loop-drill/package-lock.json@@ -0,0 +1,103 @@+{+ "name": "@cobusgreyling/loop-drill",+ "version": "1.0.0",+ "lockfileVersion": 3,+ "requires": true,+ "packages": {+ "": {+ "name": "@cobusgreyling/loop-drill",+ "version": "1.0.0",+ "license": "MIT",+ "dependencies": {+ "@cobusgreyling/loop-context": "file:../loop-context",+ "@cobusgreyling/loop-gate": "file:../loop-gate"+ },+ "bin": {+ "loop-drill": "dist/cli.js"+ },+ "devDependencies": {+ "@types/node": "^20.0.0",+ "typescript": "^5.0.0"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "../loop-context": {+ "name": "@cobusgreyling/loop-context",+ "version": "1.5.0",+ "license": "MIT",+ "dependencies": {+ "@cobusgreyling/loop-cost": "^1.1.0"+ },+ "bin": {+ "loop-context": "dist/cli.js"+ },+ "devDependencies": {+ "@types/node": "^20.0.0",+ "typescript": "^5.0.0"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "../loop-gate": {+ "name": "@cobusgreyling/loop-gate",+ "version": "1.0.0",+ "license": "MIT",+ "dependencies": {+ "minimatch": "^10.0.0",+ "yaml": "^2.8.0"+ },+ "bin": {+ "loop-gate": "dist/cli.js"+ },+ "devDependencies": {+ "@types/node": "^20.0.0",+ "typescript": "^5.0.0"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/@cobusgreyling/loop-context": {+ "resolved": "../loop-context",+ "link": true+ },+ "node_modules/@cobusgreyling/loop-gate": {+ "resolved": "../loop-gate",+ "link": true+ },+ "node_modules/@types/node": {+ "version": "20.19.43",+ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz",+ "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "undici-types": "~6.21.0"+ }+ },+ "node_modules/typescript": {+ "version": "5.9.3",+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",+ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",+ "dev": true,+ "license": "Apache-2.0",+ "bin": {+ "tsc": "bin/tsc",+ "tsserver": "bin/tsserver"+ },+ "engines": {+ "node": ">=14.17"+ }+ },+ "node_modules/undici-types": {+ "version": "6.21.0",+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",+ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",+ "dev": true,+ "license": "MIT"+ }+ }+}diff --git a/tools/loop-drill/package.json b/tools/loop-drill/package.jsonnew file mode 100644index 0000000..6d88756--- /dev/null+++ b/tools/loop-drill/package.json@@ -0,0 +1,54 @@+{+ "name": "@cobusgreyling/loop-drill",+ "version": "1.0.0",+ "description": "Fire drills for loop guardrails. Injects a known fault and asserts the guardrail actually fires — turning L3 readiness from \"the files exist\" into \"the guardrails demonstrably work\".",+ "type": "module",+ "main": "dist/drill.js",+ "exports": {+ ".": "./dist/drill.js"+ },+ "bin": {+ "loop-drill": "dist/cli.js"+ },+ "files": [+ "dist",+ "README.md"+ ],+ "scripts": {+ "build": "tsc",+ "test": "npm run build && node --test test/*.test.mjs",+ "prepublishOnly": "npm test",+ "start": "node dist/cli.js"+ },+ "engines": {+ "node": ">=18"+ },+ "keywords": [+ "loop-engineering",+ "ai-agents",+ "mutation-testing",+ "chaos-engineering",+ "verifier",+ "guardrails",+ "safety"+ ],+ "author": "Cobus Greyling",+ "license": "MIT",+ "repository": {+ "type": "git",+ "url": "git+https://github.com/cobusgreyling/loop-engineering.git",+ "directory": "tools/loop-drill"+ },+ "homepage": "https://github.com/cobusgreyling/loop-engineering/tree/main/tools/loop-drill#readme",+ "bugs": {+ "url": "https://github.com/cobusgreyling/loop-engineering/issues"+ },+ "dependencies": {+ "@cobusgreyling/loop-context": "file:../loop-context",+ "@cobusgreyling/loop-gate": "file:../loop-gate"+ },+ "devDependencies": {+ "@types/node": "^20.0.0",+ "typescript": "^5.0.0"+ }+}diff --git a/tools/loop-drill/src/canary.ts b/tools/loop-drill/src/canary.tsnew file mode 100644index 0000000..0069cb5--- /dev/null+++ b/tools/loop-drill/src/canary.ts@@ -0,0 +1,296 @@+/**+ * The verifier canary — the drill that gave this package its reason to exist.+ *+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and+ * gates L3 on it, by checking whether a file whose name contains "verifier"+ * exists. An empty file passes. docs/failure-modes.md calls the resulting+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls+ * maker/checker "the single most important structural pattern for reliable+ * loops". Nothing tested whether the verifier verifies.+ *+ * This runs the real verifier command against real seeded defects in an+ * isolated worktree, and scores it on what it caught:+ *+ * control — clean tree -> the verifier must ACCEPT (exit 0)+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)+ *+ * Both directions are required. A verifier that rejects everything catches+ * every mutant while being worthless, and the control run is what exposes it.+ */++import { exec as execShellCb, execFile as execFileCb } from 'node:child_process';+import { readFile, writeFile, rm } from 'node:fs/promises';+import path from 'node:path';+import { promisify } from 'node:util';+import { isMutationCandidate, mutate, type Mutant } from './mutations.js';+import { skip, type DrillResult } from './drill.js';++/**+ * Two spawn helpers, kept honest about their signatures.+ *+ * `execShell` runs a caller-supplied command string (--verifier-cmd, --setup)+ * through a shell, which is what those flags mean. `execFile` runs git with an+ * argv array and no shell, so repo paths never reach a shell for re-parsing.+ *+ * Earlier this file called promisify(execFile) with an options object in the+ * `args` position behind an `as never` cast -- it happened to work because Node+ * detects a non-array second argument, but it hid the real signature from tsc+ * and made timeout/killed handling harder to reason about.+ */+const execShell = promisify(execShellCb);+const exec = promisify(execFileCb);++export interface CanaryOptions {+ /** Repo root to drill. */+ root: string;+ /** The verifier command. Non-zero exit is read as "rejected". */+ command: string;+ /** How many mutants to seed. More is slower but a sharper score. */+ count: number;+ /** Per-run timeout in ms. A verifier that hangs is a failed verifier. */+ timeoutMs: number;+ /** Restrict mutation to files under this repo-relative path. */+ scope?: string;+ /** Command run inside each worktree before verifying (e.g. "npm ci"). */+ setup?: string;+}++export interface VerifierRun {+ accepted: boolean;+ exitCode: number | null;+ timedOut: boolean;+ output: string;+}++/**+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything+ * else — including a timeout — means "rejected". A verifier that cannot be+ * launched at all is a distinct failure, surfaced by the caller.+ */+export async function runVerifier(+ command: string,+ cwd: string,+ timeoutMs: number,+): Promise<VerifierRun> {+ try {+ const { stdout, stderr } = await execShell(command, {+ cwd,+ timeout: timeoutMs,+ maxBuffer: 8 * 1024 * 1024,+ });+ return { accepted: true, exitCode: 0, timedOut: false, output: `${stdout}${stderr}`.trim() };+ } catch (err) {+ const e = err as { code?: number | string; killed?: boolean; stdout?: string; stderr?: string };+ const timedOut = e.killed === true || e.code === 'ETIMEDOUT';+ return {+ accepted: false,+ exitCode: typeof e.code === 'number' ? e.code : null,+ timedOut,+ output: `${e.stdout ?? ''}${e.stderr ?? ''}`.trim(),+ };+ }+}++/** Files git tracks, filtered to plausible mutation targets. */+export async function candidateFiles(root: string, scope?: string): Promise<string[]> {+ const args = ['-C', root, 'ls-files'];+ if (scope) args.push('--', scope);+ const { stdout } = await exec('git', args, { maxBuffer: 32 * 1024 * 1024 });+ return stdout+ .split('\n')+ .map((l) => l.trim())+ .filter((l) => l.length > 0 && isMutationCandidate(l));+}++/**+ * Build up to `count` mutants from distinct files. Distinct files matter:+ * seeding three mutants into one module mostly measures whether the verifier+ * reads that module.+ */+export async function collectMutants(+ root: string,+ files: string[],+ count: number,+): Promise<Mutant[]> {+ const mutants: Mutant[] = [];+ for (const file of files) {+ if (mutants.length >= count) break;+ let content: string;+ try {+ content = await readFile(path.join(root, file), 'utf8');+ } catch {+ continue;+ }+ const mutant = mutate(content, file);+ if (mutant) mutants.push(mutant);+ }+ return mutants;+}++/**+ * Run the verifier inside an ephemeral git worktree, so a verifier that writes,+ * builds, or fixes cannot touch the real checkout. `mutant` is null for the+ * worktree control run. The worktree is always removed, including on throw.+ */+async function runInWorktree(+ root: string,+ mutant: Mutant | null,+ command: string,+ timeoutMs: number,+ setup?: string,+): Promise<VerifierRun> {+ const worktree = path.join(root, `.loop-drill-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);+ await exec('git', ['-C', root, 'worktree', 'add', '--detach', '--quiet', worktree], {+ maxBuffer: 8 * 1024 * 1024,+ });+ try {+ if (setup) {+ // Setup failure is not a verifier verdict — surface it as such.+ const setupRun = await runVerifier(setup, worktree, timeoutMs);+ if (!setupRun.accepted) {+ return { ...setupRun, output: `[setup failed] ${setupRun.output}` };+ }+ }+ if (mutant) await writeFile(path.join(worktree, mutant.file), mutant.mutated, 'utf8');+ return await runVerifier(command, worktree, timeoutMs);+ } finally {+ // --force because the verifier may have left build output behind.+ await exec('git', ['-C', root, 'worktree', 'remove', '--force', worktree]).catch(() => {});+ await rm(worktree, { recursive: true, force: true }).catch(() => {});+ }+}++export interface CanaryReport {+ results: DrillResult[];+ /** Mutants rejected / mutants run. Null when no mutant could be built. */+ mutationScore: number | null;+ caught: number;+ escaped: number;+}++export async function runCanary(options: CanaryOptions): Promise<CanaryReport> {+ const { root, command, count, timeoutMs, scope, setup } = options;+ const results: DrillResult[] = [];+ const mode = 'Verifier Theater' as const;++ // Control first, and bail if it fails: every mutant result is meaningless+ // when the verifier rejects a clean tree, and running them would report a+ // reject-everything verifier as a perfect score.+ const control = await runVerifier(command, root, timeoutMs);+ if (control.accepted) {+ results.push({+ id: 'verifier.control',+ name: 'verifier accepts an unmodified tree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'passed',+ expected: 'exit 0 on clean tree',+ actual: 'exit 0',+ });+ } else {+ results.push({+ id: 'verifier.control',+ name: 'verifier accepts an unmodified tree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'failed',+ expected: 'exit 0 on clean tree',+ actual: control.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${control.exitCode}`,+ detail:+ 'The verifier rejects code with no seeded defect, so it cannot distinguish good from bad. Mutant drills were skipped — their results would be meaningless. Fix the verifier command (or the tree it runs against) first.',+ });+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }++ // Second control, in a clean worktree. A fresh checkout has no node_modules+ // and no build output, so a verifier like "npm test" fails there for reasons+ // that have nothing to do with code quality. Without this run, every mutant+ // would be scored as "caught" and a completely broken setup would report a+ // perfect mutation score — the precise false confidence this tool exists to+ // prevent.+ const worktreeControl = await runInWorktree(root, null, command, timeoutMs, setup);+ if (!worktreeControl.accepted) {+ results.push({+ id: 'verifier.worktree-control',+ name: 'verifier accepts a clean worktree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'skipped',+ expected: 'exit 0 in a clean worktree',+ actual: worktreeControl.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${worktreeControl.exitCode}`,+ detail:+ 'The verifier passes in your checkout but fails in a fresh worktree, so mutant results would measure the environment rather than the verifier. Pass --setup to prepare the worktree first (e.g. --setup "npm ci"). ' ++ (worktreeControl.output ? `Output: ${truncate(worktreeControl.output)}` : ''),+ });+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }++ results.push({+ id: 'verifier.worktree-control',+ name: 'verifier accepts a clean worktree',+ failureMode: mode,+ direction: 'specificity',+ outcome: 'passed',+ expected: 'exit 0 in a clean worktree',+ actual: 'exit 0',+ });++ const files = await candidateFiles(root, scope);+ const mutants = await collectMutants(root, files, count);++ if (mutants.length === 0) {+ results.push(+ skip(+ 'verifier.mutants',+ 'verifier rejects seeded defects',+ mode,+ 'sensitivity',+ `No mutable source found${scope ? ` under ${scope}` : ''}. loop-drill mutates tracked, non-test, non-generated source in common languages — pass --scope to point it at yours.`,+ ),+ );+ return { results, mutationScore: null, caught: 0, escaped: 0 };+ }++ let caught = 0;+ for (const mutant of mutants) {+ const run = await runInWorktree(root, mutant, command, timeoutMs, setup);+ const where = `${mutant.file}:${mutant.line}`;+ const edit = `${mutant.operator.description} (${mutant.before} -> ${mutant.after})`;++ if (!run.accepted) {+ caught++;+ results.push({+ id: `verifier.mutant[${mutant.operator.id}]`,+ name: `verifier rejects ${mutant.operator.id} in ${where}`,+ failureMode: mode,+ direction: 'sensitivity',+ outcome: 'passed',+ expected: 'non-zero exit (rejected)',+ actual: run.timedOut ? 'timed out (treated as rejected)' : `exit ${run.exitCode}`,+ detail: edit,+ });+ } else {+ results.push({+ id: `verifier.mutant[${mutant.operator.id}]`,+ name: `verifier rejects ${mutant.operator.id} in ${where}`,+ failureMode: mode,+ direction: 'sensitivity',+ outcome: 'failed',+ expected: 'non-zero exit (rejected)',+ actual: 'exit 0 (approved a seeded defect)',+ detail: `${edit}. The verifier approved this change. That is Verifier Theater — see docs/failure-modes.md.`,+ });+ }+ }++ return {+ results,+ mutationScore: caught / mutants.length,+ caught,+ escaped: mutants.length - caught,+ };+}++function truncate(text: string, max = 400): string {+ return text.length <= max ? text : `${text.slice(0, max)}…`;+}diff --git a/tools/loop-drill/src/cli.ts b/tools/loop-drill/src/cli.tsnew file mode 100644index 0000000..24dbc6f--- /dev/null+++ b/tools/loop-drill/src/cli.ts@@ -0,0 +1,222 @@+#!/usr/bin/env node+/**+ * loop-drill CLI.+ *+ * Exit codes follow loop-gate / loop-context so control scripts can chain all+ * three: 0 proceed, 1 warnings, 2 escalate.+ */++import path from 'node:path';+import { loadGateConfig } from '@cobusgreyling/loop-gate';+import { DEFAULT_BREAKER, type CircuitBreakerConfig } from '@cobusgreyling/loop-context';+import {+ buildReport,+ exitCodeFor,+ runBreakerDrills,+ runGateDrills,+ skip,+ type DrillResult,+} from './drill.js';+import { runCanary } from './canary.js';+import { formatReport } from './report.js';++interface Flags {+ root: string;+ json: boolean;+ help: boolean;+ gateFile?: string;+ benignPath?: string;+ only?: string;+ verifierCmd?: string;+ mutants: number;+ timeoutMs: number;+ scope?: string;+ setup?: string;+ tokenBudget?: number;+}++const HELP = `loop-drill — fire drills for loop guardrails++Injects a known fault and asserts the guardrail actually fires. Turns L3+readiness from "the files exist" into "the guardrails demonstrably work".++Usage: loop-drill [path] [options]++Options:+ --only <drills> Comma-separated: gate, breaker, verifier (default: gate,breaker)+ --verifier-cmd <cmd> Verifier command to drill. Non-zero exit = rejected.+ Required to run the verifier canary.+ --mutants <n> Seeded defects for the canary (default: 3)+ --scope <path> Restrict mutation to this repo-relative path+ --setup <cmd> Command run in each worktree before verifying+ (e.g. "npm ci") — a fresh worktree has no node_modules+ --timeout <ms> Per-verifier-run timeout (default: 120000)+ --gate-file <path> Policy file (default: gate.yaml)+ --benign-path <path> Path the gate specificity drill treats as ordinary+ (default: docs/README.md)+ --token-budget <n> Token budget for the breaker drill+ --json Machine-readable output+ --help, -h Show this message++Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.++Examples:+ loop-drill .+ loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"+ loop-drill . --only gate,breaker --json+`;++function parseArgs(argv: string[]): Flags {+ const flags: Flags = { root: '.', json: false, help: false, mutants: 3, timeoutMs: 120_000 };+ const positional: string[] = [];++ for (let i = 0; i < argv.length; i++) {+ const arg = argv[i];+ const next = () => {+ const v = argv[++i];+ if (v === undefined) throw new Error(`${arg} requires a value`);+ return v;+ };+ switch (arg) {+ case '--help':+ case '-h':+ flags.help = true;+ break;+ case '--json':+ flags.json = true;+ break;+ case '--only':+ flags.only = next();+ break;+ case '--verifier-cmd':+ flags.verifierCmd = next();+ break;+ case '--mutants':+ flags.mutants = parsePositiveInt(next(), '--mutants');+ break;+ case '--scope':+ flags.scope = next();+ break;+ case '--setup':+ flags.setup = next();+ break;+ case '--timeout':+ flags.timeoutMs = parsePositiveInt(next(), '--timeout');+ break;+ case '--gate-file':+ flags.gateFile = next();+ break;+ case '--benign-path':+ flags.benignPath = next();+ break;+ case '--token-budget':+ flags.tokenBudget = parsePositiveInt(next(), '--token-budget');+ break;+ default:+ if (arg.startsWith('-')) throw new Error(`Unknown option: ${arg}`);+ positional.push(arg);+ }+ }++ if (positional.length > 0) flags.root = positional[0];+ return flags;+}++function parsePositiveInt(value: string, flag: string): number {+ const n = Number(value);+ if (!Number.isInteger(n) || n <= 0) throw new Error(`${flag} expects a positive integer, got: ${value}`);+ return n;+}++async function main(): Promise<void> {+ let flags: Flags;+ try {+ flags = parseArgs(process.argv.slice(2));+ } catch (err) {+ console.error(`${(err as Error).message}\n`);+ console.error(HELP);+ process.exit(2);+ }++ if (flags.help) {+ console.log(HELP);+ return;+ }++ const root = path.resolve(flags.root);+ const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));+ const results: DrillResult[] = [];+ let mutationScore: number | null = null;++ if (selected.has('gate')) {+ const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');+ try {+ const config = await loadGateConfig(gateFile);+ results.push(...runGateDrills({ config, benignPath: flags.benignPath }));+ } catch (err) {+ results.push(+ skip(+ 'gate',+ 'gate drills',+ 'Over-Reach (Wrong Scope)',+ 'sensitivity',+ `${(err as Error).message} Nothing mechanically stops a loop from editing a sensitive path.`,+ ),+ );+ }+ }++ if (selected.has('breaker')) {+ const config: CircuitBreakerConfig =+ flags.tokenBudget === undefined+ ? DEFAULT_BREAKER+ : { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };+ results.push(...runBreakerDrills(config));+ }++ if (selected.has('verifier')) {+ if (!flags.verifierCmd) {+ results.push(+ skip(+ 'verifier',+ 'verifier canary',+ 'Verifier Theater',+ 'sensitivity',+ 'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.',+ ),+ );+ } else {+ const canary = await runCanary({+ root,+ command: flags.verifierCmd,+ count: flags.mutants,+ timeoutMs: flags.timeoutMs,+ scope: flags.scope,+ setup: flags.setup,+ });+ results.push(...canary.results);+ mutationScore = canary.mutationScore;+ }+ }++ if (results.length === 0) {+ console.error(`No drills selected. --only accepts: gate, breaker, verifier\n`);+ process.exit(2);+ }++ const report = buildReport(results);+ const code = exitCodeFor(report);++ if (flags.json) {+ console.log(JSON.stringify({ ...report, mutationScore, exitCode: code }, null, 2));+ } else {+ console.log(formatReport(report, mutationScore));+ }++ process.exit(code);+}++main().catch((err) => {+ console.error(`loop-drill failed: ${(err as Error).message}`);+ process.exit(2);+});diff --git a/tools/loop-drill/src/drill.ts b/tools/loop-drill/src/drill.tsnew file mode 100644index 0000000..d8991d1--- /dev/null+++ b/tools/loop-drill/src/drill.ts@@ -0,0 +1,468 @@+/**+ * loop-drill — fire drills for loop guardrails.+ *+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores+ * whether those counterparts are *present*. Nothing checked whether they+ * actually fire.+ *+ * A drill injects a known fault and asserts the guardrail responds. Every+ * drill carries the failure mode it exercises, so the report reads as coverage+ * of docs/failure-modes.md rather than a list of anonymous assertions.+ *+ * Each guardrail gets both directions, because only one of them is easy:+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)+ * - specificity: benign input is NOT caught (a guardrail that blocks+ * everything passes every sensitivity drill while being useless)+ * A guardrail is only credited when it passes both.+ */++import { checkGate, type GateConfig } from '@cobusgreyling/loop-gate';+import {+ checkCircuitBreaker,+ DEFAULT_BREAKER,+ type CircuitBreakerConfig,+ type Ledger,+} from '@cobusgreyling/loop-context';++export type DrillOutcome = 'passed' | 'failed' | 'skipped';++/** Named failure modes from docs/failure-modes.md. */+export type FailureMode =+ | 'Infinite Fix Loop'+ | 'Token Burn'+ | 'Over-Reach (Wrong Scope)'+ | 'Verifier Theater'+ | 'Escalation Failure';++export interface DrillResult {+ /** Stable id, e.g. 'gate.denylist'. */+ id: string;+ name: string;+ /** The docs/failure-modes.md entry this drill exercises. */+ failureMode: FailureMode;+ /** 'sensitivity' = fault must be caught; 'specificity' = benign must pass. */+ direction: 'sensitivity' | 'specificity';+ outcome: DrillOutcome;+ expected: string;+ actual: string;+ /** Why a drill was skipped, or extra context on a failure. */+ detail?: string;+}++export interface DrillReport {+ results: DrillResult[];+ passed: number;+ failed: number;+ skipped: number;+ /** Failure modes with at least one passing drill and no failing drill. */+ covered: FailureMode[];+ /** Failure modes with no passing drill (unexercised or broken). */+ uncovered: FailureMode[];+ timestamp: string;+}++function pass(+ id: string,+ name: string,+ failureMode: FailureMode,+ direction: DrillResult['direction'],+ expected: string,+ actual: string,+): DrillResult {+ return { id, name, failureMode, direction, outcome: 'passed', expected, actual };+}++function fail(+ id: string,+ name: string,+ failureMode: FailureMode,+ direction: DrillResult['direction'],+ expected: string,+ actual: string,+ detail?: string,+): DrillResult {+ return { id, name, failureMode, direction, outcome: 'failed', expected, actual, detail };+}++export function skip(+ id: string,+ name: string,+ failureMode: FailureMode,+ direction: DrillResult['direction'],+ detail: string,+): DrillResult {+ return {+ id,+ name,+ failureMode,+ direction,+ outcome: 'skipped',+ expected: '—',+ actual: '—',+ detail,+ };+}++// ── Glob → concrete sample path ────────────────────────────────────++/**+ * Turn a denylist glob into a concrete path that should match it, so the drill+ * exercises the real matcher rather than re-implementing glob semantics.+ *+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a+ * literal, and the caller verifies the synthesized path actually matches+ * before asserting anything about it (see runGateDrills) — a glob we+ * synthesize badly is reported as skipped, never as a guardrail failure.+ */+export function samplePathFor(glob: string): string {+ const concrete = glob+ .split('/')+ .map((segment) => {+ if (segment === '**') return 'src';+ if (segment === '*') return 'sample';+ // '*_key*' -> 'drill_key_sample'; '.env.*' -> '.env.sample'+ return segment.replace(/\*/g, (_m, offset: number) => (offset === 0 ? 'drill' : 'sample'));+ })+ .join('/');++ // A trailing '**' segment became 'src', which is a directory, not a file.+ return /(^|\/)src$/.test(concrete) ? `${concrete}/drill-file.txt` : concrete;+}++// ── Gate drills (Over-Reach) ───────────────────────────────────────++export interface GateDrillOptions {+ config: GateConfig;+ /** Path assumed benign for the specificity drill. */+ benignPath?: string;+}++/**+ * Exercise loop-gate against its own gate.yaml: every denylist entry must+ * block, the file-count cap must block, auto-merge must reject a+ * non-allowlisted path, and an ordinary doc change must still be allowed.+ */+export function runGateDrills(options: GateDrillOptions): DrillResult[] {+ const { config, benignPath = 'docs/README.md' } = options;+ const results: DrillResult[] = [];+ const mode: FailureMode = 'Over-Reach (Wrong Scope)';++ // Sensitivity: each denylist glob blocks a path that matches it.+ for (const glob of config.denylist) {+ const id = `gate.denylist[${glob}]`;+ const name = `denylist blocks ${glob}`;+ const sample = samplePathFor(glob);+ const decision = checkGate({ config, action: 'commit', paths: [sample] });++ if (decision.allowed) {+ results.push(+ fail(+ id,+ name,+ mode,+ 'sensitivity',+ `commit touching ${sample} is blocked`,+ `allowed (${decision.trigger})`,+ `Denylist entry "${glob}" did not match the sample path this drill synthesized for it. Either the glob is unreachable in practice, or the sample is unrepresentative — check it by hand with: loop-gate check --action commit --paths ${sample}`,+ ),+ );+ continue;+ }++ if (decision.trigger !== 'denylist') {+ // Blocked, but by file-count or the allowlist — the denylist itself is+ // unproven, so this is not a pass.+ results.push(+ fail(+ id,+ name,+ mode,+ 'sensitivity',+ `blocked by the denylist`,+ `blocked by ${decision.trigger}`,+ 'The path was stopped, but not by the rule under test.',+ ),+ );+ continue;+ }++ results.push(pass(id, name, mode, 'sensitivity', 'blocked by denylist', decision.reason));+ }++ // Sensitivity: the file-count cap fires one file past the threshold.+ if (config.maxFiles !== undefined) {+ const id = 'gate.file-count';+ const name = `file-count cap fires above ${config.maxFiles} files`;+ const paths = Array.from({ length: config.maxFiles + 1 }, (_, i) => `docs/drill-${i}.md`);+ const decision = checkGate({ config, action: 'commit', paths });+ results.push(+ decision.trigger === 'file-count'+ ? pass(id, name, mode, 'sensitivity', 'blocked by file-count', decision.reason)+ : fail(+ id,+ name,+ mode,+ 'sensitivity',+ `${paths.length} files is blocked by file-count`,+ decision.allowed ? 'allowed' : `blocked by ${decision.trigger}`,+ ),+ );+ } else {+ results.push(+ skip(+ 'gate.file-count',+ 'file-count cap',+ mode,+ 'sensitivity',+ 'gate.yaml sets no maxFiles — a loop may propose an arbitrarily large diff.',+ ),+ );+ }++ // Sensitivity: auto-merge rejects a path outside the allowlist.+ if (config.autoMergeAllowlist && config.autoMergeAllowlist.length > 0) {+ const id = 'gate.auto-merge';+ const name = 'auto-merge rejects a non-allowlisted path';+ const outside = 'src/drill-not-allowlisted.bin';+ const decision = checkGate({ config, action: 'auto-merge', paths: [outside] });+ results.push(+ !decision.allowed+ ? pass(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, decision.reason)+ : fail(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, 'allowed'),+ );+ }++ // Specificity: an ordinary change is still allowed. A gate that blocks+ // everything would pass every drill above while being useless in practice.+ const benign = checkGate({ config, action: 'commit', paths: [benignPath] });+ results.push(+ benign.allowed+ ? pass(+ 'gate.benign',+ 'ordinary change is still allowed',+ mode,+ 'specificity',+ `commit touching ${benignPath} is allowed`,+ benign.reason,+ )+ : fail(+ 'gate.benign',+ 'ordinary change is still allowed',+ mode,+ 'specificity',+ `commit touching ${benignPath} is allowed`,+ `blocked by ${benign.trigger}`,+ 'The gate blocks a benign path. A guardrail that stops everything will pass every sensitivity drill while making the loop unusable — pass --benign-path if this path is genuinely sensitive in your project.',+ ),+ );++ return results;+}++// ── Circuit breaker drills (Infinite Fix Loop / Token Burn) ────────++function ledgerOf(attempts: Ledger['attempts']): Ledger {+ return { goal: 'loop-drill synthetic run', attempts };+}++/**+ * Errors with no shared structure, used for the no-progress drill.+ *+ * They must stay dissimilar *after* errorSignature() normalization, which+ * collapses every number to '#'. Errors that differ only by a number ("module 1+ * not found", "module 2 not found") normalize to the same signature and trip+ * stagnation instead — the drill would pass while leaving no-progress+ * unexercised.+ */+const DISTINCT_ERRORS = [+ 'TypeError: undefined is not a function',+ 'ECONNREFUSED connecting to the database',+ 'SyntaxError: unexpected token in JSON',+ 'AssertionError: expected ok but got null',+ 'Permission denied while writing the artifact',+ 'OutOfMemoryError during the bundle step',+ 'Segmentation fault in the native addon',+];++/**+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated+ * identical failures must trip stagnation, a long run of unrelated failures+ * must trip no-progress, blowing the token budget must trip, and a healthy run+ * must not.+ *+ * Each drill asserts the *specific* trigger, not merely that the breaker+ * escalated. Escalating for another reason means the rule under test is still+ * unproven — the same standard the gate drills apply to `trigger !==+ * 'denylist'`.+ */+export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER): DrillResult[] {+ const results: DrillResult[] = [];++ // Sensitivity: the same error repeated trips stagnation.+ {+ const id = 'breaker.stagnation';+ const name = `${config.stagnationThreshold} identical failures trip stagnation`;+ const mode: FailureMode = 'Infinite Fix Loop';+ const attempts = Array.from({ length: config.stagnationThreshold }, (_, i) => ({+ iteration: i + 1,+ action: 'retry the failing test',+ outcome: 'failure' as const,+ error: "TypeError: Cannot read properties of undefined (reading 'id')",+ }));+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(+ decision.trigger === 'stagnation'+ ? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)+ : fail(+ id,+ name,+ mode,+ 'sensitivity',+ 'escalate via stagnation',+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',+ decision.escalate+ ? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.',+ ),+ );+ }++ // Sensitivity: a long run of unrelated failures trips no-progress.+ {+ const id = 'breaker.no-progress';+ const name = `${config.noProgressThreshold} unrelated failures trip no-progress`;+ const mode: FailureMode = 'Infinite Fix Loop';+ const attempts = Array.from({ length: config.noProgressThreshold }, (_, i) => ({+ iteration: i + 1,+ action: `attempt strategy ${i + 1}`,+ outcome: 'failure' as const,+ error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],+ }));+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(+ decision.trigger === 'no-progress'+ ? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)+ : fail(+ id,+ name,+ mode,+ 'sensitivity',+ 'escalate via no-progress',+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',+ decision.escalate+ ? 'The breaker stopped the loop, but not by the rule under test — no-progress stays unproven.'+ : undefined,+ ),+ );+ }++ // Sensitivity: exceeding the token budget trips.+ if (config.tokenBudget !== undefined) {+ const id = 'breaker.token-budget';+ const name = 'exceeding the token budget trips the breaker';+ const mode: FailureMode = 'Token Burn';+ const attempts = [+ {+ iteration: 1,+ action: 'a very expensive sub-agent chain',+ outcome: 'success' as const,+ tokensUsed: config.tokenBudget + 1,+ },+ ];+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(+ decision.trigger === 'token-budget'+ ? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)+ : fail(+ id,+ name,+ mode,+ 'sensitivity',+ 'escalate via token-budget',+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',+ ),+ );+ } else {+ results.push(+ skip(+ 'breaker.token-budget',+ 'token budget cap',+ 'Token Burn',+ 'sensitivity',+ 'No tokenBudget configured — nothing caps spend mid-run. See loop-budget.md.',+ ),+ );+ }++ // Specificity: a healthy run is NOT escalated.+ {+ const id = 'breaker.healthy';+ const name = 'a healthy run is not escalated';+ const mode: FailureMode = 'Infinite Fix Loop';+ const attempts = [+ { iteration: 1, action: 'read the failing test', outcome: 'success' as const },+ { iteration: 2, action: 'apply a minimal fix', outcome: 'success' as const },+ ];+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ results.push(+ !decision.escalate+ ? pass(id, name, mode, 'specificity', 'continue', 'continued')+ : fail(+ id,+ name,+ mode,+ 'specificity',+ 'continue',+ `escalated (${decision.trigger})`,+ 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.',+ ),+ );+ }++ return results;+}++// ── Report assembly ────────────────────────────────────────────────++const ALL_MODES: FailureMode[] = [+ 'Infinite Fix Loop',+ 'Token Burn',+ 'Over-Reach (Wrong Scope)',+ 'Verifier Theater',+ 'Escalation Failure',+];++export function buildReport(results: DrillResult[]): DrillReport {+ const covered: FailureMode[] = [];+ const uncovered: FailureMode[] = [];++ for (const mode of ALL_MODES) {+ const forMode = results.filter((r) => r.failureMode === mode);+ const anyPassed = forMode.some((r) => r.outcome === 'passed');+ const anyFailed = forMode.some((r) => r.outcome === 'failed');+ if (anyPassed && !anyFailed) covered.push(mode);+ else uncovered.push(mode);+ }++ return {+ results,+ passed: results.filter((r) => r.outcome === 'passed').length,+ failed: results.filter((r) => r.outcome === 'failed').length,+ skipped: results.filter((r) => r.outcome === 'skipped').length,+ covered,+ uncovered,+ timestamp: new Date().toISOString(),+ };+}++/**+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's+ * "2 means escalate" convention so control scripts can chain all three.+ */+export function exitCodeFor(report: DrillReport): 0 | 1 | 2 {+ if (report.failed > 0) return 2;+ if (report.skipped > 0) return 1;+ return 0;+}diff --git a/tools/loop-drill/src/mutations.ts b/tools/loop-drill/src/mutations.tsnew file mode 100644index 0000000..34d634a--- /dev/null+++ b/tools/loop-drill/src/mutations.ts@@ -0,0 +1,145 @@+/**+ * Mutation operators for the verifier canary.+ *+ * The canary needs a defect a competent verifier should reject. Rather than+ * asking a model to invent one (which makes the drill non-deterministic and+ * costs tokens), we borrow mutation testing: apply a small, mechanical,+ * behaviour-changing edit to real source and see whether the verifier notices.+ *+ * Operators are deliberately restricted to edits that change runtime behaviour+ * in every C-family language, and that a reviewer would call a bug rather than+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:+ * a false mutant would report a verifier as broken when it was right to+ * approve.+ */++export interface MutationOperator {+ id: string;+ description: string;+ /** Matches the token to mutate. Must be global. */+ pattern: RegExp;+ replace: string;+}++/**+ * Order matters: the first operator with a match in a file wins, so the+ * least ambiguous edits come first.+ */+export const OPERATORS: MutationOperator[] = [+ {+ id: 'strict-equality-flip',+ description: 'flip === to !==',+ pattern: /===/g,+ replace: '!==',+ },+ {+ id: 'strict-inequality-flip',+ description: 'flip !== to ===',+ pattern: /!==/g,+ replace: '===',+ },+ {+ id: 'boundary-off-by-one',+ description: 'weaken <= to <',+ pattern: /<=/g,+ replace: '<',+ },+ {+ id: 'boundary-off-by-one-gte',+ description: 'weaken >= to >',+ pattern: />=/g,+ replace: '>',+ },+ {+ id: 'logical-and-to-or',+ description: 'flip && to ||',+ pattern: /&&/g,+ replace: '||',+ },+ {+ id: 'return-true-to-false',+ description: 'flip `return true` to `return false`',+ pattern: /\breturn true\b/g,+ replace: 'return false',+ },+ {+ id: 'return-false-to-true',+ description: 'flip `return false` to `return true`',+ pattern: /\breturn false\b/g,+ replace: 'return true',+ },+];++export interface Mutant {+ operator: MutationOperator;+ /** Repo-relative path of the mutated file. */+ file: string;+ mutated: string;+ /** 1-based line number of the edit, for the report. */+ line: number;+ /** The line before and after, for the report. */+ before: string;+ after: string;+}++/**+ * Apply the first applicable operator to `content`, mutating only the first+ * occurrence. One edit per mutant keeps the signal clean: if the verifier+ * rejects, it rejected *this* defect.+ *+ * Skips occurrences inside line comments — a verifier is right not to care+ * about a flipped operator in a comment, and crediting it for catching one+ * would overstate the result.+ */+export function mutate(content: string, file: string, operators = OPERATORS): Mutant | null {+ for (const operator of operators) {+ const pattern = new RegExp(operator.pattern.source, 'g');+ let match: RegExpExecArray | null;++ while ((match = pattern.exec(content)) !== null) {+ const index = match.index;+ if (isInLineComment(content, index)) continue;++ const mutated = content.slice(0, index) + operator.replace + content.slice(index + match[0].length);+ const line = content.slice(0, index).split('\n').length;+ return {+ operator,+ file,+ mutated,+ line,+ before: lineAt(content, line).trim(),+ after: lineAt(mutated, line).trim(),+ };+ }+ }+ return null;+}++function lineAt(content: string, line: number): string {+ return content.split('\n')[line - 1] ?? '';+}++/** True when `index` sits after a `//` or `#` on its own line. */+function isInLineComment(content: string, index: number): boolean {+ const lineStart = content.lastIndexOf('\n', index - 1) + 1;+ const prefix = content.slice(lineStart, index);+ return prefix.includes('//') || /(^|\s)#/.test(prefix);+}++/**+ * Files worth mutating: real source, not build output, vendored code, tests,+ * or generated artifacts. Mutating a test file would make the canary trivially+ * easy (the suite fails loudly); mutating dist/ would make it meaningless+ * (nothing reads it).+ */+const SKIP_DIRECTORIES = /(^|\/)(node_modules|dist|build|out|coverage|vendor|\.git|__snapshots__)(\/|$)/;+const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|java|go|rs|c|cc|cpp|cs|kt|swift|php|rb)$/;+const TEST_FILE = /(^|\/)(test|tests|spec|__tests__)(\/|$)|\.(test|spec)\.[a-z]+$/;++export function isMutationCandidate(relativePath: string): boolean {+ const normalized = relativePath.replace(/\\/g, '/');+ if (SKIP_DIRECTORIES.test(normalized)) return false;+ if (TEST_FILE.test(normalized)) return false;+ return SOURCE_EXTENSIONS.test(normalized);+}diff --git a/tools/loop-drill/src/report.ts b/tools/loop-drill/src/report.tsnew file mode 100644index 0000000..f5ab096--- /dev/null+++ b/tools/loop-drill/src/report.ts@@ -0,0 +1,52 @@+import type { DrillReport, DrillResult } from './drill.js';++const MARK: Record<DrillResult['outcome'], string> = {+ passed: '✅',+ failed: '❌',+ skipped: '⚠️',+};++export function formatReport(report: DrillReport, mutationScore: number | null): string {+ const lines: string[] = [];+ lines.push('Loop Drill — guardrail fire drill');+ lines.push('═'.repeat(50));+ lines.push('');++ // Group by failure mode so the report reads as coverage of+ // docs/failure-modes.md rather than a flat list of assertions.+ const modes = [...new Set(report.results.map((r) => r.failureMode))];+ for (const mode of modes) {+ const forMode = report.results.filter((r) => r.failureMode === mode);+ const failed = forMode.filter((r) => r.outcome === 'failed').length;+ const status = failed > 0 ? '❌ NOT PROVEN' : forMode.some((r) => r.outcome === 'skipped') ? '⚠️ PARTIAL' : '✅ PROVEN';+ lines.push(`${mode} — ${status}`);+ for (const r of forMode) {+ lines.push(` ${MARK[r.outcome]} ${r.name}`);+ if (r.outcome !== 'passed') {+ lines.push(` expected: ${r.expected}`);+ lines.push(` actual: ${r.actual}`);+ if (r.detail) lines.push(` ${r.detail}`);+ }+ }+ lines.push('');+ }++ if (mutationScore !== null) {+ const pct = Math.round(mutationScore * 100);+ lines.push(`Mutation score: ${pct}% of seeded defects rejected`);+ if (pct < 100) {+ lines.push('A verifier that approves seeded defects is Verifier Theater (docs/failure-modes.md).');+ }+ lines.push('');+ }++ lines.push(`Passed: ${report.passed} Failed: ${report.failed} Skipped: ${report.skipped}`);++ if (report.uncovered.length > 0) {+ lines.push('');+ lines.push('Failure modes not proven by this run:');+ for (const mode of report.uncovered) lines.push(` - ${mode}`);+ }++ return lines.join('\n');+}diff --git a/tools/loop-drill/test/canary.test.mjs b/tools/loop-drill/test/canary.test.mjsnew file mode 100644index 0000000..d5dcbdc--- /dev/null+++ b/tools/loop-drill/test/canary.test.mjs@@ -0,0 +1,166 @@+import { test, describe, before, after } from 'node:test';+import assert from 'node:assert/strict';+import path from 'node:path';+import os from 'node:os';+import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises';+import { execFile } from 'node:child_process';+import { promisify } from 'node:util';+import { runCanary, runVerifier, candidateFiles, collectMutants } from '../dist/canary.js';++const exec = promisify(execFile);++/**+ * A throwaway git repo with one mutable source file and a real test script.+ * The "verifier" is `node verify.mjs`, which asserts isAdult(18) === true —+ * a check the boundary mutation (>= to >) genuinely breaks.+ */+let repo;++before(async () => {+ repo = await mkdtemp(path.join(os.tmpdir(), 'loop-drill-'));+ await mkdir(path.join(repo, 'src'), { recursive: true });++ await writeFile(+ path.join(repo, 'src', 'rules.mjs'),+ 'export function isAdult(age) {\n return age >= 18;\n}\n',+ );+ await writeFile(+ path.join(repo, 'verify.mjs'),+ [+ "import { isAdult } from './src/rules.mjs';",+ 'if (isAdult(18) !== true) { console.error("18 must be adult"); process.exit(1); }',+ 'if (isAdult(17) !== false) { console.error("17 must not be adult"); process.exit(1); }',+ 'console.log("verified");',+ ].join('\n'),+ );++ await exec('git', ['-C', repo, 'init', '-q']);+ await exec('git', ['-C', repo, 'config', 'user.email', '[email protected]']);+ await exec('git', ['-C', repo, 'config', 'user.name', 'drill']);+ await exec('git', ['-C', repo, 'add', '.']);+ await exec('git', ['-C', repo, 'commit', '-q', '-m', 'seed']);+});++after(async () => {+ if (repo) await rm(repo, { recursive: true, force: true });+});++describe('runVerifier', () => {+ test('exit 0 is read as accepted', async () => {+ const run = await runVerifier('node -e "process.exit(0)"', process.cwd(), 20_000);+ assert.equal(run.accepted, true);+ });++ test('non-zero exit is read as rejected, and the code is reported', async () => {+ const run = await runVerifier('node -e "process.exit(3)"', process.cwd(), 20_000);+ assert.equal(run.accepted, false);+ assert.equal(run.exitCode, 3);+ });++ test('a hanging verifier is rejected rather than hanging the drill', async () => {+ const run = await runVerifier('node -e "setTimeout(()=>{}, 60000)"', process.cwd(), 800);+ assert.equal(run.accepted, false);+ assert.equal(run.timedOut, true);+ });+});++describe('candidateFiles / collectMutants', () => {+ test('lists tracked source and skips the verifier harness itself', async () => {+ const files = await candidateFiles(repo);+ assert.ok(files.includes('src/rules.mjs'));+ });++ test('builds a mutant from real source', async () => {+ const mutants = await collectMutants(repo, ['src/rules.mjs'], 1);+ assert.equal(mutants.length, 1);+ assert.equal(mutants[0].file, 'src/rules.mjs');+ assert.match(mutants[0].mutated, /age > 18/);+ });+});++describe('runCanary', () => {+ test('a real verifier catches the seeded defect and scores 100%', async () => {+ const report = await runCanary({+ root: repo,+ command: 'node verify.mjs',+ count: 1,+ timeoutMs: 60_000,+ scope: 'src',+ });+ assert.equal(report.mutationScore, 1);+ assert.equal(report.caught, 1);+ assert.equal(report.escaped, 0);+ assert.ok(report.results.every((r) => r.outcome === 'passed'));+ });++ test('a rubber-stamp verifier scores 0% and is named as Verifier Theater', async () => {+ const report = await runCanary({+ root: repo,+ command: 'node -e "console.log(\'LGTM\')"',+ count: 1,+ timeoutMs: 60_000,+ scope: 'src',+ });+ assert.equal(report.mutationScore, 0);+ assert.equal(report.escaped, 1);+ const escaped = report.results.find((r) => r.outcome === 'failed');+ assert.ok(escaped, 'expected a failed drill');+ assert.equal(escaped.failureMode, 'Verifier Theater');+ assert.match(escaped.actual, /approved a seeded defect/);+ });++ test('a reject-everything verifier fails the control and skips the mutants', async () => {+ const report = await runCanary({+ root: repo,+ command: 'node -e "process.exit(1)"',+ count: 1,+ timeoutMs: 60_000,+ scope: 'src',+ });+ // Mutant results would be meaningless here, so none were produced.+ assert.equal(report.mutationScore, null);+ assert.equal(report.results.length, 1);+ assert.equal(report.results[0].id, 'verifier.control');+ assert.equal(report.results[0].outcome, 'failed');+ });++ test('a verifier that only passes with prepared deps is skipped, not scored', async () => {+ // Passes at the repo root (the marker is untracked, so it survives there)+ // but not in a fresh worktree -- the environment-drift case that would+ // otherwise report every mutant as caught.+ await writeFile(path.join(repo, 'untracked-marker.txt'), 'present\n');+ const report = await runCanary({+ root: repo,+ command: 'node -e "require(\'fs\').accessSync(\'untracked-marker.txt\')"',+ count: 1,+ timeoutMs: 60_000,+ scope: 'src',+ });+ assert.equal(report.mutationScore, null, 'must not score mutants on a broken worktree');+ const control = report.results.find((r) => r.id === 'verifier.worktree-control');+ assert.equal(control.outcome, 'skipped');+ assert.match(control.detail, /--setup/);+ await rm(path.join(repo, 'untracked-marker.txt'), { force: true });+ });++ test('--setup repairs the worktree so mutants can be scored', async () => {+ await writeFile(path.join(repo, 'untracked-marker.txt'), 'present\n');+ const report = await runCanary({+ root: repo,+ command: 'node -e "require(\'fs\').accessSync(\'untracked-marker.txt\')" && node verify.mjs',+ setup: 'node -e "require(\'fs\').writeFileSync(\'untracked-marker.txt\',\'x\')"',+ count: 1,+ timeoutMs: 60_000,+ scope: 'src',+ });+ assert.equal(report.mutationScore, 1);+ await rm(path.join(repo, 'untracked-marker.txt'), { force: true });+ });++ test('leaves no worktree behind', async () => {+ const entries = await readdir(repo);+ assert.deepEqual(entries.filter((e) => e.startsWith('.loop-drill-')), []);+ const { stdout } = await exec('git', ['-C', repo, 'worktree', 'list']);+ assert.equal(stdout.trim().split('\n').length, 1, 'only the main worktree should remain');+ });+});diff --git a/tools/loop-drill/test/drill.test.mjs b/tools/loop-drill/test/drill.test.mjsnew file mode 100644index 0000000..31f2d27--- /dev/null+++ b/tools/loop-drill/test/drill.test.mjs@@ -0,0 +1,239 @@+import { test, describe } from 'node:test';+import assert from 'node:assert/strict';+import {+ samplePathFor,+ runGateDrills,+ runBreakerDrills,+ buildReport,+ exitCodeFor,+} from '../dist/drill.js';+import { DEFAULT_BREAKER } from '@cobusgreyling/loop-context';++const GOOD_GATE = {+ version: 1,+ denylist: ['**/.env', '**/secrets/**', '**/*_key*', '**/k8s/production/**'],+ maxFiles: 10,+ autoMergeAllowlist: ['docs/**', '**/*.md'],+};++const outcomes = (results, id) => results.filter((r) => r.id.startsWith(id)).map((r) => r.outcome);+const byId = (results, id) => results.find((r) => r.id === id);++describe('samplePathFor', () => {+ test('expands ** into a directory segment and appends a filename', () => {+ assert.equal(samplePathFor('**/secrets/**'), 'src/secrets/src/drill-file.txt');+ });++ test('keeps a literal filename under a ** prefix', () => {+ assert.equal(samplePathFor('**/.env'), 'src/.env');+ });++ test('fills inline stars so the path still matches the glob', () => {+ // '*_key*' -> 'drill' + '_key' + 'sample'; still matches the glob.+ assert.equal(samplePathFor('**/*_key*'), 'src/drill_keysample');+ });++ test('handles a dotted suffix glob', () => {+ assert.equal(samplePathFor('**/.env.*'), 'src/.env.sample');+ });+});++describe('runGateDrills', () => {+ const results = runGateDrills({ config: GOOD_GATE });++ test('every denylist entry is proven to block', () => {+ for (const glob of GOOD_GATE.denylist) {+ assert.deepEqual(outcomes(results, `gate.denylist[${glob}]`), ['passed'], `glob ${glob}`);+ }+ });++ test('the file-count cap is proven', () => {+ assert.equal(byId(results, 'gate.file-count').outcome, 'passed');+ });++ test('auto-merge rejects a non-allowlisted path', () => {+ assert.equal(byId(results, 'gate.auto-merge').outcome, 'passed');+ });++ test('a benign change is still allowed (specificity)', () => {+ const benign = byId(results, 'gate.benign');+ assert.equal(benign.outcome, 'passed');+ assert.equal(benign.direction, 'specificity');+ });++ test('every gate drill is attributed to the Over-Reach failure mode', () => {+ for (const r of results) assert.equal(r.failureMode, 'Over-Reach (Wrong Scope)');+ });++ test('skips the file-count drill when no maxFiles is configured', () => {+ const { maxFiles, ...noCap } = GOOD_GATE;+ const r = byId(runGateDrills({ config: noCap }), 'gate.file-count');+ assert.equal(r.outcome, 'skipped');+ assert.match(r.detail, /no maxFiles/);+ });++ test('fails a gate that blocks everything, even though it catches the fault', () => {+ const paranoid = { ...GOOD_GATE, denylist: ['**'] };+ const r = runGateDrills({ config: paranoid });+ // The denylist does catch the seeded fault...+ assert.equal(byId(r, 'gate.denylist[**]').outcome, 'passed');+ // ...but it blocks a benign change, so it is not a working guardrail.+ assert.equal(byId(r, 'gate.benign').outcome, 'failed');+ });++ test('a catch-all denylist leaves the file-count rule unproven', () => {+ // Every synthetic path is swallowed by the denylist before file-count is+ // reached, so the cap is never actually exercised -- reported, not hidden.+ const r = runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } });+ const fileCount = byId(r, 'gate.file-count');+ assert.equal(fileCount.outcome, 'failed');+ assert.match(fileCount.actual, /blocked by denylist/);+ });++ test('honours a caller-supplied benign path', () => {+ const r = runGateDrills({ config: GOOD_GATE, benignPath: 'src/app/index.ts' });+ assert.equal(byId(r, 'gate.benign').outcome, 'passed');+ });+});++describe('runBreakerDrills', () => {+ const results = runBreakerDrills(DEFAULT_BREAKER);++ test('stagnation is proven', () => {+ assert.equal(byId(results, 'breaker.stagnation').outcome, 'passed');+ });++ test('no-progress is proven', () => {+ assert.equal(byId(results, 'breaker.no-progress').outcome, 'passed');+ });++ test('a healthy run is not escalated (specificity)', () => {+ assert.equal(byId(results, 'breaker.healthy').outcome, 'passed');+ });++ test('token budget is skipped when unset, and proven when set', () => {+ assert.equal(byId(results, 'breaker.token-budget').outcome, 'skipped');+ const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });+ assert.equal(byId(withBudget, 'breaker.token-budget').outcome, 'passed');+ });++ // No wrong-trigger test for breaker.token-budget: its ledger is a single+ // successful attempt, so stagnation/no-progress (which need failures) and+ // max-iterations (checked after the budget) are all unreachable. There is no+ // config in which it escalates for another reason, so weakening its+ // assertion to decision.escalate is not observable from outside.++ test('token budget drill is attributed to Token Burn', () => {+ const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });+ assert.equal(byId(withBudget, 'breaker.token-budget').failureMode, 'Token Burn');+ });++ test('each drill is credited only for its own trigger', () => {+ // Escalating for another reason leaves the rule under test unproven, so the+ // reported reason must name the rule the drill exercises.+ assert.match(byId(results, 'breaker.stagnation').expected, /stagnation/);+ assert.match(byId(results, 'breaker.no-progress').expected, /no-progress/);+ assert.match(byId(results, 'breaker.no-progress').actual, /no.progress/i);+ });++ test('a percentage similarityThreshold leaves stagnation unproven', () => {+ // similarityThreshold is a 0.0-1.0 fraction. Setting it to 95 (meaning+ // "95%") means identical errors never compare as similar, so a loop retrying+ // the same failure forever is never stopped. The drill must report that.+ const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 });+ const stagnation = byId(misconfigured, 'breaker.stagnation');+ assert.equal(stagnation.outcome, 'failed');+ assert.equal(stagnation.actual, 'continued');+ assert.match(stagnation.detail, /Infinite Fix Loop/);+ assert.match(stagnation.detail, /similarityThreshold/);+ });++ test('escalating via the wrong trigger leaves stagnation unproven', () => {+ // The case that separates "the breaker fired" from "the rule under test+ // fired". Here the breaker DOES escalate, so a drill asserting only+ // decision.escalate would pass while stagnation was never exercised.+ const wrongTrigger = runBreakerDrills({+ ...DEFAULT_BREAKER,+ similarityThreshold: 95,+ noProgressThreshold: 1,+ });+ const stagnation = byId(wrongTrigger, 'breaker.stagnation');+ assert.equal(stagnation.outcome, 'failed');+ assert.equal(stagnation.actual, 'escalated via no-progress');+ assert.match(stagnation.detail, /not by the rule under test/);+ });++ test('an iteration cap that fires first also leaves stagnation unproven', () => {+ const capped = runBreakerDrills({+ ...DEFAULT_BREAKER,+ similarityThreshold: 95,+ maxIterations: 2,+ });+ const stagnation = byId(capped, 'breaker.stagnation');+ assert.equal(stagnation.outcome, 'failed');+ assert.equal(stagnation.actual, 'escalated via max-iterations');+ });++ test('stagnation swallowing every error leaves no-progress unproven', () => {+ // similarityThreshold 0 makes every error count as "the same error", so+ // stagnation fires on the no-progress drill's unrelated failures. The+ // breaker still escalates, so only a trigger-specific assertion catches it.+ const swallowed = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 0 });+ const noProgress = byId(swallowed, 'breaker.no-progress');+ assert.equal(noProgress.outcome, 'failed');+ assert.equal(noProgress.actual, 'escalated via stagnation');+ assert.match(noProgress.detail, /not by the rule under test/);+ });++ test('no-progress uses errors that stay distinct after signature normalization', () => {+ // errorSignature() collapses every number to '#', so errors differing only+ // by a number ("module 1" / "module 2") normalize to the same signature and+ // trip stagnation instead -- passing the drill while leaving no-progress+ // unexercised. This asserts the real trigger, which is what catches that.+ const r = byId(results, 'breaker.no-progress');+ assert.equal(r.outcome, 'passed');+ assert.doesNotMatch(r.actual, /stagnation/i);+ });+});++describe('buildReport / exitCodeFor', () => {+ test('a failure mode with any failing drill is not counted as covered', () => {+ const report = buildReport([+ ...runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }),+ ]);+ assert.ok(!report.covered.includes('Over-Reach (Wrong Scope)'));+ assert.ok(report.uncovered.includes('Over-Reach (Wrong Scope)'));+ });++ test('a fully passing gate run covers Over-Reach', () => {+ const report = buildReport(runGateDrills({ config: GOOD_GATE }));+ assert.ok(report.covered.includes('Over-Reach (Wrong Scope)'));+ });++ test('unexercised failure modes are reported as uncovered', () => {+ const report = buildReport(runGateDrills({ config: GOOD_GATE }));+ assert.ok(report.uncovered.includes('Verifier Theater'));+ assert.ok(report.uncovered.includes('Escalation Failure'));+ });++ test('exit code 0 when everything passes', () => {+ assert.equal(exitCodeFor(buildReport(runGateDrills({ config: GOOD_GATE }))), 0);+ });++ test('exit code 1 when a drill is skipped', () => {+ const { maxFiles, ...noCap } = GOOD_GATE;+ assert.equal(exitCodeFor(buildReport(runGateDrills({ config: noCap }))), 1);+ });++ test('exit code 2 when a guardrail fails to fire', () => {+ const report = buildReport(runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }));+ assert.equal(exitCodeFor(report), 2);+ });++ test('failure beats skip in the exit code', () => {+ const { maxFiles, ...noCap } = GOOD_GATE;+ const report = buildReport(runGateDrills({ config: { ...noCap, denylist: ['**'] } }));+ assert.ok(report.skipped > 0 && report.failed > 0);+ assert.equal(exitCodeFor(report), 2);+ });+});diff --git a/tools/loop-drill/test/mutations.test.mjs b/tools/loop-drill/test/mutations.test.mjsnew file mode 100644index 0000000..cf5336c--- /dev/null+++ b/tools/loop-drill/test/mutations.test.mjs@@ -0,0 +1,97 @@+import { test, describe } from 'node:test';+import assert from 'node:assert/strict';+import { mutate, isMutationCandidate, OPERATORS } from '../dist/mutations.js';++describe('mutate', () => {+ test('flips the first === it finds', () => {+ const m = mutate('const ok = a === b;\n', 'src/a.ts');+ assert.equal(m.operator.id, 'strict-equality-flip');+ assert.equal(m.mutated, 'const ok = a !== b;\n');+ assert.equal(m.line, 1);+ });++ test('mutates only the first occurrence, so one mutant means one defect', () => {+ const m = mutate('a === b;\nc === d;\n', 'src/a.ts');+ assert.equal(m.mutated, 'a !== b;\nc === d;\n');+ });++ test('reports the line number and the before/after text', () => {+ const m = mutate('line one\nconst n = x <= y;\n', 'src/a.ts');+ assert.equal(m.line, 2);+ assert.equal(m.before, 'const n = x <= y;');+ assert.equal(m.after, 'const n = x < y;');+ });++ test('skips a match inside a // comment', () => {+ const m = mutate('// a === b is fine\nreturn c === d;\n', 'src/a.ts');+ assert.equal(m.line, 2);+ assert.equal(m.mutated, '// a === b is fine\nreturn c !== d;\n');+ });++ test('skips a match inside a # comment', () => {+ const m = mutate('# x <= y\nif a <= b:\n', 'src/a.py');+ assert.equal(m.line, 2);+ });++ test('returns null when nothing is mutable', () => {+ assert.equal(mutate('const greeting = "hello";\n', 'src/a.ts'), null);+ });++ test('flips boolean returns', () => {+ const m = mutate('function isValid() { return true; }\n', 'src/a.ts');+ assert.equal(m.operator.id, 'return-true-to-false');+ assert.match(m.mutated, /return false/);+ });++ test('flips && to ||', () => {+ const m = mutate('if (a && b) run();\n', 'src/a.ts');+ assert.equal(m.operator.id, 'logical-and-to-or');+ assert.equal(m.mutated, 'if (a || b) run();\n');+ });++ test('every operator actually changes the content it matches', () => {+ for (const op of OPERATORS) {+ assert.notEqual(op.replace, op.pattern.source, `operator ${op.id} is a no-op`);+ }+ });++ test('excludes ambiguous operators that would produce false mutants', () => {+ // `<` -> `>` breaks TS generics and JSX; `+` -> `-` mangles string+ // concatenation. Either would make a correct verifier look broken.+ const ids = OPERATORS.map((o) => o.id);+ assert.ok(!ids.includes('less-than-flip'));+ assert.ok(!ids.includes('arithmetic-flip'));+ });+});++describe('isMutationCandidate', () => {+ test('accepts ordinary source', () => {+ assert.ok(isMutationCandidate('src/index.ts'));+ assert.ok(isMutationCandidate('app/main/Service.java'));+ assert.ok(isMutationCandidate('pkg/server.go'));+ });++ test('rejects build output and dependencies', () => {+ assert.ok(!isMutationCandidate('dist/index.js'));+ assert.ok(!isMutationCandidate('node_modules/left-pad/index.js'));+ assert.ok(!isMutationCandidate('coverage/lcov-report/app.js'));+ assert.ok(!isMutationCandidate('vendor/lib.go'));+ });++ test('rejects tests — mutating them would make the canary trivial', () => {+ assert.ok(!isMutationCandidate('test/app.test.mjs'));+ assert.ok(!isMutationCandidate('src/__tests__/app.js'));+ assert.ok(!isMutationCandidate('spec/thing.spec.ts'));+ });++ test('rejects non-source files', () => {+ assert.ok(!isMutationCandidate('README.md'));+ assert.ok(!isMutationCandidate('package.json'));+ assert.ok(!isMutationCandidate('gate.yaml'));+ });++ test('normalizes Windows separators', () => {+ assert.ok(!isMutationCandidate('dist\\index.js'));+ assert.ok(isMutationCandidate('src\\index.ts'));+ });+});diff --git a/tools/loop-drill/tsconfig.json b/tools/loop-drill/tsconfig.jsonnew file mode 100644index 0000000..ef0f184--- /dev/null+++ b/tools/loop-drill/tsconfig.json@@ -0,0 +1,13 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "NodeNext",+ "moduleResolution": "NodeNext",+ "outDir": "dist",+ "rootDir": "src",+ "strict": true,+ "skipLibCheck": true,+ "declaration": true+ },+ "include": ["src/**/*"]+}