Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

feat(loop-drill): fire drills that prove guardrails actually fire (#600)

THRISHAL DOMA336c3296bcd5
diff --git a/package.json b/package.json
index b7108a9..4f0cd3e 100644
--- a/package.json
+++ b/package.json
@@ -20,13 +20,14 @@
"test:goal-init": "cd tools/goal-init && npm test",
"test:readiness-core": "cd tools/readiness-core && npm test",
"test:goal-audit": "cd tools/goal-audit && npm test",
- "test:tools": "npm run test:readiness-core && npm run test:loop-audit && npm run test:loop-init && npm run test:loop-cost && npm run test:loop-sync && npm run test:loop-context && npm run test:loop-gate && npm run test:mcp-server && npm run test:loop-worktree && npm run test:loop-sandbox && npm run test:loop-swarm && npm run test:loop && npm run test:loop-metrics && npm run test:goal-audit && npm run test:goal-init",
- "build:tools": "cd tools/readiness-core && npm run build && cd ../loop-audit && npm run build && cd ../loop-init && npm run build && cd ../loop-cost && npm run build && cd ../loop-sync && npm run build && cd ../loop-context && npm run build && cd ../loop-gate && npm run build && cd ../mcp-server && npm run build && cd ../loop-worktree && npm run build && cd ../loop-sandbox && npm run build && cd ../loop-swarm && npm run build && cd ../loop && npm run build && cd ../loop-metrics && npm run build && cd ../goal-audit && npm run build && cd ../goal-init && npm run build",
+ "test:tools": "npm run test:readiness-core && npm run test:loop-audit && npm run test:loop-init && npm run test:loop-cost && npm run test:loop-sync && npm run test:loop-context && npm run test:loop-gate && npm run test:mcp-server && npm run test:loop-worktree && npm run test:loop-sandbox && npm run test:loop-swarm && npm run test:loop && npm run test:loop-metrics && npm run test:loop-drill && npm run test:goal-audit && npm run test:goal-init",
+ "build:tools": "cd tools/readiness-core && npm run build && cd ../loop-audit && npm run build && cd ../loop-init && npm run build && cd ../loop-cost && npm run build && cd ../loop-sync && npm run build && cd ../loop-context && npm run build && cd ../loop-gate && npm run build && cd ../mcp-server && npm run build && cd ../loop-worktree && npm run build && cd ../loop-sandbox && npm run build && cd ../loop-swarm && npm run build && cd ../loop && npm run build && cd ../loop-metrics && npm run build && cd ../loop-drill && npm run build && cd ../goal-audit && npm run build && cd ../goal-init && npm run build",
"contributors:generate": "node scripts/generate-contributors.mjs",
- "contributors:check": "node scripts/generate-contributors.mjs --check"
+ "contributors:check": "node scripts/generate-contributors.mjs --check",
+ "test:loop-drill": "cd tools/loop-drill && npm test"
},
"devDependencies": {
"ajv": "^8.17.1",
"yaml": "^2.8.0"
}
-}
\ No newline at end of file
+}
diff --git a/scripts/ci-validate-gates.sh b/scripts/ci-validate-gates.sh
index 65cd29f..15f35fa 100755
--- a/scripts/ci-validate-gates.sh
+++ b/scripts/ci-validate-gates.sh
@@ -106,4 +106,25 @@ cd ../mcp-server
npm ci
npm test
+echo "Building and testing loop-drill…"
+cd ../loop-drill
+npm ci
+npm test
+
+# Dogfood: prove this repo's own gate.yaml and circuit breaker actually fire.
+# Exit 1 means a drill was skipped (an unconfigured guardrail) and is tolerated;
+# exit 2 means a guardrail failed to fire and must fail CI.
+#
+# Written as an explicit if rather than `cmd || [ $? -eq 1 ]`: under `set -e`
+# a failing command on the right of `||` does not exit the shell, so that
+# shorter form silently passes an exit-2 drill.
+echo "Running loop-drill against the reference repo…"
+cd ../..
+drill_status=0
+node tools/loop-drill/dist/cli.js . || drill_status=$?
+if [ "$drill_status" -ge 2 ]; then
+ echo "loop-drill: a guardrail failed to fire (exit $drill_status)" >&2
+ exit "$drill_status"
+fi
+
echo "validate gates passed ✓"
diff --git a/tools/loop-drill/README.md b/tools/loop-drill/README.md
new file mode 100644
index 0000000..92adc64
--- /dev/null
+++ b/tools/loop-drill/README.md
@@ -0,0 +1,127 @@
+# loop-drill
+
+**Fire drills for loop guardrails.** Injects a known fault and asserts the guardrail actually fires.
+
+`loop-audit` scores whether your guardrails are **present**. `loop-drill` tests whether they **work**.
+
+## Why
+
+[`docs/failure-modes.md`](../../docs/failure-modes.md) names ten ways loops fail. Several have mechanical counterparts — [`loop-gate`](../loop-gate) for path scope, [`loop-context`](../loop-context)'s circuit breaker for runaway retries. `loop-audit` awards points for having them.
+
+Nothing checked whether they fire.
+
+The sharpest case is the verifier. `loop-audit` gives its joint-largest signal (14 points) for a verifier and gates L3 on it, by checking whether a file whose *name* contains `verifier` exists:
+
+```ts
+if (base.includes('verifier') || base === 'loop-verifier') // auditor.ts:181
+```
+
+An empty file passes. On a scratch repo, `touch .claude/agents/verifier.md` moves the score from **34 (L0)** to **55 (L1)**. `docs/failure-modes.md` calls the resulting failure **Verifier Theater** and rates it S2; `docs/primitives.md` calls maker/checker *"the single most important structural pattern for reliable loops"*.
+
+loop-drill closes that gap by running the real verifier against real seeded defects.
+
+## Usage
+
+```bash
+# Offline drills — no agent, no tokens, no network
+npx @cobusgreyling/loop-drill .
+
+# The verifier canary
+npx @cobusgreyling/loop-drill . --only verifier \
+ --verifier-cmd "npm test" --setup "npm ci"
+```
+
+### Options
+
+| Flag | Meaning |
+|------|---------|
+| `--only <drills>` | `gate`, `breaker`, `verifier` (default: `gate,breaker`) |
+| `--verifier-cmd <cmd>` | Verifier command. Non-zero exit = rejected. Required for the canary |
+| `--setup <cmd>` | Run in each worktree before verifying (e.g. `npm ci`) |
+| `--mutants <n>` | Seeded defects (default: 3) |
+| `--scope <path>` | Restrict mutation to a repo-relative path |
+| `--timeout <ms>` | Per-run timeout (default: 120000) |
+| `--gate-file <path>` | Policy file (default: `gate.yaml`) |
+| `--benign-path <path>` | Path the specificity drill treats as ordinary |
+| `--token-budget <n>` | Token budget for the breaker drill |
+| `--json` | Machine-readable output |
+
+Exit codes match `loop-gate` and `loop-context` so control scripts chain all three: **0** all passed, **1** some skipped, **2** a guardrail failed to fire.
+
+## The drills
+
+Each drill is tagged with the `docs/failure-modes.md` entry it exercises, so the report reads as coverage of that document.
+
+| Drill | Failure mode | Injected fault | Passes if |
+|-------|--------------|----------------|-----------|
+| `gate.denylist[glob]` | Over-Reach | A path matching each denylist entry | Blocked by the denylist |
+| `gate.file-count` | Over-Reach | `maxFiles + 1` benign paths | Blocked by file-count |
+| `gate.auto-merge` | Over-Reach | Non-allowlisted path, `--action auto-merge` | Blocked |
+| `gate.benign` | Over-Reach | An ordinary doc change | **Allowed** |
+| `breaker.stagnation` | Infinite Fix Loop | N identical failures | Escalates |
+| `breaker.no-progress` | Infinite Fix Loop | N consecutive failures | Escalates |
+| `breaker.token-budget` | Token Burn | Attempt over budget | Escalates |
+| `breaker.healthy` | Infinite Fix Loop | A healthy run | **Does not escalate** |
+| `verifier.control` | Verifier Theater | Nothing (clean tree) | **Accepts** |
+| `verifier.mutant[op]` | Verifier Theater | One seeded defect | Rejects |
+
+### Both directions, always
+
+Every guardrail is drilled twice, because only one direction is easy:
+
+- **sensitivity** — the fault is caught. A guardrail that never fires is theater.
+- **specificity** — benign input is *not* caught. A guardrail that blocks everything passes every sensitivity drill while being useless.
+
+A `denylist: ["**"]` catches every seeded fault and still fails, because it blocks an ordinary docs change too.
+
+## How the verifier canary works
+
+Borrowed from mutation testing. Rather than asking a model to invent a defect — non-deterministic and token-hungry — it applies a small mechanical edit to real source and checks whether the verifier notices:
+
+```
+control (repo) no defect -> must ACCEPT (exit 0)
+control (worktree) no defect -> must ACCEPT (exit 0)
+mutant × N one defect -> must REJECT (non-zero)
+```
+
+**Mutation score** = mutants rejected / mutants run.
+
+Three properties make the result trustworthy:
+
+1. **Isolation.** Each run happens in an ephemeral `git worktree`, removed even on throw, so a verifier that writes or builds cannot touch your checkout.
+2. **The worktree control.** A fresh worktree has no `node_modules`. Without this second control, `npm test` would fail there for reasons unrelated to code quality and *every mutant would score as caught* — a broken setup reporting a perfect score. When it fails, mutants are **skipped, not scored**, and you're pointed at `--setup`.
+3. **Bail on a failed control.** A verifier that rejects a clean tree catches every mutant while being worthless. The mutants never run, so it can't report 100%.
+
+Operators are restricted to edits that change behaviour in any C-family language and that a reviewer would call a bug: `===`↔`!==`, `<=`→`<`, `>=`→`>`, `&&`→`||`, `return true`↔`return false`. Ambiguous edits are deliberately excluded — `<`→`>` breaks TS generics and JSX, `+`→`-` mangles string concatenation, and either would make a *correct* verifier look broken. Matches inside comments are skipped, and test files, `dist/`, and `node_modules/` are never mutated.
+
+## Example
+
+A rubber-stamp verifier, caught:
+
+```
+Verifier Theater — ❌ NOT PROVEN
+ ✅ verifier accepts an unmodified tree
+ ✅ verifier accepts a clean worktree
+ ❌ verifier rejects strict-equality-flip in src/gate.ts:34
+ expected: non-zero exit (rejected)
+ actual: exit 0 (approved a seeded defect)
+ flip === to !== (typeof v === 'string' -> typeof v !== 'string')
+
+Mutation score: 0% of seeded defects rejected
+```
+
+## Limitations
+
+- **Mutation is regex-based, not AST-based.** It cannot construct a semantically interesting bug, only a mechanically detectable one. A verifier that catches every mutant is not proven to catch subtle logic errors — this establishes a floor, not a ceiling.
+- **The canary costs whatever your verifier costs**, once per mutant plus two controls. Start with `--mutants 1` and `--scope`.
+- **`Escalation Failure` and `Notification Fatigue`** have no drills yet — they need a notification sink to observe.
+
+## Development
+
+```bash
+npm ci && npm test
+```
+
+## License
+
+MIT
diff --git a/tools/loop-drill/dist/canary.d.ts b/tools/loop-drill/dist/canary.d.ts
new file mode 100644
index 0000000..59460f2
--- /dev/null
+++ b/tools/loop-drill/dist/canary.d.ts
@@ -0,0 +1,63 @@
+/**
+ * The verifier canary — the drill that gave this package its reason to exist.
+ *
+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and
+ * gates L3 on it, by checking whether a file whose name contains "verifier"
+ * exists. An empty file passes. docs/failure-modes.md calls the resulting
+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls
+ * maker/checker "the single most important structural pattern for reliable
+ * loops". Nothing tested whether the verifier verifies.
+ *
+ * This runs the real verifier command against real seeded defects in an
+ * isolated worktree, and scores it on what it caught:
+ *
+ * control — clean tree -> the verifier must ACCEPT (exit 0)
+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)
+ *
+ * Both directions are required. A verifier that rejects everything catches
+ * every mutant while being worthless, and the control run is what exposes it.
+ */
+import { type Mutant } from './mutations.js';
+import { type DrillResult } from './drill.js';
+export interface CanaryOptions {
+ /** Repo root to drill. */
+ root: string;
+ /** The verifier command. Non-zero exit is read as "rejected". */
+ command: string;
+ /** How many mutants to seed. More is slower but a sharper score. */
+ count: number;
+ /** Per-run timeout in ms. A verifier that hangs is a failed verifier. */
+ timeoutMs: number;
+ /** Restrict mutation to files under this repo-relative path. */
+ scope?: string;
+ /** Command run inside each worktree before verifying (e.g. "npm ci"). */
+ setup?: string;
+}
+export interface VerifierRun {
+ accepted: boolean;
+ exitCode: number | null;
+ timedOut: boolean;
+ output: string;
+}
+/**
+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything
+ * else — including a timeout — means "rejected". A verifier that cannot be
+ * launched at all is a distinct failure, surfaced by the caller.
+ */
+export declare function runVerifier(command: string, cwd: string, timeoutMs: number): Promise<VerifierRun>;
+/** Files git tracks, filtered to plausible mutation targets. */
+export declare function candidateFiles(root: string, scope?: string): Promise<string[]>;
+/**
+ * Build up to `count` mutants from distinct files. Distinct files matter:
+ * seeding three mutants into one module mostly measures whether the verifier
+ * reads that module.
+ */
+export declare function collectMutants(root: string, files: string[], count: number): Promise<Mutant[]>;
+export interface CanaryReport {
+ results: DrillResult[];
+ /** Mutants rejected / mutants run. Null when no mutant could be built. */
+ mutationScore: number | null;
+ caught: number;
+ escaped: number;
+}
+export declare function runCanary(options: CanaryOptions): Promise<CanaryReport>;
diff --git a/tools/loop-drill/dist/canary.js b/tools/loop-drill/dist/canary.js
new file mode 100644
index 0000000..afd0f8e
--- /dev/null
+++ b/tools/loop-drill/dist/canary.js
@@ -0,0 +1,235 @@
+/**
+ * The verifier canary — the drill that gave this package its reason to exist.
+ *
+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and
+ * gates L3 on it, by checking whether a file whose name contains "verifier"
+ * exists. An empty file passes. docs/failure-modes.md calls the resulting
+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls
+ * maker/checker "the single most important structural pattern for reliable
+ * loops". Nothing tested whether the verifier verifies.
+ *
+ * This runs the real verifier command against real seeded defects in an
+ * isolated worktree, and scores it on what it caught:
+ *
+ * control — clean tree -> the verifier must ACCEPT (exit 0)
+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)
+ *
+ * Both directions are required. A verifier that rejects everything catches
+ * every mutant while being worthless, and the control run is what exposes it.
+ */
+import { exec as execShellCb, execFile as execFileCb } from 'node:child_process';
+import { readFile, writeFile, rm } from 'node:fs/promises';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { isMutationCandidate, mutate } from './mutations.js';
+import { skip } from './drill.js';
+/**
+ * Two spawn helpers, kept honest about their signatures.
+ *
+ * `execShell` runs a caller-supplied command string (--verifier-cmd, --setup)
+ * through a shell, which is what those flags mean. `execFile` runs git with an
+ * argv array and no shell, so repo paths never reach a shell for re-parsing.
+ *
+ * Earlier this file called promisify(execFile) with an options object in the
+ * `args` position behind an `as never` cast -- it happened to work because Node
+ * detects a non-array second argument, but it hid the real signature from tsc
+ * and made timeout/killed handling harder to reason about.
+ */
+const execShell = promisify(execShellCb);
+const exec = promisify(execFileCb);
+/**
+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything
+ * else — including a timeout — means "rejected". A verifier that cannot be
+ * launched at all is a distinct failure, surfaced by the caller.
+ */
+export async function runVerifier(command, cwd, timeoutMs) {
+ try {
+ const { stdout, stderr } = await execShell(command, {
+ cwd,
+ timeout: timeoutMs,
+ maxBuffer: 8 * 1024 * 1024,
+ });
+ return { accepted: true, exitCode: 0, timedOut: false, output: `${stdout}${stderr}`.trim() };
+ }
+ catch (err) {
+ const e = err;
+ const timedOut = e.killed === true || e.code === 'ETIMEDOUT';
+ return {
+ accepted: false,
+ exitCode: typeof e.code === 'number' ? e.code : null,
+ timedOut,
+ output: `${e.stdout ?? ''}${e.stderr ?? ''}`.trim(),
+ };
+ }
+}
+/** Files git tracks, filtered to plausible mutation targets. */
+export async function candidateFiles(root, scope) {
+ const args = ['-C', root, 'ls-files'];
+ if (scope)
+ args.push('--', scope);
+ const { stdout } = await exec('git', args, { maxBuffer: 32 * 1024 * 1024 });
+ return stdout
+ .split('\n')
+ .map((l) => l.trim())
+ .filter((l) => l.length > 0 && isMutationCandidate(l));
+}
+/**
+ * Build up to `count` mutants from distinct files. Distinct files matter:
+ * seeding three mutants into one module mostly measures whether the verifier
+ * reads that module.
+ */
+export async function collectMutants(root, files, count) {
+ const mutants = [];
+ for (const file of files) {
+ if (mutants.length >= count)
+ break;
+ let content;
+ try {
+ content = await readFile(path.join(root, file), 'utf8');
+ }
+ catch {
+ continue;
+ }
+ const mutant = mutate(content, file);
+ if (mutant)
+ mutants.push(mutant);
+ }
+ return mutants;
+}
+/**
+ * Run the verifier inside an ephemeral git worktree, so a verifier that writes,
+ * builds, or fixes cannot touch the real checkout. `mutant` is null for the
+ * worktree control run. The worktree is always removed, including on throw.
+ */
+async function runInWorktree(root, mutant, command, timeoutMs, setup) {
+ const worktree = path.join(root, `.loop-drill-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+ await exec('git', ['-C', root, 'worktree', 'add', '--detach', '--quiet', worktree], {
+ maxBuffer: 8 * 1024 * 1024,
+ });
+ try {
+ if (setup) {
+ // Setup failure is not a verifier verdict — surface it as such.
+ const setupRun = await runVerifier(setup, worktree, timeoutMs);
+ if (!setupRun.accepted) {
+ return { ...setupRun, output: `[setup failed] ${setupRun.output}` };
+ }
+ }
+ if (mutant)
+ await writeFile(path.join(worktree, mutant.file), mutant.mutated, 'utf8');
+ return await runVerifier(command, worktree, timeoutMs);
+ }
+ finally {
+ // --force because the verifier may have left build output behind.
+ await exec('git', ['-C', root, 'worktree', 'remove', '--force', worktree]).catch(() => { });
+ await rm(worktree, { recursive: true, force: true }).catch(() => { });
+ }
+}
+export async function runCanary(options) {
+ const { root, command, count, timeoutMs, scope, setup } = options;
+ const results = [];
+ const mode = 'Verifier Theater';
+ // Control first, and bail if it fails: every mutant result is meaningless
+ // when the verifier rejects a clean tree, and running them would report a
+ // reject-everything verifier as a perfect score.
+ const control = await runVerifier(command, root, timeoutMs);
+ if (control.accepted) {
+ results.push({
+ id: 'verifier.control',
+ name: 'verifier accepts an unmodified tree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'passed',
+ expected: 'exit 0 on clean tree',
+ actual: 'exit 0',
+ });
+ }
+ else {
+ results.push({
+ id: 'verifier.control',
+ name: 'verifier accepts an unmodified tree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'failed',
+ expected: 'exit 0 on clean tree',
+ actual: control.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${control.exitCode}`,
+ detail: 'The verifier rejects code with no seeded defect, so it cannot distinguish good from bad. Mutant drills were skipped — their results would be meaningless. Fix the verifier command (or the tree it runs against) first.',
+ });
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+ // Second control, in a clean worktree. A fresh checkout has no node_modules
+ // and no build output, so a verifier like "npm test" fails there for reasons
+ // that have nothing to do with code quality. Without this run, every mutant
+ // would be scored as "caught" and a completely broken setup would report a
+ // perfect mutation score — the precise false confidence this tool exists to
+ // prevent.
+ const worktreeControl = await runInWorktree(root, null, command, timeoutMs, setup);
+ if (!worktreeControl.accepted) {
+ results.push({
+ id: 'verifier.worktree-control',
+ name: 'verifier accepts a clean worktree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'skipped',
+ expected: 'exit 0 in a clean worktree',
+ actual: worktreeControl.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${worktreeControl.exitCode}`,
+ detail: 'The verifier passes in your checkout but fails in a fresh worktree, so mutant results would measure the environment rather than the verifier. Pass --setup to prepare the worktree first (e.g. --setup "npm ci"). ' +
+ (worktreeControl.output ? `Output: ${truncate(worktreeControl.output)}` : ''),
+ });
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+ results.push({
+ id: 'verifier.worktree-control',
+ name: 'verifier accepts a clean worktree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'passed',
+ expected: 'exit 0 in a clean worktree',
+ actual: 'exit 0',
+ });
+ const files = await candidateFiles(root, scope);
+ const mutants = await collectMutants(root, files, count);
+ if (mutants.length === 0) {
+ results.push(skip('verifier.mutants', 'verifier rejects seeded defects', mode, 'sensitivity', `No mutable source found${scope ? ` under ${scope}` : ''}. loop-drill mutates tracked, non-test, non-generated source in common languages — pass --scope to point it at yours.`));
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+ let caught = 0;
+ for (const mutant of mutants) {
+ const run = await runInWorktree(root, mutant, command, timeoutMs, setup);
+ const where = `${mutant.file}:${mutant.line}`;
+ const edit = `${mutant.operator.description} (${mutant.before} -> ${mutant.after})`;
+ if (!run.accepted) {
+ caught++;
+ results.push({
+ id: `verifier.mutant[${mutant.operator.id}]`,
+ name: `verifier rejects ${mutant.operator.id} in ${where}`,
+ failureMode: mode,
+ direction: 'sensitivity',
+ outcome: 'passed',
+ expected: 'non-zero exit (rejected)',
+ actual: run.timedOut ? 'timed out (treated as rejected)' : `exit ${run.exitCode}`,
+ detail: edit,
+ });
+ }
+ else {
+ results.push({
+ id: `verifier.mutant[${mutant.operator.id}]`,
+ name: `verifier rejects ${mutant.operator.id} in ${where}`,
+ failureMode: mode,
+ direction: 'sensitivity',
+ outcome: 'failed',
+ expected: 'non-zero exit (rejected)',
+ actual: 'exit 0 (approved a seeded defect)',
+ detail: `${edit}. The verifier approved this change. That is Verifier Theater — see docs/failure-modes.md.`,
+ });
+ }
+ }
+ return {
+ results,
+ mutationScore: caught / mutants.length,
+ caught,
+ escaped: mutants.length - caught,
+ };
+}
+function truncate(text, max = 400) {
+ return text.length <= max ? text : `${text.slice(0, max)}…`;
+}
diff --git a/tools/loop-drill/dist/cli.d.ts b/tools/loop-drill/dist/cli.d.ts
new file mode 100644
index 0000000..4f55271
--- /dev/null
+++ b/tools/loop-drill/dist/cli.d.ts
@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+/**
+ * loop-drill CLI.
+ *
+ * Exit codes follow loop-gate / loop-context so control scripts can chain all
+ * three: 0 proceed, 1 warnings, 2 escalate.
+ */
+export {};
diff --git a/tools/loop-drill/dist/cli.js b/tools/loop-drill/dist/cli.js
new file mode 100644
index 0000000..857392c
--- /dev/null
+++ b/tools/loop-drill/dist/cli.js
@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+/**
+ * loop-drill CLI.
+ *
+ * Exit codes follow loop-gate / loop-context so control scripts can chain all
+ * three: 0 proceed, 1 warnings, 2 escalate.
+ */
+import path from 'node:path';
+import { loadGateConfig } from '@cobusgreyling/loop-gate';
+import { DEFAULT_BREAKER } from '@cobusgreyling/loop-context';
+import { buildReport, exitCodeFor, runBreakerDrills, runGateDrills, skip, } from './drill.js';
+import { runCanary } from './canary.js';
+import { formatReport } from './report.js';
+const HELP = `loop-drill — fire drills for loop guardrails
+
+Injects a known fault and asserts the guardrail actually fires. Turns L3
+readiness from "the files exist" into "the guardrails demonstrably work".
+
+Usage: loop-drill [path] [options]
+
+Options:
+ --only <drills> Comma-separated: gate, breaker, verifier (default: gate,breaker)
+ --verifier-cmd <cmd> Verifier command to drill. Non-zero exit = rejected.
+ Required to run the verifier canary.
+ --mutants <n> Seeded defects for the canary (default: 3)
+ --scope <path> Restrict mutation to this repo-relative path
+ --setup <cmd> Command run in each worktree before verifying
+ (e.g. "npm ci") — a fresh worktree has no node_modules
+ --timeout <ms> Per-verifier-run timeout (default: 120000)
+ --gate-file <path> Policy file (default: gate.yaml)
+ --benign-path <path> Path the gate specificity drill treats as ordinary
+ (default: docs/README.md)
+ --token-budget <n> Token budget for the breaker drill
+ --json Machine-readable output
+ --help, -h Show this message
+
+Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.
+
+Examples:
+ loop-drill .
+ loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"
+ loop-drill . --only gate,breaker --json
+`;
+function parseArgs(argv) {
+ const flags = { root: '.', json: false, help: false, mutants: 3, timeoutMs: 120_000 };
+ const positional = [];
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ const next = () => {
+ const v = argv[++i];
+ if (v === undefined)
+ throw new Error(`${arg} requires a value`);
+ return v;
+ };
+ switch (arg) {
+ case '--help':
+ case '-h':
+ flags.help = true;
+ break;
+ case '--json':
+ flags.json = true;
+ break;
+ case '--only':
+ flags.only = next();
+ break;
+ case '--verifier-cmd':
+ flags.verifierCmd = next();
+ break;
+ case '--mutants':
+ flags.mutants = parsePositiveInt(next(), '--mutants');
+ break;
+ case '--scope':
+ flags.scope = next();
+ break;
+ case '--setup':
+ flags.setup = next();
+ break;
+ case '--timeout':
+ flags.timeoutMs = parsePositiveInt(next(), '--timeout');
+ break;
+ case '--gate-file':
+ flags.gateFile = next();
+ break;
+ case '--benign-path':
+ flags.benignPath = next();
+ break;
+ case '--token-budget':
+ flags.tokenBudget = parsePositiveInt(next(), '--token-budget');
+ break;
+ default:
+ if (arg.startsWith('-'))
+ throw new Error(`Unknown option: ${arg}`);
+ positional.push(arg);
+ }
+ }
+ if (positional.length > 0)
+ flags.root = positional[0];
+ return flags;
+}
+function parsePositiveInt(value, flag) {
+ const n = Number(value);
+ if (!Number.isInteger(n) || n <= 0)
+ throw new Error(`${flag} expects a positive integer, got: ${value}`);
+ return n;
+}
+async function main() {
+ let flags;
+ try {
+ flags = parseArgs(process.argv.slice(2));
+ }
+ catch (err) {
+ console.error(`${err.message}\n`);
+ console.error(HELP);
+ process.exit(2);
+ }
+ if (flags.help) {
+ console.log(HELP);
+ return;
+ }
+ const root = path.resolve(flags.root);
+ const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));
+ const results = [];
+ let mutationScore = null;
+ if (selected.has('gate')) {
+ const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');
+ try {
+ const config = await loadGateConfig(gateFile);
+ results.push(...runGateDrills({ config, benignPath: flags.benignPath }));
+ }
+ catch (err) {
+ results.push(skip('gate', 'gate drills', 'Over-Reach (Wrong Scope)', 'sensitivity', `${err.message} Nothing mechanically stops a loop from editing a sensitive path.`));
+ }
+ }
+ if (selected.has('breaker')) {
+ const config = flags.tokenBudget === undefined
+ ? DEFAULT_BREAKER
+ : { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };
+ results.push(...runBreakerDrills(config));
+ }
+ if (selected.has('verifier')) {
+ if (!flags.verifierCmd) {
+ results.push(skip('verifier', 'verifier canary', 'Verifier Theater', 'sensitivity', 'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.'));
+ }
+ else {
+ const canary = await runCanary({
+ root,
+ command: flags.verifierCmd,
+ count: flags.mutants,
+ timeoutMs: flags.timeoutMs,
+ scope: flags.scope,
+ setup: flags.setup,
+ });
+ results.push(...canary.results);
+ mutationScore = canary.mutationScore;
+ }
+ }
+ if (results.length === 0) {
+ console.error(`No drills selected. --only accepts: gate, breaker, verifier\n`);
+ process.exit(2);
+ }
+ const report = buildReport(results);
+ const code = exitCodeFor(report);
+ if (flags.json) {
+ console.log(JSON.stringify({ ...report, mutationScore, exitCode: code }, null, 2));
+ }
+ else {
+ console.log(formatReport(report, mutationScore));
+ }
+ process.exit(code);
+}
+main().catch((err) => {
+ console.error(`loop-drill failed: ${err.message}`);
+ process.exit(2);
+});
diff --git a/tools/loop-drill/dist/drill.d.ts b/tools/loop-drill/dist/drill.d.ts
new file mode 100644
index 0000000..09020c3
--- /dev/null
+++ b/tools/loop-drill/dist/drill.d.ts
@@ -0,0 +1,89 @@
+/**
+ * loop-drill — fire drills for loop guardrails.
+ *
+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical
+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores
+ * whether those counterparts are *present*. Nothing checked whether they
+ * actually fire.
+ *
+ * A drill injects a known fault and asserts the guardrail responds. Every
+ * drill carries the failure mode it exercises, so the report reads as coverage
+ * of docs/failure-modes.md rather than a list of anonymous assertions.
+ *
+ * Each guardrail gets both directions, because only one of them is easy:
+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)
+ * - specificity: benign input is NOT caught (a guardrail that blocks
+ * everything passes every sensitivity drill while being useless)
+ * A guardrail is only credited when it passes both.
+ */
+import { type GateConfig } from '@cobusgreyling/loop-gate';
+import { type CircuitBreakerConfig } from '@cobusgreyling/loop-context';
+export type DrillOutcome = 'passed' | 'failed' | 'skipped';
+/** Named failure modes from docs/failure-modes.md. */
+export type FailureMode = 'Infinite Fix Loop' | 'Token Burn' | 'Over-Reach (Wrong Scope)' | 'Verifier Theater' | 'Escalation Failure';
+export interface DrillResult {
+ /** Stable id, e.g. 'gate.denylist'. */
+ id: string;
+ name: string;
+ /** The docs/failure-modes.md entry this drill exercises. */
+ failureMode: FailureMode;
+ /** 'sensitivity' = fault must be caught; 'specificity' = benign must pass. */
+ direction: 'sensitivity' | 'specificity';
+ outcome: DrillOutcome;
+ expected: string;
+ actual: string;
+ /** Why a drill was skipped, or extra context on a failure. */
+ detail?: string;
+}
+export interface DrillReport {
+ results: DrillResult[];
+ passed: number;
+ failed: number;
+ skipped: number;
+ /** Failure modes with at least one passing drill and no failing drill. */
+ covered: FailureMode[];
+ /** Failure modes with no passing drill (unexercised or broken). */
+ uncovered: FailureMode[];
+ timestamp: string;
+}
+export declare function skip(id: string, name: string, failureMode: FailureMode, direction: DrillResult['direction'], detail: string): DrillResult;
+/**
+ * Turn a denylist glob into a concrete path that should match it, so the drill
+ * exercises the real matcher rather than re-implementing glob semantics.
+ *
+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a
+ * literal, and the caller verifies the synthesized path actually matches
+ * before asserting anything about it (see runGateDrills) — a glob we
+ * synthesize badly is reported as skipped, never as a guardrail failure.
+ */
+export declare function samplePathFor(glob: string): string;
+export interface GateDrillOptions {
+ config: GateConfig;
+ /** Path assumed benign for the specificity drill. */
+ benignPath?: string;
+}
+/**
+ * Exercise loop-gate against its own gate.yaml: every denylist entry must
+ * block, the file-count cap must block, auto-merge must reject a
+ * non-allowlisted path, and an ordinary doc change must still be allowed.
+ */
+export declare function runGateDrills(options: GateDrillOptions): DrillResult[];
+/**
+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated
+ * identical failures must trip stagnation, a long run of unrelated failures
+ * must trip no-progress, blowing the token budget must trip, and a healthy run
+ * must not.
+ *
+ * Each drill asserts the *specific* trigger, not merely that the breaker
+ * escalated. Escalating for another reason means the rule under test is still
+ * unproven — the same standard the gate drills apply to `trigger !==
+ * 'denylist'`.
+ */
+export declare function runBreakerDrills(config?: CircuitBreakerConfig): DrillResult[];
+export declare function buildReport(results: DrillResult[]): DrillReport;
+/**
+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),
+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's
+ * "2 means escalate" convention so control scripts can chain all three.
+ */
+export declare function exitCodeFor(report: DrillReport): 0 | 1 | 2;
diff --git a/tools/loop-drill/dist/drill.js b/tools/loop-drill/dist/drill.js
new file mode 100644
index 0000000..3b3a384
--- /dev/null
+++ b/tools/loop-drill/dist/drill.js
@@ -0,0 +1,271 @@
+/**
+ * loop-drill — fire drills for loop guardrails.
+ *
+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical
+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores
+ * whether those counterparts are *present*. Nothing checked whether they
+ * actually fire.
+ *
+ * A drill injects a known fault and asserts the guardrail responds. Every
+ * drill carries the failure mode it exercises, so the report reads as coverage
+ * of docs/failure-modes.md rather than a list of anonymous assertions.
+ *
+ * Each guardrail gets both directions, because only one of them is easy:
+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)
+ * - specificity: benign input is NOT caught (a guardrail that blocks
+ * everything passes every sensitivity drill while being useless)
+ * A guardrail is only credited when it passes both.
+ */
+import { checkGate } from '@cobusgreyling/loop-gate';
+import { checkCircuitBreaker, DEFAULT_BREAKER, } from '@cobusgreyling/loop-context';
+function pass(id, name, failureMode, direction, expected, actual) {
+ return { id, name, failureMode, direction, outcome: 'passed', expected, actual };
+}
+function fail(id, name, failureMode, direction, expected, actual, detail) {
+ return { id, name, failureMode, direction, outcome: 'failed', expected, actual, detail };
+}
+export function skip(id, name, failureMode, direction, detail) {
+ return {
+ id,
+ name,
+ failureMode,
+ direction,
+ outcome: 'skipped',
+ expected: '—',
+ actual: '—',
+ detail,
+ };
+}
+// ── Glob → concrete sample path ────────────────────────────────────
+/**
+ * Turn a denylist glob into a concrete path that should match it, so the drill
+ * exercises the real matcher rather than re-implementing glob semantics.
+ *
+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a
+ * literal, and the caller verifies the synthesized path actually matches
+ * before asserting anything about it (see runGateDrills) — a glob we
+ * synthesize badly is reported as skipped, never as a guardrail failure.
+ */
+export function samplePathFor(glob) {
+ const concrete = glob
+ .split('/')
+ .map((segment) => {
+ if (segment === '**')
+ return 'src';
+ if (segment === '*')
+ return 'sample';
+ // '*_key*' -> 'drill_key_sample'; '.env.*' -> '.env.sample'
+ return segment.replace(/\*/g, (_m, offset) => (offset === 0 ? 'drill' : 'sample'));
+ })
+ .join('/');
+ // A trailing '**' segment became 'src', which is a directory, not a file.
+ return /(^|\/)src$/.test(concrete) ? `${concrete}/drill-file.txt` : concrete;
+}
+/**
+ * Exercise loop-gate against its own gate.yaml: every denylist entry must
+ * block, the file-count cap must block, auto-merge must reject a
+ * non-allowlisted path, and an ordinary doc change must still be allowed.
+ */
+export function runGateDrills(options) {
+ const { config, benignPath = 'docs/README.md' } = options;
+ const results = [];
+ const mode = 'Over-Reach (Wrong Scope)';
+ // Sensitivity: each denylist glob blocks a path that matches it.
+ for (const glob of config.denylist) {
+ const id = `gate.denylist[${glob}]`;
+ const name = `denylist blocks ${glob}`;
+ const sample = samplePathFor(glob);
+ const decision = checkGate({ config, action: 'commit', paths: [sample] });
+ if (decision.allowed) {
+ results.push(fail(id, name, mode, 'sensitivity', `commit touching ${sample} is blocked`, `allowed (${decision.trigger})`, `Denylist entry "${glob}" did not match the sample path this drill synthesized for it. Either the glob is unreachable in practice, or the sample is unrepresentative — check it by hand with: loop-gate check --action commit --paths ${sample}`));
+ continue;
+ }
+ if (decision.trigger !== 'denylist') {
+ // Blocked, but by file-count or the allowlist — the denylist itself is
+ // unproven, so this is not a pass.
+ results.push(fail(id, name, mode, 'sensitivity', `blocked by the denylist`, `blocked by ${decision.trigger}`, 'The path was stopped, but not by the rule under test.'));
+ continue;
+ }
+ results.push(pass(id, name, mode, 'sensitivity', 'blocked by denylist', decision.reason));
+ }
+ // Sensitivity: the file-count cap fires one file past the threshold.
+ if (config.maxFiles !== undefined) {
+ const id = 'gate.file-count';
+ const name = `file-count cap fires above ${config.maxFiles} files`;
+ const paths = Array.from({ length: config.maxFiles + 1 }, (_, i) => `docs/drill-${i}.md`);
+ const decision = checkGate({ config, action: 'commit', paths });
+ results.push(decision.trigger === 'file-count'
+ ? pass(id, name, mode, 'sensitivity', 'blocked by file-count', decision.reason)
+ : fail(id, name, mode, 'sensitivity', `${paths.length} files is blocked by file-count`, decision.allowed ? 'allowed' : `blocked by ${decision.trigger}`));
+ }
+ else {
+ results.push(skip('gate.file-count', 'file-count cap', mode, 'sensitivity', 'gate.yaml sets no maxFiles — a loop may propose an arbitrarily large diff.'));
+ }
+ // Sensitivity: auto-merge rejects a path outside the allowlist.
+ if (config.autoMergeAllowlist && config.autoMergeAllowlist.length > 0) {
+ const id = 'gate.auto-merge';
+ const name = 'auto-merge rejects a non-allowlisted path';
+ const outside = 'src/drill-not-allowlisted.bin';
+ const decision = checkGate({ config, action: 'auto-merge', paths: [outside] });
+ results.push(!decision.allowed
+ ? pass(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, decision.reason)
+ : fail(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, 'allowed'));
+ }
+ // Specificity: an ordinary change is still allowed. A gate that blocks
+ // everything would pass every drill above while being useless in practice.
+ const benign = checkGate({ config, action: 'commit', paths: [benignPath] });
+ results.push(benign.allowed
+ ? pass('gate.benign', 'ordinary change is still allowed', mode, 'specificity', `commit touching ${benignPath} is allowed`, benign.reason)
+ : fail('gate.benign', 'ordinary change is still allowed', mode, 'specificity', `commit touching ${benignPath} is allowed`, `blocked by ${benign.trigger}`, 'The gate blocks a benign path. A guardrail that stops everything will pass every sensitivity drill while making the loop unusable — pass --benign-path if this path is genuinely sensitive in your project.'));
+ return results;
+}
+// ── Circuit breaker drills (Infinite Fix Loop / Token Burn) ────────
+function ledgerOf(attempts) {
+ return { goal: 'loop-drill synthetic run', attempts };
+}
+/**
+ * Errors with no shared structure, used for the no-progress drill.
+ *
+ * They must stay dissimilar *after* errorSignature() normalization, which
+ * collapses every number to '#'. Errors that differ only by a number ("module 1
+ * not found", "module 2 not found") normalize to the same signature and trip
+ * stagnation instead — the drill would pass while leaving no-progress
+ * unexercised.
+ */
+const DISTINCT_ERRORS = [
+ 'TypeError: undefined is not a function',
+ 'ECONNREFUSED connecting to the database',
+ 'SyntaxError: unexpected token in JSON',
+ 'AssertionError: expected ok but got null',
+ 'Permission denied while writing the artifact',
+ 'OutOfMemoryError during the bundle step',
+ 'Segmentation fault in the native addon',
+];
+/**
+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated
+ * identical failures must trip stagnation, a long run of unrelated failures
+ * must trip no-progress, blowing the token budget must trip, and a healthy run
+ * must not.
+ *
+ * Each drill asserts the *specific* trigger, not merely that the breaker
+ * escalated. Escalating for another reason means the rule under test is still
+ * unproven — the same standard the gate drills apply to `trigger !==
+ * 'denylist'`.
+ */
+export function runBreakerDrills(config = DEFAULT_BREAKER) {
+ const results = [];
+ // Sensitivity: the same error repeated trips stagnation.
+ {
+ const id = 'breaker.stagnation';
+ const name = `${config.stagnationThreshold} identical failures trip stagnation`;
+ const mode = 'Infinite Fix Loop';
+ const attempts = Array.from({ length: config.stagnationThreshold }, (_, i) => ({
+ iteration: i + 1,
+ action: 'retry the failing test',
+ outcome: 'failure',
+ error: "TypeError: Cannot read properties of undefined (reading 'id')",
+ }));
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(decision.trigger === 'stagnation'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)
+ : fail(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate
+ ? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'
+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.'));
+ }
+ // Sensitivity: a long run of unrelated failures trips no-progress.
+ {
+ const id = 'breaker.no-progress';
+ const name = `${config.noProgressThreshold} unrelated failures trip no-progress`;
+ const mode = 'Infinite Fix Loop';
+ const attempts = Array.from({ length: config.noProgressThreshold }, (_, i) => ({
+ iteration: i + 1,
+ action: `attempt strategy ${i + 1}`,
+ outcome: 'failure',
+ error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],
+ }));
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(decision.trigger === 'no-progress'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)
+ : fail(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate
+ ? 'The breaker stopped the loop, but not by the rule under test — no-progress stays unproven.'
+ : undefined));
+ }
+ // Sensitivity: exceeding the token budget trips.
+ if (config.tokenBudget !== undefined) {
+ const id = 'breaker.token-budget';
+ const name = 'exceeding the token budget trips the breaker';
+ const mode = 'Token Burn';
+ const attempts = [
+ {
+ iteration: 1,
+ action: 'a very expensive sub-agent chain',
+ outcome: 'success',
+ tokensUsed: config.tokenBudget + 1,
+ },
+ ];
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(decision.trigger === 'token-budget'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)
+ : fail(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.escalate ? `escalated via ${decision.trigger}` : 'continued'));
+ }
+ else {
+ results.push(skip('breaker.token-budget', 'token budget cap', 'Token Burn', 'sensitivity', 'No tokenBudget configured — nothing caps spend mid-run. See loop-budget.md.'));
+ }
+ // Specificity: a healthy run is NOT escalated.
+ {
+ const id = 'breaker.healthy';
+ const name = 'a healthy run is not escalated';
+ const mode = 'Infinite Fix Loop';
+ const attempts = [
+ { iteration: 1, action: 'read the failing test', outcome: 'success' },
+ { iteration: 2, action: 'apply a minimal fix', outcome: 'success' },
+ ];
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(!decision.escalate
+ ? pass(id, name, mode, 'specificity', 'continue', 'continued')
+ : fail(id, name, mode, 'specificity', 'continue', `escalated (${decision.trigger})`, 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.'));
+ }
+ return results;
+}
+// ── Report assembly ────────────────────────────────────────────────
+const ALL_MODES = [
+ 'Infinite Fix Loop',
+ 'Token Burn',
+ 'Over-Reach (Wrong Scope)',
+ 'Verifier Theater',
+ 'Escalation Failure',
+];
+export function buildReport(results) {
+ const covered = [];
+ const uncovered = [];
+ for (const mode of ALL_MODES) {
+ const forMode = results.filter((r) => r.failureMode === mode);
+ const anyPassed = forMode.some((r) => r.outcome === 'passed');
+ const anyFailed = forMode.some((r) => r.outcome === 'failed');
+ if (anyPassed && !anyFailed)
+ covered.push(mode);
+ else
+ uncovered.push(mode);
+ }
+ return {
+ results,
+ passed: results.filter((r) => r.outcome === 'passed').length,
+ failed: results.filter((r) => r.outcome === 'failed').length,
+ skipped: results.filter((r) => r.outcome === 'skipped').length,
+ covered,
+ uncovered,
+ timestamp: new Date().toISOString(),
+ };
+}
+/**
+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),
+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's
+ * "2 means escalate" convention so control scripts can chain all three.
+ */
+export function exitCodeFor(report) {
+ if (report.failed > 0)
+ return 2;
+ if (report.skipped > 0)
+ return 1;
+ return 0;
+}
diff --git a/tools/loop-drill/dist/mutations.d.ts b/tools/loop-drill/dist/mutations.d.ts
new file mode 100644
index 0000000..513caeb
--- /dev/null
+++ b/tools/loop-drill/dist/mutations.d.ts
@@ -0,0 +1,49 @@
+/**
+ * Mutation operators for the verifier canary.
+ *
+ * The canary needs a defect a competent verifier should reject. Rather than
+ * asking a model to invent one (which makes the drill non-deterministic and
+ * costs tokens), we borrow mutation testing: apply a small, mechanical,
+ * behaviour-changing edit to real source and see whether the verifier notices.
+ *
+ * Operators are deliberately restricted to edits that change runtime behaviour
+ * in every C-family language, and that a reviewer would call a bug rather than
+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and
+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:
+ * a false mutant would report a verifier as broken when it was right to
+ * approve.
+ */
+export interface MutationOperator {
+ id: string;
+ description: string;
+ /** Matches the token to mutate. Must be global. */
+ pattern: RegExp;
+ replace: string;
+}
+/**
+ * Order matters: the first operator with a match in a file wins, so the
+ * least ambiguous edits come first.
+ */
+export declare const OPERATORS: MutationOperator[];
+export interface Mutant {
+ operator: MutationOperator;
+ /** Repo-relative path of the mutated file. */
+ file: string;
+ mutated: string;
+ /** 1-based line number of the edit, for the report. */
+ line: number;
+ /** The line before and after, for the report. */
+ before: string;
+ after: string;
+}
+/**
+ * Apply the first applicable operator to `content`, mutating only the first
+ * occurrence. One edit per mutant keeps the signal clean: if the verifier
+ * rejects, it rejected *this* defect.
+ *
+ * Skips occurrences inside line comments — a verifier is right not to care
+ * about a flipped operator in a comment, and crediting it for catching one
+ * would overstate the result.
+ */
+export declare function mutate(content: string, file: string, operators?: MutationOperator[]): Mutant | null;
+export declare function isMutationCandidate(relativePath: string): boolean;
diff --git a/tools/loop-drill/dist/mutations.js b/tools/loop-drill/dist/mutations.js
new file mode 100644
index 0000000..0ffc3bc
--- /dev/null
+++ b/tools/loop-drill/dist/mutations.js
@@ -0,0 +1,120 @@
+/**
+ * Mutation operators for the verifier canary.
+ *
+ * The canary needs a defect a competent verifier should reject. Rather than
+ * asking a model to invent one (which makes the drill non-deterministic and
+ * costs tokens), we borrow mutation testing: apply a small, mechanical,
+ * behaviour-changing edit to real source and see whether the verifier notices.
+ *
+ * Operators are deliberately restricted to edits that change runtime behaviour
+ * in every C-family language, and that a reviewer would call a bug rather than
+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and
+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:
+ * a false mutant would report a verifier as broken when it was right to
+ * approve.
+ */
+/**
+ * Order matters: the first operator with a match in a file wins, so the
+ * least ambiguous edits come first.
+ */
+export const OPERATORS = [
+ {
+ id: 'strict-equality-flip',
+ description: 'flip === to !==',
+ pattern: /===/g,
+ replace: '!==',
+ },
+ {
+ id: 'strict-inequality-flip',
+ description: 'flip !== to ===',
+ pattern: /!==/g,
+ replace: '===',
+ },
+ {
+ id: 'boundary-off-by-one',
+ description: 'weaken <= to <',
+ pattern: /<=/g,
+ replace: '<',
+ },
+ {
+ id: 'boundary-off-by-one-gte',
+ description: 'weaken >= to >',
+ pattern: />=/g,
+ replace: '>',
+ },
+ {
+ id: 'logical-and-to-or',
+ description: 'flip && to ||',
+ pattern: /&&/g,
+ replace: '||',
+ },
+ {
+ id: 'return-true-to-false',
+ description: 'flip `return true` to `return false`',
+ pattern: /\breturn true\b/g,
+ replace: 'return false',
+ },
+ {
+ id: 'return-false-to-true',
+ description: 'flip `return false` to `return true`',
+ pattern: /\breturn false\b/g,
+ replace: 'return true',
+ },
+];
+/**
+ * Apply the first applicable operator to `content`, mutating only the first
+ * occurrence. One edit per mutant keeps the signal clean: if the verifier
+ * rejects, it rejected *this* defect.
+ *
+ * Skips occurrences inside line comments — a verifier is right not to care
+ * about a flipped operator in a comment, and crediting it for catching one
+ * would overstate the result.
+ */
+export function mutate(content, file, operators = OPERATORS) {
+ for (const operator of operators) {
+ const pattern = new RegExp(operator.pattern.source, 'g');
+ let match;
+ while ((match = pattern.exec(content)) !== null) {
+ const index = match.index;
+ if (isInLineComment(content, index))
+ continue;
+ const mutated = content.slice(0, index) + operator.replace + content.slice(index + match[0].length);
+ const line = content.slice(0, index).split('\n').length;
+ return {
+ operator,
+ file,
+ mutated,
+ line,
+ before: lineAt(content, line).trim(),
+ after: lineAt(mutated, line).trim(),
+ };
+ }
+ }
+ return null;
+}
+function lineAt(content, line) {
+ return content.split('\n')[line - 1] ?? '';
+}
+/** True when `index` sits after a `//` or `#` on its own line. */
+function isInLineComment(content, index) {
+ const lineStart = content.lastIndexOf('\n', index - 1) + 1;
+ const prefix = content.slice(lineStart, index);
+ return prefix.includes('//') || /(^|\s)#/.test(prefix);
+}
+/**
+ * Files worth mutating: real source, not build output, vendored code, tests,
+ * or generated artifacts. Mutating a test file would make the canary trivially
+ * easy (the suite fails loudly); mutating dist/ would make it meaningless
+ * (nothing reads it).
+ */
+const SKIP_DIRECTORIES = /(^|\/)(node_modules|dist|build|out|coverage|vendor|\.git|__snapshots__)(\/|$)/;
+const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|java|go|rs|c|cc|cpp|cs|kt|swift|php|rb)$/;
+const TEST_FILE = /(^|\/)(test|tests|spec|__tests__)(\/|$)|\.(test|spec)\.[a-z]+$/;
+export function isMutationCandidate(relativePath) {
+ const normalized = relativePath.replace(/\\/g, '/');
+ if (SKIP_DIRECTORIES.test(normalized))
+ return false;
+ if (TEST_FILE.test(normalized))
+ return false;
+ return SOURCE_EXTENSIONS.test(normalized);
+}
diff --git a/tools/loop-drill/dist/report.d.ts b/tools/loop-drill/dist/report.d.ts
new file mode 100644
index 0000000..8d038f1
--- /dev/null
+++ b/tools/loop-drill/dist/report.d.ts
@@ -0,0 +1,2 @@
+import type { DrillReport } from './drill.js';
+export declare function formatReport(report: DrillReport, mutationScore: number | null): string;
diff --git a/tools/loop-drill/dist/report.js b/tools/loop-drill/dist/report.js
new file mode 100644
index 0000000..3b12478
--- /dev/null
+++ b/tools/loop-drill/dist/report.js
@@ -0,0 +1,46 @@
+const MARK = {
+ passed: '✅',
+ failed: '❌',
+ skipped: '⚠️',
+};
+export function formatReport(report, mutationScore) {
+ const lines = [];
+ lines.push('Loop Drill — guardrail fire drill');
+ lines.push('═'.repeat(50));
+ lines.push('');
+ // Group by failure mode so the report reads as coverage of
+ // docs/failure-modes.md rather than a flat list of assertions.
+ const modes = [...new Set(report.results.map((r) => r.failureMode))];
+ for (const mode of modes) {
+ const forMode = report.results.filter((r) => r.failureMode === mode);
+ const failed = forMode.filter((r) => r.outcome === 'failed').length;
+ const status = failed > 0 ? '❌ NOT PROVEN' : forMode.some((r) => r.outcome === 'skipped') ? '⚠️ PARTIAL' : '✅ PROVEN';
+ lines.push(`${mode} — ${status}`);
+ for (const r of forMode) {
+ lines.push(` ${MARK[r.outcome]} ${r.name}`);
+ if (r.outcome !== 'passed') {
+ lines.push(` expected: ${r.expected}`);
+ lines.push(` actual: ${r.actual}`);
+ if (r.detail)
+ lines.push(` ${r.detail}`);
+ }
+ }
+ lines.push('');
+ }
+ if (mutationScore !== null) {
+ const pct = Math.round(mutationScore * 100);
+ lines.push(`Mutation score: ${pct}% of seeded defects rejected`);
+ if (pct < 100) {
+ lines.push('A verifier that approves seeded defects is Verifier Theater (docs/failure-modes.md).');
+ }
+ lines.push('');
+ }
+ lines.push(`Passed: ${report.passed} Failed: ${report.failed} Skipped: ${report.skipped}`);
+ if (report.uncovered.length > 0) {
+ lines.push('');
+ lines.push('Failure modes not proven by this run:');
+ for (const mode of report.uncovered)
+ lines.push(` - ${mode}`);
+ }
+ return lines.join('\n');
+}
diff --git a/tools/loop-drill/package-lock.json b/tools/loop-drill/package-lock.json
new file mode 100644
index 0000000..196d435
--- /dev/null
+++ b/tools/loop-drill/package-lock.json
@@ -0,0 +1,103 @@
+{
+ "name": "@cobusgreyling/loop-drill",
+ "version": "1.0.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "@cobusgreyling/loop-drill",
+ "version": "1.0.0",
+ "license": "MIT",
+ "dependencies": {
+ "@cobusgreyling/loop-context": "file:../loop-context",
+ "@cobusgreyling/loop-gate": "file:../loop-gate"
+ },
+ "bin": {
+ "loop-drill": "dist/cli.js"
+ },
+ "devDependencies": {
+ "@types/node": "^20.0.0",
+ "typescript": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "../loop-context": {
+ "name": "@cobusgreyling/loop-context",
+ "version": "1.5.0",
+ "license": "MIT",
+ "dependencies": {
+ "@cobusgreyling/loop-cost": "^1.1.0"
+ },
+ "bin": {
+ "loop-context": "dist/cli.js"
+ },
+ "devDependencies": {
+ "@types/node": "^20.0.0",
+ "typescript": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "../loop-gate": {
+ "name": "@cobusgreyling/loop-gate",
+ "version": "1.0.0",
+ "license": "MIT",
+ "dependencies": {
+ "minimatch": "^10.0.0",
+ "yaml": "^2.8.0"
+ },
+ "bin": {
+ "loop-gate": "dist/cli.js"
+ },
+ "devDependencies": {
+ "@types/node": "^20.0.0",
+ "typescript": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@cobusgreyling/loop-context": {
+ "resolved": "../loop-context",
+ "link": true
+ },
+ "node_modules/@cobusgreyling/loop-gate": {
+ "resolved": "../loop-gate",
+ "link": true
+ },
+ "node_modules/@types/node": {
+ "version": "20.19.43",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz",
+ "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~6.21.0"
+ }
+ },
+ "node_modules/typescript": {
+ "version": "5.9.3",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
+ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "bin": {
+ "tsc": "bin/tsc",
+ "tsserver": "bin/tsserver"
+ },
+ "engines": {
+ "node": ">=14.17"
+ }
+ },
+ "node_modules/undici-types": {
+ "version": "6.21.0",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
+ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+ "dev": true,
+ "license": "MIT"
+ }
+ }
+}
diff --git a/tools/loop-drill/package.json b/tools/loop-drill/package.json
new file mode 100644
index 0000000..6d88756
--- /dev/null
+++ b/tools/loop-drill/package.json
@@ -0,0 +1,54 @@
+{
+ "name": "@cobusgreyling/loop-drill",
+ "version": "1.0.0",
+ "description": "Fire drills for loop guardrails. Injects a known fault and asserts the guardrail actually fires — turning L3 readiness from \"the files exist\" into \"the guardrails demonstrably work\".",
+ "type": "module",
+ "main": "dist/drill.js",
+ "exports": {
+ ".": "./dist/drill.js"
+ },
+ "bin": {
+ "loop-drill": "dist/cli.js"
+ },
+ "files": [
+ "dist",
+ "README.md"
+ ],
+ "scripts": {
+ "build": "tsc",
+ "test": "npm run build && node --test test/*.test.mjs",
+ "prepublishOnly": "npm test",
+ "start": "node dist/cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "keywords": [
+ "loop-engineering",
+ "ai-agents",
+ "mutation-testing",
+ "chaos-engineering",
+ "verifier",
+ "guardrails",
+ "safety"
+ ],
+ "author": "Cobus Greyling",
+ "license": "MIT",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/cobusgreyling/loop-engineering.git",
+ "directory": "tools/loop-drill"
+ },
+ "homepage": "https://github.com/cobusgreyling/loop-engineering/tree/main/tools/loop-drill#readme",
+ "bugs": {
+ "url": "https://github.com/cobusgreyling/loop-engineering/issues"
+ },
+ "dependencies": {
+ "@cobusgreyling/loop-context": "file:../loop-context",
+ "@cobusgreyling/loop-gate": "file:../loop-gate"
+ },
+ "devDependencies": {
+ "@types/node": "^20.0.0",
+ "typescript": "^5.0.0"
+ }
+}
diff --git a/tools/loop-drill/src/canary.ts b/tools/loop-drill/src/canary.ts
new file mode 100644
index 0000000..0069cb5
--- /dev/null
+++ b/tools/loop-drill/src/canary.ts
@@ -0,0 +1,296 @@
+/**
+ * The verifier canary — the drill that gave this package its reason to exist.
+ *
+ * loop-audit awards its joint-largest signal (14 points) for a verifier, and
+ * gates L3 on it, by checking whether a file whose name contains "verifier"
+ * exists. An empty file passes. docs/failure-modes.md calls the resulting
+ * failure "Verifier Theater" and rates it S2; docs/primitives.md calls
+ * maker/checker "the single most important structural pattern for reliable
+ * loops". Nothing tested whether the verifier verifies.
+ *
+ * This runs the real verifier command against real seeded defects in an
+ * isolated worktree, and scores it on what it caught:
+ *
+ * control — clean tree -> the verifier must ACCEPT (exit 0)
+ * mutants — one seeded bug -> the verifier must REJECT (non-zero exit)
+ *
+ * Both directions are required. A verifier that rejects everything catches
+ * every mutant while being worthless, and the control run is what exposes it.
+ */
+
+import { exec as execShellCb, execFile as execFileCb } from 'node:child_process';
+import { readFile, writeFile, rm } from 'node:fs/promises';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { isMutationCandidate, mutate, type Mutant } from './mutations.js';
+import { skip, type DrillResult } from './drill.js';
+
+/**
+ * Two spawn helpers, kept honest about their signatures.
+ *
+ * `execShell` runs a caller-supplied command string (--verifier-cmd, --setup)
+ * through a shell, which is what those flags mean. `execFile` runs git with an
+ * argv array and no shell, so repo paths never reach a shell for re-parsing.
+ *
+ * Earlier this file called promisify(execFile) with an options object in the
+ * `args` position behind an `as never` cast -- it happened to work because Node
+ * detects a non-array second argument, but it hid the real signature from tsc
+ * and made timeout/killed handling harder to reason about.
+ */
+const execShell = promisify(execShellCb);
+const exec = promisify(execFileCb);
+
+export interface CanaryOptions {
+ /** Repo root to drill. */
+ root: string;
+ /** The verifier command. Non-zero exit is read as "rejected". */
+ command: string;
+ /** How many mutants to seed. More is slower but a sharper score. */
+ count: number;
+ /** Per-run timeout in ms. A verifier that hangs is a failed verifier. */
+ timeoutMs: number;
+ /** Restrict mutation to files under this repo-relative path. */
+ scope?: string;
+ /** Command run inside each worktree before verifying (e.g. "npm ci"). */
+ setup?: string;
+}
+
+export interface VerifierRun {
+ accepted: boolean;
+ exitCode: number | null;
+ timedOut: boolean;
+ output: string;
+}
+
+/**
+ * Run the verifier in `cwd`. Exit 0 means "accepted this change"; anything
+ * else — including a timeout — means "rejected". A verifier that cannot be
+ * launched at all is a distinct failure, surfaced by the caller.
+ */
+export async function runVerifier(
+ command: string,
+ cwd: string,
+ timeoutMs: number,
+): Promise<VerifierRun> {
+ try {
+ const { stdout, stderr } = await execShell(command, {
+ cwd,
+ timeout: timeoutMs,
+ maxBuffer: 8 * 1024 * 1024,
+ });
+ return { accepted: true, exitCode: 0, timedOut: false, output: `${stdout}${stderr}`.trim() };
+ } catch (err) {
+ const e = err as { code?: number | string; killed?: boolean; stdout?: string; stderr?: string };
+ const timedOut = e.killed === true || e.code === 'ETIMEDOUT';
+ return {
+ accepted: false,
+ exitCode: typeof e.code === 'number' ? e.code : null,
+ timedOut,
+ output: `${e.stdout ?? ''}${e.stderr ?? ''}`.trim(),
+ };
+ }
+}
+
+/** Files git tracks, filtered to plausible mutation targets. */
+export async function candidateFiles(root: string, scope?: string): Promise<string[]> {
+ const args = ['-C', root, 'ls-files'];
+ if (scope) args.push('--', scope);
+ const { stdout } = await exec('git', args, { maxBuffer: 32 * 1024 * 1024 });
+ return stdout
+ .split('\n')
+ .map((l) => l.trim())
+ .filter((l) => l.length > 0 && isMutationCandidate(l));
+}
+
+/**
+ * Build up to `count` mutants from distinct files. Distinct files matter:
+ * seeding three mutants into one module mostly measures whether the verifier
+ * reads that module.
+ */
+export async function collectMutants(
+ root: string,
+ files: string[],
+ count: number,
+): Promise<Mutant[]> {
+ const mutants: Mutant[] = [];
+ for (const file of files) {
+ if (mutants.length >= count) break;
+ let content: string;
+ try {
+ content = await readFile(path.join(root, file), 'utf8');
+ } catch {
+ continue;
+ }
+ const mutant = mutate(content, file);
+ if (mutant) mutants.push(mutant);
+ }
+ return mutants;
+}
+
+/**
+ * Run the verifier inside an ephemeral git worktree, so a verifier that writes,
+ * builds, or fixes cannot touch the real checkout. `mutant` is null for the
+ * worktree control run. The worktree is always removed, including on throw.
+ */
+async function runInWorktree(
+ root: string,
+ mutant: Mutant | null,
+ command: string,
+ timeoutMs: number,
+ setup?: string,
+): Promise<VerifierRun> {
+ const worktree = path.join(root, `.loop-drill-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+ await exec('git', ['-C', root, 'worktree', 'add', '--detach', '--quiet', worktree], {
+ maxBuffer: 8 * 1024 * 1024,
+ });
+ try {
+ if (setup) {
+ // Setup failure is not a verifier verdict — surface it as such.
+ const setupRun = await runVerifier(setup, worktree, timeoutMs);
+ if (!setupRun.accepted) {
+ return { ...setupRun, output: `[setup failed] ${setupRun.output}` };
+ }
+ }
+ if (mutant) await writeFile(path.join(worktree, mutant.file), mutant.mutated, 'utf8');
+ return await runVerifier(command, worktree, timeoutMs);
+ } finally {
+ // --force because the verifier may have left build output behind.
+ await exec('git', ['-C', root, 'worktree', 'remove', '--force', worktree]).catch(() => {});
+ await rm(worktree, { recursive: true, force: true }).catch(() => {});
+ }
+}
+
+export interface CanaryReport {
+ results: DrillResult[];
+ /** Mutants rejected / mutants run. Null when no mutant could be built. */
+ mutationScore: number | null;
+ caught: number;
+ escaped: number;
+}
+
+export async function runCanary(options: CanaryOptions): Promise<CanaryReport> {
+ const { root, command, count, timeoutMs, scope, setup } = options;
+ const results: DrillResult[] = [];
+ const mode = 'Verifier Theater' as const;
+
+ // Control first, and bail if it fails: every mutant result is meaningless
+ // when the verifier rejects a clean tree, and running them would report a
+ // reject-everything verifier as a perfect score.
+ const control = await runVerifier(command, root, timeoutMs);
+ if (control.accepted) {
+ results.push({
+ id: 'verifier.control',
+ name: 'verifier accepts an unmodified tree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'passed',
+ expected: 'exit 0 on clean tree',
+ actual: 'exit 0',
+ });
+ } else {
+ results.push({
+ id: 'verifier.control',
+ name: 'verifier accepts an unmodified tree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'failed',
+ expected: 'exit 0 on clean tree',
+ actual: control.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${control.exitCode}`,
+ detail:
+ 'The verifier rejects code with no seeded defect, so it cannot distinguish good from bad. Mutant drills were skipped — their results would be meaningless. Fix the verifier command (or the tree it runs against) first.',
+ });
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+
+ // Second control, in a clean worktree. A fresh checkout has no node_modules
+ // and no build output, so a verifier like "npm test" fails there for reasons
+ // that have nothing to do with code quality. Without this run, every mutant
+ // would be scored as "caught" and a completely broken setup would report a
+ // perfect mutation score — the precise false confidence this tool exists to
+ // prevent.
+ const worktreeControl = await runInWorktree(root, null, command, timeoutMs, setup);
+ if (!worktreeControl.accepted) {
+ results.push({
+ id: 'verifier.worktree-control',
+ name: 'verifier accepts a clean worktree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'skipped',
+ expected: 'exit 0 in a clean worktree',
+ actual: worktreeControl.timedOut ? `timed out after ${timeoutMs}ms` : `exit ${worktreeControl.exitCode}`,
+ detail:
+ 'The verifier passes in your checkout but fails in a fresh worktree, so mutant results would measure the environment rather than the verifier. Pass --setup to prepare the worktree first (e.g. --setup "npm ci"). ' +
+ (worktreeControl.output ? `Output: ${truncate(worktreeControl.output)}` : ''),
+ });
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+
+ results.push({
+ id: 'verifier.worktree-control',
+ name: 'verifier accepts a clean worktree',
+ failureMode: mode,
+ direction: 'specificity',
+ outcome: 'passed',
+ expected: 'exit 0 in a clean worktree',
+ actual: 'exit 0',
+ });
+
+ const files = await candidateFiles(root, scope);
+ const mutants = await collectMutants(root, files, count);
+
+ if (mutants.length === 0) {
+ results.push(
+ skip(
+ 'verifier.mutants',
+ 'verifier rejects seeded defects',
+ mode,
+ 'sensitivity',
+ `No mutable source found${scope ? ` under ${scope}` : ''}. loop-drill mutates tracked, non-test, non-generated source in common languages — pass --scope to point it at yours.`,
+ ),
+ );
+ return { results, mutationScore: null, caught: 0, escaped: 0 };
+ }
+
+ let caught = 0;
+ for (const mutant of mutants) {
+ const run = await runInWorktree(root, mutant, command, timeoutMs, setup);
+ const where = `${mutant.file}:${mutant.line}`;
+ const edit = `${mutant.operator.description} (${mutant.before} -> ${mutant.after})`;
+
+ if (!run.accepted) {
+ caught++;
+ results.push({
+ id: `verifier.mutant[${mutant.operator.id}]`,
+ name: `verifier rejects ${mutant.operator.id} in ${where}`,
+ failureMode: mode,
+ direction: 'sensitivity',
+ outcome: 'passed',
+ expected: 'non-zero exit (rejected)',
+ actual: run.timedOut ? 'timed out (treated as rejected)' : `exit ${run.exitCode}`,
+ detail: edit,
+ });
+ } else {
+ results.push({
+ id: `verifier.mutant[${mutant.operator.id}]`,
+ name: `verifier rejects ${mutant.operator.id} in ${where}`,
+ failureMode: mode,
+ direction: 'sensitivity',
+ outcome: 'failed',
+ expected: 'non-zero exit (rejected)',
+ actual: 'exit 0 (approved a seeded defect)',
+ detail: `${edit}. The verifier approved this change. That is Verifier Theater — see docs/failure-modes.md.`,
+ });
+ }
+ }
+
+ return {
+ results,
+ mutationScore: caught / mutants.length,
+ caught,
+ escaped: mutants.length - caught,
+ };
+}
+
+function truncate(text: string, max = 400): string {
+ return text.length <= max ? text : `${text.slice(0, max)}…`;
+}
diff --git a/tools/loop-drill/src/cli.ts b/tools/loop-drill/src/cli.ts
new file mode 100644
index 0000000..24dbc6f
--- /dev/null
+++ b/tools/loop-drill/src/cli.ts
@@ -0,0 +1,222 @@
+#!/usr/bin/env node
+/**
+ * loop-drill CLI.
+ *
+ * Exit codes follow loop-gate / loop-context so control scripts can chain all
+ * three: 0 proceed, 1 warnings, 2 escalate.
+ */
+
+import path from 'node:path';
+import { loadGateConfig } from '@cobusgreyling/loop-gate';
+import { DEFAULT_BREAKER, type CircuitBreakerConfig } from '@cobusgreyling/loop-context';
+import {
+ buildReport,
+ exitCodeFor,
+ runBreakerDrills,
+ runGateDrills,
+ skip,
+ type DrillResult,
+} from './drill.js';
+import { runCanary } from './canary.js';
+import { formatReport } from './report.js';
+
+interface Flags {
+ root: string;
+ json: boolean;
+ help: boolean;
+ gateFile?: string;
+ benignPath?: string;
+ only?: string;
+ verifierCmd?: string;
+ mutants: number;
+ timeoutMs: number;
+ scope?: string;
+ setup?: string;
+ tokenBudget?: number;
+}
+
+const HELP = `loop-drill — fire drills for loop guardrails
+
+Injects a known fault and asserts the guardrail actually fires. Turns L3
+readiness from "the files exist" into "the guardrails demonstrably work".
+
+Usage: loop-drill [path] [options]
+
+Options:
+ --only <drills> Comma-separated: gate, breaker, verifier (default: gate,breaker)
+ --verifier-cmd <cmd> Verifier command to drill. Non-zero exit = rejected.
+ Required to run the verifier canary.
+ --mutants <n> Seeded defects for the canary (default: 3)
+ --scope <path> Restrict mutation to this repo-relative path
+ --setup <cmd> Command run in each worktree before verifying
+ (e.g. "npm ci") — a fresh worktree has no node_modules
+ --timeout <ms> Per-verifier-run timeout (default: 120000)
+ --gate-file <path> Policy file (default: gate.yaml)
+ --benign-path <path> Path the gate specificity drill treats as ordinary
+ (default: docs/README.md)
+ --token-budget <n> Token budget for the breaker drill
+ --json Machine-readable output
+ --help, -h Show this message
+
+Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.
+
+Examples:
+ loop-drill .
+ loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"
+ loop-drill . --only gate,breaker --json
+`;
+
+function parseArgs(argv: string[]): Flags {
+ const flags: Flags = { root: '.', json: false, help: false, mutants: 3, timeoutMs: 120_000 };
+ const positional: string[] = [];
+
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ const next = () => {
+ const v = argv[++i];
+ if (v === undefined) throw new Error(`${arg} requires a value`);
+ return v;
+ };
+ switch (arg) {
+ case '--help':
+ case '-h':
+ flags.help = true;
+ break;
+ case '--json':
+ flags.json = true;
+ break;
+ case '--only':
+ flags.only = next();
+ break;
+ case '--verifier-cmd':
+ flags.verifierCmd = next();
+ break;
+ case '--mutants':
+ flags.mutants = parsePositiveInt(next(), '--mutants');
+ break;
+ case '--scope':
+ flags.scope = next();
+ break;
+ case '--setup':
+ flags.setup = next();
+ break;
+ case '--timeout':
+ flags.timeoutMs = parsePositiveInt(next(), '--timeout');
+ break;
+ case '--gate-file':
+ flags.gateFile = next();
+ break;
+ case '--benign-path':
+ flags.benignPath = next();
+ break;
+ case '--token-budget':
+ flags.tokenBudget = parsePositiveInt(next(), '--token-budget');
+ break;
+ default:
+ if (arg.startsWith('-')) throw new Error(`Unknown option: ${arg}`);
+ positional.push(arg);
+ }
+ }
+
+ if (positional.length > 0) flags.root = positional[0];
+ return flags;
+}
+
+function parsePositiveInt(value: string, flag: string): number {
+ const n = Number(value);
+ if (!Number.isInteger(n) || n <= 0) throw new Error(`${flag} expects a positive integer, got: ${value}`);
+ return n;
+}
+
+async function main(): Promise<void> {
+ let flags: Flags;
+ try {
+ flags = parseArgs(process.argv.slice(2));
+ } catch (err) {
+ console.error(`${(err as Error).message}\n`);
+ console.error(HELP);
+ process.exit(2);
+ }
+
+ if (flags.help) {
+ console.log(HELP);
+ return;
+ }
+
+ const root = path.resolve(flags.root);
+ const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));
+ const results: DrillResult[] = [];
+ let mutationScore: number | null = null;
+
+ if (selected.has('gate')) {
+ const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');
+ try {
+ const config = await loadGateConfig(gateFile);
+ results.push(...runGateDrills({ config, benignPath: flags.benignPath }));
+ } catch (err) {
+ results.push(
+ skip(
+ 'gate',
+ 'gate drills',
+ 'Over-Reach (Wrong Scope)',
+ 'sensitivity',
+ `${(err as Error).message} Nothing mechanically stops a loop from editing a sensitive path.`,
+ ),
+ );
+ }
+ }
+
+ if (selected.has('breaker')) {
+ const config: CircuitBreakerConfig =
+ flags.tokenBudget === undefined
+ ? DEFAULT_BREAKER
+ : { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };
+ results.push(...runBreakerDrills(config));
+ }
+
+ if (selected.has('verifier')) {
+ if (!flags.verifierCmd) {
+ results.push(
+ skip(
+ 'verifier',
+ 'verifier canary',
+ 'Verifier Theater',
+ 'sensitivity',
+ 'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.',
+ ),
+ );
+ } else {
+ const canary = await runCanary({
+ root,
+ command: flags.verifierCmd,
+ count: flags.mutants,
+ timeoutMs: flags.timeoutMs,
+ scope: flags.scope,
+ setup: flags.setup,
+ });
+ results.push(...canary.results);
+ mutationScore = canary.mutationScore;
+ }
+ }
+
+ if (results.length === 0) {
+ console.error(`No drills selected. --only accepts: gate, breaker, verifier\n`);
+ process.exit(2);
+ }
+
+ const report = buildReport(results);
+ const code = exitCodeFor(report);
+
+ if (flags.json) {
+ console.log(JSON.stringify({ ...report, mutationScore, exitCode: code }, null, 2));
+ } else {
+ console.log(formatReport(report, mutationScore));
+ }
+
+ process.exit(code);
+}
+
+main().catch((err) => {
+ console.error(`loop-drill failed: ${(err as Error).message}`);
+ process.exit(2);
+});
diff --git a/tools/loop-drill/src/drill.ts b/tools/loop-drill/src/drill.ts
new file mode 100644
index 0000000..d8991d1
--- /dev/null
+++ b/tools/loop-drill/src/drill.ts
@@ -0,0 +1,468 @@
+/**
+ * loop-drill — fire drills for loop guardrails.
+ *
+ * docs/failure-modes.md names ten ways loops fail. Some have mechanical
+ * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores
+ * whether those counterparts are *present*. Nothing checked whether they
+ * actually fire.
+ *
+ * A drill injects a known fault and asserts the guardrail responds. Every
+ * drill carries the failure mode it exercises, so the report reads as coverage
+ * of docs/failure-modes.md rather than a list of anonymous assertions.
+ *
+ * Each guardrail gets both directions, because only one of them is easy:
+ * - sensitivity: the fault is caught (a guardrail that never fires is theater)
+ * - specificity: benign input is NOT caught (a guardrail that blocks
+ * everything passes every sensitivity drill while being useless)
+ * A guardrail is only credited when it passes both.
+ */
+
+import { checkGate, type GateConfig } from '@cobusgreyling/loop-gate';
+import {
+ checkCircuitBreaker,
+ DEFAULT_BREAKER,
+ type CircuitBreakerConfig,
+ type Ledger,
+} from '@cobusgreyling/loop-context';
+
+export type DrillOutcome = 'passed' | 'failed' | 'skipped';
+
+/** Named failure modes from docs/failure-modes.md. */
+export type FailureMode =
+ | 'Infinite Fix Loop'
+ | 'Token Burn'
+ | 'Over-Reach (Wrong Scope)'
+ | 'Verifier Theater'
+ | 'Escalation Failure';
+
+export interface DrillResult {
+ /** Stable id, e.g. 'gate.denylist'. */
+ id: string;
+ name: string;
+ /** The docs/failure-modes.md entry this drill exercises. */
+ failureMode: FailureMode;
+ /** 'sensitivity' = fault must be caught; 'specificity' = benign must pass. */
+ direction: 'sensitivity' | 'specificity';
+ outcome: DrillOutcome;
+ expected: string;
+ actual: string;
+ /** Why a drill was skipped, or extra context on a failure. */
+ detail?: string;
+}
+
+export interface DrillReport {
+ results: DrillResult[];
+ passed: number;
+ failed: number;
+ skipped: number;
+ /** Failure modes with at least one passing drill and no failing drill. */
+ covered: FailureMode[];
+ /** Failure modes with no passing drill (unexercised or broken). */
+ uncovered: FailureMode[];
+ timestamp: string;
+}
+
+function pass(
+ id: string,
+ name: string,
+ failureMode: FailureMode,
+ direction: DrillResult['direction'],
+ expected: string,
+ actual: string,
+): DrillResult {
+ return { id, name, failureMode, direction, outcome: 'passed', expected, actual };
+}
+
+function fail(
+ id: string,
+ name: string,
+ failureMode: FailureMode,
+ direction: DrillResult['direction'],
+ expected: string,
+ actual: string,
+ detail?: string,
+): DrillResult {
+ return { id, name, failureMode, direction, outcome: 'failed', expected, actual, detail };
+}
+
+export function skip(
+ id: string,
+ name: string,
+ failureMode: FailureMode,
+ direction: DrillResult['direction'],
+ detail: string,
+): DrillResult {
+ return {
+ id,
+ name,
+ failureMode,
+ direction,
+ outcome: 'skipped',
+ expected: '—',
+ actual: '—',
+ detail,
+ };
+}
+
+// ── Glob → concrete sample path ────────────────────────────────────
+
+/**
+ * Turn a denylist glob into a concrete path that should match it, so the drill
+ * exercises the real matcher rather than re-implementing glob semantics.
+ *
+ * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a
+ * literal, and the caller verifies the synthesized path actually matches
+ * before asserting anything about it (see runGateDrills) — a glob we
+ * synthesize badly is reported as skipped, never as a guardrail failure.
+ */
+export function samplePathFor(glob: string): string {
+ const concrete = glob
+ .split('/')
+ .map((segment) => {
+ if (segment === '**') return 'src';
+ if (segment === '*') return 'sample';
+ // '*_key*' -> 'drill_key_sample'; '.env.*' -> '.env.sample'
+ return segment.replace(/\*/g, (_m, offset: number) => (offset === 0 ? 'drill' : 'sample'));
+ })
+ .join('/');
+
+ // A trailing '**' segment became 'src', which is a directory, not a file.
+ return /(^|\/)src$/.test(concrete) ? `${concrete}/drill-file.txt` : concrete;
+}
+
+// ── Gate drills (Over-Reach) ───────────────────────────────────────
+
+export interface GateDrillOptions {
+ config: GateConfig;
+ /** Path assumed benign for the specificity drill. */
+ benignPath?: string;
+}
+
+/**
+ * Exercise loop-gate against its own gate.yaml: every denylist entry must
+ * block, the file-count cap must block, auto-merge must reject a
+ * non-allowlisted path, and an ordinary doc change must still be allowed.
+ */
+export function runGateDrills(options: GateDrillOptions): DrillResult[] {
+ const { config, benignPath = 'docs/README.md' } = options;
+ const results: DrillResult[] = [];
+ const mode: FailureMode = 'Over-Reach (Wrong Scope)';
+
+ // Sensitivity: each denylist glob blocks a path that matches it.
+ for (const glob of config.denylist) {
+ const id = `gate.denylist[${glob}]`;
+ const name = `denylist blocks ${glob}`;
+ const sample = samplePathFor(glob);
+ const decision = checkGate({ config, action: 'commit', paths: [sample] });
+
+ if (decision.allowed) {
+ results.push(
+ fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ `commit touching ${sample} is blocked`,
+ `allowed (${decision.trigger})`,
+ `Denylist entry "${glob}" did not match the sample path this drill synthesized for it. Either the glob is unreachable in practice, or the sample is unrepresentative — check it by hand with: loop-gate check --action commit --paths ${sample}`,
+ ),
+ );
+ continue;
+ }
+
+ if (decision.trigger !== 'denylist') {
+ // Blocked, but by file-count or the allowlist — the denylist itself is
+ // unproven, so this is not a pass.
+ results.push(
+ fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ `blocked by the denylist`,
+ `blocked by ${decision.trigger}`,
+ 'The path was stopped, but not by the rule under test.',
+ ),
+ );
+ continue;
+ }
+
+ results.push(pass(id, name, mode, 'sensitivity', 'blocked by denylist', decision.reason));
+ }
+
+ // Sensitivity: the file-count cap fires one file past the threshold.
+ if (config.maxFiles !== undefined) {
+ const id = 'gate.file-count';
+ const name = `file-count cap fires above ${config.maxFiles} files`;
+ const paths = Array.from({ length: config.maxFiles + 1 }, (_, i) => `docs/drill-${i}.md`);
+ const decision = checkGate({ config, action: 'commit', paths });
+ results.push(
+ decision.trigger === 'file-count'
+ ? pass(id, name, mode, 'sensitivity', 'blocked by file-count', decision.reason)
+ : fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ `${paths.length} files is blocked by file-count`,
+ decision.allowed ? 'allowed' : `blocked by ${decision.trigger}`,
+ ),
+ );
+ } else {
+ results.push(
+ skip(
+ 'gate.file-count',
+ 'file-count cap',
+ mode,
+ 'sensitivity',
+ 'gate.yaml sets no maxFiles — a loop may propose an arbitrarily large diff.',
+ ),
+ );
+ }
+
+ // Sensitivity: auto-merge rejects a path outside the allowlist.
+ if (config.autoMergeAllowlist && config.autoMergeAllowlist.length > 0) {
+ const id = 'gate.auto-merge';
+ const name = 'auto-merge rejects a non-allowlisted path';
+ const outside = 'src/drill-not-allowlisted.bin';
+ const decision = checkGate({ config, action: 'auto-merge', paths: [outside] });
+ results.push(
+ !decision.allowed
+ ? pass(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, decision.reason)
+ : fail(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, 'allowed'),
+ );
+ }
+
+ // Specificity: an ordinary change is still allowed. A gate that blocks
+ // everything would pass every drill above while being useless in practice.
+ const benign = checkGate({ config, action: 'commit', paths: [benignPath] });
+ results.push(
+ benign.allowed
+ ? pass(
+ 'gate.benign',
+ 'ordinary change is still allowed',
+ mode,
+ 'specificity',
+ `commit touching ${benignPath} is allowed`,
+ benign.reason,
+ )
+ : fail(
+ 'gate.benign',
+ 'ordinary change is still allowed',
+ mode,
+ 'specificity',
+ `commit touching ${benignPath} is allowed`,
+ `blocked by ${benign.trigger}`,
+ 'The gate blocks a benign path. A guardrail that stops everything will pass every sensitivity drill while making the loop unusable — pass --benign-path if this path is genuinely sensitive in your project.',
+ ),
+ );
+
+ return results;
+}
+
+// ── Circuit breaker drills (Infinite Fix Loop / Token Burn) ────────
+
+function ledgerOf(attempts: Ledger['attempts']): Ledger {
+ return { goal: 'loop-drill synthetic run', attempts };
+}
+
+/**
+ * Errors with no shared structure, used for the no-progress drill.
+ *
+ * They must stay dissimilar *after* errorSignature() normalization, which
+ * collapses every number to '#'. Errors that differ only by a number ("module 1
+ * not found", "module 2 not found") normalize to the same signature and trip
+ * stagnation instead — the drill would pass while leaving no-progress
+ * unexercised.
+ */
+const DISTINCT_ERRORS = [
+ 'TypeError: undefined is not a function',
+ 'ECONNREFUSED connecting to the database',
+ 'SyntaxError: unexpected token in JSON',
+ 'AssertionError: expected ok but got null',
+ 'Permission denied while writing the artifact',
+ 'OutOfMemoryError during the bundle step',
+ 'Segmentation fault in the native addon',
+];
+
+/**
+ * Exercise loop-context's circuit breaker with synthetic ledgers: repeated
+ * identical failures must trip stagnation, a long run of unrelated failures
+ * must trip no-progress, blowing the token budget must trip, and a healthy run
+ * must not.
+ *
+ * Each drill asserts the *specific* trigger, not merely that the breaker
+ * escalated. Escalating for another reason means the rule under test is still
+ * unproven — the same standard the gate drills apply to `trigger !==
+ * 'denylist'`.
+ */
+export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER): DrillResult[] {
+ const results: DrillResult[] = [];
+
+ // Sensitivity: the same error repeated trips stagnation.
+ {
+ const id = 'breaker.stagnation';
+ const name = `${config.stagnationThreshold} identical failures trip stagnation`;
+ const mode: FailureMode = 'Infinite Fix Loop';
+ const attempts = Array.from({ length: config.stagnationThreshold }, (_, i) => ({
+ iteration: i + 1,
+ action: 'retry the failing test',
+ outcome: 'failure' as const,
+ error: "TypeError: Cannot read properties of undefined (reading 'id')",
+ }));
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(
+ decision.trigger === 'stagnation'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)
+ : fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ 'escalate via stagnation',
+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
+ decision.escalate
+ ? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'
+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.',
+ ),
+ );
+ }
+
+ // Sensitivity: a long run of unrelated failures trips no-progress.
+ {
+ const id = 'breaker.no-progress';
+ const name = `${config.noProgressThreshold} unrelated failures trip no-progress`;
+ const mode: FailureMode = 'Infinite Fix Loop';
+ const attempts = Array.from({ length: config.noProgressThreshold }, (_, i) => ({
+ iteration: i + 1,
+ action: `attempt strategy ${i + 1}`,
+ outcome: 'failure' as const,
+ error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],
+ }));
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(
+ decision.trigger === 'no-progress'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)
+ : fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ 'escalate via no-progress',
+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
+ decision.escalate
+ ? 'The breaker stopped the loop, but not by the rule under test — no-progress stays unproven.'
+ : undefined,
+ ),
+ );
+ }
+
+ // Sensitivity: exceeding the token budget trips.
+ if (config.tokenBudget !== undefined) {
+ const id = 'breaker.token-budget';
+ const name = 'exceeding the token budget trips the breaker';
+ const mode: FailureMode = 'Token Burn';
+ const attempts = [
+ {
+ iteration: 1,
+ action: 'a very expensive sub-agent chain',
+ outcome: 'success' as const,
+ tokensUsed: config.tokenBudget + 1,
+ },
+ ];
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(
+ decision.trigger === 'token-budget'
+ ? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)
+ : fail(
+ id,
+ name,
+ mode,
+ 'sensitivity',
+ 'escalate via token-budget',
+ decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
+ ),
+ );
+ } else {
+ results.push(
+ skip(
+ 'breaker.token-budget',
+ 'token budget cap',
+ 'Token Burn',
+ 'sensitivity',
+ 'No tokenBudget configured — nothing caps spend mid-run. See loop-budget.md.',
+ ),
+ );
+ }
+
+ // Specificity: a healthy run is NOT escalated.
+ {
+ const id = 'breaker.healthy';
+ const name = 'a healthy run is not escalated';
+ const mode: FailureMode = 'Infinite Fix Loop';
+ const attempts = [
+ { iteration: 1, action: 'read the failing test', outcome: 'success' as const },
+ { iteration: 2, action: 'apply a minimal fix', outcome: 'success' as const },
+ ];
+ const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ results.push(
+ !decision.escalate
+ ? pass(id, name, mode, 'specificity', 'continue', 'continued')
+ : fail(
+ id,
+ name,
+ mode,
+ 'specificity',
+ 'continue',
+ `escalated (${decision.trigger})`,
+ 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.',
+ ),
+ );
+ }
+
+ return results;
+}
+
+// ── Report assembly ────────────────────────────────────────────────
+
+const ALL_MODES: FailureMode[] = [
+ 'Infinite Fix Loop',
+ 'Token Burn',
+ 'Over-Reach (Wrong Scope)',
+ 'Verifier Theater',
+ 'Escalation Failure',
+];
+
+export function buildReport(results: DrillResult[]): DrillReport {
+ const covered: FailureMode[] = [];
+ const uncovered: FailureMode[] = [];
+
+ for (const mode of ALL_MODES) {
+ const forMode = results.filter((r) => r.failureMode === mode);
+ const anyPassed = forMode.some((r) => r.outcome === 'passed');
+ const anyFailed = forMode.some((r) => r.outcome === 'failed');
+ if (anyPassed && !anyFailed) covered.push(mode);
+ else uncovered.push(mode);
+ }
+
+ return {
+ results,
+ passed: results.filter((r) => r.outcome === 'passed').length,
+ failed: results.filter((r) => r.outcome === 'failed').length,
+ skipped: results.filter((r) => r.outcome === 'skipped').length,
+ covered,
+ uncovered,
+ timestamp: new Date().toISOString(),
+ };
+}
+
+/**
+ * 0 = every drill passed, 1 = some skipped (guardrail not configured),
+ * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's
+ * "2 means escalate" convention so control scripts can chain all three.
+ */
+export function exitCodeFor(report: DrillReport): 0 | 1 | 2 {
+ if (report.failed > 0) return 2;
+ if (report.skipped > 0) return 1;
+ return 0;
+}
diff --git a/tools/loop-drill/src/mutations.ts b/tools/loop-drill/src/mutations.ts
new file mode 100644
index 0000000..34d634a
--- /dev/null
+++ b/tools/loop-drill/src/mutations.ts
@@ -0,0 +1,145 @@
+/**
+ * Mutation operators for the verifier canary.
+ *
+ * The canary needs a defect a competent verifier should reject. Rather than
+ * asking a model to invent one (which makes the drill non-deterministic and
+ * costs tokens), we borrow mutation testing: apply a small, mechanical,
+ * behaviour-changing edit to real source and see whether the verifier notices.
+ *
+ * Operators are deliberately restricted to edits that change runtime behaviour
+ * in every C-family language, and that a reviewer would call a bug rather than
+ * a style choice. Anything ambiguous (`<` -> `>`, which breaks TS generics and
+ * JSX; `+` -> `-`, which silently mangles string concatenation) is excluded:
+ * a false mutant would report a verifier as broken when it was right to
+ * approve.
+ */
+
+export interface MutationOperator {
+ id: string;
+ description: string;
+ /** Matches the token to mutate. Must be global. */
+ pattern: RegExp;
+ replace: string;
+}
+
+/**
+ * Order matters: the first operator with a match in a file wins, so the
+ * least ambiguous edits come first.
+ */
+export const OPERATORS: MutationOperator[] = [
+ {
+ id: 'strict-equality-flip',
+ description: 'flip === to !==',
+ pattern: /===/g,
+ replace: '!==',
+ },
+ {
+ id: 'strict-inequality-flip',
+ description: 'flip !== to ===',
+ pattern: /!==/g,
+ replace: '===',
+ },
+ {
+ id: 'boundary-off-by-one',
+ description: 'weaken <= to <',
+ pattern: /<=/g,
+ replace: '<',
+ },
+ {
+ id: 'boundary-off-by-one-gte',
+ description: 'weaken >= to >',
+ pattern: />=/g,
+ replace: '>',
+ },
+ {
+ id: 'logical-and-to-or',
+ description: 'flip && to ||',
+ pattern: /&&/g,
+ replace: '||',
+ },
+ {
+ id: 'return-true-to-false',
+ description: 'flip `return true` to `return false`',
+ pattern: /\breturn true\b/g,
+ replace: 'return false',
+ },
+ {
+ id: 'return-false-to-true',
+ description: 'flip `return false` to `return true`',
+ pattern: /\breturn false\b/g,
+ replace: 'return true',
+ },
+];
+
+export interface Mutant {
+ operator: MutationOperator;
+ /** Repo-relative path of the mutated file. */
+ file: string;
+ mutated: string;
+ /** 1-based line number of the edit, for the report. */
+ line: number;
+ /** The line before and after, for the report. */
+ before: string;
+ after: string;
+}
+
+/**
+ * Apply the first applicable operator to `content`, mutating only the first
+ * occurrence. One edit per mutant keeps the signal clean: if the verifier
+ * rejects, it rejected *this* defect.
+ *
+ * Skips occurrences inside line comments — a verifier is right not to care
+ * about a flipped operator in a comment, and crediting it for catching one
+ * would overstate the result.
+ */
+export function mutate(content: string, file: string, operators = OPERATORS): Mutant | null {
+ for (const operator of operators) {
+ const pattern = new RegExp(operator.pattern.source, 'g');
+ let match: RegExpExecArray | null;
+
+ while ((match = pattern.exec(content)) !== null) {
+ const index = match.index;
+ if (isInLineComment(content, index)) continue;
+
+ const mutated = content.slice(0, index) + operator.replace + content.slice(index + match[0].length);
+ const line = content.slice(0, index).split('\n').length;
+ return {
+ operator,
+ file,
+ mutated,
+ line,
+ before: lineAt(content, line).trim(),
+ after: lineAt(mutated, line).trim(),
+ };
+ }
+ }
+ return null;
+}
+
+function lineAt(content: string, line: number): string {
+ return content.split('\n')[line - 1] ?? '';
+}
+
+/** True when `index` sits after a `//` or `#` on its own line. */
+function isInLineComment(content: string, index: number): boolean {
+ const lineStart = content.lastIndexOf('\n', index - 1) + 1;
+ const prefix = content.slice(lineStart, index);
+ return prefix.includes('//') || /(^|\s)#/.test(prefix);
+}
+
+/**
+ * Files worth mutating: real source, not build output, vendored code, tests,
+ * or generated artifacts. Mutating a test file would make the canary trivially
+ * easy (the suite fails loudly); mutating dist/ would make it meaningless
+ * (nothing reads it).
+ */
+const SKIP_DIRECTORIES = /(^|\/)(node_modules|dist|build|out|coverage|vendor|\.git|__snapshots__)(\/|$)/;
+const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|java|go|rs|c|cc|cpp|cs|kt|swift|php|rb)$/;
+const TEST_FILE = /(^|\/)(test|tests|spec|__tests__)(\/|$)|\.(test|spec)\.[a-z]+$/;
+
+export function isMutationCandidate(relativePath: string): boolean {
+ const normalized = relativePath.replace(/\\/g, '/');
+ if (SKIP_DIRECTORIES.test(normalized)) return false;
+ if (TEST_FILE.test(normalized)) return false;
+ return SOURCE_EXTENSIONS.test(normalized);
+}
diff --git a/tools/loop-drill/src/report.ts b/tools/loop-drill/src/report.ts
new file mode 100644
index 0000000..f5ab096
--- /dev/null
+++ b/tools/loop-drill/src/report.ts
@@ -0,0 +1,52 @@
+import type { DrillReport, DrillResult } from './drill.js';
+
+const MARK: Record<DrillResult['outcome'], string> = {
+ passed: '✅',
+ failed: '❌',
+ skipped: '⚠️',
+};
+
+export function formatReport(report: DrillReport, mutationScore: number | null): string {
+ const lines: string[] = [];
+ lines.push('Loop Drill — guardrail fire drill');
+ lines.push('═'.repeat(50));
+ lines.push('');
+
+ // Group by failure mode so the report reads as coverage of
+ // docs/failure-modes.md rather than a flat list of assertions.
+ const modes = [...new Set(report.results.map((r) => r.failureMode))];
+ for (const mode of modes) {
+ const forMode = report.results.filter((r) => r.failureMode === mode);
+ const failed = forMode.filter((r) => r.outcome === 'failed').length;
+ const status = failed > 0 ? '❌ NOT PROVEN' : forMode.some((r) => r.outcome === 'skipped') ? '⚠️ PARTIAL' : '✅ PROVEN';
+ lines.push(`${mode} — ${status}`);
+ for (const r of forMode) {
+ lines.push(` ${MARK[r.outcome]} ${r.name}`);
+ if (r.outcome !== 'passed') {
+ lines.push(` expected: ${r.expected}`);
+ lines.push(` actual: ${r.actual}`);
+ if (r.detail) lines.push(` ${r.detail}`);
+ }
+ }
+ lines.push('');
+ }
+
+ if (mutationScore !== null) {
+ const pct = Math.round(mutationScore * 100);
+ lines.push(`Mutation score: ${pct}% of seeded defects rejected`);
+ if (pct < 100) {
+ lines.push('A verifier that approves seeded defects is Verifier Theater (docs/failure-modes.md).');
+ }
+ lines.push('');
+ }
+
+ lines.push(`Passed: ${report.passed} Failed: ${report.failed} Skipped: ${report.skipped}`);
+
+ if (report.uncovered.length > 0) {
+ lines.push('');
+ lines.push('Failure modes not proven by this run:');
+ for (const mode of report.uncovered) lines.push(` - ${mode}`);
+ }
+
+ return lines.join('\n');
+}
diff --git a/tools/loop-drill/test/canary.test.mjs b/tools/loop-drill/test/canary.test.mjs
new file mode 100644
index 0000000..d5dcbdc
--- /dev/null
+++ b/tools/loop-drill/test/canary.test.mjs
@@ -0,0 +1,166 @@
+import { test, describe, before, after } from 'node:test';
+import assert from 'node:assert/strict';
+import path from 'node:path';
+import os from 'node:os';
+import { mkdtemp, mkdir, writeFile, rm, readdir } from 'node:fs/promises';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { runCanary, runVerifier, candidateFiles, collectMutants } from '../dist/canary.js';
+
+const exec = promisify(execFile);
+
+/**
+ * A throwaway git repo with one mutable source file and a real test script.
+ * The "verifier" is `node verify.mjs`, which asserts isAdult(18) === true —
+ * a check the boundary mutation (>= to >) genuinely breaks.
+ */
+let repo;
+
+before(async () => {
+ repo = await mkdtemp(path.join(os.tmpdir(), 'loop-drill-'));
+ await mkdir(path.join(repo, 'src'), { recursive: true });
+
+ await writeFile(
+ path.join(repo, 'src', 'rules.mjs'),
+ 'export function isAdult(age) {\n return age >= 18;\n}\n',
+ );
+ await writeFile(
+ path.join(repo, 'verify.mjs'),
+ [
+ "import { isAdult } from './src/rules.mjs';",
+ 'if (isAdult(18) !== true) { console.error("18 must be adult"); process.exit(1); }',
+ 'if (isAdult(17) !== false) { console.error("17 must not be adult"); process.exit(1); }',
+ 'console.log("verified");',
+ ].join('\n'),
+ );
+
+ await exec('git', ['-C', repo, 'init', '-q']);
+ await exec('git', ['-C', repo, 'config', 'user.email', '[email protected]']);
+ await exec('git', ['-C', repo, 'config', 'user.name', 'drill']);
+ await exec('git', ['-C', repo, 'add', '.']);
+ await exec('git', ['-C', repo, 'commit', '-q', '-m', 'seed']);
+});
+
+after(async () => {
+ if (repo) await rm(repo, { recursive: true, force: true });
+});
+
+describe('runVerifier', () => {
+ test('exit 0 is read as accepted', async () => {
+ const run = await runVerifier('node -e "process.exit(0)"', process.cwd(), 20_000);
+ assert.equal(run.accepted, true);
+ });
+
+ test('non-zero exit is read as rejected, and the code is reported', async () => {
+ const run = await runVerifier('node -e "process.exit(3)"', process.cwd(), 20_000);
+ assert.equal(run.accepted, false);
+ assert.equal(run.exitCode, 3);
+ });
+
+ test('a hanging verifier is rejected rather than hanging the drill', async () => {
+ const run = await runVerifier('node -e "setTimeout(()=>{}, 60000)"', process.cwd(), 800);
+ assert.equal(run.accepted, false);
+ assert.equal(run.timedOut, true);
+ });
+});
+
+describe('candidateFiles / collectMutants', () => {
+ test('lists tracked source and skips the verifier harness itself', async () => {
+ const files = await candidateFiles(repo);
+ assert.ok(files.includes('src/rules.mjs'));
+ });
+
+ test('builds a mutant from real source', async () => {
+ const mutants = await collectMutants(repo, ['src/rules.mjs'], 1);
+ assert.equal(mutants.length, 1);
+ assert.equal(mutants[0].file, 'src/rules.mjs');
+ assert.match(mutants[0].mutated, /age > 18/);
+ });
+});
+
+describe('runCanary', () => {
+ test('a real verifier catches the seeded defect and scores 100%', async () => {
+ const report = await runCanary({
+ root: repo,
+ command: 'node verify.mjs',
+ count: 1,
+ timeoutMs: 60_000,
+ scope: 'src',
+ });
+ assert.equal(report.mutationScore, 1);
+ assert.equal(report.caught, 1);
+ assert.equal(report.escaped, 0);
+ assert.ok(report.results.every((r) => r.outcome === 'passed'));
+ });
+
+ test('a rubber-stamp verifier scores 0% and is named as Verifier Theater', async () => {
+ const report = await runCanary({
+ root: repo,
+ command: 'node -e "console.log(\'LGTM\')"',
+ count: 1,
+ timeoutMs: 60_000,
+ scope: 'src',
+ });
+ assert.equal(report.mutationScore, 0);
+ assert.equal(report.escaped, 1);
+ const escaped = report.results.find((r) => r.outcome === 'failed');
+ assert.ok(escaped, 'expected a failed drill');
+ assert.equal(escaped.failureMode, 'Verifier Theater');
+ assert.match(escaped.actual, /approved a seeded defect/);
+ });
+
+ test('a reject-everything verifier fails the control and skips the mutants', async () => {
+ const report = await runCanary({
+ root: repo,
+ command: 'node -e "process.exit(1)"',
+ count: 1,
+ timeoutMs: 60_000,
+ scope: 'src',
+ });
+ // Mutant results would be meaningless here, so none were produced.
+ assert.equal(report.mutationScore, null);
+ assert.equal(report.results.length, 1);
+ assert.equal(report.results[0].id, 'verifier.control');
+ assert.equal(report.results[0].outcome, 'failed');
+ });
+
+ test('a verifier that only passes with prepared deps is skipped, not scored', async () => {
+ // Passes at the repo root (the marker is untracked, so it survives there)
+ // but not in a fresh worktree -- the environment-drift case that would
+ // otherwise report every mutant as caught.
+ await writeFile(path.join(repo, 'untracked-marker.txt'), 'present\n');
+ const report = await runCanary({
+ root: repo,
+ command: 'node -e "require(\'fs\').accessSync(\'untracked-marker.txt\')"',
+ count: 1,
+ timeoutMs: 60_000,
+ scope: 'src',
+ });
+ assert.equal(report.mutationScore, null, 'must not score mutants on a broken worktree');
+ const control = report.results.find((r) => r.id === 'verifier.worktree-control');
+ assert.equal(control.outcome, 'skipped');
+ assert.match(control.detail, /--setup/);
+ await rm(path.join(repo, 'untracked-marker.txt'), { force: true });
+ });
+
+ test('--setup repairs the worktree so mutants can be scored', async () => {
+ await writeFile(path.join(repo, 'untracked-marker.txt'), 'present\n');
+ const report = await runCanary({
+ root: repo,
+ command: 'node -e "require(\'fs\').accessSync(\'untracked-marker.txt\')" && node verify.mjs',
+ setup: 'node -e "require(\'fs\').writeFileSync(\'untracked-marker.txt\',\'x\')"',
+ count: 1,
+ timeoutMs: 60_000,
+ scope: 'src',
+ });
+ assert.equal(report.mutationScore, 1);
+ await rm(path.join(repo, 'untracked-marker.txt'), { force: true });
+ });
+
+ test('leaves no worktree behind', async () => {
+ const entries = await readdir(repo);
+ assert.deepEqual(entries.filter((e) => e.startsWith('.loop-drill-')), []);
+ const { stdout } = await exec('git', ['-C', repo, 'worktree', 'list']);
+ assert.equal(stdout.trim().split('\n').length, 1, 'only the main worktree should remain');
+ });
+});
diff --git a/tools/loop-drill/test/drill.test.mjs b/tools/loop-drill/test/drill.test.mjs
new file mode 100644
index 0000000..31f2d27
--- /dev/null
+++ b/tools/loop-drill/test/drill.test.mjs
@@ -0,0 +1,239 @@
+import { test, describe } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+ samplePathFor,
+ runGateDrills,
+ runBreakerDrills,
+ buildReport,
+ exitCodeFor,
+} from '../dist/drill.js';
+import { DEFAULT_BREAKER } from '@cobusgreyling/loop-context';
+
+const GOOD_GATE = {
+ version: 1,
+ denylist: ['**/.env', '**/secrets/**', '**/*_key*', '**/k8s/production/**'],
+ maxFiles: 10,
+ autoMergeAllowlist: ['docs/**', '**/*.md'],
+};
+
+const outcomes = (results, id) => results.filter((r) => r.id.startsWith(id)).map((r) => r.outcome);
+const byId = (results, id) => results.find((r) => r.id === id);
+
+describe('samplePathFor', () => {
+ test('expands ** into a directory segment and appends a filename', () => {
+ assert.equal(samplePathFor('**/secrets/**'), 'src/secrets/src/drill-file.txt');
+ });
+
+ test('keeps a literal filename under a ** prefix', () => {
+ assert.equal(samplePathFor('**/.env'), 'src/.env');
+ });
+
+ test('fills inline stars so the path still matches the glob', () => {
+ // '*_key*' -> 'drill' + '_key' + 'sample'; still matches the glob.
+ assert.equal(samplePathFor('**/*_key*'), 'src/drill_keysample');
+ });
+
+ test('handles a dotted suffix glob', () => {
+ assert.equal(samplePathFor('**/.env.*'), 'src/.env.sample');
+ });
+});
+
+describe('runGateDrills', () => {
+ const results = runGateDrills({ config: GOOD_GATE });
+
+ test('every denylist entry is proven to block', () => {
+ for (const glob of GOOD_GATE.denylist) {
+ assert.deepEqual(outcomes(results, `gate.denylist[${glob}]`), ['passed'], `glob ${glob}`);
+ }
+ });
+
+ test('the file-count cap is proven', () => {
+ assert.equal(byId(results, 'gate.file-count').outcome, 'passed');
+ });
+
+ test('auto-merge rejects a non-allowlisted path', () => {
+ assert.equal(byId(results, 'gate.auto-merge').outcome, 'passed');
+ });
+
+ test('a benign change is still allowed (specificity)', () => {
+ const benign = byId(results, 'gate.benign');
+ assert.equal(benign.outcome, 'passed');
+ assert.equal(benign.direction, 'specificity');
+ });
+
+ test('every gate drill is attributed to the Over-Reach failure mode', () => {
+ for (const r of results) assert.equal(r.failureMode, 'Over-Reach (Wrong Scope)');
+ });
+
+ test('skips the file-count drill when no maxFiles is configured', () => {
+ const { maxFiles, ...noCap } = GOOD_GATE;
+ const r = byId(runGateDrills({ config: noCap }), 'gate.file-count');
+ assert.equal(r.outcome, 'skipped');
+ assert.match(r.detail, /no maxFiles/);
+ });
+
+ test('fails a gate that blocks everything, even though it catches the fault', () => {
+ const paranoid = { ...GOOD_GATE, denylist: ['**'] };
+ const r = runGateDrills({ config: paranoid });
+ // The denylist does catch the seeded fault...
+ assert.equal(byId(r, 'gate.denylist[**]').outcome, 'passed');
+ // ...but it blocks a benign change, so it is not a working guardrail.
+ assert.equal(byId(r, 'gate.benign').outcome, 'failed');
+ });
+
+ test('a catch-all denylist leaves the file-count rule unproven', () => {
+ // Every synthetic path is swallowed by the denylist before file-count is
+ // reached, so the cap is never actually exercised -- reported, not hidden.
+ const r = runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } });
+ const fileCount = byId(r, 'gate.file-count');
+ assert.equal(fileCount.outcome, 'failed');
+ assert.match(fileCount.actual, /blocked by denylist/);
+ });
+
+ test('honours a caller-supplied benign path', () => {
+ const r = runGateDrills({ config: GOOD_GATE, benignPath: 'src/app/index.ts' });
+ assert.equal(byId(r, 'gate.benign').outcome, 'passed');
+ });
+});
+
+describe('runBreakerDrills', () => {
+ const results = runBreakerDrills(DEFAULT_BREAKER);
+
+ test('stagnation is proven', () => {
+ assert.equal(byId(results, 'breaker.stagnation').outcome, 'passed');
+ });
+
+ test('no-progress is proven', () => {
+ assert.equal(byId(results, 'breaker.no-progress').outcome, 'passed');
+ });
+
+ test('a healthy run is not escalated (specificity)', () => {
+ assert.equal(byId(results, 'breaker.healthy').outcome, 'passed');
+ });
+
+ test('token budget is skipped when unset, and proven when set', () => {
+ assert.equal(byId(results, 'breaker.token-budget').outcome, 'skipped');
+ const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });
+ assert.equal(byId(withBudget, 'breaker.token-budget').outcome, 'passed');
+ });
+
+ // No wrong-trigger test for breaker.token-budget: its ledger is a single
+ // successful attempt, so stagnation/no-progress (which need failures) and
+ // max-iterations (checked after the budget) are all unreachable. There is no
+ // config in which it escalates for another reason, so weakening its
+ // assertion to decision.escalate is not observable from outside.
+
+ test('token budget drill is attributed to Token Burn', () => {
+ const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });
+ assert.equal(byId(withBudget, 'breaker.token-budget').failureMode, 'Token Burn');
+ });
+
+ test('each drill is credited only for its own trigger', () => {
+ // Escalating for another reason leaves the rule under test unproven, so the
+ // reported reason must name the rule the drill exercises.
+ assert.match(byId(results, 'breaker.stagnation').expected, /stagnation/);
+ assert.match(byId(results, 'breaker.no-progress').expected, /no-progress/);
+ assert.match(byId(results, 'breaker.no-progress').actual, /no.progress/i);
+ });
+
+ test('a percentage similarityThreshold leaves stagnation unproven', () => {
+ // similarityThreshold is a 0.0-1.0 fraction. Setting it to 95 (meaning
+ // "95%") means identical errors never compare as similar, so a loop retrying
+ // the same failure forever is never stopped. The drill must report that.
+ const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 });
+ const stagnation = byId(misconfigured, 'breaker.stagnation');
+ assert.equal(stagnation.outcome, 'failed');
+ assert.equal(stagnation.actual, 'continued');
+ assert.match(stagnation.detail, /Infinite Fix Loop/);
+ assert.match(stagnation.detail, /similarityThreshold/);
+ });
+
+ test('escalating via the wrong trigger leaves stagnation unproven', () => {
+ // The case that separates "the breaker fired" from "the rule under test
+ // fired". Here the breaker DOES escalate, so a drill asserting only
+ // decision.escalate would pass while stagnation was never exercised.
+ const wrongTrigger = runBreakerDrills({
+ ...DEFAULT_BREAKER,
+ similarityThreshold: 95,
+ noProgressThreshold: 1,
+ });
+ const stagnation = byId(wrongTrigger, 'breaker.stagnation');
+ assert.equal(stagnation.outcome, 'failed');
+ assert.equal(stagnation.actual, 'escalated via no-progress');
+ assert.match(stagnation.detail, /not by the rule under test/);
+ });
+
+ test('an iteration cap that fires first also leaves stagnation unproven', () => {
+ const capped = runBreakerDrills({
+ ...DEFAULT_BREAKER,
+ similarityThreshold: 95,
+ maxIterations: 2,
+ });
+ const stagnation = byId(capped, 'breaker.stagnation');
+ assert.equal(stagnation.outcome, 'failed');
+ assert.equal(stagnation.actual, 'escalated via max-iterations');
+ });
+
+ test('stagnation swallowing every error leaves no-progress unproven', () => {
+ // similarityThreshold 0 makes every error count as "the same error", so
+ // stagnation fires on the no-progress drill's unrelated failures. The
+ // breaker still escalates, so only a trigger-specific assertion catches it.
+ const swallowed = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 0 });
+ const noProgress = byId(swallowed, 'breaker.no-progress');
+ assert.equal(noProgress.outcome, 'failed');
+ assert.equal(noProgress.actual, 'escalated via stagnation');
+ assert.match(noProgress.detail, /not by the rule under test/);
+ });
+
+ test('no-progress uses errors that stay distinct after signature normalization', () => {
+ // errorSignature() collapses every number to '#', so errors differing only
+ // by a number ("module 1" / "module 2") normalize to the same signature and
+ // trip stagnation instead -- passing the drill while leaving no-progress
+ // unexercised. This asserts the real trigger, which is what catches that.
+ const r = byId(results, 'breaker.no-progress');
+ assert.equal(r.outcome, 'passed');
+ assert.doesNotMatch(r.actual, /stagnation/i);
+ });
+});
+
+describe('buildReport / exitCodeFor', () => {
+ test('a failure mode with any failing drill is not counted as covered', () => {
+ const report = buildReport([
+ ...runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }),
+ ]);
+ assert.ok(!report.covered.includes('Over-Reach (Wrong Scope)'));
+ assert.ok(report.uncovered.includes('Over-Reach (Wrong Scope)'));
+ });
+
+ test('a fully passing gate run covers Over-Reach', () => {
+ const report = buildReport(runGateDrills({ config: GOOD_GATE }));
+ assert.ok(report.covered.includes('Over-Reach (Wrong Scope)'));
+ });
+
+ test('unexercised failure modes are reported as uncovered', () => {
+ const report = buildReport(runGateDrills({ config: GOOD_GATE }));
+ assert.ok(report.uncovered.includes('Verifier Theater'));
+ assert.ok(report.uncovered.includes('Escalation Failure'));
+ });
+
+ test('exit code 0 when everything passes', () => {
+ assert.equal(exitCodeFor(buildReport(runGateDrills({ config: GOOD_GATE }))), 0);
+ });
+
+ test('exit code 1 when a drill is skipped', () => {
+ const { maxFiles, ...noCap } = GOOD_GATE;
+ assert.equal(exitCodeFor(buildReport(runGateDrills({ config: noCap }))), 1);
+ });
+
+ test('exit code 2 when a guardrail fails to fire', () => {
+ const report = buildReport(runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }));
+ assert.equal(exitCodeFor(report), 2);
+ });
+
+ test('failure beats skip in the exit code', () => {
+ const { maxFiles, ...noCap } = GOOD_GATE;
+ const report = buildReport(runGateDrills({ config: { ...noCap, denylist: ['**'] } }));
+ assert.ok(report.skipped > 0 && report.failed > 0);
+ assert.equal(exitCodeFor(report), 2);
+ });
+});
diff --git a/tools/loop-drill/test/mutations.test.mjs b/tools/loop-drill/test/mutations.test.mjs
new file mode 100644
index 0000000..cf5336c
--- /dev/null
+++ b/tools/loop-drill/test/mutations.test.mjs
@@ -0,0 +1,97 @@
+import { test, describe } from 'node:test';
+import assert from 'node:assert/strict';
+import { mutate, isMutationCandidate, OPERATORS } from '../dist/mutations.js';
+
+describe('mutate', () => {
+ test('flips the first === it finds', () => {
+ const m = mutate('const ok = a === b;\n', 'src/a.ts');
+ assert.equal(m.operator.id, 'strict-equality-flip');
+ assert.equal(m.mutated, 'const ok = a !== b;\n');
+ assert.equal(m.line, 1);
+ });
+
+ test('mutates only the first occurrence, so one mutant means one defect', () => {
+ const m = mutate('a === b;\nc === d;\n', 'src/a.ts');
+ assert.equal(m.mutated, 'a !== b;\nc === d;\n');
+ });
+
+ test('reports the line number and the before/after text', () => {
+ const m = mutate('line one\nconst n = x <= y;\n', 'src/a.ts');
+ assert.equal(m.line, 2);
+ assert.equal(m.before, 'const n = x <= y;');
+ assert.equal(m.after, 'const n = x < y;');
+ });
+
+ test('skips a match inside a // comment', () => {
+ const m = mutate('// a === b is fine\nreturn c === d;\n', 'src/a.ts');
+ assert.equal(m.line, 2);
+ assert.equal(m.mutated, '// a === b is fine\nreturn c !== d;\n');
+ });
+
+ test('skips a match inside a # comment', () => {
+ const m = mutate('# x <= y\nif a <= b:\n', 'src/a.py');
+ assert.equal(m.line, 2);
+ });
+
+ test('returns null when nothing is mutable', () => {
+ assert.equal(mutate('const greeting = "hello";\n', 'src/a.ts'), null);
+ });
+
+ test('flips boolean returns', () => {
+ const m = mutate('function isValid() { return true; }\n', 'src/a.ts');
+ assert.equal(m.operator.id, 'return-true-to-false');
+ assert.match(m.mutated, /return false/);
+ });
+
+ test('flips && to ||', () => {
+ const m = mutate('if (a && b) run();\n', 'src/a.ts');
+ assert.equal(m.operator.id, 'logical-and-to-or');
+ assert.equal(m.mutated, 'if (a || b) run();\n');
+ });
+
+ test('every operator actually changes the content it matches', () => {
+ for (const op of OPERATORS) {
+ assert.notEqual(op.replace, op.pattern.source, `operator ${op.id} is a no-op`);
+ }
+ });
+
+ test('excludes ambiguous operators that would produce false mutants', () => {
+ // `<` -> `>` breaks TS generics and JSX; `+` -> `-` mangles string
+ // concatenation. Either would make a correct verifier look broken.
+ const ids = OPERATORS.map((o) => o.id);
+ assert.ok(!ids.includes('less-than-flip'));
+ assert.ok(!ids.includes('arithmetic-flip'));
+ });
+});
+
+describe('isMutationCandidate', () => {
+ test('accepts ordinary source', () => {
+ assert.ok(isMutationCandidate('src/index.ts'));
+ assert.ok(isMutationCandidate('app/main/Service.java'));
+ assert.ok(isMutationCandidate('pkg/server.go'));
+ });
+
+ test('rejects build output and dependencies', () => {
+ assert.ok(!isMutationCandidate('dist/index.js'));
+ assert.ok(!isMutationCandidate('node_modules/left-pad/index.js'));
+ assert.ok(!isMutationCandidate('coverage/lcov-report/app.js'));
+ assert.ok(!isMutationCandidate('vendor/lib.go'));
+ });
+
+ test('rejects tests — mutating them would make the canary trivial', () => {
+ assert.ok(!isMutationCandidate('test/app.test.mjs'));
+ assert.ok(!isMutationCandidate('src/__tests__/app.js'));
+ assert.ok(!isMutationCandidate('spec/thing.spec.ts'));
+ });
+
+ test('rejects non-source files', () => {
+ assert.ok(!isMutationCandidate('README.md'));
+ assert.ok(!isMutationCandidate('package.json'));
+ assert.ok(!isMutationCandidate('gate.yaml'));
+ });
+
+ test('normalizes Windows separators', () => {
+ assert.ok(!isMutationCandidate('dist\\index.js'));
+ assert.ok(isMutationCandidate('src\\index.ts'));
+ });
+});
diff --git a/tools/loop-drill/tsconfig.json b/tools/loop-drill/tsconfig.json
new file mode 100644
index 0000000..ef0f184
--- /dev/null
+++ b/tools/loop-drill/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
+ "outDir": "dist",
+ "rootDir": "src",
+ "strict": true,
+ "skipLibCheck": true,
+ "declaration": true
+ },
+ "include": ["src/**/*"]
+}