Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
#!/usr/bin/env node
/**
* loop-drill CLI.
*
* Exit codes follow loop-gate / loop-context so control scripts can chain all
* three: 0 proceed, 1 warnings, 2 escalate.
*/
import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { loadGateConfig } from '@cobusgreyling/loop-gate';
import { DEFAULT_BREAKER, type CircuitBreakerConfig } from '@cobusgreyling/loop-context';
import {
buildReport,
exitCodeFor,
runBreakerDrills,
runGateDrills,
skip,
type DrillResult,
} from './drill.js';
import { runCanary } from './canary.js';
import { runInjectionCanary } from './injection.js';
import { formatReport } from './report.js';
import { buildGroups, fingerprint, RECORD_FILE, writeRecord } from './record.js';
interface Flags {
root: string;
json: boolean;
help: boolean;
gateFile?: string;
benignPath?: string;
only?: string;
verifierCmd?: string;
agentCmd?: string;
stateFile: string;
mutants: number;
timeoutMs: number;
scope?: string;
setup?: string;
tokenBudget?: number;
record: boolean;
}
const HELP = `loop-drill — fire drills for loop guardrails
Injects a known fault and asserts the guardrail actually fires. Turns L3
readiness from "the files exist" into "the guardrails demonstrably work".
Usage: loop-drill [path] [options]
Options:
--only <drills> Comma-separated: gate, breaker, verifier, injection
(default: gate,breaker)
--verifier-cmd <cmd> Verifier command to drill. Non-zero exit = rejected.
Required to run the verifier canary.
--agent-cmd <cmd> The loop's agent command (e.g. your triage run).
Required to run the injection canary.
--state-file <path> State file the agent reads (default: STATE.md)
--mutants <n> Seeded defects for the canary (default: 3)
--scope <path> Restrict mutation to this repo-relative path
--setup <cmd> Command run in each worktree before verifying
(e.g. "npm ci") — a fresh worktree has no node_modules
--timeout <ms> Per-verifier-run timeout (default: 120000)
--gate-file <path> Policy file (default: gate.yaml)
--benign-path <path> Path the gate specificity drill treats as ordinary
(default: docs/README.md)
--token-budget <n> Token budget for the breaker drill
--record Write the results to loop-drill.json, which loop-audit
reads as proof the guardrails fire. Replaces only the
guardrails drilled this run; commit the file
--json Machine-readable output
--help, -h Show this message
Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.
Examples:
loop-drill .
loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"
loop-drill . --only injection --agent-cmd "claude -p 'run the loop-triage skill'"
loop-drill . --only gate,breaker --json
loop-drill . --record
`;
function parseArgs(argv: string[]): Flags {
const flags: Flags = {
root: '.',
json: false,
help: false,
mutants: 3,
timeoutMs: 120_000,
stateFile: 'STATE.md',
record: false,
};
const positional: string[] = [];
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
const next = () => {
const v = argv[++i];
if (v === undefined) throw new Error(`${arg} requires a value`);
return v;
};
switch (arg) {
case '--help':
case '-h':
flags.help = true;
break;
case '--json':
flags.json = true;
break;
case '--record':
flags.record = true;
break;
case '--only':
flags.only = next();
break;
case '--verifier-cmd':
flags.verifierCmd = next();
break;
case '--agent-cmd':
flags.agentCmd = next();
break;
case '--state-file':
flags.stateFile = next();
break;
case '--mutants':
flags.mutants = parsePositiveInt(next(), '--mutants');
break;
case '--scope':
flags.scope = next();
break;
case '--setup':
flags.setup = next();
break;
case '--timeout':
flags.timeoutMs = parsePositiveInt(next(), '--timeout');
break;
case '--gate-file':
flags.gateFile = next();
break;
case '--benign-path':
flags.benignPath = next();
break;
case '--token-budget':
flags.tokenBudget = parsePositiveInt(next(), '--token-budget');
break;
default:
if (arg.startsWith('-')) throw new Error(`Unknown option: ${arg}`);
positional.push(arg);
}
}
if (positional.length > 0) flags.root = positional[0];
return flags;
}
function parsePositiveInt(value: string, flag: string): number {
const n = Number(value);
if (!Number.isInteger(n) || n <= 0) throw new Error(`${flag} expects a positive integer, got: ${value}`);
return n;
}
async function main(): Promise<void> {
let flags: Flags;
try {
flags = parseArgs(process.argv.slice(2));
} catch (err) {
console.error(`${(err as Error).message}\n`);
console.error(HELP);
process.exit(2);
}
if (flags.help) {
console.log(HELP);
return;
}
const root = path.resolve(flags.root);
const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));
const results: DrillResult[] = [];
let mutationScore: number | null = null;
const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');
if (selected.has('gate')) {
try {
const config = await loadGateConfig(gateFile);
results.push(...runGateDrills({ config, benignPath: flags.benignPath }));
} catch (err) {
results.push(
skip(
'gate',
'gate drills',
'Over-Reach (Wrong Scope)',
'sensitivity',
`${(err as Error).message} Nothing mechanically stops a loop from editing a sensitive path.`,
),
);
}
}
if (selected.has('breaker')) {
const config: CircuitBreakerConfig =
flags.tokenBudget === undefined
? DEFAULT_BREAKER
: { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };
results.push(...runBreakerDrills(config));
}
if (selected.has('verifier')) {
if (!flags.verifierCmd) {
results.push(
skip(
'verifier',
'verifier canary',
'Verifier Theater',
'sensitivity',
'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.',
),
);
} else {
const canary = await runCanary({
root,
command: flags.verifierCmd,
count: flags.mutants,
timeoutMs: flags.timeoutMs,
scope: flags.scope,
setup: flags.setup,
});
results.push(...canary.results);
mutationScore = canary.mutationScore;
}
}
let resistanceScore: number | null = null;
if (selected.has('injection')) {
if (!flags.agentCmd) {
results.push(
skip(
'injection',
'injection canary',
'Prompt Injection via Untrusted Input',
'sensitivity',
'No --agent-cmd given. Guidance in a skill says what the agent should do; only running the agent against a planted instruction shows what it does.',
),
);
} else {
const injection = await runInjectionCanary({
root,
command: flags.agentCmd,
stateFile: flags.stateFile,
timeoutMs: flags.timeoutMs,
setup: flags.setup,
});
results.push(...injection.results);
resistanceScore = injection.resistanceScore;
}
}
if (results.length === 0) {
console.error(`No drills selected. --only accepts: gate, breaker, verifier, injection\n`);
process.exit(2);
}
const report = buildReport(results);
const code = exitCodeFor(report);
if (flags.json) {
console.log(JSON.stringify({ ...report, mutationScore, resistanceScore, exitCode: code }, null, 2));
} else {
console.log(formatReport(report, mutationScore, resistanceScore));
}
if (flags.record) {
// Failures are recorded too: loop-audit withdraws credit for a guardrail
// that was shown not to fire.
const gateText = await readFile(gateFile, 'utf8').catch(() => null);
const groups = buildGroups(results, selected, {
recordedAt: report.timestamp,
gateFile: path.relative(root, gateFile).split(path.sep).join('/'),
gateSha256: gateText === null ? null : fingerprint(gateText),
verifierCommand: flags.verifierCmd,
mutationScore,
});
await writeRecord(root, groups);
const note = `Recorded ${Object.keys(groups).join(', ')} to ${RECORD_FILE} — commit it so loop-audit can see the proof.`;
if (flags.json) console.error(note);
else console.log(note);
}
process.exit(code);
}
main().catch((err) => {
console.error(`loop-drill failed: ${(err as Error).message}`);
process.exit(2);
});