Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

9.0 KB
Raw Blame
#!/usr/bin/env node
/**
 * loop-drill CLI.
 *
 * Exit codes follow loop-gate / loop-context so control scripts can chain all
 * three: 0 proceed, 1 warnings, 2 escalate.
 */

import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { loadGateConfig } from '@cobusgreyling/loop-gate';
import { DEFAULT_BREAKER, type CircuitBreakerConfig } from '@cobusgreyling/loop-context';
import {
  buildReport,
  exitCodeFor,
  runBreakerDrills,
  runGateDrills,
  skip,
  type DrillResult,
} from './drill.js';
import { runCanary } from './canary.js';
import { runInjectionCanary } from './injection.js';
import { formatReport } from './report.js';
import { buildGroups, fingerprint, RECORD_FILE, writeRecord } from './record.js';

interface Flags {
  root: string;
  json: boolean;
  help: boolean;
  gateFile?: string;
  benignPath?: string;
  only?: string;
  verifierCmd?: string;
  agentCmd?: string;
  stateFile: string;
  mutants: number;
  timeoutMs: number;
  scope?: string;
  setup?: string;
  tokenBudget?: number;
  record: boolean;
}

const HELP = `loop-drill — fire drills for loop guardrails

Injects a known fault and asserts the guardrail actually fires. Turns L3
readiness from "the files exist" into "the guardrails demonstrably work".

Usage: loop-drill [path] [options]

Options:
  --only <drills>        Comma-separated: gate, breaker, verifier, injection
                         (default: gate,breaker)
  --verifier-cmd <cmd>   Verifier command to drill. Non-zero exit = rejected.
                         Required to run the verifier canary.
  --agent-cmd <cmd>      The loop's agent command (e.g. your triage run).
                         Required to run the injection canary.
  --state-file <path>    State file the agent reads (default: STATE.md)
  --mutants <n>          Seeded defects for the canary (default: 3)
  --scope <path>         Restrict mutation to this repo-relative path
  --setup <cmd>          Command run in each worktree before verifying
                         (e.g. "npm ci") — a fresh worktree has no node_modules
  --timeout <ms>         Per-verifier-run timeout (default: 120000)
  --gate-file <path>     Policy file (default: gate.yaml)
  --benign-path <path>   Path the gate specificity drill treats as ordinary
                         (default: docs/README.md)
  --token-budget <n>     Token budget for the breaker drill
  --record               Write the results to loop-drill.json, which loop-audit
                         reads as proof the guardrails fire. Replaces only the
                         guardrails drilled this run; commit the file
  --json                 Machine-readable output
  --help, -h             Show this message

Exit codes: 0 all drills passed, 1 some skipped, 2 a guardrail failed to fire.

Examples:
  loop-drill .
  loop-drill . --only verifier --verifier-cmd "npm test" --setup "npm ci"
  loop-drill . --only injection --agent-cmd "claude -p 'run the loop-triage skill'"
  loop-drill . --only gate,breaker --json
  loop-drill . --record
`;

function parseArgs(argv: string[]): Flags {
  const flags: Flags = {
    root: '.',
    json: false,
    help: false,
    mutants: 3,
    timeoutMs: 120_000,
    stateFile: 'STATE.md',
    record: false,
  };
  const positional: string[] = [];

  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    const next = () => {
      const v = argv[++i];
      if (v === undefined) throw new Error(`${arg} requires a value`);
      return v;
    };
    switch (arg) {
      case '--help':
      case '-h':
        flags.help = true;
        break;
      case '--json':
        flags.json = true;
        break;
      case '--record':
        flags.record = true;
        break;
      case '--only':
        flags.only = next();
        break;
      case '--verifier-cmd':
        flags.verifierCmd = next();
        break;
      case '--agent-cmd':
        flags.agentCmd = next();
        break;
      case '--state-file':
        flags.stateFile = next();
        break;
      case '--mutants':
        flags.mutants = parsePositiveInt(next(), '--mutants');
        break;
      case '--scope':
        flags.scope = next();
        break;
      case '--setup':
        flags.setup = next();
        break;
      case '--timeout':
        flags.timeoutMs = parsePositiveInt(next(), '--timeout');
        break;
      case '--gate-file':
        flags.gateFile = next();
        break;
      case '--benign-path':
        flags.benignPath = next();
        break;
      case '--token-budget':
        flags.tokenBudget = parsePositiveInt(next(), '--token-budget');
        break;
      default:
        if (arg.startsWith('-')) throw new Error(`Unknown option: ${arg}`);
        positional.push(arg);
    }
  }

  if (positional.length > 0) flags.root = positional[0];
  return flags;
}

function parsePositiveInt(value: string, flag: string): number {
  const n = Number(value);
  if (!Number.isInteger(n) || n <= 0) throw new Error(`${flag} expects a positive integer, got: ${value}`);
  return n;
}

async function main(): Promise<void> {
  let flags: Flags;
  try {
    flags = parseArgs(process.argv.slice(2));
  } catch (err) {
    console.error(`${(err as Error).message}\n`);
    console.error(HELP);
    process.exit(2);
  }

  if (flags.help) {
    console.log(HELP);
    return;
  }

  const root = path.resolve(flags.root);
  const selected = new Set((flags.only ?? 'gate,breaker').split(',').map((s) => s.trim()).filter(Boolean));
  const results: DrillResult[] = [];
  let mutationScore: number | null = null;

  const gateFile = path.resolve(root, flags.gateFile ?? 'gate.yaml');
  if (selected.has('gate')) {
    try {
      const config = await loadGateConfig(gateFile);
      results.push(...runGateDrills({ config, benignPath: flags.benignPath }));
    } catch (err) {
      results.push(
        skip(
          'gate',
          'gate drills',
          'Over-Reach (Wrong Scope)',
          'sensitivity',
          `${(err as Error).message} Nothing mechanically stops a loop from editing a sensitive path.`,
        ),
      );
    }
  }

  if (selected.has('breaker')) {
    const config: CircuitBreakerConfig =
      flags.tokenBudget === undefined
        ? DEFAULT_BREAKER
        : { ...DEFAULT_BREAKER, tokenBudget: flags.tokenBudget };
    results.push(...runBreakerDrills(config));
  }

  if (selected.has('verifier')) {
    if (!flags.verifierCmd) {
      results.push(
        skip(
          'verifier',
          'verifier canary',
          'Verifier Theater',
          'sensitivity',
          'No --verifier-cmd given. loop-audit scores a verifier by filename alone; without a command to run, this drill cannot tell a real verifier from an empty file.',
        ),
      );
    } else {
      const canary = await runCanary({
        root,
        command: flags.verifierCmd,
        count: flags.mutants,
        timeoutMs: flags.timeoutMs,
        scope: flags.scope,
        setup: flags.setup,
      });
      results.push(...canary.results);
      mutationScore = canary.mutationScore;
    }
  }

  let resistanceScore: number | null = null;
  if (selected.has('injection')) {
    if (!flags.agentCmd) {
      results.push(
        skip(
          'injection',
          'injection canary',
          'Prompt Injection via Untrusted Input',
          'sensitivity',
          'No --agent-cmd given. Guidance in a skill says what the agent should do; only running the agent against a planted instruction shows what it does.',
        ),
      );
    } else {
      const injection = await runInjectionCanary({
        root,
        command: flags.agentCmd,
        stateFile: flags.stateFile,
        timeoutMs: flags.timeoutMs,
        setup: flags.setup,
      });
      results.push(...injection.results);
      resistanceScore = injection.resistanceScore;
    }
  }

  if (results.length === 0) {
    console.error(`No drills selected. --only accepts: gate, breaker, verifier, injection\n`);
    process.exit(2);
  }

  const report = buildReport(results);
  const code = exitCodeFor(report);

  if (flags.json) {
    console.log(JSON.stringify({ ...report, mutationScore, resistanceScore, exitCode: code }, null, 2));
  } else {
    console.log(formatReport(report, mutationScore, resistanceScore));
  }

  if (flags.record) {
    // Failures are recorded too: loop-audit withdraws credit for a guardrail
    // that was shown not to fire.
    const gateText = await readFile(gateFile, 'utf8').catch(() => null);
    const groups = buildGroups(results, selected, {
      recordedAt: report.timestamp,
      gateFile: path.relative(root, gateFile).split(path.sep).join('/'),
      gateSha256: gateText === null ? null : fingerprint(gateText),
      verifierCommand: flags.verifierCmd,
      mutationScore,
    });
    await writeRecord(root, groups);
    const note = `Recorded ${Object.keys(groups).join(', ')} to ${RECORD_FILE} — commit it so loop-audit can see the proof.`;
    if (flags.json) console.error(note);
    else console.log(note);
  }

  process.exit(code);
}

main().catch((err) => {
  console.error(`loop-drill failed: ${(err as Error).message}`);
  process.exit(2);
});