Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

17.1 KB
Raw Blame
/**
 * loop-drill — fire drills for loop guardrails.
 *
 * docs/failure-modes.md names the ways loops fail. Some have mechanical
 * counterparts (loop-gate, loop-context's circuit breaker); loop-audit scores
 * whether those counterparts are *present*. Nothing checked whether they
 * actually fire.
 *
 * A drill injects a known fault and asserts the guardrail responds. Every
 * drill carries the failure mode it exercises, so the report reads as coverage
 * of docs/failure-modes.md rather than a list of anonymous assertions.
 *
 * Each guardrail gets both directions, because only one of them is easy:
 *   - sensitivity: the fault is caught (a guardrail that never fires is theater)
 *   - specificity: benign input is NOT caught (a guardrail that blocks
 *     everything passes every sensitivity drill while being useless)
 * A guardrail is only credited when it passes both.
 */

import { checkGate, type GateConfig } from '@cobusgreyling/loop-gate';
import {
  checkCircuitBreaker,
  DEFAULT_BREAKER,
  type BreakerDecision,
  type CircuitBreakerConfig,
  type Ledger,
} from '@cobusgreyling/loop-context';

export type DrillOutcome = 'passed' | 'failed' | 'skipped';

/** Named failure modes from docs/failure-modes.md. */
export type FailureMode =
  | 'Infinite Fix Loop'
  | 'Token Burn'
  | 'Over-Reach (Wrong Scope)'
  | 'Verifier Theater'
  | 'Escalation Failure'
  | 'Prompt Injection via Untrusted Input';

export interface DrillResult {
  /** Stable id, e.g. 'gate.denylist'. */
  id: string;
  name: string;
  /** The docs/failure-modes.md entry this drill exercises. */
  failureMode: FailureMode;
  /** 'sensitivity' = fault must be caught; 'specificity' = benign must pass. */
  direction: 'sensitivity' | 'specificity';
  outcome: DrillOutcome;
  expected: string;
  actual: string;
  /** Why a drill was skipped, or extra context on a failure. */
  detail?: string;
}

export interface DrillReport {
  results: DrillResult[];
  passed: number;
  failed: number;
  skipped: number;
  /** Failure modes with at least one passing drill and no failing drill. */
  covered: FailureMode[];
  /** Failure modes with no passing drill (unexercised or broken). */
  uncovered: FailureMode[];
  timestamp: string;
}

function pass(
  id: string,
  name: string,
  failureMode: FailureMode,
  direction: DrillResult['direction'],
  expected: string,
  actual: string,
): DrillResult {
  return { id, name, failureMode, direction, outcome: 'passed', expected, actual };
}

function fail(
  id: string,
  name: string,
  failureMode: FailureMode,
  direction: DrillResult['direction'],
  expected: string,
  actual: string,
  detail?: string,
): DrillResult {
  return { id, name, failureMode, direction, outcome: 'failed', expected, actual, detail };
}

export function skip(
  id: string,
  name: string,
  failureMode: FailureMode,
  direction: DrillResult['direction'],
  detail: string,
): DrillResult {
  return {
    id,
    name,
    failureMode,
    direction,
    outcome: 'skipped',
    expected: '—',
    actual: '—',
    detail,
  };
}

// ── Glob → concrete sample path ────────────────────────────────────

/**
 * Turn a denylist glob into a concrete path that should match it, so the drill
 * exercises the real matcher rather than re-implementing glob semantics.
 *
 * Deliberately conservative: `**` becomes a fixed segment, `*` becomes a
 * literal, and the caller verifies the synthesized path actually matches
 * before asserting anything about it (see runGateDrills) — a glob we
 * synthesize badly is reported as skipped, never as a guardrail failure.
 */
export function samplePathFor(glob: string): string {
  const concrete = glob
    .split('/')
    .map((segment) => {
      if (segment === '**') return 'src';
      if (segment === '*') return 'sample';
      // '*_key*' -> 'drill_key_sample'; '.env.*' -> '.env.sample'
      return segment.replace(/\*/g, (_m, offset: number) => (offset === 0 ? 'drill' : 'sample'));
    })
    .join('/');

  // A trailing '**' segment became 'src', which is a directory, not a file.
  return /(^|\/)src$/.test(concrete) ? `${concrete}/drill-file.txt` : concrete;
}

// ── Gate drills (Over-Reach) ───────────────────────────────────────

export interface GateDrillOptions {
  config: GateConfig;
  /** Path assumed benign for the specificity drill. */
  benignPath?: string;
}

/**
 * Exercise loop-gate against its own gate.yaml: every denylist entry must
 * block, the file-count cap must block, auto-merge must reject a
 * non-allowlisted path, and an ordinary doc change must still be allowed.
 */
export function runGateDrills(options: GateDrillOptions): DrillResult[] {
  const { config, benignPath = 'docs/README.md' } = options;
  const results: DrillResult[] = [];
  const mode: FailureMode = 'Over-Reach (Wrong Scope)';

  // Sensitivity: each denylist glob blocks a path that matches it.
  for (const glob of config.denylist) {
    const id = `gate.denylist[${glob}]`;
    const name = `denylist blocks ${glob}`;
    const sample = samplePathFor(glob);
    const decision = checkGate({ config, action: 'commit', paths: [sample] });

    if (decision.allowed) {
      results.push(
        fail(
          id,
          name,
          mode,
          'sensitivity',
          `commit touching ${sample} is blocked`,
          `allowed (${decision.trigger})`,
          `Denylist entry "${glob}" did not match the sample path this drill synthesized for it. Either the glob is unreachable in practice, or the sample is unrepresentative — check it by hand with: loop-gate check --action commit --paths ${sample}`,
        ),
      );
      continue;
    }

    if (decision.trigger !== 'denylist') {
      // Blocked, but by file-count or the allowlist — the denylist itself is
      // unproven, so this is not a pass.
      results.push(
        fail(
          id,
          name,
          mode,
          'sensitivity',
          `blocked by the denylist`,
          `blocked by ${decision.trigger}`,
          'The path was stopped, but not by the rule under test.',
        ),
      );
      continue;
    }

    results.push(pass(id, name, mode, 'sensitivity', 'blocked by denylist', decision.reason));
  }

  // Sensitivity: the file-count cap fires one file past the threshold.
  if (config.maxFiles !== undefined) {
    const id = 'gate.file-count';
    const name = `file-count cap fires above ${config.maxFiles} files`;
    const paths = Array.from({ length: config.maxFiles + 1 }, (_, i) => `docs/drill-${i}.md`);
    const decision = checkGate({ config, action: 'commit', paths });
    results.push(
      decision.trigger === 'file-count'
        ? pass(id, name, mode, 'sensitivity', 'blocked by file-count', decision.reason)
        : fail(
            id,
            name,
            mode,
            'sensitivity',
            `${paths.length} files is blocked by file-count`,
            decision.allowed ? 'allowed' : `blocked by ${decision.trigger}`,
          ),
    );
  } else {
    results.push(
      skip(
        'gate.file-count',
        'file-count cap',
        mode,
        'sensitivity',
        'gate.yaml sets no maxFiles — a loop may propose an arbitrarily large diff.',
      ),
    );
  }

  // Sensitivity: auto-merge rejects a path outside the allowlist.
  if (config.autoMergeAllowlist && config.autoMergeAllowlist.length > 0) {
    const id = 'gate.auto-merge';
    const name = 'auto-merge rejects a non-allowlisted path';
    const outside = 'src/drill-not-allowlisted.bin';
    const decision = checkGate({ config, action: 'auto-merge', paths: [outside] });
    results.push(
      !decision.allowed
        ? pass(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, decision.reason)
        : fail(id, name, mode, 'sensitivity', `auto-merge of ${outside} is blocked`, 'allowed'),
    );
  }

  // Specificity: an ordinary change is still allowed. A gate that blocks
  // everything would pass every drill above while being useless in practice.
  const benign = checkGate({ config, action: 'commit', paths: [benignPath] });
  results.push(
    benign.allowed
      ? pass(
          'gate.benign',
          'ordinary change is still allowed',
          mode,
          'specificity',
          `commit touching ${benignPath} is allowed`,
          benign.reason,
        )
      : fail(
          'gate.benign',
          'ordinary change is still allowed',
          mode,
          'specificity',
          `commit touching ${benignPath} is allowed`,
          `blocked by ${benign.trigger}`,
          'The gate blocks a benign path. A guardrail that stops everything will pass every sensitivity drill while making the loop unusable — pass --benign-path if this path is genuinely sensitive in your project.',
        ),
  );

  return results;
}

// ── Circuit breaker drills (Infinite Fix Loop / Token Burn) ────────

function ledgerOf(attempts: Ledger['attempts']): Ledger {
  return { goal: 'loop-drill synthetic run', attempts };
}

/**
 * Errors with no shared structure, used for the no-progress drill.
 *
 * They must stay dissimilar *after* errorSignature() normalization, which
 * collapses every number to '#'. Errors that differ only by a number ("module 1
 * not found", "module 2 not found") normalize to the same signature and trip
 * stagnation instead — the drill would pass while leaving no-progress
 * unexercised.
 */
const DISTINCT_ERRORS = [
  'TypeError: undefined is not a function',
  'ECONNREFUSED connecting to the database',
  'SyntaxError: unexpected token in JSON',
  'AssertionError: expected ok but got null',
  'Permission denied while writing the artifact',
  'OutOfMemoryError during the bundle step',
  'Segmentation fault in the native addon',
];

/**
 * Exercise loop-context's circuit breaker with synthetic ledgers: repeated
 * identical failures must trip stagnation, a long run of unrelated failures
 * must trip no-progress, blowing the token budget must trip, and a healthy run
 * must not.
 *
 * Each drill asserts the *specific* trigger, not merely that the breaker
 * escalated. Escalating for another reason means the rule under test is still
 * unproven — the same standard the gate drills apply to `trigger !==
 * 'denylist'`.
 *
 * loop-context throws on a config that would switch a rule off (a similarity
 * threshold of 95 meant as 95%, say). That is reported as a failed drill, not
 * a crash: a breaker that refuses to start protects nothing either.
 *
 * `check` defaults to loop-context's breaker; tests pass a stand-in to
 * exercise the trigger-attribution logic against a breaker that misbehaves.
 */
export type BreakerCheck = (ledger: Ledger, config: CircuitBreakerConfig) => BreakerDecision;

export function runBreakerDrills(
  config: CircuitBreakerConfig = DEFAULT_BREAKER,
  check: BreakerCheck = checkCircuitBreaker,
): DrillResult[] {
  try {
    return breakerDrills(config, check);
  } catch (err) {
    return [
      fail(
        'breaker.config',
        'loop-context accepts the breaker config',
        'Infinite Fix Loop',
        'sensitivity',
        'a config loop-context accepts',
        `rejected: ${(err as Error).message}`,
        'loop-context refuses to run with a rule switched off, so no breaker rule was drilled. Fix the value and re-run.',
      ),
    ];
  }
}

function breakerDrills(config: CircuitBreakerConfig, check: BreakerCheck): DrillResult[] {
  const results: DrillResult[] = [];

  // Sensitivity: the same error repeated trips stagnation.
  {
    const id = 'breaker.stagnation';
    const name = `${config.stagnationThreshold} identical failures trip stagnation`;
    const mode: FailureMode = 'Infinite Fix Loop';
    const attempts = Array.from({ length: config.stagnationThreshold }, (_, i) => ({
      iteration: i + 1,
      action: 'retry the failing test',
      outcome: 'failure' as const,
      error: "TypeError: Cannot read properties of undefined (reading 'id')",
    }));
    const decision = check(ledgerOf(attempts), config);
    results.push(
      decision.trigger === 'stagnation'
        ? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)
        : fail(
            id,
            name,
            mode,
            'sensitivity',
            'escalate via stagnation',
            decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
            decision.escalate
              ? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'
              : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop.',
          ),
    );
  }

  // Sensitivity: a long run of unrelated failures trips no-progress.
  {
    const id = 'breaker.no-progress';
    const name = `${config.noProgressThreshold} unrelated failures trip no-progress`;
    const mode: FailureMode = 'Infinite Fix Loop';
    const attempts = Array.from({ length: config.noProgressThreshold }, (_, i) => ({
      iteration: i + 1,
      action: `attempt strategy ${i + 1}`,
      outcome: 'failure' as const,
      error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],
    }));
    const decision = check(ledgerOf(attempts), config);
    results.push(
      decision.trigger === 'no-progress'
        ? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)
        : fail(
            id,
            name,
            mode,
            'sensitivity',
            'escalate via no-progress',
            decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
            decision.escalate
              ? 'The breaker stopped the loop, but not by the rule under test — no-progress stays unproven.'
              : undefined,
          ),
    );
  }

  // Sensitivity: exceeding the token budget trips.
  if (config.tokenBudget !== undefined) {
    const id = 'breaker.token-budget';
    const name = 'exceeding the token budget trips the breaker';
    const mode: FailureMode = 'Token Burn';
    const attempts = [
      {
        iteration: 1,
        action: 'a very expensive sub-agent chain',
        outcome: 'success' as const,
        tokensUsed: config.tokenBudget + 1,
      },
    ];
    const decision = check(ledgerOf(attempts), config);
    results.push(
      decision.trigger === 'token-budget'
        ? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)
        : fail(
            id,
            name,
            mode,
            'sensitivity',
            'escalate via token-budget',
            decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
          ),
    );
  } else {
    results.push(
      skip(
        'breaker.token-budget',
        'token budget cap',
        'Token Burn',
        'sensitivity',
        'No tokenBudget configured — nothing caps spend mid-run. See loop-budget.md.',
      ),
    );
  }

  // Specificity: a healthy run is NOT escalated.
  {
    const id = 'breaker.healthy';
    const name = 'a healthy run is not escalated';
    const mode: FailureMode = 'Infinite Fix Loop';
    const attempts = [
      { iteration: 1, action: 'read the failing test', outcome: 'success' as const },
      { iteration: 2, action: 'apply a minimal fix', outcome: 'success' as const },
    ];
    const decision = check(ledgerOf(attempts), config);
    results.push(
      !decision.escalate
        ? pass(id, name, mode, 'specificity', 'continue', 'continued')
        : fail(
            id,
            name,
            mode,
            'specificity',
            'continue',
            `escalated (${decision.trigger})`,
            'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.',
          ),
    );
  }

  return results;
}

// ── Report assembly ────────────────────────────────────────────────

const ALL_MODES: FailureMode[] = [
  'Infinite Fix Loop',
  'Token Burn',
  'Over-Reach (Wrong Scope)',
  'Verifier Theater',
  'Escalation Failure',
  'Prompt Injection via Untrusted Input',
];

export function buildReport(results: DrillResult[]): DrillReport {
  const covered: FailureMode[] = [];
  const uncovered: FailureMode[] = [];

  for (const mode of ALL_MODES) {
    const forMode = results.filter((r) => r.failureMode === mode);
    const anyPassed = forMode.some((r) => r.outcome === 'passed');
    const anyFailed = forMode.some((r) => r.outcome === 'failed');
    if (anyPassed && !anyFailed) covered.push(mode);
    else uncovered.push(mode);
  }

  return {
    results,
    passed: results.filter((r) => r.outcome === 'passed').length,
    failed: results.filter((r) => r.outcome === 'failed').length,
    skipped: results.filter((r) => r.outcome === 'skipped').length,
    covered,
    uncovered,
    timestamp: new Date().toISOString(),
  };
}

/**
 * 0 = every drill passed, 1 = some skipped (guardrail not configured),
 * 2 = a guardrail failed to fire. Mirrors loop-gate / loop-context's
 * "2 means escalate" convention so control scripts can chain all three.
 */
export function exitCodeFor(report: DrillReport): 0 | 1 | 2 {
  if (report.failed > 0) return 2;
  if (report.skipped > 0) return 1;
  return 0;
}