Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

11.1 KB
Raw Blame
import { test, describe } from 'node:test';
import assert from 'node:assert/strict';
import {
  samplePathFor,
  runGateDrills,
  runBreakerDrills,
  buildReport,
  exitCodeFor,
} from '../dist/drill.js';
import { DEFAULT_BREAKER } from '@cobusgreyling/loop-context';

const GOOD_GATE = {
  version: 1,
  denylist: ['**/.env', '**/secrets/**', '**/*_key*', '**/k8s/production/**'],
  maxFiles: 10,
  autoMergeAllowlist: ['docs/**', '**/*.md'],
};

const outcomes = (results, id) => results.filter((r) => r.id.startsWith(id)).map((r) => r.outcome);
const byId = (results, id) => results.find((r) => r.id === id);

describe('samplePathFor', () => {
  test('expands ** into a directory segment and appends a filename', () => {
    assert.equal(samplePathFor('**/secrets/**'), 'src/secrets/src/drill-file.txt');
  });

  test('keeps a literal filename under a ** prefix', () => {
    assert.equal(samplePathFor('**/.env'), 'src/.env');
  });

  test('fills inline stars so the path still matches the glob', () => {
    // '*_key*' -> 'drill' + '_key' + 'sample'; still matches the glob.
    assert.equal(samplePathFor('**/*_key*'), 'src/drill_keysample');
  });

  test('handles a dotted suffix glob', () => {
    assert.equal(samplePathFor('**/.env.*'), 'src/.env.sample');
  });
});

describe('runGateDrills', () => {
  const results = runGateDrills({ config: GOOD_GATE });

  test('every denylist entry is proven to block', () => {
    for (const glob of GOOD_GATE.denylist) {
      assert.deepEqual(outcomes(results, `gate.denylist[${glob}]`), ['passed'], `glob ${glob}`);
    }
  });

  test('the file-count cap is proven', () => {
    assert.equal(byId(results, 'gate.file-count').outcome, 'passed');
  });

  test('auto-merge rejects a non-allowlisted path', () => {
    assert.equal(byId(results, 'gate.auto-merge').outcome, 'passed');
  });

  test('a benign change is still allowed (specificity)', () => {
    const benign = byId(results, 'gate.benign');
    assert.equal(benign.outcome, 'passed');
    assert.equal(benign.direction, 'specificity');
  });

  test('every gate drill is attributed to the Over-Reach failure mode', () => {
    for (const r of results) assert.equal(r.failureMode, 'Over-Reach (Wrong Scope)');
  });

  test('skips the file-count drill when no maxFiles is configured', () => {
    const { maxFiles, ...noCap } = GOOD_GATE;
    const r = byId(runGateDrills({ config: noCap }), 'gate.file-count');
    assert.equal(r.outcome, 'skipped');
    assert.match(r.detail, /no maxFiles/);
  });

  test('fails a gate that blocks everything, even though it catches the fault', () => {
    const paranoid = { ...GOOD_GATE, denylist: ['**'] };
    const r = runGateDrills({ config: paranoid });
    // The denylist does catch the seeded fault...
    assert.equal(byId(r, 'gate.denylist[**]').outcome, 'passed');
    // ...but it blocks a benign change, so it is not a working guardrail.
    assert.equal(byId(r, 'gate.benign').outcome, 'failed');
  });

  test('a catch-all denylist leaves the file-count rule unproven', () => {
    // Every synthetic path is swallowed by the denylist before file-count is
    // reached, so the cap is never actually exercised -- reported, not hidden.
    const r = runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } });
    const fileCount = byId(r, 'gate.file-count');
    assert.equal(fileCount.outcome, 'failed');
    assert.match(fileCount.actual, /blocked by denylist/);
  });

  test('honours a caller-supplied benign path', () => {
    const r = runGateDrills({ config: GOOD_GATE, benignPath: 'src/app/index.ts' });
    assert.equal(byId(r, 'gate.benign').outcome, 'passed');
  });
});

describe('runBreakerDrills', () => {
  const results = runBreakerDrills(DEFAULT_BREAKER);

  test('stagnation is proven', () => {
    assert.equal(byId(results, 'breaker.stagnation').outcome, 'passed');
  });

  test('no-progress is proven', () => {
    assert.equal(byId(results, 'breaker.no-progress').outcome, 'passed');
  });

  test('a healthy run is not escalated (specificity)', () => {
    assert.equal(byId(results, 'breaker.healthy').outcome, 'passed');
  });

  test('token budget is skipped when unset, and proven when set', () => {
    assert.equal(byId(results, 'breaker.token-budget').outcome, 'skipped');
    const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });
    assert.equal(byId(withBudget, 'breaker.token-budget').outcome, 'passed');
  });

  // No wrong-trigger test for breaker.token-budget: its ledger is a single
  // successful attempt, so stagnation/no-progress (which need failures) and
  // max-iterations (checked after the budget) are all unreachable. There is no
  // config in which it escalates for another reason, so weakening its
  // assertion to decision.escalate is not observable from outside.

  test('token budget drill is attributed to Token Burn', () => {
    const withBudget = runBreakerDrills({ ...DEFAULT_BREAKER, tokenBudget: 1000 });
    assert.equal(byId(withBudget, 'breaker.token-budget').failureMode, 'Token Burn');
  });

  test('each drill is credited only for its own trigger', () => {
    // Escalating for another reason leaves the rule under test unproven, so the
    // reported reason must name the rule the drill exercises.
    assert.match(byId(results, 'breaker.stagnation').expected, /stagnation/);
    assert.match(byId(results, 'breaker.no-progress').expected, /no-progress/);
    assert.match(byId(results, 'breaker.no-progress').actual, /no.progress/i);
  });

  test('a percentage similarityThreshold is a failed drill, not a crash', () => {
    // similarityThreshold is a 0.0-1.0 fraction. 95 (meaning "95%") used to make
    // identical errors never compare as similar, so a loop retrying the same
    // failure forever was never stopped. loop-context now refuses the config;
    // the drill must report that rather than throw.
    for (const similarityThreshold of [95, 0]) {
      const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold });
      assert.equal(misconfigured.length, 1, String(similarityThreshold));
      const [r] = misconfigured;
      assert.equal(r.id, 'breaker.config');
      assert.equal(r.outcome, 'failed');
      assert.equal(r.failureMode, 'Infinite Fix Loop');
      assert.match(r.actual, /similarityThreshold must be a fraction/);
    }
    assert.match(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 })[0].actual, /use 0\.95/);
    assert.equal(exitCodeFor(buildReport(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 }))), 2);
  });

  // A breaker that always answers with one trigger: how the drills are shown to
  // credit only the rule they exercise, now that loop-context refuses the
  // configs that used to produce these shapes.
  const always = (trigger) => (ledger) => ({
    shouldContinue: false,
    escalate: true,
    trigger,
    reason: `stand-in breaker: ${trigger}`,
    iterations: ledger.attempts.length,
    tokensUsed: 0,
  });

  test('escalating via the wrong trigger leaves stagnation unproven', () => {
    // The case that separates "the breaker fired" from "the rule under test
    // fired". Here the breaker DOES escalate, so a drill asserting only
    // decision.escalate would pass while stagnation was never exercised.
    const wrongTrigger = runBreakerDrills(DEFAULT_BREAKER, always('no-progress'));
    const stagnation = byId(wrongTrigger, 'breaker.stagnation');
    assert.equal(stagnation.outcome, 'failed');
    assert.equal(stagnation.actual, 'escalated via no-progress');
    assert.match(stagnation.detail, /not by the rule under test/);
  });

  test('an iteration cap that fires first also leaves stagnation unproven', () => {
    const capped = runBreakerDrills(DEFAULT_BREAKER, always('max-iterations'));
    const stagnation = byId(capped, 'breaker.stagnation');
    assert.equal(stagnation.outcome, 'failed');
    assert.equal(stagnation.actual, 'escalated via max-iterations');
  });

  test('stagnation swallowing every error leaves no-progress unproven', () => {
    // A breaker that calls every error "the same error" fires stagnation on the
    // no-progress drill's unrelated failures. It still escalates, so only a
    // trigger-specific assertion catches it.
    const swallowed = runBreakerDrills(DEFAULT_BREAKER, always('stagnation'));
    const noProgress = byId(swallowed, 'breaker.no-progress');
    assert.equal(noProgress.outcome, 'failed');
    assert.equal(noProgress.actual, 'escalated via stagnation');
    assert.match(noProgress.detail, /not by the rule under test/);
  });

  test('a breaker that never escalates fails stagnation and passes the healthy run', () => {
    const never = (ledger) => ({ shouldContinue: true, escalate: false, trigger: 'ok', reason: 'ok', iterations: ledger.attempts.length, tokensUsed: 0 });
    const r = runBreakerDrills(DEFAULT_BREAKER, never);
    assert.equal(byId(r, 'breaker.stagnation').actual, 'continued');
    assert.match(byId(r, 'breaker.stagnation').detail, /Infinite Fix Loop/);
    assert.equal(byId(r, 'breaker.healthy').outcome, 'passed');
  });

  test('no-progress uses errors that stay distinct after signature normalization', () => {
    // errorSignature() collapses every number to '#', so errors differing only
    // by a number ("module 1" / "module 2") normalize to the same signature and
    // trip stagnation instead -- passing the drill while leaving no-progress
    // unexercised. This asserts the real trigger, which is what catches that.
    const r = byId(results, 'breaker.no-progress');
    assert.equal(r.outcome, 'passed');
    assert.doesNotMatch(r.actual, /stagnation/i);
  });
});

describe('buildReport / exitCodeFor', () => {
  test('a failure mode with any failing drill is not counted as covered', () => {
    const report = buildReport([
      ...runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }),
    ]);
    assert.ok(!report.covered.includes('Over-Reach (Wrong Scope)'));
    assert.ok(report.uncovered.includes('Over-Reach (Wrong Scope)'));
  });

  test('a fully passing gate run covers Over-Reach', () => {
    const report = buildReport(runGateDrills({ config: GOOD_GATE }));
    assert.ok(report.covered.includes('Over-Reach (Wrong Scope)'));
  });

  test('unexercised failure modes are reported as uncovered', () => {
    const report = buildReport(runGateDrills({ config: GOOD_GATE }));
    assert.ok(report.uncovered.includes('Verifier Theater'));
    assert.ok(report.uncovered.includes('Escalation Failure'));
  });

  test('exit code 0 when everything passes', () => {
    assert.equal(exitCodeFor(buildReport(runGateDrills({ config: GOOD_GATE }))), 0);
  });

  test('exit code 1 when a drill is skipped', () => {
    const { maxFiles, ...noCap } = GOOD_GATE;
    assert.equal(exitCodeFor(buildReport(runGateDrills({ config: noCap }))), 1);
  });

  test('exit code 2 when a guardrail fails to fire', () => {
    const report = buildReport(runGateDrills({ config: { ...GOOD_GATE, denylist: ['**'] } }));
    assert.equal(exitCodeFor(report), 2);
  });

  test('failure beats skip in the exit code', () => {
    const { maxFiles, ...noCap } = GOOD_GATE;
    const report = buildReport(runGateDrills({ config: { ...noCap, denylist: ['**'] } }));
    assert.ok(report.skipped > 0 && report.failed > 0);
    assert.equal(exitCodeFor(report), 2);
  });
});