Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resolveDailyBudgetFromPattern } from '../dist/budget-resolver.js';
const testDir = path.dirname(fileURLToPath(import.meta.url));
const cli = path.join(testDir, '../dist/cli.js');
const onExceedCaptureScript = path.join(testDir, 'fixtures/on-exceed-capture.mjs');
const onExceedIgnoreStdinScript = path.join(testDir, 'fixtures/on-exceed-ignore-stdin.mjs');
async function freshDir() {
return mkdtemp(path.join(tmpdir(), 'loop-context-cli-daily-'));
}
const stagnationLedger = JSON.stringify({
goal: 'x',
attempts: [
{ iteration: 1, action: 'a', outcome: 'failure', error: 'boom' },
{ iteration: 2, action: 'a', outcome: 'failure', error: 'boom' },
{ iteration: 3, action: 'a', outcome: 'failure', error: 'boom' },
],
});
function runCli(args, input = stagnationLedger) {
return spawnSync(process.execPath, [cli, ...args], {
input,
encoding: 'utf8',
// Default 1MB is too small once a decision's JSON echoes back a large
// error string (see the --on-exceed EPIPE regression test below).
maxBuffer: 20 * 1024 * 1024,
});
}
test('cli --check trips stagnation with default threshold', () => {
const r = runCli(['--check', '--json']);
assert.equal(r.status, 2);
const decision = JSON.parse(r.stdout);
assert.equal(decision.trigger, 'stagnation');
});
test('cli rejects invalid --stagnation values before running the breaker', () => {
const r = runCli(['--check', '--stagnation', 'nope', '--json']);
assert.equal(r.status, 1);
assert.match(r.stderr, /--stagnation must be a positive integer/);
});
test('cli rejects invalid --no-progress values', () => {
const r = runCli(['--check', '--no-progress', '0', '--json']);
assert.equal(r.status, 1);
assert.match(r.stderr, /--no-progress must be a positive integer/);
});
test('cli rejects invalid --max-iterations values', () => {
const r = runCli(['--check', '--max-iterations', '1.5', '--json']);
assert.equal(r.status, 1);
assert.match(r.stderr, /--max-iterations must be a positive integer/);
});
test('cli rejects a percentage-style --similarity-threshold instead of disabling stagnation', () => {
// 95 was accepted and meant no two errors could ever match: the same failure
// repeated forever came back CONTINUE with exit 0.
for (const value of ['95', '100', '1.5', 'Infinity']) {
const r = runCli(['--check', '--similarity-threshold', value, '--json']);
assert.equal(r.status, 1, value);
assert.equal(r.stdout, '', `${value}: no decision is printed`);
assert.match(r.stderr, /--similarity-threshold must be a fraction greater than 0 and at most 1/, value);
assert.match(r.stderr, /stagnation rule would never fire/, value);
}
assert.match(runCli(['--check', '--similarity-threshold', '95']).stderr, /If you meant 95%, use 0\.95/);
});
test('cli rejects zero, negative and non-numeric --similarity-threshold', () => {
for (const value of ['0', '-0.5', 'abc']) {
const r = runCli(['--check', '--similarity-threshold', value, '--json']);
assert.equal(r.status, 1, value);
assert.ok(r.stderr.includes(`got ${value}.`), value);
}
});
test('cli accepts --similarity-threshold across (0, 1] and still trips stagnation', () => {
for (const value of ['1', '0.95', '0.5', '0.01']) {
const r = runCli(['--check', '--similarity-threshold', value, '--json']);
assert.equal(r.status, 2, value);
assert.equal(JSON.parse(r.stdout).trigger, 'stagnation', value);
}
});
test('cli validates --similarity-threshold for --prune too', () => {
const r = runCli(['--prune', '--similarity-threshold', '85']);
assert.equal(r.status, 1);
assert.match(r.stderr, /If you meant 85%, use 0\.85/);
});
test('cli rejects missing numeric flag values', () => {
const r = runCli(['--check', '--stagnation']);
assert.equal(r.status, 1);
assert.match(r.stderr, /--stagnation requires a positive integer value/);
});
const cheapLedger = JSON.stringify({
goal: 'x',
attempts: [{ iteration: 1, action: 'a', outcome: 'success', tokensUsed: 100 }],
});
test('cli resolves --token-budget from --budget-from-pattern', () => {
const r = runCli(
['--check', '--budget-from-pattern', 'ci-sweeper', '--budget-level', 'L2', '--json'],
cheapLedger,
);
assert.equal(r.status, 0);
const decision = JSON.parse(r.stdout);
assert.equal(decision.escalate, false);
});
test('cli --token-budget wins over --budget-from-pattern when both are given', () => {
const r = runCli(
['--check', '--budget-from-pattern', 'ci-sweeper', '--token-budget', '50', '--json'],
cheapLedger,
);
assert.equal(r.status, 2);
const decision = JSON.parse(r.stdout);
assert.equal(decision.trigger, 'token-budget');
});
test('cli --budget-from-pattern surfaces loop-cost\'s unknown-pattern error', () => {
const r = runCli(['--check', '--budget-from-pattern', 'not-a-pattern'], cheapLedger);
assert.equal(r.status, 1);
assert.match(r.stderr, /Unknown pattern: not-a-pattern/);
});
test('cli --daily-budget-from-pattern escalates once the daily cap is reached', async () => {
const dir = await freshDir();
const dailyCap = await resolveDailyBudgetFromPattern({ pattern: 'ci-sweeper', level: 'L2' });
// Pre-seed today's spend right at the cap so this run's single-token delta tips it over.
await writeFile(
path.join(dir, 'daily-spend.ci-sweeper.json'),
JSON.stringify({ date: new Date().toISOString().slice(0, 10), tokensUsedToday: dailyCap }),
);
const r = runCli(
[
'--check', '--daily-budget-from-pattern', 'ci-sweeper', '--budget-level', 'L2',
'--daily-state-dir', dir, '--json',
],
cheapLedger,
);
assert.equal(r.status, 2);
const decision = JSON.parse(r.stdout);
assert.equal(decision.trigger, 'daily-budget');
});
test('cli --daily-budget-from-pattern does not override an existing per-run trigger', async () => {
const dir = await freshDir();
const dailyCap = await resolveDailyBudgetFromPattern({ pattern: 'ci-sweeper', level: 'L2' });
await writeFile(
path.join(dir, 'daily-spend.ci-sweeper.json'),
JSON.stringify({ date: new Date().toISOString().slice(0, 10), tokensUsedToday: dailyCap * 10 }),
);
const r = runCli([
'--check', '--daily-budget-from-pattern', 'ci-sweeper', '--budget-level', 'L2',
'--daily-state-dir', dir, '--json',
]);
assert.equal(r.status, 2);
const decision = JSON.parse(r.stdout);
assert.equal(decision.trigger, 'stagnation');
});
test('cli --on-exceed pipes the escalation decision to the script as JSON', async () => {
const dir = await freshDir();
const outFile = path.join(dir, 'captured.json');
const r = runCli(['--check', '--on-exceed', `node ${onExceedCaptureScript} ${outFile}`, '--json']);
assert.equal(r.status, 2);
const captured = JSON.parse(await readFile(outFile, 'utf8'));
assert.equal(captured.trigger, 'stagnation');
assert.equal(captured.escalate, true);
});
test('cli --on-exceed is not invoked when the run continues', async () => {
const dir = await freshDir();
const outFile = path.join(dir, 'should-not-exist.json');
const r = runCli(
['--check', '--on-exceed', `node ${onExceedCaptureScript} ${outFile}`, '--json'],
cheapLedger,
);
assert.equal(r.status, 0);
await assert.rejects(() => readFile(outFile, 'utf8'));
});
test('cli --on-exceed does not crash when the hook script exits without reading stdin', async () => {
const dir = await freshDir();
const outFile = path.join(dir, 'ran.marker');
// A large first-line error survives errorSignature() unshortened, so the
// piped decision JSON is big enough to exceed the OS pipe buffer -- this
// makes the write-after-close failure (EPIPE on POSIX, EOF on Windows)
// land reliably instead of racing the hook process's own startup time.
const bigError = 'e'.repeat(5_000_000);
const bigLedger = JSON.stringify({
goal: 'x',
attempts: [
{ iteration: 1, action: 'a', outcome: 'failure', error: bigError },
{ iteration: 2, action: 'a', outcome: 'failure', error: bigError },
{ iteration: 3, action: 'a', outcome: 'failure', error: bigError },
],
});
const r = runCli(
['--check', '--on-exceed', `node ${onExceedIgnoreStdinScript} ${outFile}`, '--json'],
bigLedger,
);
assert.equal(r.status, 2, `expected a clean escalation exit, not a crash. stderr: ${r.stderr}`);
assert.doesNotMatch(r.stderr, /Uncaught|EPIPE|EOF/);
assert.equal(await readFile(outFile, 'utf8'), 'ran');
});