Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
errorSignature,
checkCircuitBreaker,
pruneStackTrace,
pruneLedger,
summarizeAttempts,
buildContextInjection,
calculateSimilarity,
DEFAULT_BREAKER,
DEFAULT_PRUNE,
validateBreakerConfig,
validatePruneConfig,
assertSimilarityThreshold,
} from '../dist/context-manager.js';
function attempt(iteration, outcome, opts = {}) {
return { iteration, action: opts.action ?? `try ${iteration}`, outcome, ...opts };
}
function ledger(attempts, goal = 'make the build green') {
return { goal, attempts };
}
// ── errorSignature ─────────────────────────────────────────────────
test('errorSignature normalizes volatile details to a stable key', () => {
const a = errorSignature('TypeError: cannot read x at /home/u/app/foo.js:12:5');
const b = errorSignature('TypeError: cannot read x at /tmp/build/foo.js:88:9');
assert.equal(a, b);
});
test('errorSignature collapses ports and addresses', () => {
const a = errorSignature('Error: connect ECONNREFUSED 127.0.0.1:5432');
const b = errorSignature('Error: connect ECONNREFUSED 127.0.0.1:5433');
assert.equal(a, b);
});
test('errorSignature keeps genuinely different errors distinct', () => {
const a = errorSignature('TypeError: undefined is not a function');
const b = errorSignature('ReferenceError: foo is not defined');
assert.notEqual(a, b);
});
// ── calculateSimilarity ────────────────────────────────────────────
test('calculateSimilarity returns 1.0 for identical strings', () => {
assert.equal(calculateSimilarity('foo', 'foo'), 1.0);
});
test('calculateSimilarity detects high overlap in phrasing', () => {
const a = 'Error: connect ECONNREFUSED 127.0.0.1:5432';
const b = 'Error: connection refused on 127.0.0.1:5432';
const sim = calculateSimilarity(a, b);
assert.ok(sim > 0.4 && sim < 1.0, `Similarity was ${sim}`);
});
test('calculateSimilarity returns 0.0 for completely disjoint strings', () => {
assert.equal(calculateSimilarity('abc', 'xyz'), 0.0);
});
// ── circuit breaker: stagnation ────────────────────────────────────
test('breaker trips on stagnation (same error 3× in a row)', () => {
const err = 'Error: connect ECONNREFUSED 127.0.0.1:5432';
const l = ledger([
attempt(1, 'failure', { error: err }),
attempt(2, 'failure', { error: err }),
attempt(3, 'failure', { error: err }),
]);
const d = checkCircuitBreaker(l);
assert.equal(d.escalate, true);
assert.equal(d.trigger, 'stagnation');
assert.equal(d.shouldContinue, false);
});
test('breaker trips on stagnation with slightly different wording', () => {
const l = ledger([
attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),
]);
const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, similarityThreshold: 0.5 });
assert.equal(d.escalate, true);
assert.equal(d.trigger, 'stagnation');
});
test('breaker does not trip when errors differ each time', () => {
const l = ledger([
attempt(1, 'failure', { error: 'TypeError: a' }),
attempt(2, 'failure', { error: 'ReferenceError: b' }),
]);
const d = checkCircuitBreaker(l);
assert.equal(d.escalate, false);
assert.equal(d.trigger, 'ok');
});
test('breaker stagnation resets after a success', () => {
const err = 'Error: boom';
const l = ledger([
attempt(1, 'failure', { error: err }),
attempt(2, 'failure', { error: err }),
attempt(3, 'success'),
attempt(4, 'failure', { error: err }),
]);
const d = checkCircuitBreaker(l);
assert.equal(d.escalate, false, 'trailing failure run is only 1 after the success');
});
// ── circuit breaker: frustration ───────────────────────────────────
test('breaker trips on frustration (semantically similar actions but different errors)', () => {
const l = ledger([
attempt(1, 'failure', { action: 'Try to fix the syntax error', error: 'e1' }),
attempt(2, 'failure', { action: 'I will try to fix the syntax error', error: 'e2' }),
attempt(3, 'failure', { action: 'Trying to fix the syntax error', error: 'e3' }),
]);
const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, similarityThreshold: 0.5 });
assert.equal(d.escalate, true);
assert.equal(d.trigger, 'frustration');
assert.equal(d.shouldContinue, false);
});
test('breaker does not trip on frustration if actions differ', () => {
const l = ledger([
attempt(1, 'failure', { action: 'Install missing dependencies', error: 'e1' }),
attempt(2, 'failure', { action: 'Fix the failing test case', error: 'e2' }),
attempt(3, 'failure', { action: 'Revert the previous change', error: 'e3' }),
]);
const d = checkCircuitBreaker(l);
assert.equal(d.escalate, false);
});
// ── circuit breaker: no-progress ───────────────────────────────────
test('breaker trips on no-progress (consecutive failures, distinct errors)', () => {
const l = ledger([
attempt(1, 'failure', { error: 'e1' }),
attempt(2, 'failure', { error: 'e2' }),
attempt(3, 'failure', { error: 'e3' }),
attempt(4, 'failure', { error: 'e4' }),
attempt(5, 'failure', { error: 'e5' }),
]);
const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, stagnationThreshold: 99 });
assert.equal(d.escalate, true);
assert.equal(d.trigger, 'no-progress');
});
// ── circuit breaker: caps ──────────────────────────────────────────
test('breaker trips on token budget', () => {
const l = ledger([
attempt(1, 'noop', { tokensUsed: 600 }),
attempt(2, 'noop', { tokensUsed: 600 }),
]);
const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, tokenBudget: 1000 });
assert.equal(d.trigger, 'token-budget');
assert.equal(d.tokensUsed, 1200);
});
test('breaker trips on iteration cap', () => {
const attempts = [];
for (let i = 1; i <= 10; i++) attempts.push(attempt(i, 'noop'));
const d = checkCircuitBreaker(ledger(attempts), { ...DEFAULT_BREAKER, maxIterations: 10 });
assert.equal(d.trigger, 'max-iterations');
});
test('breaker clears an empty ledger', () => {
const d = checkCircuitBreaker(ledger([]));
assert.equal(d.shouldContinue, true);
assert.equal(d.iterations, 0);
});
// ── pruning ────────────────────────────────────────────────────────
test('pruneStackTrace keeps head and notes omissions', () => {
const trace = Array.from({ length: 20 }, (_, i) => `line ${i}`).join('\n');
const pruned = pruneStackTrace(trace, 5);
assert.ok(pruned.includes('line 0'));
assert.ok(pruned.includes('15 more lines pruned'));
assert.ok(!pruned.includes('line 19'));
});
test('pruneStackTrace leaves short traces intact', () => {
const trace = 'line 0\nline 1';
assert.equal(pruneStackTrace(trace, 5), trace);
});
test('pruneLedger keeps only the recent window', () => {
const attempts = [];
for (let i = 1; i <= 12; i++) attempts.push(attempt(i, 'noop'));
const pruned = pruneLedger(ledger(attempts), { ...DEFAULT_PRUNE, window: 3 });
assert.equal(pruned.attempts.length, 3);
assert.equal(pruned.attempts[2].iteration, 12);
});
test('pruneLedger collapses consecutive identical failures with a count', () => {
const err = 'Error: same thing';
const l = ledger([
attempt(1, 'failure', { error: err }),
attempt(2, 'failure', { error: err }),
attempt(3, 'failure', { error: err }),
]);
const pruned = pruneLedger(l, { ...DEFAULT_PRUNE, window: 5 });
assert.equal(pruned.attempts.length, 1);
assert.equal(pruned.attempts[0].repeated, 3);
assert.equal(pruned.attempts[0].iteration, 3);
});
test('pruneLedger collapses near-duplicate failures based on similarity', () => {
const l = ledger([
attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),
]);
const pruned = pruneLedger(l, { ...DEFAULT_PRUNE, window: 5, similarityThreshold: 0.5 });
assert.equal(pruned.attempts.length, 1);
assert.equal(pruned.attempts[0].repeated, 3);
});
test('pruneLedger does not mutate the input ledger', () => {
const l = ledger([attempt(1, 'failure', { error: 'x'.repeat(10) + '\n'.repeat(20) })]);
const before = JSON.stringify(l);
pruneLedger(l);
assert.equal(JSON.stringify(l), before);
});
// ── summarization ──────────────────────────────────────────────────
test('summarizeAttempts groups errors by signature, most frequent first', () => {
const l = ledger([
attempt(1, 'failure', { error: 'Error: connect ECONNREFUSED 127.0.0.1:5432', action: 'run migration' }),
attempt(2, 'failure', { error: 'Error: connect ECONNREFUSED 127.0.0.1:5433', action: 'run migration' }),
attempt(3, 'failure', { error: 'TypeError: x', action: 'patch code' }),
attempt(4, 'success', { action: 'patch code' }),
]);
const s = summarizeAttempts(l);
assert.equal(s.totalAttempts, 4);
assert.equal(s.failures, 3);
assert.equal(s.successes, 1);
assert.equal(s.distinctErrors[0].count, 2, 'ECONNREFUSED collapses to one group of 2');
assert.deepEqual(s.actionsTried, ['run migration', 'patch code']);
});
test('summarizeAttempts groups errors by similarity', () => {
const l = ledger([
attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
attempt(3, 'failure', { error: 'Error: totally different error' }),
]);
const s = summarizeAttempts(l, 0.5);
assert.equal(s.distinctErrors.length, 2);
assert.equal(s.distinctErrors[0].count, 2);
});
// ── injection ──────────────────────────────────────────────────────
test('buildContextInjection includes goal, tried actions, and breaker status', () => {
const err = 'Error: boom';
const l = ledger([
attempt(1, 'failure', { error: err, action: 'approach A' }),
attempt(2, 'failure', { error: err, action: 'approach A' }),
attempt(3, 'failure', { error: err, action: 'approach A' }),
]);
const block = buildContextInjection(l);
assert.ok(block.includes('make the build green'));
assert.ok(block.includes('approach A'));
assert.ok(block.includes('do NOT repeat'));
assert.ok(block.includes('STOP'), 'stagnation should surface a STOP directive');
});
test('buildContextInjection shows OK status for a healthy run', () => {
const l = ledger([attempt(1, 'noop'), attempt(2, 'success')]);
const block = buildContextInjection(l);
assert.ok(block.includes('Circuit breaker: OK'));
});
// ── config validation ──────────────────────────────────────────────
const identicalFailures = () =>
ledger([1, 2, 3, 4].map((i) => attempt(i, 'failure', { action: 'same fix', error: 'TypeError: boom' })));
test('checkCircuitBreaker throws on a similarityThreshold above 1 instead of never stagnating', () => {
// Similarity tops out at 1.0, so 95 (a percentage) silently switched stagnation off.
for (const similarityThreshold of [95, 1.0001, Infinity]) {
assert.throws(
() => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold }),
/similarityThreshold must be a fraction/,
String(similarityThreshold),
);
}
});
test('checkCircuitBreaker throws on configs that would switch a rule off', () => {
const bad = [
{ similarityThreshold: NaN },
{ similarityThreshold: 0 },
{ similarityThreshold: -0.1 },
{ similarityThreshold: '0.85' },
{ maxIterations: 0 },
{ stagnationThreshold: NaN },
{ frustrationThreshold: 2.5 },
{ noProgressThreshold: Infinity },
{ tokenBudget: 0 },
{ tokenBudget: -100 },
];
for (const override of bad) {
assert.throws(() => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, ...override }), undefined, JSON.stringify(override));
}
});
test('valid configs, including the boundaries, still trip stagnation', () => {
for (const similarityThreshold of [1, 0.5, Number.MIN_VALUE]) {
const d = checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold });
assert.equal(d.trigger, 'stagnation', String(similarityThreshold));
}
assert.doesNotThrow(() => validateBreakerConfig({ ...DEFAULT_BREAKER, tokenBudget: 1 }));
assert.doesNotThrow(() => validateBreakerConfig(DEFAULT_BREAKER));
assert.doesNotThrow(() => validatePruneConfig(DEFAULT_PRUNE));
});
test('pruneLedger, summarizeAttempts and buildContextInjection reject a percentage threshold', () => {
assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, similarityThreshold: 85 }), /If you meant 85%, use 0\.85/);
assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, window: 0 }), /window must be a positive integer/);
assert.throws(() => summarizeAttempts(identicalFailures(), 85), /similarityThreshold must be a fraction/);
assert.throws(
() => buildContextInjection(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold: 85 }),
/similarityThreshold must be a fraction/,
);
});
test('assertSimilarityThreshold only gives the percentage hint for whole-number percentages', () => {
assert.throws(() => assertSimilarityThreshold(95), /If you meant 95%, use 0\.95/);
assert.throws(() => assertSimilarityThreshold(1.5), (err) => !/If you meant/.test(err.message));
assert.throws(() => assertSimilarityThreshold(0), (err) => !/never fire/.test(err.message));
assert.throws(() => assertSimilarityThreshold(150), (err) => /never fire/.test(err.message) && !/If you meant/.test(err.message));
assert.throws(() => assertSimilarityThreshold(95, '--similarity-threshold'), /^Error: --similarity-threshold must/);
});