Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

feat(loop-context): similarity-based stagnation detection (#296)

THRISHAL DOMAc1608072562c
diff --git a/tools/loop-context/src/cli.ts b/tools/loop-context/src/cli.ts
index 51f2161..3f88449 100644
--- a/tools/loop-context/src/cli.ts
+++ b/tools/loop-context/src/cli.ts
@@ -52,6 +52,17 @@ function parsePositiveIntFlag(raw: string | undefined, flag: string): number {
return n;
}
+function parsePositiveFloatFlag(raw: string | undefined, flag: string): number {
+ if (raw === undefined || raw === '') {
+ throw new Error(`${flag} requires a positive number value.`);
+ }
+ const n = Number(raw);
+ if (Number.isNaN(n) || n <= 0) {
+ throw new Error(`${flag} must be a positive number; got "${raw}".`);
+ }
+ return n;
+}
+
function parseArgs(argv: string[]): Args {
const breaker: CircuitBreakerConfig = { ...DEFAULT_BREAKER };
const prune: PruneConfig = { ...DEFAULT_PRUNE };
@@ -97,6 +108,11 @@ function parseArgs(argv: string[]): Args {
else if (a === '--on-exceed') onExceed = argv[++i];
else if (a === '--window') prune.window = parsePositiveIntFlag(argv[++i], '--window');
else if (a === '--max-trace-lines') prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');
+ else if (a === '--similarity-threshold') {
+ const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');
+ breaker.similarityThreshold = val;
+ prune.similarityThreshold = val;
+ }
}
return { help: false, ledger, ...base() };
@@ -171,6 +187,7 @@ Options:
--on-exceed <script> On escalate, pipe the decision as JSON to this
script's stdin (fire-and-forget; its exit code
is not checked and does not change --check's own).
+ --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})
--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})
--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})
-h, --help This help
diff --git a/tools/loop-context/src/context-manager.ts b/tools/loop-context/src/context-manager.ts
index 8e734e6..f343b34 100644
--- a/tools/loop-context/src/context-manager.ts
+++ b/tools/loop-context/src/context-manager.ts
@@ -53,6 +53,8 @@ export interface CircuitBreakerConfig {
noProgressThreshold: number;
/** Optional hard cap on cumulative tokens across the run. */
tokenBudget?: number;
+ /** Float 0.0-1.0. If consecutive errors are this similar, they count as stagnant. */
+ similarityThreshold: number;
}
export interface PruneConfig {
@@ -60,17 +62,21 @@ export interface PruneConfig {
maxTraceLines: number;
/** Number of most-recent attempts to retain in the pruned ledger. */
window: number;
+ /** Float 0.0-1.0. If consecutive errors are this similar, they are collapsed. */
+ similarityThreshold: number;
}
export const DEFAULT_BREAKER: CircuitBreakerConfig = {
maxIterations: 10,
stagnationThreshold: 3,
noProgressThreshold: 5,
+ similarityThreshold: 0.85,
};
export const DEFAULT_PRUNE: PruneConfig = {
maxTraceLines: 8,
window: 5,
+ similarityThreshold: 0.85,
};
// ── Error normalization ────────────────────────────────────────────
@@ -96,6 +102,34 @@ export function errorSignature(error: string): string {
.trim();
}
+/**
+ * Calculate Jaccard similarity (0.0 to 1.0) using character trigrams.
+ * Highly robust to minor phrasing variations.
+ */
+function getTrigrams(str: string): Set<string> {
+ const trigrams = new Set<string>();
+ const padded = ` ${str.toLowerCase()} `;
+ for (let i = 0; i < padded.length - 2; i++) {
+ trigrams.add(padded.substring(i, i + 3));
+ }
+ return trigrams;
+}
+
+export function calculateSimilarity(a: string, b: string): number {
+ if (a === b) return 1.0;
+ const setA = getTrigrams(a);
+ const setB = getTrigrams(b);
+ if (setA.size === 0 && setB.size === 0) return 1.0;
+ if (setA.size === 0 || setB.size === 0) return 0.0;
+
+ let intersection = 0;
+ for (const item of setA) {
+ if (setB.has(item)) intersection++;
+ }
+ const union = setA.size + setB.size - intersection;
+ return intersection / union;
+}
+
// ── Circuit breaker ────────────────────────────────────────────────
export type BreakerTrigger =
@@ -154,7 +188,8 @@ export function checkCircuitBreaker(
const lastSig = errorSignature(failRun[failRun.length - 1].error ?? '');
let same = 0;
for (let i = failRun.length - 1; i >= 0; i--) {
- if (errorSignature(failRun[i].error ?? '') === lastSig) same++;
+ const curSig = errorSignature(failRun[i].error ?? '');
+ if (calculateSimilarity(curSig, lastSig) >= config.similarityThreshold) same++;
else break;
}
if (same >= config.stagnationThreshold) {
@@ -244,7 +279,7 @@ export function pruneLedger(ledger: Ledger, config: PruneConfig = DEFAULT_PRUNE)
prev !== undefined &&
prev.outcome === 'failure' &&
pruned.outcome === 'failure' &&
- errorSignature(prev.error ?? '') === errorSignature(pruned.error ?? '');
+ calculateSimilarity(errorSignature(prev.error ?? ''), errorSignature(pruned.error ?? '')) >= config.similarityThreshold;
if (sameFailure) {
prev.repeated = (prev.repeated ?? 1) + 1;
@@ -278,7 +313,7 @@ export interface AttemptSummary {
}
/** Deterministic factual rollup of the whole run — no LLM required. */
-export function summarizeAttempts(ledger: Ledger): AttemptSummary {
+export function summarizeAttempts(ledger: Ledger, similarityThreshold: number = 0.85): AttemptSummary {
const groups = new Map<string, ErrorGroup>();
const actions = new Set<string>();
let successes = 0;
@@ -293,9 +328,17 @@ export function summarizeAttempts(ledger: Ledger): AttemptSummary {
failures++;
if (a.error) {
const sig = errorSignature(a.error);
- const existing = groups.get(sig);
- if (existing) existing.count++;
- else groups.set(sig, { signature: sig, count: 1, sample: a.error.split('\n')[0].trim() });
+ let found = false;
+ for (const existing of groups.values()) {
+ if (calculateSimilarity(existing.signature, sig) >= similarityThreshold) {
+ existing.count++;
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ groups.set(sig, { signature: sig, count: 1, sample: a.error.split('\n')[0].trim() });
+ }
}
}
}
@@ -323,7 +366,7 @@ export function buildContextInjection(
breaker: CircuitBreakerConfig = DEFAULT_BREAKER,
prune: PruneConfig = DEFAULT_PRUNE,
): string {
- const summary = summarizeAttempts(ledger);
+ const summary = summarizeAttempts(ledger, breaker.similarityThreshold);
const decision = checkCircuitBreaker(ledger, breaker);
const pruned = pruneLedger(ledger, prune);
const lines: string[] = [];
diff --git a/tools/loop-context/test/context-manager.test.mjs b/tools/loop-context/test/context-manager.test.mjs
index 43b3c95..e609e68 100644
--- a/tools/loop-context/test/context-manager.test.mjs
+++ b/tools/loop-context/test/context-manager.test.mjs
@@ -7,6 +7,7 @@ import {
pruneLedger,
summarizeAttempts,
buildContextInjection,
+ calculateSimilarity,
DEFAULT_BREAKER,
DEFAULT_PRUNE,
} from '../dist/context-manager.js';
@@ -39,6 +40,23 @@ test('errorSignature keeps genuinely different errors distinct', () => {
assert.notEqual(a, b);
});
+// ── calculateSimilarity ────────────────────────────────────────────
+
+test('calculateSimilarity returns 1.0 for identical strings', () => {
+ assert.equal(calculateSimilarity('foo', 'foo'), 1.0);
+});
+
+test('calculateSimilarity detects high overlap in phrasing', () => {
+ const a = 'Error: connect ECONNREFUSED 127.0.0.1:5432';
+ const b = 'Error: connection refused on 127.0.0.1:5432';
+ const sim = calculateSimilarity(a, b);
+ assert.ok(sim > 0.4 && sim < 1.0, `Similarity was ${sim}`);
+});
+
+test('calculateSimilarity returns 0.0 for completely disjoint strings', () => {
+ assert.equal(calculateSimilarity('abc', 'xyz'), 0.0);
+});
+
// ── circuit breaker: stagnation ────────────────────────────────────
test('breaker trips on stagnation (same error 3× in a row)', () => {
@@ -54,6 +72,17 @@ test('breaker trips on stagnation (same error 3× in a row)', () => {
assert.equal(d.shouldContinue, false);
});
+test('breaker trips on stagnation with slightly different wording', () => {
+ const l = ledger([
+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
+ attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),
+ ]);
+ const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, similarityThreshold: 0.5 });
+ assert.equal(d.escalate, true);
+ assert.equal(d.trigger, 'stagnation');
+});
+
test('breaker does not trip when errors differ each time', () => {
const l = ledger([
attempt(1, 'failure', { error: 'TypeError: a' }),
@@ -152,6 +181,17 @@ test('pruneLedger collapses consecutive identical failures with a count', () =>
assert.equal(pruned.attempts[0].iteration, 3);
});
+test('pruneLedger collapses near-duplicate failures based on similarity', () => {
+ const l = ledger([
+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
+ attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),
+ ]);
+ const pruned = pruneLedger(l, { ...DEFAULT_PRUNE, window: 5, similarityThreshold: 0.5 });
+ assert.equal(pruned.attempts.length, 1);
+ assert.equal(pruned.attempts[0].repeated, 3);
+});
+
test('pruneLedger does not mutate the input ledger', () => {
const l = ledger([attempt(1, 'failure', { error: 'x'.repeat(10) + '\n'.repeat(20) })]);
const before = JSON.stringify(l);
@@ -176,6 +216,17 @@ test('summarizeAttempts groups errors by signature, most frequent first', () =>
assert.deepEqual(s.actionsTried, ['run migration', 'patch code']);
});
+test('summarizeAttempts groups errors by similarity', () => {
+ const l = ledger([
+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),
+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),
+ attempt(3, 'failure', { error: 'Error: totally different error' }),
+ ]);
+ const s = summarizeAttempts(l, 0.5);
+ assert.equal(s.distinctErrors.length, 2);
+ assert.equal(s.distinctErrors[0].count, 2);
+});
+
// ── injection ──────────────────────────────────────────────────────
test('buildContextInjection includes goal, tried actions, and breaker status', () => {