Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
feat(loop-context): similarity-based stagnation detection (#296)
THRISHAL DOMAc1608072562c
diff --git a/tools/loop-context/src/cli.ts b/tools/loop-context/src/cli.tsindex 51f2161..3f88449 100644--- a/tools/loop-context/src/cli.ts+++ b/tools/loop-context/src/cli.ts@@ -52,6 +52,17 @@ function parsePositiveIntFlag(raw: string | undefined, flag: string): number {return n;}+function parsePositiveFloatFlag(raw: string | undefined, flag: string): number {+ if (raw === undefined || raw === '') {+ throw new Error(`${flag} requires a positive number value.`);+ }+ const n = Number(raw);+ if (Number.isNaN(n) || n <= 0) {+ throw new Error(`${flag} must be a positive number; got "${raw}".`);+ }+ return n;+}+function parseArgs(argv: string[]): Args {const breaker: CircuitBreakerConfig = { ...DEFAULT_BREAKER };const prune: PruneConfig = { ...DEFAULT_PRUNE };@@ -97,6 +108,11 @@ function parseArgs(argv: string[]): Args {else if (a === '--on-exceed') onExceed = argv[++i];else if (a === '--window') prune.window = parsePositiveIntFlag(argv[++i], '--window');else if (a === '--max-trace-lines') prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');+ else if (a === '--similarity-threshold') {+ const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');+ breaker.similarityThreshold = val;+ prune.similarityThreshold = val;+ }}return { help: false, ledger, ...base() };@@ -171,6 +187,7 @@ Options:--on-exceed <script> On escalate, pipe the decision as JSON to thisscript's stdin (fire-and-forget; its exit codeis not checked and does not change --check's own).+ --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})-h, --help This helpdiff --git a/tools/loop-context/src/context-manager.ts b/tools/loop-context/src/context-manager.tsindex 8e734e6..f343b34 100644--- a/tools/loop-context/src/context-manager.ts+++ b/tools/loop-context/src/context-manager.ts@@ -53,6 +53,8 @@ export interface CircuitBreakerConfig {noProgressThreshold: number;/** Optional hard cap on cumulative tokens across the run. */tokenBudget?: number;+ /** Float 0.0-1.0. If consecutive errors are this similar, they count as stagnant. */+ similarityThreshold: number;}export interface PruneConfig {@@ -60,17 +62,21 @@ export interface PruneConfig {maxTraceLines: number;/** Number of most-recent attempts to retain in the pruned ledger. */window: number;+ /** Float 0.0-1.0. If consecutive errors are this similar, they are collapsed. */+ similarityThreshold: number;}export const DEFAULT_BREAKER: CircuitBreakerConfig = {maxIterations: 10,stagnationThreshold: 3,noProgressThreshold: 5,+ similarityThreshold: 0.85,};export const DEFAULT_PRUNE: PruneConfig = {maxTraceLines: 8,window: 5,+ similarityThreshold: 0.85,};// ── Error normalization ────────────────────────────────────────────@@ -96,6 +102,34 @@ export function errorSignature(error: string): string {.trim();}+/**+ * Calculate Jaccard similarity (0.0 to 1.0) using character trigrams.+ * Highly robust to minor phrasing variations.+ */+function getTrigrams(str: string): Set<string> {+ const trigrams = new Set<string>();+ const padded = ` ${str.toLowerCase()} `;+ for (let i = 0; i < padded.length - 2; i++) {+ trigrams.add(padded.substring(i, i + 3));+ }+ return trigrams;+}++export function calculateSimilarity(a: string, b: string): number {+ if (a === b) return 1.0;+ const setA = getTrigrams(a);+ const setB = getTrigrams(b);+ if (setA.size === 0 && setB.size === 0) return 1.0;+ if (setA.size === 0 || setB.size === 0) return 0.0;++ let intersection = 0;+ for (const item of setA) {+ if (setB.has(item)) intersection++;+ }+ const union = setA.size + setB.size - intersection;+ return intersection / union;+}+// ── Circuit breaker ────────────────────────────────────────────────export type BreakerTrigger =@@ -154,7 +188,8 @@ export function checkCircuitBreaker(const lastSig = errorSignature(failRun[failRun.length - 1].error ?? '');let same = 0;for (let i = failRun.length - 1; i >= 0; i--) {- if (errorSignature(failRun[i].error ?? '') === lastSig) same++;+ const curSig = errorSignature(failRun[i].error ?? '');+ if (calculateSimilarity(curSig, lastSig) >= config.similarityThreshold) same++;else break;}if (same >= config.stagnationThreshold) {@@ -244,7 +279,7 @@ export function pruneLedger(ledger: Ledger, config: PruneConfig = DEFAULT_PRUNE)prev !== undefined &&prev.outcome === 'failure' &&pruned.outcome === 'failure' &&- errorSignature(prev.error ?? '') === errorSignature(pruned.error ?? '');+ calculateSimilarity(errorSignature(prev.error ?? ''), errorSignature(pruned.error ?? '')) >= config.similarityThreshold;if (sameFailure) {prev.repeated = (prev.repeated ?? 1) + 1;@@ -278,7 +313,7 @@ export interface AttemptSummary {}/** Deterministic factual rollup of the whole run — no LLM required. */-export function summarizeAttempts(ledger: Ledger): AttemptSummary {+export function summarizeAttempts(ledger: Ledger, similarityThreshold: number = 0.85): AttemptSummary {const groups = new Map<string, ErrorGroup>();const actions = new Set<string>();let successes = 0;@@ -293,9 +328,17 @@ export function summarizeAttempts(ledger: Ledger): AttemptSummary {failures++;if (a.error) {const sig = errorSignature(a.error);- const existing = groups.get(sig);- if (existing) existing.count++;- else groups.set(sig, { signature: sig, count: 1, sample: a.error.split('\n')[0].trim() });+ let found = false;+ for (const existing of groups.values()) {+ if (calculateSimilarity(existing.signature, sig) >= similarityThreshold) {+ existing.count++;+ found = true;+ break;+ }+ }+ if (!found) {+ groups.set(sig, { signature: sig, count: 1, sample: a.error.split('\n')[0].trim() });+ }}}}@@ -323,7 +366,7 @@ export function buildContextInjection(breaker: CircuitBreakerConfig = DEFAULT_BREAKER,prune: PruneConfig = DEFAULT_PRUNE,): string {- const summary = summarizeAttempts(ledger);+ const summary = summarizeAttempts(ledger, breaker.similarityThreshold);const decision = checkCircuitBreaker(ledger, breaker);const pruned = pruneLedger(ledger, prune);const lines: string[] = [];diff --git a/tools/loop-context/test/context-manager.test.mjs b/tools/loop-context/test/context-manager.test.mjsindex 43b3c95..e609e68 100644--- a/tools/loop-context/test/context-manager.test.mjs+++ b/tools/loop-context/test/context-manager.test.mjs@@ -7,6 +7,7 @@ import {pruneLedger,summarizeAttempts,buildContextInjection,+ calculateSimilarity,DEFAULT_BREAKER,DEFAULT_PRUNE,} from '../dist/context-manager.js';@@ -39,6 +40,23 @@ test('errorSignature keeps genuinely different errors distinct', () => {assert.notEqual(a, b);});+// ── calculateSimilarity ────────────────────────────────────────────++test('calculateSimilarity returns 1.0 for identical strings', () => {+ assert.equal(calculateSimilarity('foo', 'foo'), 1.0);+});++test('calculateSimilarity detects high overlap in phrasing', () => {+ const a = 'Error: connect ECONNREFUSED 127.0.0.1:5432';+ const b = 'Error: connection refused on 127.0.0.1:5432';+ const sim = calculateSimilarity(a, b);+ assert.ok(sim > 0.4 && sim < 1.0, `Similarity was ${sim}`);+});++test('calculateSimilarity returns 0.0 for completely disjoint strings', () => {+ assert.equal(calculateSimilarity('abc', 'xyz'), 0.0);+});+// ── circuit breaker: stagnation ────────────────────────────────────test('breaker trips on stagnation (same error 3× in a row)', () => {@@ -54,6 +72,17 @@ test('breaker trips on stagnation (same error 3× in a row)', () => {assert.equal(d.shouldContinue, false);});+test('breaker trips on stagnation with slightly different wording', () => {+ const l = ledger([+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),+ attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),+ ]);+ const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, similarityThreshold: 0.5 });+ assert.equal(d.escalate, true);+ assert.equal(d.trigger, 'stagnation');+});+test('breaker does not trip when errors differ each time', () => {const l = ledger([attempt(1, 'failure', { error: 'TypeError: a' }),@@ -152,6 +181,17 @@ test('pruneLedger collapses consecutive identical failures with a count', () =>assert.equal(pruned.attempts[0].iteration, 3);});+test('pruneLedger collapses near-duplicate failures based on similarity', () => {+ const l = ledger([+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),+ attempt(3, 'failure', { error: 'Error: socket timeout on port 8080' }),+ ]);+ const pruned = pruneLedger(l, { ...DEFAULT_PRUNE, window: 5, similarityThreshold: 0.5 });+ assert.equal(pruned.attempts.length, 1);+ assert.equal(pruned.attempts[0].repeated, 3);+});+test('pruneLedger does not mutate the input ledger', () => {const l = ledger([attempt(1, 'failure', { error: 'x'.repeat(10) + '\n'.repeat(20) })]);const before = JSON.stringify(l);@@ -176,6 +216,17 @@ test('summarizeAttempts groups errors by signature, most frequent first', () =>assert.deepEqual(s.actionsTried, ['run migration', 'patch code']);});+test('summarizeAttempts groups errors by similarity', () => {+ const l = ledger([+ attempt(1, 'failure', { error: 'Error: connection timeout on port 8080' }),+ attempt(2, 'failure', { error: 'Error: timeout on port 8080' }),+ attempt(3, 'failure', { error: 'Error: totally different error' }),+ ]);+ const s = summarizeAttempts(l, 0.5);+ assert.equal(s.distinctErrors.length, 2);+ assert.equal(s.distinctErrors[0].count, 2);+});+// ── injection ──────────────────────────────────────────────────────test('buildContextInjection includes goal, tried actions, and breaker status', () => {