Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
feat(loop-context): semantic frustration circuit breaker (#337)
THRISHAL DOMA9b90b48ac5ff
diff --git a/tools/loop-context/dist/cli.js b/tools/loop-context/dist/cli.jsindex 03b0c54..6eada01 100755--- a/tools/loop-context/dist/cli.js+++ b/tools/loop-context/dist/cli.js@@ -15,6 +15,16 @@ function parsePositiveIntFlag(raw, flag) {}return n;}+function parsePositiveFloatFlag(raw, flag) {+ if (raw === undefined || raw === '') {+ throw new Error(`${flag} requires a positive number value.`);+ }+ const n = Number(raw);+ if (Number.isNaN(n) || n <= 0) {+ throw new Error(`${flag} must be a positive number; got "${raw}".`);+ }+ return n;+}function parseArgs(argv) {const breaker = { ...DEFAULT_BREAKER };const prune = { ...DEFAULT_PRUNE };@@ -80,6 +90,11 @@ function parseArgs(argv) {prune.window = parsePositiveIntFlag(argv[++i], '--window');else if (a === '--max-trace-lines')prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');+ else if (a === '--similarity-threshold') {+ const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');+ breaker.similarityThreshold = val;+ prune.similarityThreshold = val;+ }}return { help: false, ledger, ...base() };}@@ -150,6 +165,7 @@ Options:--on-exceed <script> On escalate, pipe the decision as JSON to thisscript's stdin (fire-and-forget; its exit codeis not checked and does not change --check's own).+ --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})-h, --help This helpdiff --git a/tools/loop-context/dist/context-manager.d.ts b/tools/loop-context/dist/context-manager.d.tsindex 39c69e3..8d1d57d 100644--- a/tools/loop-context/dist/context-manager.d.ts+++ b/tools/loop-context/dist/context-manager.d.ts@@ -45,16 +45,22 @@ export interface CircuitBreakerConfig {maxIterations: number;/** Escalate when the same error signature repeats this many times in a row. */stagnationThreshold: number;+ /** Escalate when the agent attempts semantically similar actions this many times in a row. */+ frustrationThreshold: number;/** Escalate after this many consecutive failures with no success in between. */noProgressThreshold: number;/** Optional hard cap on cumulative tokens across the run. */tokenBudget?: number;+ /** Float 0.0-1.0. If consecutive errors are this similar, they count as stagnant. */+ similarityThreshold: number;}export interface PruneConfig {/** Max lines to keep from any single stack trace. */maxTraceLines: number;/** Number of most-recent attempts to retain in the pruned ledger. */window: number;+ /** Float 0.0-1.0. If consecutive errors are this similar, they are collapsed. */+ similarityThreshold: number;}export declare const DEFAULT_BREAKER: CircuitBreakerConfig;export declare const DEFAULT_PRUNE: PruneConfig;@@ -64,7 +70,8 @@ export declare const DEFAULT_PRUNE: PruneConfig;* (line numbers, addresses, timestamps, ports, temp paths) differ.*/export declare function errorSignature(error: string): string;-export type BreakerTrigger = 'ok' | 'stagnation' | 'no-progress' | 'token-budget' | 'daily-budget' | 'max-iterations';+export declare function calculateSimilarity(a: string, b: string): number;+export type BreakerTrigger = 'ok' | 'stagnation' | 'frustration' | 'no-progress' | 'token-budget' | 'daily-budget' | 'max-iterations';export interface BreakerDecision {/** Whether the loop is cleared to run another iteration. */shouldContinue: boolean;@@ -110,7 +117,7 @@ export interface AttemptSummary {actionsTried: string[];}/** Deterministic factual rollup of the whole run — no LLM required. */-export declare function summarizeAttempts(ledger: Ledger): AttemptSummary;+export declare function summarizeAttempts(ledger: Ledger, similarityThreshold?: number): AttemptSummary;/*** Build the compact context block to prepend to the next prompt. Contains only* what the agent needs to make progress: the goal, what has already been trieddiff --git a/tools/loop-context/dist/context-manager.js b/tools/loop-context/dist/context-manager.jsindex a229f70..c3b602e 100644--- a/tools/loop-context/dist/context-manager.js+++ b/tools/loop-context/dist/context-manager.js@@ -18,11 +18,14 @@export const DEFAULT_BREAKER = {maxIterations: 10,stagnationThreshold: 3,+ frustrationThreshold: 3,noProgressThreshold: 5,+ similarityThreshold: 0.85,};export const DEFAULT_PRUNE = {maxTraceLines: 8,window: 5,+ similarityThreshold: 0.85,};// ── Error normalization ────────────────────────────────────────────/**@@ -45,6 +48,35 @@ export function errorSignature(error) {.replace(/\s+/g, ' ').trim();}+/**+ * Calculate Jaccard similarity (0.0 to 1.0) using character trigrams.+ * Highly robust to minor phrasing variations.+ */+function getTrigrams(str) {+ const trigrams = new Set();+ const padded = ` ${str.toLowerCase()} `;+ for (let i = 0; i < padded.length - 2; i++) {+ trigrams.add(padded.substring(i, i + 3));+ }+ return trigrams;+}+export function calculateSimilarity(a, b) {+ if (a === b)+ return 1.0;+ const setA = getTrigrams(a);+ const setB = getTrigrams(b);+ if (setA.size === 0 && setB.size === 0)+ return 1.0;+ if (setA.size === 0 || setB.size === 0)+ return 0.0;+ let intersection = 0;+ for (const item of setA) {+ if (setB.has(item))+ intersection++;+ }+ const union = setA.size + setB.size - intersection;+ return intersection / union;+}function totalTokens(ledger) {return ledger.attempts.reduce((sum, a) => sum + (a.tokensUsed ?? 0), 0);}@@ -74,7 +106,8 @@ export function checkCircuitBreaker(ledger, config = DEFAULT_BREAKER) {const lastSig = errorSignature(failRun[failRun.length - 1].error ?? '');let same = 0;for (let i = failRun.length - 1; i >= 0; i--) {- if (errorSignature(failRun[i].error ?? '') === lastSig)+ const curSig = errorSignature(failRun[i].error ?? '');+ if (calculateSimilarity(curSig, lastSig) >= config.similarityThreshold)same++;elsebreak;@@ -89,6 +122,26 @@ export function checkCircuitBreaker(ledger, config = DEFAULT_BREAKER) {};}}+ // Frustration: semantically similar actions repeated without success.+ if (failRun.length >= config.frustrationThreshold) {+ const lastAction = failRun[failRun.length - 1].action;+ let sameAction = 0;+ for (let i = failRun.length - 1; i >= 0; i--) {+ if (calculateSimilarity(failRun[i].action, lastAction) >= config.similarityThreshold)+ sameAction++;+ else+ break;+ }+ if (sameAction >= config.frustrationThreshold) {+ return {+ ...base,+ shouldContinue: false,+ escalate: true,+ trigger: 'frustration',+ reason: `Semantic looping detected: agent repeated highly similar action ${sameAction}× in a row (threshold ${config.frustrationThreshold}): "${lastAction}". Escalating.`,+ };+ }+ }// No-progress: many consecutive failures without a success.if (failRun.length >= config.noProgressThreshold) {return {@@ -157,7 +210,7 @@ export function pruneLedger(ledger, config = DEFAULT_PRUNE) {const sameFailure = prev !== undefined &&prev.outcome === 'failure' &&pruned.outcome === 'failure' &&- errorSignature(prev.error ?? '') === errorSignature(pruned.error ?? '');+ calculateSimilarity(errorSignature(prev.error ?? ''), errorSignature(pruned.error ?? '')) >= config.similarityThreshold;if (sameFailure) {prev.repeated = (prev.repeated ?? 1) + 1;prev.iteration = pruned.iteration; // advance to the latest iteration@@ -169,7 +222,7 @@ export function pruneLedger(ledger, config = DEFAULT_PRUNE) {return { goal: ledger.goal, startedAt: ledger.startedAt, attempts: collapsed };}/** Deterministic factual rollup of the whole run — no LLM required. */-export function summarizeAttempts(ledger) {+export function summarizeAttempts(ledger, similarityThreshold = 0.85) {const groups = new Map();const actions = new Set();let successes = 0;@@ -185,11 +238,17 @@ export function summarizeAttempts(ledger) {failures++;if (a.error) {const sig = errorSignature(a.error);- const existing = groups.get(sig);- if (existing)- existing.count++;- else+ let found = false;+ for (const existing of groups.values()) {+ if (calculateSimilarity(existing.signature, sig) >= similarityThreshold) {+ existing.count++;+ found = true;+ break;+ }+ }+ if (!found) {groups.set(sig, { signature: sig, count: 1, sample: a.error.split('\n')[0].trim() });+ }}}}@@ -210,7 +269,7 @@ export function summarizeAttempts(ledger) {* (so it does not repeat itself), the last pruned error, and the breaker status.*/export function buildContextInjection(ledger, breaker = DEFAULT_BREAKER, prune = DEFAULT_PRUNE) {- const summary = summarizeAttempts(ledger);+ const summary = summarizeAttempts(ledger, breaker.similarityThreshold);const decision = checkCircuitBreaker(ledger, breaker);const pruned = pruneLedger(ledger, prune);const lines = [];diff --git a/tools/loop-context/src/context-manager.ts b/tools/loop-context/src/context-manager.tsindex f343b34..5d20967 100644--- a/tools/loop-context/src/context-manager.ts+++ b/tools/loop-context/src/context-manager.ts@@ -49,6 +49,8 @@ export interface CircuitBreakerConfig {maxIterations: number;/** Escalate when the same error signature repeats this many times in a row. */stagnationThreshold: number;+ /** Escalate when the agent attempts semantically similar actions this many times in a row. */+ frustrationThreshold: number;/** Escalate after this many consecutive failures with no success in between. */noProgressThreshold: number;/** Optional hard cap on cumulative tokens across the run. */@@ -69,6 +71,7 @@ export interface PruneConfig {export const DEFAULT_BREAKER: CircuitBreakerConfig = {maxIterations: 10,stagnationThreshold: 3,+ frustrationThreshold: 3,noProgressThreshold: 5,similarityThreshold: 0.85,};@@ -135,6 +138,7 @@ export function calculateSimilarity(a: string, b: string): number {export type BreakerTrigger =| 'ok'| 'stagnation'+ | 'frustration'| 'no-progress'| 'token-budget'| 'daily-budget'@@ -203,6 +207,25 @@ export function checkCircuitBreaker(}}+ // Frustration: semantically similar actions repeated without success.+ if (failRun.length >= config.frustrationThreshold) {+ const lastAction = failRun[failRun.length - 1].action;+ let sameAction = 0;+ for (let i = failRun.length - 1; i >= 0; i--) {+ if (calculateSimilarity(failRun[i].action, lastAction) >= config.similarityThreshold) sameAction++;+ else break;+ }+ if (sameAction >= config.frustrationThreshold) {+ return {+ ...base,+ shouldContinue: false,+ escalate: true,+ trigger: 'frustration',+ reason: `Semantic looping detected: agent repeated highly similar action ${sameAction}× in a row (threshold ${config.frustrationThreshold}): "${lastAction}". Escalating.`,+ };+ }+ }+// No-progress: many consecutive failures without a success.if (failRun.length >= config.noProgressThreshold) {return {diff --git a/tools/loop-context/test/context-manager.test.mjs b/tools/loop-context/test/context-manager.test.mjsindex e609e68..f64b0eb 100644--- a/tools/loop-context/test/context-manager.test.mjs+++ b/tools/loop-context/test/context-manager.test.mjs@@ -105,6 +105,30 @@ test('breaker stagnation resets after a success', () => {assert.equal(d.escalate, false, 'trailing failure run is only 1 after the success');});+// ── circuit breaker: frustration ───────────────────────────────────++test('breaker trips on frustration (semantically similar actions but different errors)', () => {+ const l = ledger([+ attempt(1, 'failure', { action: 'Try to fix the syntax error', error: 'e1' }),+ attempt(2, 'failure', { action: 'I will try to fix the syntax error', error: 'e2' }),+ attempt(3, 'failure', { action: 'Trying to fix the syntax error', error: 'e3' }),+ ]);+ const d = checkCircuitBreaker(l, { ...DEFAULT_BREAKER, similarityThreshold: 0.5 });+ assert.equal(d.escalate, true);+ assert.equal(d.trigger, 'frustration');+ assert.equal(d.shouldContinue, false);+});++test('breaker does not trip on frustration if actions differ', () => {+ const l = ledger([+ attempt(1, 'failure', { action: 'Install missing dependencies', error: 'e1' }),+ attempt(2, 'failure', { action: 'Fix the failing test case', error: 'e2' }),+ attempt(3, 'failure', { action: 'Revert the previous change', error: 'e3' }),+ ]);+ const d = checkCircuitBreaker(l);+ assert.equal(d.escalate, false);+});+// ── circuit breaker: no-progress ───────────────────────────────────test('breaker trips on no-progress (consecutive failures, distinct errors)', () => {