Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.
TypeScript agentic-aiai-agentsai-codinganthropicautomationclaudeclaude-codecodexcoding-agentsdevops-automationdevtoolsgithub-actionsgrokllmloop-engineeringmcpprompt-engineering
fix(loop-context): refuse a similarity threshold that switches stagnation off
THRISHAL DOMA9e1e2b017d0a
diff --git a/tools/loop-context/README.md b/tools/loop-context/README.mdindex 905089c..4439c7f 100644--- a/tools/loop-context/README.md+++ b/tools/loop-context/README.md@@ -70,11 +70,14 @@ cat run.json | loop-context --check| `--stagnation <n>` | 3 | Escalate when the same error repeats N× in a row || `--no-progress <n>` | 5 | Escalate after N consecutive failures || `--token-budget <n>` | none | Escalate when cumulative tokens reach the cap |+| `--similarity-threshold <f>` | 0.85 | How alike two errors (or actions) must be to count as a repeat. A fraction in (0, 1], **not a percentage**: `0.95`, not `95` || `--window <n>` | 5 | Attempts kept when pruning || `--max-trace-lines <n>` | 8 | Stack-trace lines kept when pruning |Exit codes: `0` continue · `2` escalate · `1` error.+A value that would switch a rule off is refused with exit `1` instead of being accepted. That covers a threshold of `0` or a fraction like `1.5`, and a similarity threshold above `1`: similarity is at most 1.0, so `95` would mean no two errors ever match and the stagnation rule never fires.+## Resolving the token budget from a patternTyping `--token-budget <n>` by hand means guessing. [`loop-cost`](../loop-cost) already@@ -179,6 +182,8 @@ if (decision.escalate) escalateToHuman(decision.reason);else runNextIteration(buildContextInjection(ledger));```+`checkCircuitBreaker`, `pruneLedger`, `summarizeAttempts` and `buildContextInjection` throw on a config that would switch a rule off (a non-positive or fractional count, or a `similarityThreshold` outside (0, 1]), rather than quietly never escalating. Call `validateBreakerConfig` / `validatePruneConfig` to check a config up front.+## Where it fitsThis is the **Memory / State** primitive of loop engineering made dynamic: `STATE.md` stores state statically; `loop-context` manages it across iterations. See [docs/primitives.md](../../docs/primitives.md) and the [operating & safety](../../docs/) guides.diff --git a/tools/loop-context/dist/cli.js b/tools/loop-context/dist/cli.jsindex 06e76e6..ccb5bb9 100755--- a/tools/loop-context/dist/cli.js+++ b/tools/loop-context/dist/cli.js@@ -1,7 +1,7 @@#!/usr/bin/env nodeimport { readFile } from 'node:fs/promises';import { spawn } from 'node:child_process';-import { buildContextInjection, checkCircuitBreaker, pruneLedger, summarizeAttempts, DEFAULT_BREAKER, DEFAULT_PRUNE, } from './context-manager.js';+import { buildContextInjection, checkCircuitBreaker, pruneLedger, summarizeAttempts, DEFAULT_BREAKER, DEFAULT_PRUNE, assertSimilarityThreshold, } from './context-manager.js';import { resolveTokenBudgetFromPattern, resolveDailyBudgetFromPattern, } from './budget-resolver.js';import { recordDailySpend } from './daily-spend.js';/** Reject NaN/0/floats so a bad flag cannot silently disable the breaker. */@@ -15,14 +15,18 @@ function parsePositiveIntFlag(raw, flag) {}return n;}-function parsePositiveFloatFlag(raw, flag) {+/**+ * A similarity threshold is a fraction in (0, 1]. "95" (meant as 95%) used to+ * be accepted and silently switched off the stagnation rule, because no two+ * errors are ever more than 1.0 similar.+ */+function parseFractionFlag(raw, flag) {if (raw === undefined || raw === '') {- throw new Error(`${flag} requires a positive number value.`);+ throw new Error(`${flag} requires a value between 0 and 1 (e.g. 0.85).`);}const n = Number(raw);- if (Number.isNaN(n) || n <= 0) {- throw new Error(`${flag} must be a positive number; got "${raw}".`);- }+ // Report "abc" rather than NaN when the value is not a number at all.+ assertSimilarityThreshold(Number.isNaN(n) ? raw : n, flag);return n;}function parseArgs(argv) {@@ -91,7 +95,7 @@ function parseArgs(argv) {else if (a === '--max-trace-lines')prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');else if (a === '--similarity-threshold') {- const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');+ const val = parseFractionFlag(argv[++i], '--similarity-threshold');breaker.similarityThreshold = val;prune.similarityThreshold = val;}@@ -167,7 +171,8 @@ Options:--on-exceed <script> On escalate, pipe the decision as JSON to thisscript's stdin (fire-and-forget; its exit codeis not checked and does not change --check's own).- --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})+ --similarity-threshold <f> Fraction in (0, 1] to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold}).+ A fraction, not a percentage: 0.95, not 95--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})-h, --help This helpdiff --git a/tools/loop-context/dist/context-manager.d.ts b/tools/loop-context/dist/context-manager.d.tsindex 8d1d57d..825786d 100644--- a/tools/loop-context/dist/context-manager.d.ts+++ b/tools/loop-context/dist/context-manager.d.ts@@ -64,6 +64,11 @@ export interface PruneConfig {}export declare const DEFAULT_BREAKER: CircuitBreakerConfig;export declare const DEFAULT_PRUNE: PruneConfig;+/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */+export declare function assertSimilarityThreshold(value: unknown, label?: string): asserts value is number;+/** Throw if any rule in the config could never fire (or would fire on nothing). */+export declare function validateBreakerConfig(config: CircuitBreakerConfig): void;+export declare function validatePruneConfig(config: PruneConfig): void;/*** Reduce a raw error / stack trace to a stable signature so that "the same* error" can be recognized across iterations even when volatile detailsdiff --git a/tools/loop-context/dist/context-manager.js b/tools/loop-context/dist/context-manager.jsindex c3b602e..289a3c5 100644--- a/tools/loop-context/dist/context-manager.js+++ b/tools/loop-context/dist/context-manager.js@@ -27,6 +27,47 @@ export const DEFAULT_PRUNE = {window: 5,similarityThreshold: 0.85,};+// ── Config validation ──────────────────────────────────────────────+//+// Every breaker rule compares a count or a similarity against a threshold, so+// an out-of-range value does not error on its own: it quietly turns the rule+// off. calculateSimilarity() tops out at 1.0, so a similarityThreshold of 95+// (meant as 95%) means no two errors ever match and stagnation never fires;+// NaN makes every comparison false. A breaker that silently stops breaking is+// worse than one that refuses to start, so bad configs throw.+/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */+export function assertSimilarityThreshold(value, label = 'similarityThreshold') {+ // NaN fails both comparisons and Infinity fails the second, so this range check covers them.+ if (typeof value === 'number' && value > 0 && value <= 1)+ return;+ let why = '';+ if (typeof value === 'number' && value > 1) {+ why = ' Similarity is at most 1.0, so no two errors would ever match and the stagnation rule would never fire.';+ if (Number.isInteger(value) && value <= 100)+ why += ` If you meant ${value}%, use ${value / 100}.`;+ }+ throw new Error(`${label} must be a fraction greater than 0 and at most 1 (default ${DEFAULT_BREAKER.similarityThreshold}); got ${String(value)}.${why}`);+}+function assertPositiveInteger(value, label) {+ if (typeof value === 'number' && Number.isInteger(value) && value >= 1)+ return;+ throw new Error(`${label} must be a positive integer; got ${String(value)}. An invalid value would switch this rule off.`);+}+/** Throw if any rule in the config could never fire (or would fire on nothing). */+export function validateBreakerConfig(config) {+ assertPositiveInteger(config.maxIterations, 'maxIterations');+ assertPositiveInteger(config.stagnationThreshold, 'stagnationThreshold');+ assertPositiveInteger(config.frustrationThreshold, 'frustrationThreshold');+ assertPositiveInteger(config.noProgressThreshold, 'noProgressThreshold');+ if (config.tokenBudget !== undefined)+ assertPositiveInteger(config.tokenBudget, 'tokenBudget');+ assertSimilarityThreshold(config.similarityThreshold);+}+export function validatePruneConfig(config) {+ assertPositiveInteger(config.maxTraceLines, 'maxTraceLines');+ assertPositiveInteger(config.window, 'window');+ assertSimilarityThreshold(config.similarityThreshold);+}// ── Error normalization ────────────────────────────────────────────/*** Reduce a raw error / stack trace to a stable signature so that "the same@@ -97,6 +138,7 @@ function trailingFailureRun(attempts) {* actionable one when several conditions hold.*/export function checkCircuitBreaker(ledger, config = DEFAULT_BREAKER) {+ validateBreakerConfig(config);const iterations = ledger.attempts.length;const tokensUsed = totalTokens(ledger);const base = { iterations, tokensUsed };@@ -197,6 +239,7 @@ export function pruneStackTrace(trace, maxLines) {* returns a new object.*/export function pruneLedger(ledger, config = DEFAULT_PRUNE) {+ validatePruneConfig(config);const recent = ledger.attempts.slice(-config.window);const collapsed = [];for (const attempt of recent) {@@ -223,6 +266,7 @@ export function pruneLedger(ledger, config = DEFAULT_PRUNE) {}/** Deterministic factual rollup of the whole run — no LLM required. */export function summarizeAttempts(ledger, similarityThreshold = 0.85) {+ assertSimilarityThreshold(similarityThreshold);const groups = new Map();const actions = new Set();let successes = 0;diff --git a/tools/loop-context/src/cli.ts b/tools/loop-context/src/cli.tsindex b0592f0..6bc097c 100644--- a/tools/loop-context/src/cli.ts+++ b/tools/loop-context/src/cli.ts@@ -8,6 +8,7 @@ import {summarizeAttempts,DEFAULT_BREAKER,DEFAULT_PRUNE,+ assertSimilarityThreshold,type Ledger,type CircuitBreakerConfig,type PruneConfig,@@ -52,14 +53,18 @@ function parsePositiveIntFlag(raw: string | undefined, flag: string): number {return n;}-function parsePositiveFloatFlag(raw: string | undefined, flag: string): number {+/**+ * A similarity threshold is a fraction in (0, 1]. "95" (meant as 95%) used to+ * be accepted and silently switched off the stagnation rule, because no two+ * errors are ever more than 1.0 similar.+ */+function parseFractionFlag(raw: string | undefined, flag: string): number {if (raw === undefined || raw === '') {- throw new Error(`${flag} requires a positive number value.`);+ throw new Error(`${flag} requires a value between 0 and 1 (e.g. 0.85).`);}const n = Number(raw);- if (Number.isNaN(n) || n <= 0) {- throw new Error(`${flag} must be a positive number; got "${raw}".`);- }+ // Report "abc" rather than NaN when the value is not a number at all.+ assertSimilarityThreshold(Number.isNaN(n) ? raw : n, flag);return n;}@@ -109,7 +114,7 @@ function parseArgs(argv: string[]): Args {else if (a === '--window') prune.window = parsePositiveIntFlag(argv[++i], '--window');else if (a === '--max-trace-lines') prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');else if (a === '--similarity-threshold') {- const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');+ const val = parseFractionFlag(argv[++i], '--similarity-threshold');breaker.similarityThreshold = val;prune.similarityThreshold = val;}@@ -189,7 +194,8 @@ Options:--on-exceed <script> On escalate, pipe the decision as JSON to thisscript's stdin (fire-and-forget; its exit codeis not checked and does not change --check's own).- --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})+ --similarity-threshold <f> Fraction in (0, 1] to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold}).+ A fraction, not a percentage: 0.95, not 95--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})-h, --help This helpdiff --git a/tools/loop-context/src/context-manager.ts b/tools/loop-context/src/context-manager.tsindex 5d20967..3bca4d2 100644--- a/tools/loop-context/src/context-manager.ts+++ b/tools/loop-context/src/context-manager.ts@@ -82,6 +82,50 @@ export const DEFAULT_PRUNE: PruneConfig = {similarityThreshold: 0.85,};+// ── Config validation ──────────────────────────────────────────────+//+// Every breaker rule compares a count or a similarity against a threshold, so+// an out-of-range value does not error on its own: it quietly turns the rule+// off. calculateSimilarity() tops out at 1.0, so a similarityThreshold of 95+// (meant as 95%) means no two errors ever match and stagnation never fires;+// NaN makes every comparison false. A breaker that silently stops breaking is+// worse than one that refuses to start, so bad configs throw.++/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */+export function assertSimilarityThreshold(value: unknown, label = 'similarityThreshold'): asserts value is number {+ // NaN fails both comparisons and Infinity fails the second, so this range check covers them.+ if (typeof value === 'number' && value > 0 && value <= 1) return;+ let why = '';+ if (typeof value === 'number' && value > 1) {+ why = ' Similarity is at most 1.0, so no two errors would ever match and the stagnation rule would never fire.';+ if (Number.isInteger(value) && value <= 100) why += ` If you meant ${value}%, use ${value / 100}.`;+ }+ throw new Error(+ `${label} must be a fraction greater than 0 and at most 1 (default ${DEFAULT_BREAKER.similarityThreshold}); got ${String(value)}.${why}`,+ );+}++function assertPositiveInteger(value: unknown, label: string): void {+ if (typeof value === 'number' && Number.isInteger(value) && value >= 1) return;+ throw new Error(`${label} must be a positive integer; got ${String(value)}. An invalid value would switch this rule off.`);+}++/** Throw if any rule in the config could never fire (or would fire on nothing). */+export function validateBreakerConfig(config: CircuitBreakerConfig): void {+ assertPositiveInteger(config.maxIterations, 'maxIterations');+ assertPositiveInteger(config.stagnationThreshold, 'stagnationThreshold');+ assertPositiveInteger(config.frustrationThreshold, 'frustrationThreshold');+ assertPositiveInteger(config.noProgressThreshold, 'noProgressThreshold');+ if (config.tokenBudget !== undefined) assertPositiveInteger(config.tokenBudget, 'tokenBudget');+ assertSimilarityThreshold(config.similarityThreshold);+}++export function validatePruneConfig(config: PruneConfig): void {+ assertPositiveInteger(config.maxTraceLines, 'maxTraceLines');+ assertPositiveInteger(config.window, 'window');+ assertSimilarityThreshold(config.similarityThreshold);+}+// ── Error normalization ────────────────────────────────────────────/**@@ -181,6 +225,7 @@ export function checkCircuitBreaker(ledger: Ledger,config: CircuitBreakerConfig = DEFAULT_BREAKER,): BreakerDecision {+ validateBreakerConfig(config);const iterations = ledger.attempts.length;const tokensUsed = totalTokens(ledger);const base = { iterations, tokensUsed };@@ -286,6 +331,7 @@ export function pruneStackTrace(trace: string, maxLines: number): string {* returns a new object.*/export function pruneLedger(ledger: Ledger, config: PruneConfig = DEFAULT_PRUNE): Ledger {+ validatePruneConfig(config);const recent = ledger.attempts.slice(-config.window);const collapsed: Attempt[] = [];@@ -337,6 +383,7 @@ export interface AttemptSummary {/** Deterministic factual rollup of the whole run — no LLM required. */export function summarizeAttempts(ledger: Ledger, similarityThreshold: number = 0.85): AttemptSummary {+ assertSimilarityThreshold(similarityThreshold);const groups = new Map<string, ErrorGroup>();const actions = new Set<string>();let successes = 0;diff --git a/tools/loop-context/test/cli.test.mjs b/tools/loop-context/test/cli.test.mjsindex 0506f45..324ab82 100644--- a/tools/loop-context/test/cli.test.mjs+++ b/tools/loop-context/test/cli.test.mjs@@ -60,6 +60,41 @@ test('cli rejects invalid --max-iterations values', () => {assert.match(r.stderr, /--max-iterations must be a positive integer/);});+test('cli rejects a percentage-style --similarity-threshold instead of disabling stagnation', () => {+ // 95 was accepted and meant no two errors could ever match: the same failure+ // repeated forever came back CONTINUE with exit 0.+ for (const value of ['95', '100', '1.5', 'Infinity']) {+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);+ assert.equal(r.status, 1, value);+ assert.equal(r.stdout, '', `${value}: no decision is printed`);+ assert.match(r.stderr, /--similarity-threshold must be a fraction greater than 0 and at most 1/, value);+ assert.match(r.stderr, /stagnation rule would never fire/, value);+ }+ assert.match(runCli(['--check', '--similarity-threshold', '95']).stderr, /If you meant 95%, use 0\.95/);+});++test('cli rejects zero, negative and non-numeric --similarity-threshold', () => {+ for (const value of ['0', '-0.5', 'abc']) {+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);+ assert.equal(r.status, 1, value);+ assert.ok(r.stderr.includes(`got ${value}.`), value);+ }+});++test('cli accepts --similarity-threshold across (0, 1] and still trips stagnation', () => {+ for (const value of ['1', '0.95', '0.5', '0.01']) {+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);+ assert.equal(r.status, 2, value);+ assert.equal(JSON.parse(r.stdout).trigger, 'stagnation', value);+ }+});++test('cli validates --similarity-threshold for --prune too', () => {+ const r = runCli(['--prune', '--similarity-threshold', '85']);+ assert.equal(r.status, 1);+ assert.match(r.stderr, /If you meant 85%, use 0\.85/);+});+test('cli rejects missing numeric flag values', () => {const r = runCli(['--check', '--stagnation']);assert.equal(r.status, 1);diff --git a/tools/loop-context/test/context-manager.test.mjs b/tools/loop-context/test/context-manager.test.mjsindex f64b0eb..a24d478 100644--- a/tools/loop-context/test/context-manager.test.mjs+++ b/tools/loop-context/test/context-manager.test.mjs@@ -10,6 +10,9 @@ import {calculateSimilarity,DEFAULT_BREAKER,DEFAULT_PRUNE,+ validateBreakerConfig,+ validatePruneConfig,+ assertSimilarityThreshold,} from '../dist/context-manager.js';function attempt(iteration, outcome, opts = {}) {@@ -272,3 +275,65 @@ test('buildContextInjection shows OK status for a healthy run', () => {const block = buildContextInjection(l);assert.ok(block.includes('Circuit breaker: OK'));});++// ── config validation ──────────────────────────────────────────────++const identicalFailures = () =>+ ledger([1, 2, 3, 4].map((i) => attempt(i, 'failure', { action: 'same fix', error: 'TypeError: boom' })));++test('checkCircuitBreaker throws on a similarityThreshold above 1 instead of never stagnating', () => {+ // Similarity tops out at 1.0, so 95 (a percentage) silently switched stagnation off.+ for (const similarityThreshold of [95, 1.0001, Infinity]) {+ assert.throws(+ () => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold }),+ /similarityThreshold must be a fraction/,+ String(similarityThreshold),+ );+ }+});++test('checkCircuitBreaker throws on configs that would switch a rule off', () => {+ const bad = [+ { similarityThreshold: NaN },+ { similarityThreshold: 0 },+ { similarityThreshold: -0.1 },+ { similarityThreshold: '0.85' },+ { maxIterations: 0 },+ { stagnationThreshold: NaN },+ { frustrationThreshold: 2.5 },+ { noProgressThreshold: Infinity },+ { tokenBudget: 0 },+ { tokenBudget: -100 },+ ];+ for (const override of bad) {+ assert.throws(() => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, ...override }), undefined, JSON.stringify(override));+ }+});++test('valid configs, including the boundaries, still trip stagnation', () => {+ for (const similarityThreshold of [1, 0.5, Number.MIN_VALUE]) {+ const d = checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold });+ assert.equal(d.trigger, 'stagnation', String(similarityThreshold));+ }+ assert.doesNotThrow(() => validateBreakerConfig({ ...DEFAULT_BREAKER, tokenBudget: 1 }));+ assert.doesNotThrow(() => validateBreakerConfig(DEFAULT_BREAKER));+ assert.doesNotThrow(() => validatePruneConfig(DEFAULT_PRUNE));+});++test('pruneLedger, summarizeAttempts and buildContextInjection reject a percentage threshold', () => {+ assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, similarityThreshold: 85 }), /If you meant 85%, use 0\.85/);+ assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, window: 0 }), /window must be a positive integer/);+ assert.throws(() => summarizeAttempts(identicalFailures(), 85), /similarityThreshold must be a fraction/);+ assert.throws(+ () => buildContextInjection(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold: 85 }),+ /similarityThreshold must be a fraction/,+ );+});++test('assertSimilarityThreshold only gives the percentage hint for whole-number percentages', () => {+ assert.throws(() => assertSimilarityThreshold(95), /If you meant 95%, use 0\.95/);+ assert.throws(() => assertSimilarityThreshold(1.5), (err) => !/If you meant/.test(err.message));+ assert.throws(() => assertSimilarityThreshold(0), (err) => !/never fire/.test(err.message));+ assert.throws(() => assertSimilarityThreshold(150), (err) => /never fire/.test(err.message) && !/If you meant/.test(err.message));+ assert.throws(() => assertSimilarityThreshold(95, '--similarity-threshold'), /^Error: --similarity-threshold must/);+});diff --git a/tools/loop-drill/README.md b/tools/loop-drill/README.mdindex 92adc64..5ad17da 100644--- a/tools/loop-drill/README.md+++ b/tools/loop-drill/README.md@@ -62,6 +62,7 @@ Each drill is tagged with the `docs/failure-modes.md` entry it exercises, so the| `breaker.no-progress` | Infinite Fix Loop | N consecutive failures | Escalates || `breaker.token-budget` | Token Burn | Attempt over budget | Escalates || `breaker.healthy` | Infinite Fix Loop | A healthy run | **Does not escalate** |+| `breaker.config` | Infinite Fix Loop | Only reported when `loop-context` rejects the config (e.g. `similarityThreshold: 95`) | Never passes; the other breaker drills can't run || `verifier.control` | Verifier Theater | Nothing (clean tree) | **Accepts** || `verifier.mutant[op]` | Verifier Theater | One seeded defect | Rejects |diff --git a/tools/loop-drill/dist/drill.d.ts b/tools/loop-drill/dist/drill.d.tsindex 09020c3..9d88dfc 100644--- a/tools/loop-drill/dist/drill.d.ts+++ b/tools/loop-drill/dist/drill.d.ts@@ -17,7 +17,7 @@* A guardrail is only credited when it passes both.*/import { type GateConfig } from '@cobusgreyling/loop-gate';-import { type CircuitBreakerConfig } from '@cobusgreyling/loop-context';+import { type BreakerDecision, type CircuitBreakerConfig, type Ledger } from '@cobusgreyling/loop-context';export type DrillOutcome = 'passed' | 'failed' | 'skipped';/** Named failure modes from docs/failure-modes.md. */export type FailureMode = 'Infinite Fix Loop' | 'Token Burn' | 'Over-Reach (Wrong Scope)' | 'Verifier Theater' | 'Escalation Failure';@@ -78,8 +78,16 @@ export declare function runGateDrills(options: GateDrillOptions): DrillResult[];* escalated. Escalating for another reason means the rule under test is still* unproven — the same standard the gate drills apply to `trigger !==* 'denylist'`.+ *+ * loop-context throws on a config that would switch a rule off (a similarity+ * threshold of 95 meant as 95%, say). That is reported as a failed drill, not+ * a crash: a breaker that refuses to start protects nothing either.+ *+ * `check` defaults to loop-context's breaker; tests pass a stand-in to+ * exercise the trigger-attribution logic against a breaker that misbehaves.*/-export declare function runBreakerDrills(config?: CircuitBreakerConfig): DrillResult[];+export type BreakerCheck = (ledger: Ledger, config: CircuitBreakerConfig) => BreakerDecision;+export declare function runBreakerDrills(config?: CircuitBreakerConfig, check?: BreakerCheck): DrillResult[];export declare function buildReport(results: DrillResult[]): DrillReport;/*** 0 = every drill passed, 1 = some skipped (guardrail not configured),diff --git a/tools/loop-drill/dist/drill.js b/tools/loop-drill/dist/drill.jsindex 3b3a384..cdd2dae 100644--- a/tools/loop-drill/dist/drill.js+++ b/tools/loop-drill/dist/drill.js@@ -141,18 +141,17 @@ const DISTINCT_ERRORS = ['OutOfMemoryError during the bundle step','Segmentation fault in the native addon',];-/**- * Exercise loop-context's circuit breaker with synthetic ledgers: repeated- * identical failures must trip stagnation, a long run of unrelated failures- * must trip no-progress, blowing the token budget must trip, and a healthy run- * must not.- *- * Each drill asserts the *specific* trigger, not merely that the breaker- * escalated. Escalating for another reason means the rule under test is still- * unproven — the same standard the gate drills apply to `trigger !==- * 'denylist'`.- */-export function runBreakerDrills(config = DEFAULT_BREAKER) {+export function runBreakerDrills(config = DEFAULT_BREAKER, check = checkCircuitBreaker) {+ try {+ return breakerDrills(config, check);+ }+ catch (err) {+ return [+ fail('breaker.config', 'loop-context accepts the breaker config', 'Infinite Fix Loop', 'sensitivity', 'a config loop-context accepts', `rejected: ${err.message}`, 'loop-context refuses to run with a rule switched off, so no breaker rule was drilled. Fix the value and re-run.'),+ ];+ }+}+function breakerDrills(config, check) {const results = [];// Sensitivity: the same error repeated trips stagnation.{@@ -165,12 +164,12 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {outcome: 'failure',error: "TypeError: Cannot read properties of undefined (reading 'id')",}));- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'stagnation'? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason): fail(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'- : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.'));+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop.'));}// Sensitivity: a long run of unrelated failures trips no-progress.{@@ -183,7 +182,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {outcome: 'failure',error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],}));- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'no-progress'? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason): fail(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate@@ -203,7 +202,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {tokensUsed: config.tokenBudget + 1,},];- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'token-budget'? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason): fail(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.escalate ? `escalated via ${decision.trigger}` : 'continued'));@@ -220,7 +219,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {{ iteration: 1, action: 'read the failing test', outcome: 'success' },{ iteration: 2, action: 'apply a minimal fix', outcome: 'success' },];- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(!decision.escalate? pass(id, name, mode, 'specificity', 'continue', 'continued'): fail(id, name, mode, 'specificity', 'continue', `escalated (${decision.trigger})`, 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.'));diff --git a/tools/loop-drill/src/drill.ts b/tools/loop-drill/src/drill.tsindex d8991d1..cdfdb69 100644--- a/tools/loop-drill/src/drill.ts+++ b/tools/loop-drill/src/drill.ts@@ -21,6 +21,7 @@ import { checkGate, type GateConfig } from '@cobusgreyling/loop-gate';import {checkCircuitBreaker,DEFAULT_BREAKER,+ type BreakerDecision,type CircuitBreakerConfig,type Ledger,} from '@cobusgreyling/loop-context';@@ -295,8 +296,38 @@ const DISTINCT_ERRORS = [* escalated. Escalating for another reason means the rule under test is still* unproven — the same standard the gate drills apply to `trigger !==* 'denylist'`.+ *+ * loop-context throws on a config that would switch a rule off (a similarity+ * threshold of 95 meant as 95%, say). That is reported as a failed drill, not+ * a crash: a breaker that refuses to start protects nothing either.+ *+ * `check` defaults to loop-context's breaker; tests pass a stand-in to+ * exercise the trigger-attribution logic against a breaker that misbehaves.*/-export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER): DrillResult[] {+export type BreakerCheck = (ledger: Ledger, config: CircuitBreakerConfig) => BreakerDecision;++export function runBreakerDrills(+ config: CircuitBreakerConfig = DEFAULT_BREAKER,+ check: BreakerCheck = checkCircuitBreaker,+): DrillResult[] {+ try {+ return breakerDrills(config, check);+ } catch (err) {+ return [+ fail(+ 'breaker.config',+ 'loop-context accepts the breaker config',+ 'Infinite Fix Loop',+ 'sensitivity',+ 'a config loop-context accepts',+ `rejected: ${(err as Error).message}`,+ 'loop-context refuses to run with a rule switched off, so no breaker rule was drilled. Fix the value and re-run.',+ ),+ ];+ }+}++function breakerDrills(config: CircuitBreakerConfig, check: BreakerCheck): DrillResult[] {const results: DrillResult[] = [];// Sensitivity: the same error repeated trips stagnation.@@ -310,7 +341,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)outcome: 'failure' as const,error: "TypeError: Cannot read properties of undefined (reading 'id')",}));- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'stagnation'? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)@@ -323,7 +354,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)decision.escalate ? `escalated via ${decision.trigger}` : 'continued',decision.escalate? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'- : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.',+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop.',),);}@@ -339,7 +370,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)outcome: 'failure' as const,error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],}));- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'no-progress'? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)@@ -370,7 +401,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)tokensUsed: config.tokenBudget + 1,},];- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(decision.trigger === 'token-budget'? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)@@ -404,7 +435,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER){ iteration: 1, action: 'read the failing test', outcome: 'success' as const },{ iteration: 2, action: 'apply a minimal fix', outcome: 'success' as const },];- const decision = checkCircuitBreaker(ledgerOf(attempts), config);+ const decision = check(ledgerOf(attempts), config);results.push(!decision.escalate? pass(id, name, mode, 'specificity', 'continue', 'continued')diff --git a/tools/loop-drill/test/drill.test.mjs b/tools/loop-drill/test/drill.test.mjsindex 31f2d27..6533f12 100644--- a/tools/loop-drill/test/drill.test.mjs+++ b/tools/loop-drill/test/drill.test.mjs@@ -136,27 +136,41 @@ describe('runBreakerDrills', () => {assert.match(byId(results, 'breaker.no-progress').actual, /no.progress/i);});- test('a percentage similarityThreshold leaves stagnation unproven', () => {- // similarityThreshold is a 0.0-1.0 fraction. Setting it to 95 (meaning- // "95%") means identical errors never compare as similar, so a loop retrying- // the same failure forever is never stopped. The drill must report that.- const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 });- const stagnation = byId(misconfigured, 'breaker.stagnation');- assert.equal(stagnation.outcome, 'failed');- assert.equal(stagnation.actual, 'continued');- assert.match(stagnation.detail, /Infinite Fix Loop/);- assert.match(stagnation.detail, /similarityThreshold/);+ test('a percentage similarityThreshold is a failed drill, not a crash', () => {+ // similarityThreshold is a 0.0-1.0 fraction. 95 (meaning "95%") used to make+ // identical errors never compare as similar, so a loop retrying the same+ // failure forever was never stopped. loop-context now refuses the config;+ // the drill must report that rather than throw.+ for (const similarityThreshold of [95, 0]) {+ const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold });+ assert.equal(misconfigured.length, 1, String(similarityThreshold));+ const [r] = misconfigured;+ assert.equal(r.id, 'breaker.config');+ assert.equal(r.outcome, 'failed');+ assert.equal(r.failureMode, 'Infinite Fix Loop');+ assert.match(r.actual, /similarityThreshold must be a fraction/);+ }+ assert.match(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 })[0].actual, /use 0\.95/);+ assert.equal(exitCodeFor(buildReport(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 }))), 2);+ });++ // A breaker that always answers with one trigger: how the drills are shown to+ // credit only the rule they exercise, now that loop-context refuses the+ // configs that used to produce these shapes.+ const always = (trigger) => (ledger) => ({+ shouldContinue: false,+ escalate: true,+ trigger,+ reason: `stand-in breaker: ${trigger}`,+ iterations: ledger.attempts.length,+ tokensUsed: 0,});test('escalating via the wrong trigger leaves stagnation unproven', () => {// The case that separates "the breaker fired" from "the rule under test// fired". Here the breaker DOES escalate, so a drill asserting only// decision.escalate would pass while stagnation was never exercised.- const wrongTrigger = runBreakerDrills({- ...DEFAULT_BREAKER,- similarityThreshold: 95,- noProgressThreshold: 1,- });+ const wrongTrigger = runBreakerDrills(DEFAULT_BREAKER, always('no-progress'));const stagnation = byId(wrongTrigger, 'breaker.stagnation');assert.equal(stagnation.outcome, 'failed');assert.equal(stagnation.actual, 'escalated via no-progress');@@ -164,27 +178,31 @@ describe('runBreakerDrills', () => {});test('an iteration cap that fires first also leaves stagnation unproven', () => {- const capped = runBreakerDrills({- ...DEFAULT_BREAKER,- similarityThreshold: 95,- maxIterations: 2,- });+ const capped = runBreakerDrills(DEFAULT_BREAKER, always('max-iterations'));const stagnation = byId(capped, 'breaker.stagnation');assert.equal(stagnation.outcome, 'failed');assert.equal(stagnation.actual, 'escalated via max-iterations');});test('stagnation swallowing every error leaves no-progress unproven', () => {- // similarityThreshold 0 makes every error count as "the same error", so- // stagnation fires on the no-progress drill's unrelated failures. The- // breaker still escalates, so only a trigger-specific assertion catches it.- const swallowed = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 0 });+ // A breaker that calls every error "the same error" fires stagnation on the+ // no-progress drill's unrelated failures. It still escalates, so only a+ // trigger-specific assertion catches it.+ const swallowed = runBreakerDrills(DEFAULT_BREAKER, always('stagnation'));const noProgress = byId(swallowed, 'breaker.no-progress');assert.equal(noProgress.outcome, 'failed');assert.equal(noProgress.actual, 'escalated via stagnation');assert.match(noProgress.detail, /not by the rule under test/);});+ test('a breaker that never escalates fails stagnation and passes the healthy run', () => {+ const never = (ledger) => ({ shouldContinue: true, escalate: false, trigger: 'ok', reason: 'ok', iterations: ledger.attempts.length, tokensUsed: 0 });+ const r = runBreakerDrills(DEFAULT_BREAKER, never);+ assert.equal(byId(r, 'breaker.stagnation').actual, 'continued');+ assert.match(byId(r, 'breaker.stagnation').detail, /Infinite Fix Loop/);+ assert.equal(byId(r, 'breaker.healthy').outcome, 'passed');+ });+test('no-progress uses errors that stay distinct after signature normalization', () => {// errorSignature() collapses every number to '#', so errors differing only// by a number ("module 1" / "module 2") normalize to the same signature and