Skip to content

Practical patterns, starters & CLI tools for loop engineering with AI coding agents. Design systems that prompt and orchestrate agents (inspired by Addy Osmani and Boris Cherny). Includes loop-audit, loop-init, loop-cost.

fix(loop-context): refuse a similarity threshold that switches stagnation off

THRISHAL DOMA9e1e2b017d0a
diff --git a/tools/loop-context/README.md b/tools/loop-context/README.md
index 905089c..4439c7f 100644
--- a/tools/loop-context/README.md
+++ b/tools/loop-context/README.md
@@ -70,11 +70,14 @@ cat run.json | loop-context --check
| `--stagnation <n>` | 3 | Escalate when the same error repeats N× in a row |
| `--no-progress <n>` | 5 | Escalate after N consecutive failures |
| `--token-budget <n>` | none | Escalate when cumulative tokens reach the cap |
+| `--similarity-threshold <f>` | 0.85 | How alike two errors (or actions) must be to count as a repeat. A fraction in (0, 1], **not a percentage**: `0.95`, not `95` |
| `--window <n>` | 5 | Attempts kept when pruning |
| `--max-trace-lines <n>` | 8 | Stack-trace lines kept when pruning |
Exit codes: `0` continue · `2` escalate · `1` error.
+A value that would switch a rule off is refused with exit `1` instead of being accepted. That covers a threshold of `0` or a fraction like `1.5`, and a similarity threshold above `1`: similarity is at most 1.0, so `95` would mean no two errors ever match and the stagnation rule never fires.
+
## Resolving the token budget from a pattern
Typing `--token-budget <n>` by hand means guessing. [`loop-cost`](../loop-cost) already
@@ -179,6 +182,8 @@ if (decision.escalate) escalateToHuman(decision.reason);
else runNextIteration(buildContextInjection(ledger));
```
+`checkCircuitBreaker`, `pruneLedger`, `summarizeAttempts` and `buildContextInjection` throw on a config that would switch a rule off (a non-positive or fractional count, or a `similarityThreshold` outside (0, 1]), rather than quietly never escalating. Call `validateBreakerConfig` / `validatePruneConfig` to check a config up front.
+
## Where it fits
This is the **Memory / State** primitive of loop engineering made dynamic: `STATE.md` stores state statically; `loop-context` manages it across iterations. See [docs/primitives.md](../../docs/primitives.md) and the [operating & safety](../../docs/) guides.
diff --git a/tools/loop-context/dist/cli.js b/tools/loop-context/dist/cli.js
index 06e76e6..ccb5bb9 100755
--- a/tools/loop-context/dist/cli.js
+++ b/tools/loop-context/dist/cli.js
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { readFile } from 'node:fs/promises';
import { spawn } from 'node:child_process';
-import { buildContextInjection, checkCircuitBreaker, pruneLedger, summarizeAttempts, DEFAULT_BREAKER, DEFAULT_PRUNE, } from './context-manager.js';
+import { buildContextInjection, checkCircuitBreaker, pruneLedger, summarizeAttempts, DEFAULT_BREAKER, DEFAULT_PRUNE, assertSimilarityThreshold, } from './context-manager.js';
import { resolveTokenBudgetFromPattern, resolveDailyBudgetFromPattern, } from './budget-resolver.js';
import { recordDailySpend } from './daily-spend.js';
/** Reject NaN/0/floats so a bad flag cannot silently disable the breaker. */
@@ -15,14 +15,18 @@ function parsePositiveIntFlag(raw, flag) {
}
return n;
}
-function parsePositiveFloatFlag(raw, flag) {
+/**
+ * A similarity threshold is a fraction in (0, 1]. "95" (meant as 95%) used to
+ * be accepted and silently switched off the stagnation rule, because no two
+ * errors are ever more than 1.0 similar.
+ */
+function parseFractionFlag(raw, flag) {
if (raw === undefined || raw === '') {
- throw new Error(`${flag} requires a positive number value.`);
+ throw new Error(`${flag} requires a value between 0 and 1 (e.g. 0.85).`);
}
const n = Number(raw);
- if (Number.isNaN(n) || n <= 0) {
- throw new Error(`${flag} must be a positive number; got "${raw}".`);
- }
+ // Report "abc" rather than NaN when the value is not a number at all.
+ assertSimilarityThreshold(Number.isNaN(n) ? raw : n, flag);
return n;
}
function parseArgs(argv) {
@@ -91,7 +95,7 @@ function parseArgs(argv) {
else if (a === '--max-trace-lines')
prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');
else if (a === '--similarity-threshold') {
- const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');
+ const val = parseFractionFlag(argv[++i], '--similarity-threshold');
breaker.similarityThreshold = val;
prune.similarityThreshold = val;
}
@@ -167,7 +171,8 @@ Options:
--on-exceed <script> On escalate, pipe the decision as JSON to this
script's stdin (fire-and-forget; its exit code
is not checked and does not change --check's own).
- --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})
+ --similarity-threshold <f> Fraction in (0, 1] to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold}).
+ A fraction, not a percentage: 0.95, not 95
--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})
--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})
-h, --help This help
diff --git a/tools/loop-context/dist/context-manager.d.ts b/tools/loop-context/dist/context-manager.d.ts
index 8d1d57d..825786d 100644
--- a/tools/loop-context/dist/context-manager.d.ts
+++ b/tools/loop-context/dist/context-manager.d.ts
@@ -64,6 +64,11 @@ export interface PruneConfig {
}
export declare const DEFAULT_BREAKER: CircuitBreakerConfig;
export declare const DEFAULT_PRUNE: PruneConfig;
+/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */
+export declare function assertSimilarityThreshold(value: unknown, label?: string): asserts value is number;
+/** Throw if any rule in the config could never fire (or would fire on nothing). */
+export declare function validateBreakerConfig(config: CircuitBreakerConfig): void;
+export declare function validatePruneConfig(config: PruneConfig): void;
/**
* Reduce a raw error / stack trace to a stable signature so that "the same
* error" can be recognized across iterations even when volatile details
diff --git a/tools/loop-context/dist/context-manager.js b/tools/loop-context/dist/context-manager.js
index c3b602e..289a3c5 100644
--- a/tools/loop-context/dist/context-manager.js
+++ b/tools/loop-context/dist/context-manager.js
@@ -27,6 +27,47 @@ export const DEFAULT_PRUNE = {
window: 5,
similarityThreshold: 0.85,
};
+// ── Config validation ──────────────────────────────────────────────
+//
+// Every breaker rule compares a count or a similarity against a threshold, so
+// an out-of-range value does not error on its own: it quietly turns the rule
+// off. calculateSimilarity() tops out at 1.0, so a similarityThreshold of 95
+// (meant as 95%) means no two errors ever match and stagnation never fires;
+// NaN makes every comparison false. A breaker that silently stops breaking is
+// worse than one that refuses to start, so bad configs throw.
+/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */
+export function assertSimilarityThreshold(value, label = 'similarityThreshold') {
+ // NaN fails both comparisons and Infinity fails the second, so this range check covers them.
+ if (typeof value === 'number' && value > 0 && value <= 1)
+ return;
+ let why = '';
+ if (typeof value === 'number' && value > 1) {
+ why = ' Similarity is at most 1.0, so no two errors would ever match and the stagnation rule would never fire.';
+ if (Number.isInteger(value) && value <= 100)
+ why += ` If you meant ${value}%, use ${value / 100}.`;
+ }
+ throw new Error(`${label} must be a fraction greater than 0 and at most 1 (default ${DEFAULT_BREAKER.similarityThreshold}); got ${String(value)}.${why}`);
+}
+function assertPositiveInteger(value, label) {
+ if (typeof value === 'number' && Number.isInteger(value) && value >= 1)
+ return;
+ throw new Error(`${label} must be a positive integer; got ${String(value)}. An invalid value would switch this rule off.`);
+}
+/** Throw if any rule in the config could never fire (or would fire on nothing). */
+export function validateBreakerConfig(config) {
+ assertPositiveInteger(config.maxIterations, 'maxIterations');
+ assertPositiveInteger(config.stagnationThreshold, 'stagnationThreshold');
+ assertPositiveInteger(config.frustrationThreshold, 'frustrationThreshold');
+ assertPositiveInteger(config.noProgressThreshold, 'noProgressThreshold');
+ if (config.tokenBudget !== undefined)
+ assertPositiveInteger(config.tokenBudget, 'tokenBudget');
+ assertSimilarityThreshold(config.similarityThreshold);
+}
+export function validatePruneConfig(config) {
+ assertPositiveInteger(config.maxTraceLines, 'maxTraceLines');
+ assertPositiveInteger(config.window, 'window');
+ assertSimilarityThreshold(config.similarityThreshold);
+}
// ── Error normalization ────────────────────────────────────────────
/**
* Reduce a raw error / stack trace to a stable signature so that "the same
@@ -97,6 +138,7 @@ function trailingFailureRun(attempts) {
* actionable one when several conditions hold.
*/
export function checkCircuitBreaker(ledger, config = DEFAULT_BREAKER) {
+ validateBreakerConfig(config);
const iterations = ledger.attempts.length;
const tokensUsed = totalTokens(ledger);
const base = { iterations, tokensUsed };
@@ -197,6 +239,7 @@ export function pruneStackTrace(trace, maxLines) {
* returns a new object.
*/
export function pruneLedger(ledger, config = DEFAULT_PRUNE) {
+ validatePruneConfig(config);
const recent = ledger.attempts.slice(-config.window);
const collapsed = [];
for (const attempt of recent) {
@@ -223,6 +266,7 @@ export function pruneLedger(ledger, config = DEFAULT_PRUNE) {
}
/** Deterministic factual rollup of the whole run — no LLM required. */
export function summarizeAttempts(ledger, similarityThreshold = 0.85) {
+ assertSimilarityThreshold(similarityThreshold);
const groups = new Map();
const actions = new Set();
let successes = 0;
diff --git a/tools/loop-context/src/cli.ts b/tools/loop-context/src/cli.ts
index b0592f0..6bc097c 100644
--- a/tools/loop-context/src/cli.ts
+++ b/tools/loop-context/src/cli.ts
@@ -8,6 +8,7 @@ import {
summarizeAttempts,
DEFAULT_BREAKER,
DEFAULT_PRUNE,
+ assertSimilarityThreshold,
type Ledger,
type CircuitBreakerConfig,
type PruneConfig,
@@ -52,14 +53,18 @@ function parsePositiveIntFlag(raw: string | undefined, flag: string): number {
return n;
}
-function parsePositiveFloatFlag(raw: string | undefined, flag: string): number {
+/**
+ * A similarity threshold is a fraction in (0, 1]. "95" (meant as 95%) used to
+ * be accepted and silently switched off the stagnation rule, because no two
+ * errors are ever more than 1.0 similar.
+ */
+function parseFractionFlag(raw: string | undefined, flag: string): number {
if (raw === undefined || raw === '') {
- throw new Error(`${flag} requires a positive number value.`);
+ throw new Error(`${flag} requires a value between 0 and 1 (e.g. 0.85).`);
}
const n = Number(raw);
- if (Number.isNaN(n) || n <= 0) {
- throw new Error(`${flag} must be a positive number; got "${raw}".`);
- }
+ // Report "abc" rather than NaN when the value is not a number at all.
+ assertSimilarityThreshold(Number.isNaN(n) ? raw : n, flag);
return n;
}
@@ -109,7 +114,7 @@ function parseArgs(argv: string[]): Args {
else if (a === '--window') prune.window = parsePositiveIntFlag(argv[++i], '--window');
else if (a === '--max-trace-lines') prune.maxTraceLines = parsePositiveIntFlag(argv[++i], '--max-trace-lines');
else if (a === '--similarity-threshold') {
- const val = parsePositiveFloatFlag(argv[++i], '--similarity-threshold');
+ const val = parseFractionFlag(argv[++i], '--similarity-threshold');
breaker.similarityThreshold = val;
prune.similarityThreshold = val;
}
@@ -189,7 +194,8 @@ Options:
--on-exceed <script> On escalate, pipe the decision as JSON to this
script's stdin (fire-and-forget; its exit code
is not checked and does not change --check's own).
- --similarity-threshold <f> Float 0.0-1.0 to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold})
+ --similarity-threshold <f> Fraction in (0, 1] to cluster similar errors (default: ${DEFAULT_BREAKER.similarityThreshold}).
+ A fraction, not a percentage: 0.95, not 95
--window <n> Attempts kept when pruning (default: ${DEFAULT_PRUNE.window})
--max-trace-lines <n> Stack-trace lines kept (default: ${DEFAULT_PRUNE.maxTraceLines})
-h, --help This help
diff --git a/tools/loop-context/src/context-manager.ts b/tools/loop-context/src/context-manager.ts
index 5d20967..3bca4d2 100644
--- a/tools/loop-context/src/context-manager.ts
+++ b/tools/loop-context/src/context-manager.ts
@@ -82,6 +82,50 @@ export const DEFAULT_PRUNE: PruneConfig = {
similarityThreshold: 0.85,
};
+// ── Config validation ──────────────────────────────────────────────
+//
+// Every breaker rule compares a count or a similarity against a threshold, so
+// an out-of-range value does not error on its own: it quietly turns the rule
+// off. calculateSimilarity() tops out at 1.0, so a similarityThreshold of 95
+// (meant as 95%) means no two errors ever match and stagnation never fires;
+// NaN makes every comparison false. A breaker that silently stops breaking is
+// worse than one that refuses to start, so bad configs throw.
+
+/** Throw unless value is a similarity threshold: a fraction in (0, 1]. */
+export function assertSimilarityThreshold(value: unknown, label = 'similarityThreshold'): asserts value is number {
+ // NaN fails both comparisons and Infinity fails the second, so this range check covers them.
+ if (typeof value === 'number' && value > 0 && value <= 1) return;
+ let why = '';
+ if (typeof value === 'number' && value > 1) {
+ why = ' Similarity is at most 1.0, so no two errors would ever match and the stagnation rule would never fire.';
+ if (Number.isInteger(value) && value <= 100) why += ` If you meant ${value}%, use ${value / 100}.`;
+ }
+ throw new Error(
+ `${label} must be a fraction greater than 0 and at most 1 (default ${DEFAULT_BREAKER.similarityThreshold}); got ${String(value)}.${why}`,
+ );
+}
+
+function assertPositiveInteger(value: unknown, label: string): void {
+ if (typeof value === 'number' && Number.isInteger(value) && value >= 1) return;
+ throw new Error(`${label} must be a positive integer; got ${String(value)}. An invalid value would switch this rule off.`);
+}
+
+/** Throw if any rule in the config could never fire (or would fire on nothing). */
+export function validateBreakerConfig(config: CircuitBreakerConfig): void {
+ assertPositiveInteger(config.maxIterations, 'maxIterations');
+ assertPositiveInteger(config.stagnationThreshold, 'stagnationThreshold');
+ assertPositiveInteger(config.frustrationThreshold, 'frustrationThreshold');
+ assertPositiveInteger(config.noProgressThreshold, 'noProgressThreshold');
+ if (config.tokenBudget !== undefined) assertPositiveInteger(config.tokenBudget, 'tokenBudget');
+ assertSimilarityThreshold(config.similarityThreshold);
+}
+
+export function validatePruneConfig(config: PruneConfig): void {
+ assertPositiveInteger(config.maxTraceLines, 'maxTraceLines');
+ assertPositiveInteger(config.window, 'window');
+ assertSimilarityThreshold(config.similarityThreshold);
+}
+
// ── Error normalization ────────────────────────────────────────────
/**
@@ -181,6 +225,7 @@ export function checkCircuitBreaker(
ledger: Ledger,
config: CircuitBreakerConfig = DEFAULT_BREAKER,
): BreakerDecision {
+ validateBreakerConfig(config);
const iterations = ledger.attempts.length;
const tokensUsed = totalTokens(ledger);
const base = { iterations, tokensUsed };
@@ -286,6 +331,7 @@ export function pruneStackTrace(trace: string, maxLines: number): string {
* returns a new object.
*/
export function pruneLedger(ledger: Ledger, config: PruneConfig = DEFAULT_PRUNE): Ledger {
+ validatePruneConfig(config);
const recent = ledger.attempts.slice(-config.window);
const collapsed: Attempt[] = [];
@@ -337,6 +383,7 @@ export interface AttemptSummary {
/** Deterministic factual rollup of the whole run — no LLM required. */
export function summarizeAttempts(ledger: Ledger, similarityThreshold: number = 0.85): AttemptSummary {
+ assertSimilarityThreshold(similarityThreshold);
const groups = new Map<string, ErrorGroup>();
const actions = new Set<string>();
let successes = 0;
diff --git a/tools/loop-context/test/cli.test.mjs b/tools/loop-context/test/cli.test.mjs
index 0506f45..324ab82 100644
--- a/tools/loop-context/test/cli.test.mjs
+++ b/tools/loop-context/test/cli.test.mjs
@@ -60,6 +60,41 @@ test('cli rejects invalid --max-iterations values', () => {
assert.match(r.stderr, /--max-iterations must be a positive integer/);
});
+test('cli rejects a percentage-style --similarity-threshold instead of disabling stagnation', () => {
+ // 95 was accepted and meant no two errors could ever match: the same failure
+ // repeated forever came back CONTINUE with exit 0.
+ for (const value of ['95', '100', '1.5', 'Infinity']) {
+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);
+ assert.equal(r.status, 1, value);
+ assert.equal(r.stdout, '', `${value}: no decision is printed`);
+ assert.match(r.stderr, /--similarity-threshold must be a fraction greater than 0 and at most 1/, value);
+ assert.match(r.stderr, /stagnation rule would never fire/, value);
+ }
+ assert.match(runCli(['--check', '--similarity-threshold', '95']).stderr, /If you meant 95%, use 0\.95/);
+});
+
+test('cli rejects zero, negative and non-numeric --similarity-threshold', () => {
+ for (const value of ['0', '-0.5', 'abc']) {
+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);
+ assert.equal(r.status, 1, value);
+ assert.ok(r.stderr.includes(`got ${value}.`), value);
+ }
+});
+
+test('cli accepts --similarity-threshold across (0, 1] and still trips stagnation', () => {
+ for (const value of ['1', '0.95', '0.5', '0.01']) {
+ const r = runCli(['--check', '--similarity-threshold', value, '--json']);
+ assert.equal(r.status, 2, value);
+ assert.equal(JSON.parse(r.stdout).trigger, 'stagnation', value);
+ }
+});
+
+test('cli validates --similarity-threshold for --prune too', () => {
+ const r = runCli(['--prune', '--similarity-threshold', '85']);
+ assert.equal(r.status, 1);
+ assert.match(r.stderr, /If you meant 85%, use 0\.85/);
+});
+
test('cli rejects missing numeric flag values', () => {
const r = runCli(['--check', '--stagnation']);
assert.equal(r.status, 1);
diff --git a/tools/loop-context/test/context-manager.test.mjs b/tools/loop-context/test/context-manager.test.mjs
index f64b0eb..a24d478 100644
--- a/tools/loop-context/test/context-manager.test.mjs
+++ b/tools/loop-context/test/context-manager.test.mjs
@@ -10,6 +10,9 @@ import {
calculateSimilarity,
DEFAULT_BREAKER,
DEFAULT_PRUNE,
+ validateBreakerConfig,
+ validatePruneConfig,
+ assertSimilarityThreshold,
} from '../dist/context-manager.js';
function attempt(iteration, outcome, opts = {}) {
@@ -272,3 +275,65 @@ test('buildContextInjection shows OK status for a healthy run', () => {
const block = buildContextInjection(l);
assert.ok(block.includes('Circuit breaker: OK'));
});
+
+// ── config validation ──────────────────────────────────────────────
+
+const identicalFailures = () =>
+ ledger([1, 2, 3, 4].map((i) => attempt(i, 'failure', { action: 'same fix', error: 'TypeError: boom' })));
+
+test('checkCircuitBreaker throws on a similarityThreshold above 1 instead of never stagnating', () => {
+ // Similarity tops out at 1.0, so 95 (a percentage) silently switched stagnation off.
+ for (const similarityThreshold of [95, 1.0001, Infinity]) {
+ assert.throws(
+ () => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold }),
+ /similarityThreshold must be a fraction/,
+ String(similarityThreshold),
+ );
+ }
+});
+
+test('checkCircuitBreaker throws on configs that would switch a rule off', () => {
+ const bad = [
+ { similarityThreshold: NaN },
+ { similarityThreshold: 0 },
+ { similarityThreshold: -0.1 },
+ { similarityThreshold: '0.85' },
+ { maxIterations: 0 },
+ { stagnationThreshold: NaN },
+ { frustrationThreshold: 2.5 },
+ { noProgressThreshold: Infinity },
+ { tokenBudget: 0 },
+ { tokenBudget: -100 },
+ ];
+ for (const override of bad) {
+ assert.throws(() => checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, ...override }), undefined, JSON.stringify(override));
+ }
+});
+
+test('valid configs, including the boundaries, still trip stagnation', () => {
+ for (const similarityThreshold of [1, 0.5, Number.MIN_VALUE]) {
+ const d = checkCircuitBreaker(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold });
+ assert.equal(d.trigger, 'stagnation', String(similarityThreshold));
+ }
+ assert.doesNotThrow(() => validateBreakerConfig({ ...DEFAULT_BREAKER, tokenBudget: 1 }));
+ assert.doesNotThrow(() => validateBreakerConfig(DEFAULT_BREAKER));
+ assert.doesNotThrow(() => validatePruneConfig(DEFAULT_PRUNE));
+});
+
+test('pruneLedger, summarizeAttempts and buildContextInjection reject a percentage threshold', () => {
+ assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, similarityThreshold: 85 }), /If you meant 85%, use 0\.85/);
+ assert.throws(() => pruneLedger(identicalFailures(), { ...DEFAULT_PRUNE, window: 0 }), /window must be a positive integer/);
+ assert.throws(() => summarizeAttempts(identicalFailures(), 85), /similarityThreshold must be a fraction/);
+ assert.throws(
+ () => buildContextInjection(identicalFailures(), { ...DEFAULT_BREAKER, similarityThreshold: 85 }),
+ /similarityThreshold must be a fraction/,
+ );
+});
+
+test('assertSimilarityThreshold only gives the percentage hint for whole-number percentages', () => {
+ assert.throws(() => assertSimilarityThreshold(95), /If you meant 95%, use 0\.95/);
+ assert.throws(() => assertSimilarityThreshold(1.5), (err) => !/If you meant/.test(err.message));
+ assert.throws(() => assertSimilarityThreshold(0), (err) => !/never fire/.test(err.message));
+ assert.throws(() => assertSimilarityThreshold(150), (err) => /never fire/.test(err.message) && !/If you meant/.test(err.message));
+ assert.throws(() => assertSimilarityThreshold(95, '--similarity-threshold'), /^Error: --similarity-threshold must/);
+});
diff --git a/tools/loop-drill/README.md b/tools/loop-drill/README.md
index 92adc64..5ad17da 100644
--- a/tools/loop-drill/README.md
+++ b/tools/loop-drill/README.md
@@ -62,6 +62,7 @@ Each drill is tagged with the `docs/failure-modes.md` entry it exercises, so the
| `breaker.no-progress` | Infinite Fix Loop | N consecutive failures | Escalates |
| `breaker.token-budget` | Token Burn | Attempt over budget | Escalates |
| `breaker.healthy` | Infinite Fix Loop | A healthy run | **Does not escalate** |
+| `breaker.config` | Infinite Fix Loop | Only reported when `loop-context` rejects the config (e.g. `similarityThreshold: 95`) | Never passes; the other breaker drills can't run |
| `verifier.control` | Verifier Theater | Nothing (clean tree) | **Accepts** |
| `verifier.mutant[op]` | Verifier Theater | One seeded defect | Rejects |
diff --git a/tools/loop-drill/dist/drill.d.ts b/tools/loop-drill/dist/drill.d.ts
index 09020c3..9d88dfc 100644
--- a/tools/loop-drill/dist/drill.d.ts
+++ b/tools/loop-drill/dist/drill.d.ts
@@ -17,7 +17,7 @@
* A guardrail is only credited when it passes both.
*/
import { type GateConfig } from '@cobusgreyling/loop-gate';
-import { type CircuitBreakerConfig } from '@cobusgreyling/loop-context';
+import { type BreakerDecision, type CircuitBreakerConfig, type Ledger } from '@cobusgreyling/loop-context';
export type DrillOutcome = 'passed' | 'failed' | 'skipped';
/** Named failure modes from docs/failure-modes.md. */
export type FailureMode = 'Infinite Fix Loop' | 'Token Burn' | 'Over-Reach (Wrong Scope)' | 'Verifier Theater' | 'Escalation Failure';
@@ -78,8 +78,16 @@ export declare function runGateDrills(options: GateDrillOptions): DrillResult[];
* escalated. Escalating for another reason means the rule under test is still
* unproven — the same standard the gate drills apply to `trigger !==
* 'denylist'`.
+ *
+ * loop-context throws on a config that would switch a rule off (a similarity
+ * threshold of 95 meant as 95%, say). That is reported as a failed drill, not
+ * a crash: a breaker that refuses to start protects nothing either.
+ *
+ * `check` defaults to loop-context's breaker; tests pass a stand-in to
+ * exercise the trigger-attribution logic against a breaker that misbehaves.
*/
-export declare function runBreakerDrills(config?: CircuitBreakerConfig): DrillResult[];
+export type BreakerCheck = (ledger: Ledger, config: CircuitBreakerConfig) => BreakerDecision;
+export declare function runBreakerDrills(config?: CircuitBreakerConfig, check?: BreakerCheck): DrillResult[];
export declare function buildReport(results: DrillResult[]): DrillReport;
/**
* 0 = every drill passed, 1 = some skipped (guardrail not configured),
diff --git a/tools/loop-drill/dist/drill.js b/tools/loop-drill/dist/drill.js
index 3b3a384..cdd2dae 100644
--- a/tools/loop-drill/dist/drill.js
+++ b/tools/loop-drill/dist/drill.js
@@ -141,18 +141,17 @@ const DISTINCT_ERRORS = [
'OutOfMemoryError during the bundle step',
'Segmentation fault in the native addon',
];
-/**
- * Exercise loop-context's circuit breaker with synthetic ledgers: repeated
- * identical failures must trip stagnation, a long run of unrelated failures
- * must trip no-progress, blowing the token budget must trip, and a healthy run
- * must not.
- *
- * Each drill asserts the *specific* trigger, not merely that the breaker
- * escalated. Escalating for another reason means the rule under test is still
- * unproven — the same standard the gate drills apply to `trigger !==
- * 'denylist'`.
- */
-export function runBreakerDrills(config = DEFAULT_BREAKER) {
+export function runBreakerDrills(config = DEFAULT_BREAKER, check = checkCircuitBreaker) {
+ try {
+ return breakerDrills(config, check);
+ }
+ catch (err) {
+ return [
+ fail('breaker.config', 'loop-context accepts the breaker config', 'Infinite Fix Loop', 'sensitivity', 'a config loop-context accepts', `rejected: ${err.message}`, 'loop-context refuses to run with a rule switched off, so no breaker rule was drilled. Fix the value and re-run.'),
+ ];
+ }
+}
+function breakerDrills(config, check) {
const results = [];
// Sensitivity: the same error repeated trips stagnation.
{
@@ -165,12 +164,12 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {
outcome: 'failure',
error: "TypeError: Cannot read properties of undefined (reading 'id')",
}));
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(decision.trigger === 'stagnation'
? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)
: fail(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate
? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'
- : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.'));
+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop.'));
}
// Sensitivity: a long run of unrelated failures trips no-progress.
{
@@ -183,7 +182,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {
outcome: 'failure',
error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],
}));
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(decision.trigger === 'no-progress'
? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)
: fail(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.escalate ? `escalated via ${decision.trigger}` : 'continued', decision.escalate
@@ -203,7 +202,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {
tokensUsed: config.tokenBudget + 1,
},
];
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(decision.trigger === 'token-budget'
? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)
: fail(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.escalate ? `escalated via ${decision.trigger}` : 'continued'));
@@ -220,7 +219,7 @@ export function runBreakerDrills(config = DEFAULT_BREAKER) {
{ iteration: 1, action: 'read the failing test', outcome: 'success' },
{ iteration: 2, action: 'apply a minimal fix', outcome: 'success' },
];
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(!decision.escalate
? pass(id, name, mode, 'specificity', 'continue', 'continued')
: fail(id, name, mode, 'specificity', 'continue', `escalated (${decision.trigger})`, 'The breaker escalates a healthy run. It will halt working loops and train the team to ignore escalations.'));
diff --git a/tools/loop-drill/src/drill.ts b/tools/loop-drill/src/drill.ts
index d8991d1..cdfdb69 100644
--- a/tools/loop-drill/src/drill.ts
+++ b/tools/loop-drill/src/drill.ts
@@ -21,6 +21,7 @@ import { checkGate, type GateConfig } from '@cobusgreyling/loop-gate';
import {
checkCircuitBreaker,
DEFAULT_BREAKER,
+ type BreakerDecision,
type CircuitBreakerConfig,
type Ledger,
} from '@cobusgreyling/loop-context';
@@ -295,8 +296,38 @@ const DISTINCT_ERRORS = [
* escalated. Escalating for another reason means the rule under test is still
* unproven — the same standard the gate drills apply to `trigger !==
* 'denylist'`.
+ *
+ * loop-context throws on a config that would switch a rule off (a similarity
+ * threshold of 95 meant as 95%, say). That is reported as a failed drill, not
+ * a crash: a breaker that refuses to start protects nothing either.
+ *
+ * `check` defaults to loop-context's breaker; tests pass a stand-in to
+ * exercise the trigger-attribution logic against a breaker that misbehaves.
*/
-export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER): DrillResult[] {
+export type BreakerCheck = (ledger: Ledger, config: CircuitBreakerConfig) => BreakerDecision;
+
+export function runBreakerDrills(
+ config: CircuitBreakerConfig = DEFAULT_BREAKER,
+ check: BreakerCheck = checkCircuitBreaker,
+): DrillResult[] {
+ try {
+ return breakerDrills(config, check);
+ } catch (err) {
+ return [
+ fail(
+ 'breaker.config',
+ 'loop-context accepts the breaker config',
+ 'Infinite Fix Loop',
+ 'sensitivity',
+ 'a config loop-context accepts',
+ `rejected: ${(err as Error).message}`,
+ 'loop-context refuses to run with a rule switched off, so no breaker rule was drilled. Fix the value and re-run.',
+ ),
+ ];
+ }
+}
+
+function breakerDrills(config: CircuitBreakerConfig, check: BreakerCheck): DrillResult[] {
const results: DrillResult[] = [];
// Sensitivity: the same error repeated trips stagnation.
@@ -310,7 +341,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)
outcome: 'failure' as const,
error: "TypeError: Cannot read properties of undefined (reading 'id')",
}));
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(
decision.trigger === 'stagnation'
? pass(id, name, mode, 'sensitivity', 'escalate via stagnation', decision.reason)
@@ -323,7 +354,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)
decision.escalate ? `escalated via ${decision.trigger}` : 'continued',
decision.escalate
? 'The breaker stopped the loop, but not by the rule under test — stagnation stays unproven.'
- : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop. Check similarityThreshold: it is a 0.0-1.0 fraction, and a percentage (e.g. 95) never matches.',
+ : 'The loop would keep retrying an identical failure — the exact shape of an Infinite Fix Loop.',
),
);
}
@@ -339,7 +370,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)
outcome: 'failure' as const,
error: DISTINCT_ERRORS[i % DISTINCT_ERRORS.length],
}));
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(
decision.trigger === 'no-progress'
? pass(id, name, mode, 'sensitivity', 'escalate via no-progress', decision.reason)
@@ -370,7 +401,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)
tokensUsed: config.tokenBudget + 1,
},
];
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(
decision.trigger === 'token-budget'
? pass(id, name, mode, 'sensitivity', 'escalate via token-budget', decision.reason)
@@ -404,7 +435,7 @@ export function runBreakerDrills(config: CircuitBreakerConfig = DEFAULT_BREAKER)
{ iteration: 1, action: 'read the failing test', outcome: 'success' as const },
{ iteration: 2, action: 'apply a minimal fix', outcome: 'success' as const },
];
- const decision = checkCircuitBreaker(ledgerOf(attempts), config);
+ const decision = check(ledgerOf(attempts), config);
results.push(
!decision.escalate
? pass(id, name, mode, 'specificity', 'continue', 'continued')
diff --git a/tools/loop-drill/test/drill.test.mjs b/tools/loop-drill/test/drill.test.mjs
index 31f2d27..6533f12 100644
--- a/tools/loop-drill/test/drill.test.mjs
+++ b/tools/loop-drill/test/drill.test.mjs
@@ -136,27 +136,41 @@ describe('runBreakerDrills', () => {
assert.match(byId(results, 'breaker.no-progress').actual, /no.progress/i);
});
- test('a percentage similarityThreshold leaves stagnation unproven', () => {
- // similarityThreshold is a 0.0-1.0 fraction. Setting it to 95 (meaning
- // "95%") means identical errors never compare as similar, so a loop retrying
- // the same failure forever is never stopped. The drill must report that.
- const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 });
- const stagnation = byId(misconfigured, 'breaker.stagnation');
- assert.equal(stagnation.outcome, 'failed');
- assert.equal(stagnation.actual, 'continued');
- assert.match(stagnation.detail, /Infinite Fix Loop/);
- assert.match(stagnation.detail, /similarityThreshold/);
+ test('a percentage similarityThreshold is a failed drill, not a crash', () => {
+ // similarityThreshold is a 0.0-1.0 fraction. 95 (meaning "95%") used to make
+ // identical errors never compare as similar, so a loop retrying the same
+ // failure forever was never stopped. loop-context now refuses the config;
+ // the drill must report that rather than throw.
+ for (const similarityThreshold of [95, 0]) {
+ const misconfigured = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold });
+ assert.equal(misconfigured.length, 1, String(similarityThreshold));
+ const [r] = misconfigured;
+ assert.equal(r.id, 'breaker.config');
+ assert.equal(r.outcome, 'failed');
+ assert.equal(r.failureMode, 'Infinite Fix Loop');
+ assert.match(r.actual, /similarityThreshold must be a fraction/);
+ }
+ assert.match(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 })[0].actual, /use 0\.95/);
+ assert.equal(exitCodeFor(buildReport(runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 95 }))), 2);
+ });
+
+ // A breaker that always answers with one trigger: how the drills are shown to
+ // credit only the rule they exercise, now that loop-context refuses the
+ // configs that used to produce these shapes.
+ const always = (trigger) => (ledger) => ({
+ shouldContinue: false,
+ escalate: true,
+ trigger,
+ reason: `stand-in breaker: ${trigger}`,
+ iterations: ledger.attempts.length,
+ tokensUsed: 0,
});
test('escalating via the wrong trigger leaves stagnation unproven', () => {
// The case that separates "the breaker fired" from "the rule under test
// fired". Here the breaker DOES escalate, so a drill asserting only
// decision.escalate would pass while stagnation was never exercised.
- const wrongTrigger = runBreakerDrills({
- ...DEFAULT_BREAKER,
- similarityThreshold: 95,
- noProgressThreshold: 1,
- });
+ const wrongTrigger = runBreakerDrills(DEFAULT_BREAKER, always('no-progress'));
const stagnation = byId(wrongTrigger, 'breaker.stagnation');
assert.equal(stagnation.outcome, 'failed');
assert.equal(stagnation.actual, 'escalated via no-progress');
@@ -164,27 +178,31 @@ describe('runBreakerDrills', () => {
});
test('an iteration cap that fires first also leaves stagnation unproven', () => {
- const capped = runBreakerDrills({
- ...DEFAULT_BREAKER,
- similarityThreshold: 95,
- maxIterations: 2,
- });
+ const capped = runBreakerDrills(DEFAULT_BREAKER, always('max-iterations'));
const stagnation = byId(capped, 'breaker.stagnation');
assert.equal(stagnation.outcome, 'failed');
assert.equal(stagnation.actual, 'escalated via max-iterations');
});
test('stagnation swallowing every error leaves no-progress unproven', () => {
- // similarityThreshold 0 makes every error count as "the same error", so
- // stagnation fires on the no-progress drill's unrelated failures. The
- // breaker still escalates, so only a trigger-specific assertion catches it.
- const swallowed = runBreakerDrills({ ...DEFAULT_BREAKER, similarityThreshold: 0 });
+ // A breaker that calls every error "the same error" fires stagnation on the
+ // no-progress drill's unrelated failures. It still escalates, so only a
+ // trigger-specific assertion catches it.
+ const swallowed = runBreakerDrills(DEFAULT_BREAKER, always('stagnation'));
const noProgress = byId(swallowed, 'breaker.no-progress');
assert.equal(noProgress.outcome, 'failed');
assert.equal(noProgress.actual, 'escalated via stagnation');
assert.match(noProgress.detail, /not by the rule under test/);
});
+ test('a breaker that never escalates fails stagnation and passes the healthy run', () => {
+ const never = (ledger) => ({ shouldContinue: true, escalate: false, trigger: 'ok', reason: 'ok', iterations: ledger.attempts.length, tokensUsed: 0 });
+ const r = runBreakerDrills(DEFAULT_BREAKER, never);
+ assert.equal(byId(r, 'breaker.stagnation').actual, 'continued');
+ assert.match(byId(r, 'breaker.stagnation').detail, /Infinite Fix Loop/);
+ assert.equal(byId(r, 'breaker.healthy').outcome, 'passed');
+ });
+
test('no-progress uses errors that stay distinct after signature normalization', () => {
// errorSignature() collapses every number to '#', so errors differing only
// by a number ("module 1" / "module 2") normalize to the same signature and