an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
'use strict';
/**
* Circuit-breaker policy tests. Self-contained, no test framework — run with
* `node test/breaker.test.cjs` (mirrors test/agent-provider.test.cjs). breaker.ts
* imports only node builtins (node:crypto), so it transpiles standalone with the
* bundled `typescript` compiler and runs under plain node.
*
* Focus: the no-progress false-positive fixes (upstream issue #109 + fleet
* evidence — compaction / inbox-ack bursts and background work tripping
* "no-progress: generating tokens without coordinating"):
* 1. compaction exemption — PreCompact→PostCompact (with safety cap) skips the
* Δoutput-based trips;
* 2. recent DISTINCT tool activity counts as progress (an agent running varied
* tools is working — true loops are still caught by repeatedToolLimit);
* 3. the no-progress arm requires 2 consecutive tripping beats (debounce) so a
* one-beat blip never fires a steer.
* Plus regression guards for the pre-existing trips (loop, error storm, velocity).
*/
const assert = require('node:assert');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const ts = require('typescript');
const SRC = path.join(__dirname, '..', 'src', 'main', 'breaker.ts');
const out = fs.mkdtempSync(path.join(os.tmpdir(), 'breaker-'));
const js = ts.transpileModule(fs.readFileSync(SRC, 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }
}).outputText;
fs.writeFileSync(path.join(out, 'breaker.js'), js, 'utf8');
const { CircuitBreaker } = require(path.join(out, 'breaker.js'));
let failures = 0;
function test(name, fn) {
try { fn(); console.log(` ok ${name}`); }
catch (e) { failures++; console.error(`FAIL ${name}\n ${e.message}`); }
}
/** A breaker with fixed config (no caps, hardStop off). */
function makeBreaker(over = {}) {
return new CircuitBreaker(() => ({
enabled: true, hardStop: false, repeatedToolLimit: 8, errorStormLimit: 5,
tokenVelocityPerMin: 60000, ...over
}));
}
/** Cumulative sample helper. */
function sample(agentId, ts, output, input = 1000) {
return { agentId, sessionId: 's1', ts, input, output, cacheRead: 0, cacheCreation: 0, model: 'm', usd: 0 };
}
const T0 = 1_000_000_000_000; // fixed epoch base so tests are deterministic
const BEAT = 30_000;
/** Run one beat for a single agent; returns its decision. */
function beat(b, id, s, progressing, now, lastWorkAt) {
return b.tick([{ agentId: id, sample: s, progressing, lastWorkAt }], now)[0];
}
// ── regression: pre-existing trips still fire ────────────────────────────────
test('repeated identical tool calls trip the loop arm', () => {
const b = makeBreaker();
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /looping/);
});
test('error storm trips', () => {
const b = makeBreaker();
for (let i = 0; i < 5; i++) b.recordError('a');
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /error storm/);
});
test('token velocity spike trips on the second sample', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), true, T0);
const d = beat(b, 'a', sample('a', T0 + BEAT, 40_000), true, T0 + BEAT); // 80k/min
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /velocity/);
});
// ── fix 3: no-progress needs 2 consecutive tripping beats ───────────────────
test('no-progress does NOT trip on a single beat (debounce)', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
const d = beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('sustained no-progress still trips (second consecutive beat)', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /no-progress/);
});
test('a progressing beat resets the no-progress debounce', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT); // count 1
beat(b, 'a', sample('a', T0 + 2 * BEAT, 600), true, T0 + 2 * BEAT); // reset
const d = beat(b, 'a', sample('a', T0 + 3 * BEAT, 1100), false, T0 + 3 * BEAT); // count 1 again
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
// ── fix 1: compaction exemption ──────────────────────────────────────────────
test('compaction exempts the no-progress trip', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordCompactStart('a', T0 + 1);
// Two beats of token burst with stale coordination — would trip without the fix.
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('compaction exempts the velocity trip', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), true, T0);
b.recordCompactStart('a', T0 + 1);
const d = beat(b, 'a', sample('a', T0 + BEAT, 40_000), true, T0 + BEAT); // 80k/min burst
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('trips resume after compaction end + trailing grace', () => {
const b = makeBreaker();
const GRACE = 120_000; // must cover POST_COMPACT_GRACE_MS
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordCompactStart('a', T0 + 1);
b.recordCompactEnd('a', T0 + 2);
const t1 = T0 + GRACE + BEAT;
const t2 = t1 + BEAT;
beat(b, 'a', sample('a', t1, 5000), false, t1);
const d = beat(b, 'a', sample('a', t2, 6000), false, t2);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
});
test('compaction safety cap: exemption expires even without PostCompact', () => {
const b = makeBreaker();
const CAP = 10 * 60_000; // must exceed COMPACT_GRACE_MS
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordCompactStart('a', T0 + 1); // PostCompact never arrives
const t1 = T0 + CAP + BEAT;
const t2 = t1 + BEAT;
beat(b, 'a', sample('a', t1, 5000), false, t1);
const d = beat(b, 'a', sample('a', t2, 6000), false, t2);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
});
test('recordCompactEnd without a compaction in flight is a no-op', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordCompactEnd('a', T0 + 1); // e.g. SessionStart on a fresh session
beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`); // still trips normally
});
// ── #189: the per-agent cap measures work, not cached context ───────────────
// Cached input is re-billed on every request, so the all-kinds total grows with
// the request count against a fixed context, not with what the agent does. The
// measured incidents: two agents crossed a 4M cap 4–6 minutes into a session
// with 98.6% / 99.2% of the counted figure being cacheRead.
test('per-agent cap does not trip on cached context (the #189 incident shape)', () => {
const b = makeBreaker({ agentTokenCaps: { a: 4_000_000 } });
// Agent A at its crossing row: 56,884 tokens of work under 4,102,505 of cache reads.
const s = { ...sample('a', T0, 50_000, 6_884), cacheRead: 4_102_505 };
const d = beat(b, 'a', s, true, T0);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('per-agent cap still trips on work tokens, and the reason names the metric', () => {
const b = makeBreaker({ agentTokenCaps: { a: 4_000_000 } });
const s = { ...sample('a', T0, 1_500_000, 2_000_000), cacheCreation: 600_000, cacheRead: 9_000_000 };
const d = beat(b, 'a', s, true, T0);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
const work = (4_100_000).toLocaleString(), cap = (4_000_000).toLocaleString(), total = (13_100_000).toLocaleString();
assert.ok(d.state.reason.startsWith(`token limit: ${work} work tokens over the agent cap of ${cap}`), d.state.reason);
assert.ok(d.state.reason.includes(`${total} total`), d.state.reason); // the display figure is still reported
});
test('cache writes count toward the agent cap; cache reads do not', () => {
const reads = { ...sample('a', T0, 400, 500), cacheRead: 200 }; // 900 work, 1,100 total
const writes = { ...sample('a', T0, 400, 500), cacheCreation: 200 }; // 1,100 work
const b1 = makeBreaker({ agentTokenCaps: { a: 1_000 } });
assert.equal(beat(b1, 'a', reads, true, T0).state.level, 'healthy');
const b2 = makeBreaker({ agentTokenCaps: { a: 1_000 } });
assert.equal(beat(b2, 'a', writes, true, T0).state.level, 'steering');
});
test('the floor-wide token budget still counts all kinds (only the per-agent arm changed)', () => {
const b = makeBreaker({ costCapTokens: 4_000_000 });
const s = { ...sample('a', T0, 50_000, 6_884), cacheRead: 4_102_505 };
const d = beat(b, 'a', s, true, T0);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
assert.ok(d.state.reason.startsWith('token cap: floor total'), d.state.reason);
});
// ── fix 2: recent distinct tool activity counts as progress ─────────────────
test('recent distinct tool calls exempt the no-progress trip', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
// Varied tool stream (background workflow / interactive work) right before each beat.
b.recordToolUse('a', 'Read', { file: 'x' }, T0 + BEAT - 1000);
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);
b.recordToolUse('a', 'Read', { file: 'y' }, T0 + 2 * BEAT - 1000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
// -- fix 4: the agent's own working directory counts as progress ------------
//
// `progressing` is computed from coordination files only - inbox, outbox and
// memory.md. An agent editing, building and committing touches none of them,
// so a working agent read as wedged. These three pin the whole behaviour: it
// counts, staleness still trips, and a caller that reports nothing is
// unaffected.
test('a recently changed working directory counts as progress', () => {
const b = makeBreaker();
// No coordination, no tool events - only the workspace moving. This is the
// shape that misfired: real work, invisible to every signal the arm had.
beat(b, 'a', sample('a', T0, 0), false, T0, T0 - 1000);
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT, T0 + BEAT - 1000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT, T0 + 2 * BEAT - 1000);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('a STALE working directory still trips the no-progress arm', () => {
const b = makeBreaker();
const stale = T0 - 10 * 60_000; // older than PROGRESS_TOOL_WINDOW_MS
beat(b, 'a', sample('a', T0, 0), false, T0, stale);
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT, stale);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT, stale);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
assert.match(d.state.reason, /no-progress/);
});
test('an ABSENT lastWorkAt behaves exactly as before', () => {
const b = makeBreaker();
// A caller with no workspace notion for this agent omits the field entirely.
// Nothing about that agent may change, or the fix would alter behaviour for
// every provider that reports nothing.
beat(b, 'a', sample('a', T0, 0), false, T0);
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
assert.match(d.state.reason, /no-progress/);
});
test('REPEATED identical tool calls do not count as progress', () => {
const b = makeBreaker();
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 - 10 * 60_000); // distinct stamp long ago
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + BEAT - 1000); // repeat — no fresh stamp
beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + 2 * BEAT - 1000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
assert.match(d.state.reason, /no-progress/);
});
// ── toolKey stays cheap AND discriminating on huge inputs ────────────────────
test('huge identical Write inputs still register as repeats (loop arm)', () => {
const b = makeBreaker();
const huge = { file_path: '/x.txt', content: 'A'.repeat(1_000_000) };
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Write', huge);
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /looping/);
});
test('huge inputs differing early still count as distinct calls', () => {
const b = makeBreaker();
for (let i = 0; i < 8; i++) {
b.recordToolUse('a', 'Write', { file_path: `/f${i}.txt`, content: 'A'.repeat(500_000) });
}
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
// #377: Bash commands routinely open with a long identical preamble (absolute
// paths, a cd, an interpreter invocation). Under the old `slice(0, 200)` key,
// commands first differing past that horizon collided, and nine DIFFERENT
// measurements in a row constrained the agent doing the most careful work.
test('inputs differing LATE (past the old 200-char horizon) count as distinct calls', () => {
const b = makeBreaker();
// ~310-char shared preamble, in the shape of the reported repro
const preamble = 'SP=/private/tmp/claude-501/-Users-me-Documents-Projects/0000000-0000-0000-0000-000000000000/scratchpad\n'
+ "python3 - <<'PY'\nimport json\n"
+ 'SP="/private/tmp/claude-501/-Users-me-Documents-Projects/0000000-0000-0000-0000-000000000000/scratchpad"\n'
+ 'doc = json.load(open(SP + "/edit.otio"))\n';
for (let i = 0; i < 9; i++) {
b.recordToolUse('a', 'Bash', { command: `${preamble}print(measure_${i}(doc))\nPY` });
}
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('identical long-preamble calls still trip the loop arm (no strictness lost)', () => {
const b = makeBreaker();
const cmd = 'SP=/private/tmp/claude-501/very/long/identical/path\n' + 'x'.repeat(400);
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { command: cmd });
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
});
test('strings capped at 4096 still key equal when they differ only past the cap', () => {
// The cap bounds serialization work on the hook reply path; inputs that
// differ only beyond it are indistinguishable BY DESIGN (bounded work wins
// at that size), and must at least behave consistently: same key, counted
// as repeats — never a crash or a flapping key.
const b = makeBreaker();
const head = 'y'.repeat(5000);
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { command: head + `tail_${i}` });
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
});
// ── recovery still works ─────────────────────────────────────────────────────
test('a healthy beat de-escalates one level', () => {
const b = makeBreaker();
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });
beat(b, 'a', null, true, T0); // → steering
b.recordToolUse('a', 'Read', { file: 'new' }); // distinct call clears the loop
const d = beat(b, 'a', null, true, T0 + BEAT);
assert.equal(d.state.level, 'healthy');
});
// ── #376: a live human conversation counts as progress ──────────────────────
// A conversation is prose in, prose out — no hive files, no tool spans — and
// the measured incidents fired 19s and 30s after an operator prompt, while the
// file-based progress signal had been stale for over an hour.
test('a recent human prompt exempts the no-progress trip (mid-conversation)', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
// The incident shape: operator prompts ~30s before each beat; the agent
// answers in prose (Δoutput > 0) and touches nothing else.
b.recordUserPrompt('a', T0 + BEAT - 30_000);
beat(b, 'a', sample('a', T0 + BEAT, 800), false, T0 + BEAT);
b.recordUserPrompt('a', T0 + 2 * BEAT - 19_000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1600), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('one prompt covers a long answer for the whole progress window', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordUserPrompt('a', T0 + 1000); // one question, then a long prose answer
for (let i = 1; i <= 9; i++) { // 9 beats = 4.5 min, inside the 300s window
const t = T0 + i * BEAT;
const d = beat(b, 'a', sample('a', t, i * 700), false, t);
assert.equal(d.state.level, 'healthy', `beat ${i}: ${d.state.reason}`);
}
});
test('the conversation clock EXPIRES: a stale prompt does not blind the arm', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordUserPrompt('a', T0); // one prompt, then the agent burns tokens alone
const t1 = T0 + 320_000; // past the 300s window
const t2 = t1 + BEAT;
beat(b, 'a', sample('a', t1, 5000), false, t1);
const d = beat(b, 'a', sample('a', t2, 6000), false, t2);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
});
test('a human prompt does not exempt the loop, error-storm, or velocity arms', () => {
const b = makeBreaker();
b.recordUserPrompt('a', T0 - 1000);
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });
assert.equal(beat(b, 'a', null, true, T0).state.level, 'steering'); // loop still trips
const b2 = makeBreaker();
b2.recordUserPrompt('a', T0 - 1000);
beat(b2, 'a', sample('a', T0, 0), true, T0);
const d = beat(b2, 'a', sample('a', T0 + BEAT, 40_000), true, T0 + BEAT); // 80k/min
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`); // velocity still trips
});
// ── fix 2: recent distinct tool activity counts as progress ─────────────────
test('recent distinct tool calls exempt the no-progress trip', () => {
const b = makeBreaker();
beat(b, 'a', sample('a', T0, 0), false, T0);
// Varied tool stream (background workflow / interactive work) right before each beat.
b.recordToolUse('a', 'Read', { file: 'x' }, T0 + BEAT - 1000);
beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);
b.recordToolUse('a', 'Read', { file: 'y' }, T0 + 2 * BEAT - 1000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
test('REPEATED identical tool calls do not count as progress', () => {
const b = makeBreaker();
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 - 10 * 60_000); // distinct stamp long ago
beat(b, 'a', sample('a', T0, 0), false, T0);
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + BEAT - 1000); // repeat — no fresh stamp
beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);
b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + 2 * BEAT - 1000);
const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);
assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);
assert.match(d.state.reason, /no-progress/);
});
// ── toolKey stays cheap AND discriminating on huge inputs ────────────────────
test('huge identical Write inputs still register as repeats (loop arm)', () => {
const b = makeBreaker();
const huge = { file_path: '/x.txt', content: 'A'.repeat(1_000_000) };
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Write', huge);
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'steering');
assert.match(d.state.reason, /looping/);
});
test('huge inputs differing early still count as distinct calls', () => {
const b = makeBreaker();
for (let i = 0; i < 8; i++) {
b.recordToolUse('a', 'Write', { file_path: `/f${i}.txt`, content: 'A'.repeat(500_000) });
}
const d = beat(b, 'a', null, true, T0);
assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);
});
// ── recovery still works ─────────────────────────────────────────────────────
test('a healthy beat de-escalates one level', () => {
const b = makeBreaker();
for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });
beat(b, 'a', null, true, T0); // → steering
b.recordToolUse('a', 'Read', { file: 'new' }); // distinct call clears the loop
const d = beat(b, 'a', null, true, T0 + BEAT);
assert.equal(d.state.level, 'healthy');
});
process.exit(failures ? 1 : 0);