an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
fix(breaker): eliminate the no-progress false-positive storm (#109) (#112)
Quentin Schmicke0e2cee9c2cb
diff --git a/src/main/breaker.ts b/src/main/breaker.tsindex 71153905..a2e9718b 100644--- a/src/main/breaker.ts+++ b/src/main/breaker.ts@@ -73,6 +73,19 @@ const DEFAULTS = {tokenVelocityPerMin: 60_000 // output tokens/min — coarse backstop, deliberately high};+/** Safety cap on the PreCompact exemption: if PostCompact never arrives (crash,+ * or a Claude build that doesn't emit it), the Δoutput trips re-arm on their own. */+const COMPACT_GRACE_MS = 5 * 60_000;+/** Trailing grace after PostCompact: the compaction burst lands in the NEXT+ * beat's cumulative diff, so the exemption must outlive the compaction itself. */+const POST_COMPACT_GRACE_MS = 90_000;+/** How recent a DISTINCT tool call must be to count as progress for the+ * no-progress arm. Mirrors the beat's file-mtime progress window (300s). */+const PROGRESS_TOOL_WINDOW_MS = 300_000;+/** Consecutive tripping beats the no-progress arm needs before it fires — a+ * one-beat blip (inbox ack, statusline burst) never steers on its own. */+const NO_PROGRESS_BEATS = 2;+interface AgentBreakerState {level: BreakerLevel;reason: string;@@ -82,6 +95,17 @@ interface AgentBreakerState {repeatCount: number;/** Consecutive api_error / retry events with no intervening progress. */errorCount: number;+ /** Δoutput-based trips are exempt until this instant (compaction in flight,+ * set on PreCompact; PostCompact shortens it to a trailing grace). */+ compactingUntil: number;+ /** When the last DISTINCT (name+input) tool call ran. A varied tool stream is+ * work — background workflows / interactive sessions whose output lands+ * outside the hive files (git, Jira) must not read as "no progress". A true+ * single-call loop never refreshes this; an alternating loop that would is+ * still backstopped by the velocity trip. */+ lastDistinctToolAt: number;+ /** Consecutive beats the no-progress condition held (debounce counter). */+ noProgressBeats: number;}export class CircuitBreaker {@@ -106,7 +130,10 @@ export class CircuitBreaker {private get(agentId: string): AgentBreakerState {let s = this.agents.get(agentId);if (!s) {- s = { level: 'healthy', reason: '', lastSample: null, repeatKey: null, repeatCount: 0, errorCount: 0 };+ s = {+ level: 'healthy', reason: '', lastSample: null, repeatKey: null, repeatCount: 0,+ errorCount: 0, compactingUntil: 0, lastDistinctToolAt: 0, noProgressBeats: 0+ };this.agents.set(agentId, s);}return s;@@ -125,8 +152,9 @@ export class CircuitBreaker {// ── event-driven inputs (fed by HookServer) ──────────────────────────────/** A tool call ran. A NEW (name+input) key counts as forward progress (resets- * the repeat + error counters); the SAME key in a row is the loop signal. */- recordToolUse(agentId: string, toolName: string | undefined, toolInput: unknown): void {+ * the repeat + error counters and stamps the distinct-tool clock the+ * no-progress arm reads); the SAME key in a row is the loop signal. */+ recordToolUse(agentId: string, toolName: string | undefined, toolInput: unknown, now = Date.now()): void {const s = this.get(agentId);const key = this.toolKey(toolName, toolInput);if (key === s.repeatKey) {@@ -135,6 +163,7 @@ export class CircuitBreaker {s.repeatKey = key;s.repeatCount = 1;s.errorCount = 0; // a distinct tool call = progress; clear the error storm+ s.lastDistinctToolAt = now;}}@@ -143,9 +172,36 @@ export class CircuitBreaker {this.get(agentId).errorCount += 1;}+ /** Compaction started (PreCompact hook). Exempt the Δoutput-based trips —+ * compaction burns output tokens while touching no coordination file, which+ * is exactly the false-positive shape of upstream issue #109 (the harness's+ * own auto-compact mission tripping its own breaker on idle agents). */+ recordCompactStart(agentId: string, now = Date.now()): void {+ this.get(agentId).compactingUntil = now + COMPACT_GRACE_MS;+ }++ /** Compaction finished (PostCompact, or any SessionStart). Shortens the+ * exemption to a trailing grace — the burst still lands in the next beat's+ * cumulative diff. A no-op when no compaction is in flight, so a plain+ * session start never grants an exemption. */+ recordCompactEnd(agentId: string, now = Date.now()): void {+ const s = this.get(agentId);+ if (s.compactingUntil > now) s.compactingUntil = now + POST_COMPACT_GRACE_MS;+ }+private toolKey(toolName: string | undefined, toolInput: unknown): string {+ // Truncating replacer: a Write/Edit tool_input carries the whole file body+ // (up to MBs), and this runs synchronously inside the hook reply path on+ // EVERY PostToolUse — serializing it all only to keep 200 chars was a+ // multi-MB transient allocation per large write. Capping each string field+ // bounds the work while keeping the key semantics (a repeat of the same+ // call still yields the same key; distinct calls still differ within the+ // first 200 chars far more often than full serialization ever mattered).let inp = '';- try { inp = JSON.stringify(toolInput) ?? ''; } catch { inp = String(toolInput); }+ try {+ inp = JSON.stringify(toolInput, (_k, v) =>+ typeof v === 'string' && v.length > 250 ? v.slice(0, 250) : v) ?? '';+ } catch { inp = String(toolInput); }return `${toolName ?? '?'}:${inp.slice(0, 200)}`;}@@ -196,7 +252,7 @@ export class CircuitBreaker {for (const input of inputs) {const s = this.get(input.agentId);const trip = this.evaluate(- input, s, cfg,+ input, s, cfg, nowMs,input.agentId === topSpender, cfg.costCapUsd,input.agentId === topTokenSpender, cfg.costCapTokens);@@ -229,6 +285,7 @@ export class CircuitBreaker {input: BreakerInput,s: AgentBreakerState,cfg: ReturnType<CircuitBreaker['cfg']>,+ nowMs: number,isTopSpender: boolean,costCapUsd: number | undefined,isTopTokenSpender: boolean,@@ -255,8 +312,12 @@ export class CircuitBreaker {if (isTopTokenSpender && typeof costCapTokens === 'number') {return { tripping: true, reason: `token cap: floor total over ${costCapTokens.toLocaleString()} tokens (top spender ${tokensOf(input.sample).toLocaleString()})` };}- // (a) token-velocity spike — diff cumulative output across consecutive beats- if (input.sample && s.lastSample) {+ // (a) token-velocity spike — diff cumulative output across consecutive beats.+ // Skipped entirely while a compaction is in flight (+ trailing grace): a+ // /compact burns output tokens with no coordination writes, which is the+ // false-positive shape of issue #109 — the auto-compact mission tripping+ // the breaker on idle agents.+ if (input.sample && s.lastSample && nowMs >= s.compactingUntil) {const dOut = input.sample.output - s.lastSample.output;const dMin = (input.sample.ts - s.lastSample.ts) / 60_000;if (dOut > 0 && dMin > 0) {@@ -264,9 +325,20 @@ export class CircuitBreaker {if (velocity > cfg.tokenVelocityPerMin) {return { tripping: true, reason: `token velocity ${Math.round(velocity)}/min > ${cfg.tokenVelocityPerMin}/min` };}- // (c) no-progress: burning output tokens while not coordinating- if (!input.progressing) {- return { tripping: true, reason: 'no-progress: generating tokens without coordinating (stale log/files)' };+ // (c) no-progress: burning output tokens while not coordinating. A recent+ // DISTINCT tool call counts as progress too — background workflows and+ // interactive sessions do real work that never touches the hive files+ // (a single-call loop never refreshes that clock, and the loop/velocity+ // arms above still backstop). Debounced: fires only after+ // NO_PROGRESS_BEATS consecutive beats, so a one-beat blip never steers.+ const toolActive = nowMs - s.lastDistinctToolAt < PROGRESS_TOOL_WINDOW_MS;+ if (!input.progressing && !toolActive) {+ s.noProgressBeats += 1;+ if (s.noProgressBeats >= NO_PROGRESS_BEATS) {+ return { tripping: true, reason: 'no-progress: generating tokens without coordinating (stale log/files)' };+ }+ } else {+ s.noProgressBeats = 0;}}}diff --git a/src/main/hooks.ts b/src/main/hooks.tsindex e134f4af..3f4658ea 100644--- a/src/main/hooks.ts+++ b/src/main/hooks.ts@@ -200,6 +200,15 @@ export class HookServer {this.breaker?.recordToolUse(agentId, p.tool_name, p.tool_input);}+ // Compaction exemption (issue #109): PreCompact opens it so the compaction+ // token burst can't trip the Δoutput arms; PostCompact — or any SessionStart,+ // since a fresh session makes in-flight compaction state moot — closes it+ // down to the trailing grace (a no-op when nothing was compacting).+ if (event === 'PreCompact' && agentId) this.breaker?.recordCompactStart(agentId);+ if ((event === 'PostCompact' || event === 'SessionStart') && agentId) {+ this.breaker?.recordCompactEnd(agentId);+ }+if ((event === 'Stop' || event === 'SubagentStop') && agentId) {// Loop guard: a previous Stop hook already blocked this turn → let it stop.if (p.stop_hook_active) { this.emit(agentId, event, p); return {}; }diff --git a/src/main/index.ts b/src/main/index.tsindex d33bf5e0..253f4a03 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -612,9 +612,12 @@ function isFloorQuiet(thresholdMs: number): boolean {return Date.now() - Math.max(...times) > thresholdMs;}-/** Newest coordination-file mtime for one agent (inbox, outbox/.sent, memory.md)- * — FILES only, deliberately excluding PTY output, so "no-progress" means "not- * coordinating" even while the agent is busy printing tokens. */+/** Newest coordination-file mtime for one agent (inbox + inbox/.done, outbox ++ * outbox/.sent, memory.md) — FILES only, deliberately excluding PTY output, so+ * "no-progress" means "not coordinating" even while the agent is busy printing+ * tokens. inbox/.done and the outbox dir count because handling mail (moving a+ * message to .done, drafting an outbox message) IS coordination — without them+ * an inbox-ack turn reads as no-progress (issue #109's second trigger). */function lastCoordinationAt(agentId: string): number {const root = hive.root();if (!root) return 0;@@ -622,6 +625,8 @@ function lastCoordinationAt(agentId: string): number {const pushMtime = (p: string): void => { try { times.push(statSync(p).mtimeMs); } catch { /* missing */ } };const dir = join(root, 'agents', agentId);pushMtime(join(dir, 'inbox'));+ pushMtime(join(dir, 'inbox', '.done'));+ pushMtime(join(dir, 'outbox'));pushMtime(join(dir, 'outbox', '.sent'));pushMtime(join(dir, 'memory.md'));return Math.max(...times);@@ -747,7 +752,20 @@ function runBreakerBeat(progressWindowMs: number): void {// "is there a live session" without changing any live-agent behavior.if (sample?.sessionId) hive.appendCostLedger(sample); // ledger covers everyone incl. godif (id === reg.godId) continue; // breaker skips god- inputs.push({ agentId: id, sample, progressing: now - lastCoordinationAt(id) < progressWindowMs });+ // Progress = fresh coordination files OR a recent OTel tool span. The span+ // leg closes the background-work blind spot: subagent/Workflow tool calls+ // never reach the parent session's PostToolUse hook (so the breaker's own+ // distinct-tool clock stays stale) but their spans DO flow through the+ // collector under this agent's id — an idle parent supervising a hard-+ // working background fleet is progressing, not wedged. Observed live: the+ // one residual no-progress false positive after the #109 fixes.+ const spans = telemetry.getSpans(id);+ const lastSpanAt = spans.length ? spans[spans.length - 1].ts : 0;+ inputs.push({+ agentId: id,+ sample,+ progressing: now - lastCoordinationAt(id) < progressWindowMs || now - lastSpanAt < progressWindowMs+ });}for (const d of breaker.tick(inputs, now)) {try { liveWebContents()?.send('control:breakerState', d.state); } catch { /* window gone */ }diff --git a/test/breaker.test.cjs b/test/breaker.test.cjsnew file mode 100644index 00000000..ca08ce3a--- /dev/null+++ b/test/breaker.test.cjs@@ -0,0 +1,224 @@+'use strict';+/**+ * Circuit-breaker policy tests. Self-contained, no test framework — run with+ * `node test/breaker.test.cjs` (mirrors test/agent-provider.test.cjs). breaker.ts+ * only has type-only imports, so it transpiles standalone with the bundled+ * `typescript` compiler.+ *+ * Focus: the no-progress false-positive fixes (upstream issue #109 + fleet+ * evidence — compaction / inbox-ack bursts and background work tripping+ * "no-progress: generating tokens without coordinating"):+ * 1. compaction exemption — PreCompact→PostCompact (with safety cap) skips the+ * Δoutput-based trips;+ * 2. recent DISTINCT tool activity counts as progress (an agent running varied+ * tools is working — true loops are still caught by repeatedToolLimit);+ * 3. the no-progress arm requires 2 consecutive tripping beats (debounce) so a+ * one-beat blip never fires a steer.+ * Plus regression guards for the pre-existing trips (loop, error storm, velocity).+ */++const assert = require('node:assert');+const fs = require('node:fs');+const os = require('node:os');+const path = require('node:path');+const ts = require('typescript');++const SRC = path.join(__dirname, '..', 'src', 'main', 'breaker.ts');+const out = fs.mkdtempSync(path.join(os.tmpdir(), 'breaker-'));+const js = ts.transpileModule(fs.readFileSync(SRC, 'utf8'), {+ compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }+}).outputText;+fs.writeFileSync(path.join(out, 'breaker.js'), js, 'utf8');+const { CircuitBreaker } = require(path.join(out, 'breaker.js'));++let failures = 0;+function test(name, fn) {+ try { fn(); console.log(` ok ${name}`); }+ catch (e) { failures++; console.error(`FAIL ${name}\n ${e.message}`); }+}++/** A breaker with fixed config (no caps, hardStop off). */+function makeBreaker(over = {}) {+ return new CircuitBreaker(() => ({+ enabled: true, hardStop: false, repeatedToolLimit: 8, errorStormLimit: 5,+ tokenVelocityPerMin: 60000, ...over+ }));+}++/** Cumulative sample helper. */+function sample(agentId, ts, output, input = 1000) {+ return { agentId, sessionId: 's1', ts, input, output, cacheRead: 0, cacheCreation: 0, model: 'm', usd: 0 };+}++const T0 = 1_000_000_000_000; // fixed epoch base so tests are deterministic+const BEAT = 30_000;++/** Run one beat for a single agent; returns its decision. */+function beat(b, id, s, progressing, now) {+ return b.tick([{ agentId: id, sample: s, progressing }], now)[0];+}++// ── regression: pre-existing trips still fire ────────────────────────────────++test('repeated identical tool calls trip the loop arm', () => {+ const b = makeBreaker();+ for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });+ const d = beat(b, 'a', null, true, T0);+ assert.equal(d.state.level, 'steering');+ assert.match(d.state.reason, /looping/);+});++test('error storm trips', () => {+ const b = makeBreaker();+ for (let i = 0; i < 5; i++) b.recordError('a');+ const d = beat(b, 'a', null, true, T0);+ assert.equal(d.state.level, 'steering');+ assert.match(d.state.reason, /error storm/);+});++test('token velocity spike trips on the second sample', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), true, T0);+ const d = beat(b, 'a', sample('a', T0 + BEAT, 40_000), true, T0 + BEAT); // 80k/min+ assert.equal(d.state.level, 'steering');+ assert.match(d.state.reason, /velocity/);+});++// ── fix 3: no-progress needs 2 consecutive tripping beats ───────────────────++test('no-progress does NOT trip on a single beat (debounce)', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ const d = beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++test('sustained no-progress still trips (second consecutive beat)', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);+ const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);+ assert.equal(d.state.level, 'steering');+ assert.match(d.state.reason, /no-progress/);+});++test('a progressing beat resets the no-progress debounce', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT); // count 1+ beat(b, 'a', sample('a', T0 + 2 * BEAT, 600), true, T0 + 2 * BEAT); // reset+ const d = beat(b, 'a', sample('a', T0 + 3 * BEAT, 1100), false, T0 + 3 * BEAT); // count 1 again+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++// ── fix 1: compaction exemption ──────────────────────────────────────────────++test('compaction exempts the no-progress trip', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ b.recordCompactStart('a', T0 + 1);+ // Two beats of token burst with stale coordination — would trip without the fix.+ beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);+ const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++test('compaction exempts the velocity trip', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), true, T0);+ b.recordCompactStart('a', T0 + 1);+ const d = beat(b, 'a', sample('a', T0 + BEAT, 40_000), true, T0 + BEAT); // 80k/min burst+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++test('trips resume after compaction end + trailing grace', () => {+ const b = makeBreaker();+ const GRACE = 120_000; // must cover POST_COMPACT_GRACE_MS+ beat(b, 'a', sample('a', T0, 0), false, T0);+ b.recordCompactStart('a', T0 + 1);+ b.recordCompactEnd('a', T0 + 2);+ const t1 = T0 + GRACE + BEAT;+ const t2 = t1 + BEAT;+ beat(b, 'a', sample('a', t1, 5000), false, t1);+ const d = beat(b, 'a', sample('a', t2, 6000), false, t2);+ assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);+});++test('compaction safety cap: exemption expires even without PostCompact', () => {+ const b = makeBreaker();+ const CAP = 10 * 60_000; // must exceed COMPACT_GRACE_MS+ beat(b, 'a', sample('a', T0, 0), false, T0);+ b.recordCompactStart('a', T0 + 1); // PostCompact never arrives+ const t1 = T0 + CAP + BEAT;+ const t2 = t1 + BEAT;+ beat(b, 'a', sample('a', t1, 5000), false, t1);+ const d = beat(b, 'a', sample('a', t2, 6000), false, t2);+ assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);+});++test('recordCompactEnd without a compaction in flight is a no-op', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ b.recordCompactEnd('a', T0 + 1); // e.g. SessionStart on a fresh session+ beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);+ const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);+ assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`); // still trips normally+});++// ── fix 2: recent distinct tool activity counts as progress ─────────────────++test('recent distinct tool calls exempt the no-progress trip', () => {+ const b = makeBreaker();+ beat(b, 'a', sample('a', T0, 0), false, T0);+ // Varied tool stream (background workflow / interactive work) right before each beat.+ b.recordToolUse('a', 'Read', { file: 'x' }, T0 + BEAT - 1000);+ beat(b, 'a', sample('a', T0 + BEAT, 5000), false, T0 + BEAT);+ b.recordToolUse('a', 'Read', { file: 'y' }, T0 + 2 * BEAT - 1000);+ const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 9000), false, T0 + 2 * BEAT);+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++test('REPEATED identical tool calls do not count as progress', () => {+ const b = makeBreaker();+ b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 - 10 * 60_000); // distinct stamp long ago+ beat(b, 'a', sample('a', T0, 0), false, T0);+ b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + BEAT - 1000); // repeat — no fresh stamp+ beat(b, 'a', sample('a', T0 + BEAT, 500), false, T0 + BEAT);+ b.recordToolUse('a', 'Bash', { cmd: 'same' }, T0 + 2 * BEAT - 1000);+ const d = beat(b, 'a', sample('a', T0 + 2 * BEAT, 1000), false, T0 + 2 * BEAT);+ assert.equal(d.state.level, 'steering', `reason: ${d.state.reason}`);+ assert.match(d.state.reason, /no-progress/);+});++// ── toolKey stays cheap AND discriminating on huge inputs ────────────────────++test('huge identical Write inputs still register as repeats (loop arm)', () => {+ const b = makeBreaker();+ const huge = { file_path: '/x.txt', content: 'A'.repeat(1_000_000) };+ for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Write', huge);+ const d = beat(b, 'a', null, true, T0);+ assert.equal(d.state.level, 'steering');+ assert.match(d.state.reason, /looping/);+});++test('huge inputs differing early still count as distinct calls', () => {+ const b = makeBreaker();+ for (let i = 0; i < 8; i++) {+ b.recordToolUse('a', 'Write', { file_path: `/f${i}.txt`, content: 'A'.repeat(500_000) });+ }+ const d = beat(b, 'a', null, true, T0);+ assert.equal(d.state.level, 'healthy', `reason: ${d.state.reason}`);+});++// ── recovery still works ─────────────────────────────────────────────────────++test('a healthy beat de-escalates one level', () => {+ const b = makeBreaker();+ for (let i = 0; i < 8; i++) b.recordToolUse('a', 'Bash', { cmd: 'same' });+ beat(b, 'a', null, true, T0); // → steering+ b.recordToolUse('a', 'Read', { file: 'new' }); // distinct call clears the loop+ const d = beat(b, 'a', null, true, T0 + BEAT);+ assert.equal(d.state.level, 'healthy');+});++process.exit(failures ? 1 : 0);