Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix: three faults found by reading a day of logs, not by guessing

0xdiego2519fc9f1d612f
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index a41ead4f6..68504b1a1 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -1753,6 +1753,21 @@ async function runForeground(session: AuthSession): Promise<void> {
?? hermesReaders.get(sessionId)?.turnOpen
?? devinReaders.get(sessionId)?.turnOpen
?? commandcodeNormalizers.get(sessionId)?.turnOpen
+ // A TURN THAT IS OPEN GETS ITS TRANSCRIPT RE-READ, every beat.
+ //
+ // The engines whose turn ends in a FILE — codex writes `task_complete`,
+ // and the other JSONL readers are the same shape — depend on chokidar
+ // delivering that last write. MEASURED twice, on two sessions an hour
+ // apart: it did not. Codex finished at 15:31:38 and the turn closed at
+ // 15:35:29, to the second, because the only thing that noticed was the
+ // five-minute reconciliation sweep. The web client never saw it because
+ // it renders the terminal stream; the device waits on `turn_ended` for
+ // its recap, so it sat spinning for 3m51s on a question answered in 18s.
+ //
+ // Reading to EOF here costs one stat and a short read of a file that is
+ // already open, and it bounds that failure at one heartbeat instead of
+ // at whatever is left of a five-minute window.
+ if (turnOpen) void watcher.pollSession(sessionId)
// Device: keep the busy tile alive through the turn AND the summarize window. Returns false when idle.
const deviceBusy = mirror.heartbeat(sessionId)
// Web: unchanged — heartbeat only while the turn itself is open (summarizing uses turn_summary_pending).
@@ -1785,6 +1800,8 @@ async function runForeground(session: AuthSession): Promise<void> {
input.onTurnStarted(agentIdFor(sessionId), event.payload.userMessage)
startHeartbeat(sessionId)
questionWatcher.start(sessionId) // Claude opens its dialog INSIDE a turn
+ // ...and anything already drawn belongs to the turn BEFORE this one.
+ questionWatcher.noteTurnStart(sessionId)
} else if (event.type === 'turn_ended') {
const startedAt = turnStartedAt.get(sessionId)
turnStartedAt.delete(sessionId)
diff --git a/cli/src/lib/askQuestion.spec.ts b/cli/src/lib/askQuestion.spec.ts
index 0d504cd60..2f5aea9aa 100644
--- a/cli/src/lib/askQuestion.spec.ts
+++ b/cli/src/lib/askQuestion.spec.ts
@@ -327,6 +327,27 @@ describe('QuestionWatcher', () => {
])
})
+ it('does not announce a dialog that was already on the pane when the turn began', async () => {
+ // The daemon attaches to a running engine whose LAST prompt was answered a
+ // moment ago in another client. Its pixels are still there, and a pane that
+ // was just answered looks exactly like one still waiting.
+ const w = watcher([fixture('single'), fixture('single'), fixture('single')]);
+ w.instance.noteTurnStart('s1');
+ await Promise.resolve();
+ await Promise.resolve();
+ await w.tick();
+ expect(w.seen).toHaveLength(0);
+ });
+
+ it('...but announces the next one, once the dialog actually changes', async () => {
+ const w = watcher([fixture('single'), fixture('multi'), fixture('multi')]);
+ w.instance.noteTurnStart('s1');
+ await Promise.resolve();
+ await Promise.resolve();
+ await w.tick(); // the NEW dialog — nothing like the one that predated the turn
+ expect(w.seen).toHaveLength(1);
+ });
+
it('announces a question ONCE while it stays on screen', async () => {
const w = watcher([fixture('single')])
await w.tick(); await w.tick(); await w.tick()
diff --git a/cli/src/lib/askQuestion.ts b/cli/src/lib/askQuestion.ts
index be68a3efa..9b3c7ea0b 100644
--- a/cli/src/lib/askQuestion.ts
+++ b/cli/src/lib/askQuestion.ts
@@ -771,10 +771,46 @@ export class QuestionWatcher {
private lastId = new Map<string, string>() // sessionId → requestId of the announced question
private misses = new Map<string, number>() // sessionId → consecutive polls with no dialog
private readonly blocked = new Map<string, string>()
+ /** sessionId → the dialog that was ALREADY on the pane when this turn began. */
+ private readonly preTurn = new Map<string, string>()
constructor(private readonly deps: QuestionWatcherDeps) {}
+ /**
+ * A turn just began — remember any dialog already on the pane, so it is not
+ * announced as this turn's question.
+ *
+ * A dialog belonging to a turn cannot have been on screen before that turn's
+ * first byte. That invariant is the only way to tell "waiting for an answer"
+ * from "answered a moment ago and still drawn", because a pane looks
+ * identical either way.
+ *
+ * MEASURED: the daemon attached to a Codex that had just been asked about an
+ * update and answered in the app. Its first capture, 0.8s into the NEXT turn,
+ * still held that prompt — so the dial was shown a question nobody was being
+ * asked, and it stood there for 25s until the engine's output scrolled it out
+ * of the captured window.
+ *
+ * Deliberately NOT applied on attach: re-announcing a genuinely open dialog to
+ * a client that arrives mid-question is a feature, and there is no turn
+ * boundary there to reason from.
+ */
+ noteTurnStart(sessionId: string): void {
+ this.preTurn.delete(sessionId)
+ void (async () => {
+ const session = this.deps.getSession(sessionId)
+ const target = session?.agentId || session?.sessionId
+ if (!target) return
+ const view = parseEngineQuestionPane(session?.engine ?? 'claude', await this.deps.capture(target, CAPTURE_LINES) ?? '')
+ if (!view || view.kind !== 'question' || !view.question || view.rows.length === 0) return
+ // Only if nothing has been announced for this turn yet: the capture takes
+ // a moment, and a dialog that opened inside that window is this turn's.
+ if (this.lastId.has(sessionId)) return
+ this.preTurn.set(sessionId, fingerprintOf(view))
+ })()
+ }
+
/** Poll this session's pane while its turn is open (called on turn_started). */
start(sessionId: string): void {
if (this.timers.has(sessionId)) return
@@ -799,6 +835,7 @@ export class QuestionWatcher {
* was removed cannot answer either.
*/
stop(sessionId: string): void {
+ this.preTurn.delete(sessionId)
const timer = this.timers.get(sessionId)
if (timer) { clearInterval(timer); this.timers.delete(sessionId) }
const requestId = this.lastId.get(sessionId)
@@ -866,7 +903,11 @@ export class QuestionWatcher {
return
}
this.misses.delete(sessionId) // a dialog on screen ends any run of misses
- const fingerprint = `${view.question}|${view.rows.map((r) => r.label).join('|')}|${view.multi}`
+ const fingerprint = fingerprintOf(view)
+ // Already on the pane before this turn started → it belongs to whatever came
+ // before, and has been answered. Say nothing until it changes or leaves.
+ if (this.preTurn.get(sessionId) === fingerprint) return
+ this.preTurn.delete(sessionId)
if (this.last.get(sessionId) === fingerprint) return
this.last.set(sessionId, fingerprint)
@@ -885,6 +926,11 @@ export class QuestionWatcher {
}
}
+/** What makes two captures the SAME dialog: its words, its options, its arity. */
+function fingerprintOf(view: QuestionView): string {
+ return `${view.question}|${view.rows.map((r) => r.label).join('|')}|${view.multi}`
+}
+
function hash(value: string): string {
let h = 0x811c9dc5
for (let i = 0; i < value.length; i++) h = Math.imul(h ^ value.charCodeAt(i), 0x01000193) >>> 0
diff --git a/cli/src/lib/disposableOneShotPool.spec.ts b/cli/src/lib/disposableOneShotPool.spec.ts
index 194178819..a39496f7b 100644
--- a/cli/src/lib/disposableOneShotPool.spec.ts
+++ b/cli/src/lib/disposableOneShotPool.spec.ts
@@ -42,6 +42,60 @@ describe('DisposableOneShotPool', () => {
expect([0, 1, 2, 3, 4, 6, 7].map(poolTargetForActiveSessions)).toEqual([0, 1, 1, 1, 2, 2, 3])
})
+ it('backs off instead of forking a process every second forever', async () => {
+ // MEASURED before this existed: an engine whose CLI could not be spawned
+ // produced 69,341 log lines in ten hours — one fork per second, with
+ // nothing in the loop able to notice it was losing.
+ vi.useFakeTimers()
+ let attempts = 0
+ const lines: string[] = []
+ const pool = new DisposableOneShotPool<string, Result>(async () => {
+ attempts++
+ throw new Error('spawn opencode ENOENT')
+ }, 300_000, (line) => lines.push(line))
+
+ pool.setActiveCounts({ claude: 0, codex: 0, cursor: 0, opencode: 1, pi: 0, commandcode: 0, kilo: 0 })
+ pool.setDeviceConnected(true)
+ await settle()
+ expect(attempts).toBe(1)
+
+ // A flat one-second retry would be 60 forks here. Doubling is 6.
+ // Long enough to reach the ceiling: 1+2+4+8+16+32+60 = 123s of backoff.
+ // A flat one-second retry would have forked 200 times by here.
+ for (let i = 0; i < 200; i++) {
+ await vi.advanceTimersByTimeAsync(1_000)
+ await settle()
+ }
+ expect(attempts).toBeLessThan(12)
+
+ // ...and it stops narrating once the delay stops changing.
+ const failureLines = lines.filter((line) => line.includes('warm spawn failed'))
+ expect(failureLines.length).toBeLessThanOrEqual(8)
+ expect(failureLines.at(-1)).toContain('staying quiet')
+
+ pool.shutdown()
+ })
+
+ it('a worker that finally comes up clears the backoff', async () => {
+ vi.useFakeTimers()
+ let attempts = 0
+ const pool = new DisposableOneShotPool<string, Result>(async (engine) => {
+ attempts++
+ if (attempts < 3) throw new Error('spawn opencode ENOENT')
+ return new FakeWorker(engine, attempts)
+ }, 300_000, () => {})
+
+ pool.setActiveCounts({ claude: 0, codex: 0, cursor: 0, opencode: 1, pi: 0, commandcode: 0, kilo: 0 })
+ pool.setDeviceConnected(true)
+ await settle()
+ for (let i = 0; i < 5; i++) {
+ await vi.advanceTimersByTimeAsync(2_000)
+ await settle()
+ }
+ expect(pool.snapshot().opencode).toMatchObject({ ready: 1 })
+ pool.shutdown()
+ })
+
it('consumes a ready worker once and immediately replenishes that engine', async () => {
let id = 0
const created: FakeWorker[] = []
diff --git a/cli/src/lib/disposableOneShotPool.ts b/cli/src/lib/disposableOneShotPool.ts
index 9abe6edf6..4ac6dd25a 100644
--- a/cli/src/lib/disposableOneShotPool.ts
+++ b/cli/src/lib/disposableOneShotPool.ts
@@ -24,8 +24,24 @@ interface EngineState<Options, Result> {
starting: number
ready: Set<DisposableWorker<Options, Result>>
retryTimer: NodeJS.Timeout | null
+ /** Consecutive warm-spawn failures. Drives the backoff and silences the log. */
+ failures: number
}
+/**
+ * Warm-spawn retry backoff.
+ *
+ * A flat one-second retry is fine for a worker that crashed and will come back.
+ * It is a disaster for a condition that never changes on its own — a missing or
+ * unresolvable binary — because nothing in the loop ever notices it is losing.
+ * MEASURED before this existed: one engine whose CLI could not be spawned
+ * produced 69,341 log lines in ten hours, one process fork per second, forever.
+ */
+const RETRY_BASE_MS = 1_000
+const RETRY_CAP_MS = 60_000
+/** The attempt whose delay first reaches the cap — the last one that logs. */
+const CAP_ANNOUNCE_AT = 7
+
export interface PoolSnapshot {
enabled: boolean
connected: boolean
@@ -41,13 +57,13 @@ export interface PoolSnapshot {
/** Maintains disposable, pre-spawned workers. A checked-out worker is never returned to the pool. */
export class DisposableOneShotPool<Options, Result> {
private readonly states: Record<OneShotEngine, EngineState<Options, Result>> = {
- claude: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- codex: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- cursor: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- opencode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- pi: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- commandcode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
- kilo: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },
+ claude: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ codex: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ cursor: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ opencode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ pi: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ commandcode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
+ kilo: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },
}
private readonly all = new Set<DisposableWorker<Options, Result>>()
private connected = false
@@ -218,6 +234,10 @@ export class DisposableOneShotPool<Options, Result> {
clearTimeout(state.retryTimer)
state.retryTimer = null
}
+ // Going idle ends the streak. The next session of this engine deserves a
+ // fast first attempt — the thing that was broken may well have been fixed
+ // in between, and it is the only moment we get to find out cheaply.
+ if (state.target === 0) state.failures = 0
while (state.ready.size > state.target) {
const worker = state.ready.values().next().value as DisposableWorker<Options, Result> | undefined
@@ -242,13 +262,24 @@ export class DisposableOneShotPool<Options, Result> {
if (!this.closed && this.enabled && state.ready.size < state.target) this.scheduleRetry(engine)
} else {
state.ready.add(worker)
+ state.failures = 0
this.log(`[recap-pool] ${engine} warm ready=${state.ready.size}/${state.target}`)
this.reconcile(engine)
}
})
.catch((err) => {
state.starting--
- this.log(`[recap-pool] ${engine} warm spawn failed: ${err instanceof Error ? err.message : String(err)}`)
+ state.failures++
+ const delay = this.retryDelay(state)
+ // Every attempt while the backoff is still growing, then ONCE at the
+ // ceiling. A failure that cannot fix itself should say so and go quiet,
+ // not narrate the same sentence every second until the log is 9MB.
+ if (delay < RETRY_CAP_MS || state.failures === CAP_ANNOUNCE_AT) {
+ this.log(
+ `[recap-pool] ${engine} warm spawn failed (${state.failures}): ${err instanceof Error ? err.message : String(err)}`
+ + (delay >= RETRY_CAP_MS ? ` · backing off to ${RETRY_CAP_MS / 1000}s and staying quiet` : ` · retry in ${delay}ms`),
+ )
+ }
this.scheduleRetry(engine)
})
}
@@ -275,6 +306,12 @@ export class DisposableOneShotPool<Options, Result> {
state.retryTimer = setTimeout(() => {
state.retryTimer = null
this.reconcile(engine)
- }, 1000)
+ }, this.retryDelay(state))
+ }
+
+ /** 1s, 2s, 4s … capped at a minute. Doubling is what makes a permanent failure cost nothing. */
+ private retryDelay(state: EngineState<Options, Result>): number {
+ const shift = Math.min(Math.max(state.failures - 1, 0), 6)
+ return Math.min(RETRY_BASE_MS * 2 ** shift, RETRY_CAP_MS)
}
}
diff --git a/cli/src/lib/engineBin.ts b/cli/src/lib/engineBin.ts
index bc0bdc20a..03db357d3 100644
--- a/cli/src/lib/engineBin.ts
+++ b/cli/src/lib/engineBin.ts
@@ -115,9 +115,14 @@ function interactivePathEntries(): string[] {
}
}
+/** Only what a spawned child inherits. A hit here is safe to return as a bare name. */
+function processPathEntries(): string[] {
+ return (process.env.PATH ?? '').split(delimiter).filter(Boolean)
+}
+
function pathEntries(): string[] {
return [...new Set([
- ...(process.env.PATH ?? '').split(delimiter).filter(Boolean),
+ ...processPathEntries(),
...interactivePathEntries(),
])]
}
@@ -334,12 +339,30 @@ export function cursorRuntimeBin(snapshot = agentCommandOwnershipSnapshot()): st
*/
export function opencodeBin(): string {
if (env.OPENCODE_PATH) return env.OPENCODE_PATH
- for (const entry of pathEntries()) {
+ // THE CHILD'S PATH FIRST, and only that one may answer with a bare name.
+ //
+ // `pathEntries()` searches the interactive login shell's PATH as well as this
+ // process's, and a spawned child inherits only the latter. A hit in a
+ // shell-only directory therefore resolved here and then failed to spawn with
+ // `ENOENT` — which is how the fallback below, added for exactly this bug, was
+ // being skipped: the loop kept FINDING opencode and kept returning a name the
+ // child could not resolve. Measured on a live daemon: 44 entries in the
+ // process PATH, 48 in the interactive one, and `~/.opencode/bin` among the
+ // four that only the shell knows about.
+ for (const entry of processPathEntries()) {
try {
accessSync(join(entry, 'opencode'), constants.X_OK)
return 'opencode'
} catch { /* keep looking */ }
}
+ // Anywhere else it is reachable, it travels as an absolute path.
+ for (const entry of pathEntries()) {
+ const candidate = join(entry, 'opencode')
+ try {
+ accessSync(candidate, constants.X_OK)
+ return candidate
+ } catch { /* keep looking */ }
+ }
const installed = join(homedir(), '.opencode', 'bin', 'opencode')
try {
accessSync(installed, constants.X_OK)
diff --git a/cli/src/lib/engineBinOpencode.spec.ts b/cli/src/lib/engineBinOpencode.spec.ts
index 589016a68..10bb48f51 100644
--- a/cli/src/lib/engineBinOpencode.spec.ts
+++ b/cli/src/lib/engineBinOpencode.spec.ts
@@ -5,10 +5,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest'
const REAL_PATH = process.env.PATH
const REAL_HOME = process.env.HOME
+const REAL_SHELL = process.env.SHELL
afterEach(() => {
process.env.PATH = REAL_PATH
if (REAL_HOME) process.env.HOME = REAL_HOME
+ if (REAL_SHELL) process.env.SHELL = REAL_SHELL; else delete process.env.SHELL
vi.resetModules()
})
@@ -48,6 +50,39 @@ describe('opencodeBin', () => {
expect(opencodeBin()).toBe('opencode')
})
+ it('returns an absolute path for an install only the login shell can see', async () => {
+ // THE CASE THAT DEFEATED THE FALLBACK ABOVE. The resolver also searches the
+ // interactive shell's PATH, and a child inherits only this process's — so a
+ // hit there used to come back as the bare name and die on `spawn ENOENT`,
+ // without the ~/.opencode fallback ever being reached. Measured on a live
+ // daemon: 44 entries in the process PATH, 48 in the interactive one.
+ const home = mkdtempSync(join(tmpdir(), 'oc-empty-'));
+ const shellOnly = mkdtempSync(join(tmpdir(), 'oc-shell-'));
+ const file = join(shellOnly, 'opencode');
+ writeFileSync(file, '#!/bin/sh\nexit 0\n');
+ chmodSync(file, 0o755);
+
+ // A shell that answers the resolver's probe with a PATH this process lacks.
+ const fakeShell = join(mkdtempSync(join(tmpdir(), 'oc-sh-')), 'shell');
+ writeFileSync(
+ fakeShell,
+ `#!/bin/sh\nprintf '\\n__HARNESS_ENGINE_PATH__=%s\\n' "${shellOnly}"\n`,
+ );
+ chmodSync(fakeShell, 0o755);
+
+ process.env.HOME = home;
+ process.env.PATH = '/usr/bin:/bin';
+ process.env.SHELL = fakeShell;
+ const realNodeEnv = process.env.NODE_ENV;
+ delete process.env.NODE_ENV; // the interactive probe is skipped under test
+ try {
+ const { opencodeBin } = await import('./engineBin.js');
+ expect(opencodeBin()).toBe(file);
+ } finally {
+ if (realNodeEnv !== undefined) process.env.NODE_ENV = realNodeEnv;
+ }
+ });
+
it('still says `opencode` when nothing is installed, so the error is the usual one', async () => {
process.env.HOME = mkdtempSync(join(tmpdir(), 'oc-none-'))
process.env.PATH = '/nonexistent'