The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix: three faults found by reading a day of logs, not by guessing
0xdiego2519fc9f1d612f
diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex a41ead4f6..68504b1a1 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -1753,6 +1753,21 @@ async function runForeground(session: AuthSession): Promise<void> {?? hermesReaders.get(sessionId)?.turnOpen?? devinReaders.get(sessionId)?.turnOpen?? commandcodeNormalizers.get(sessionId)?.turnOpen+ // A TURN THAT IS OPEN GETS ITS TRANSCRIPT RE-READ, every beat.+ //+ // The engines whose turn ends in a FILE — codex writes `task_complete`,+ // and the other JSONL readers are the same shape — depend on chokidar+ // delivering that last write. MEASURED twice, on two sessions an hour+ // apart: it did not. Codex finished at 15:31:38 and the turn closed at+ // 15:35:29, to the second, because the only thing that noticed was the+ // five-minute reconciliation sweep. The web client never saw it because+ // it renders the terminal stream; the device waits on `turn_ended` for+ // its recap, so it sat spinning for 3m51s on a question answered in 18s.+ //+ // Reading to EOF here costs one stat and a short read of a file that is+ // already open, and it bounds that failure at one heartbeat instead of+ // at whatever is left of a five-minute window.+ if (turnOpen) void watcher.pollSession(sessionId)// Device: keep the busy tile alive through the turn AND the summarize window. Returns false when idle.const deviceBusy = mirror.heartbeat(sessionId)// Web: unchanged — heartbeat only while the turn itself is open (summarizing uses turn_summary_pending).@@ -1785,6 +1800,8 @@ async function runForeground(session: AuthSession): Promise<void> {input.onTurnStarted(agentIdFor(sessionId), event.payload.userMessage)startHeartbeat(sessionId)questionWatcher.start(sessionId) // Claude opens its dialog INSIDE a turn+ // ...and anything already drawn belongs to the turn BEFORE this one.+ questionWatcher.noteTurnStart(sessionId)} else if (event.type === 'turn_ended') {const startedAt = turnStartedAt.get(sessionId)turnStartedAt.delete(sessionId)diff --git a/cli/src/lib/askQuestion.spec.ts b/cli/src/lib/askQuestion.spec.tsindex 0d504cd60..2f5aea9aa 100644--- a/cli/src/lib/askQuestion.spec.ts+++ b/cli/src/lib/askQuestion.spec.ts@@ -327,6 +327,27 @@ describe('QuestionWatcher', () => {])})+ it('does not announce a dialog that was already on the pane when the turn began', async () => {+ // The daemon attaches to a running engine whose LAST prompt was answered a+ // moment ago in another client. Its pixels are still there, and a pane that+ // was just answered looks exactly like one still waiting.+ const w = watcher([fixture('single'), fixture('single'), fixture('single')]);+ w.instance.noteTurnStart('s1');+ await Promise.resolve();+ await Promise.resolve();+ await w.tick();+ expect(w.seen).toHaveLength(0);+ });++ it('...but announces the next one, once the dialog actually changes', async () => {+ const w = watcher([fixture('single'), fixture('multi'), fixture('multi')]);+ w.instance.noteTurnStart('s1');+ await Promise.resolve();+ await Promise.resolve();+ await w.tick(); // the NEW dialog — nothing like the one that predated the turn+ expect(w.seen).toHaveLength(1);+ });+it('announces a question ONCE while it stays on screen', async () => {const w = watcher([fixture('single')])await w.tick(); await w.tick(); await w.tick()diff --git a/cli/src/lib/askQuestion.ts b/cli/src/lib/askQuestion.tsindex be68a3efa..9b3c7ea0b 100644--- a/cli/src/lib/askQuestion.ts+++ b/cli/src/lib/askQuestion.ts@@ -771,10 +771,46 @@ export class QuestionWatcher {private lastId = new Map<string, string>() // sessionId → requestId of the announced questionprivate misses = new Map<string, number>() // sessionId → consecutive polls with no dialogprivate readonly blocked = new Map<string, string>()+ /** sessionId → the dialog that was ALREADY on the pane when this turn began. */+ private readonly preTurn = new Map<string, string>()constructor(private readonly deps: QuestionWatcherDeps) {}+ /**+ * A turn just began — remember any dialog already on the pane, so it is not+ * announced as this turn's question.+ *+ * A dialog belonging to a turn cannot have been on screen before that turn's+ * first byte. That invariant is the only way to tell "waiting for an answer"+ * from "answered a moment ago and still drawn", because a pane looks+ * identical either way.+ *+ * MEASURED: the daemon attached to a Codex that had just been asked about an+ * update and answered in the app. Its first capture, 0.8s into the NEXT turn,+ * still held that prompt — so the dial was shown a question nobody was being+ * asked, and it stood there for 25s until the engine's output scrolled it out+ * of the captured window.+ *+ * Deliberately NOT applied on attach: re-announcing a genuinely open dialog to+ * a client that arrives mid-question is a feature, and there is no turn+ * boundary there to reason from.+ */+ noteTurnStart(sessionId: string): void {+ this.preTurn.delete(sessionId)+ void (async () => {+ const session = this.deps.getSession(sessionId)+ const target = session?.agentId || session?.sessionId+ if (!target) return+ const view = parseEngineQuestionPane(session?.engine ?? 'claude', await this.deps.capture(target, CAPTURE_LINES) ?? '')+ if (!view || view.kind !== 'question' || !view.question || view.rows.length === 0) return+ // Only if nothing has been announced for this turn yet: the capture takes+ // a moment, and a dialog that opened inside that window is this turn's.+ if (this.lastId.has(sessionId)) return+ this.preTurn.set(sessionId, fingerprintOf(view))+ })()+ }+/** Poll this session's pane while its turn is open (called on turn_started). */start(sessionId: string): void {if (this.timers.has(sessionId)) return@@ -799,6 +835,7 @@ export class QuestionWatcher {* was removed cannot answer either.*/stop(sessionId: string): void {+ this.preTurn.delete(sessionId)const timer = this.timers.get(sessionId)if (timer) { clearInterval(timer); this.timers.delete(sessionId) }const requestId = this.lastId.get(sessionId)@@ -866,7 +903,11 @@ export class QuestionWatcher {return}this.misses.delete(sessionId) // a dialog on screen ends any run of misses- const fingerprint = `${view.question}|${view.rows.map((r) => r.label).join('|')}|${view.multi}`+ const fingerprint = fingerprintOf(view)+ // Already on the pane before this turn started → it belongs to whatever came+ // before, and has been answered. Say nothing until it changes or leaves.+ if (this.preTurn.get(sessionId) === fingerprint) return+ this.preTurn.delete(sessionId)if (this.last.get(sessionId) === fingerprint) returnthis.last.set(sessionId, fingerprint)@@ -885,6 +926,11 @@ export class QuestionWatcher {}}+/** What makes two captures the SAME dialog: its words, its options, its arity. */+function fingerprintOf(view: QuestionView): string {+ return `${view.question}|${view.rows.map((r) => r.label).join('|')}|${view.multi}`+}+function hash(value: string): string {let h = 0x811c9dc5for (let i = 0; i < value.length; i++) h = Math.imul(h ^ value.charCodeAt(i), 0x01000193) >>> 0diff --git a/cli/src/lib/disposableOneShotPool.spec.ts b/cli/src/lib/disposableOneShotPool.spec.tsindex 194178819..a39496f7b 100644--- a/cli/src/lib/disposableOneShotPool.spec.ts+++ b/cli/src/lib/disposableOneShotPool.spec.ts@@ -42,6 +42,60 @@ describe('DisposableOneShotPool', () => {expect([0, 1, 2, 3, 4, 6, 7].map(poolTargetForActiveSessions)).toEqual([0, 1, 1, 1, 2, 2, 3])})+ it('backs off instead of forking a process every second forever', async () => {+ // MEASURED before this existed: an engine whose CLI could not be spawned+ // produced 69,341 log lines in ten hours — one fork per second, with+ // nothing in the loop able to notice it was losing.+ vi.useFakeTimers()+ let attempts = 0+ const lines: string[] = []+ const pool = new DisposableOneShotPool<string, Result>(async () => {+ attempts+++ throw new Error('spawn opencode ENOENT')+ }, 300_000, (line) => lines.push(line))++ pool.setActiveCounts({ claude: 0, codex: 0, cursor: 0, opencode: 1, pi: 0, commandcode: 0, kilo: 0 })+ pool.setDeviceConnected(true)+ await settle()+ expect(attempts).toBe(1)++ // A flat one-second retry would be 60 forks here. Doubling is 6.+ // Long enough to reach the ceiling: 1+2+4+8+16+32+60 = 123s of backoff.+ // A flat one-second retry would have forked 200 times by here.+ for (let i = 0; i < 200; i++) {+ await vi.advanceTimersByTimeAsync(1_000)+ await settle()+ }+ expect(attempts).toBeLessThan(12)++ // ...and it stops narrating once the delay stops changing.+ const failureLines = lines.filter((line) => line.includes('warm spawn failed'))+ expect(failureLines.length).toBeLessThanOrEqual(8)+ expect(failureLines.at(-1)).toContain('staying quiet')++ pool.shutdown()+ })++ it('a worker that finally comes up clears the backoff', async () => {+ vi.useFakeTimers()+ let attempts = 0+ const pool = new DisposableOneShotPool<string, Result>(async (engine) => {+ attempts+++ if (attempts < 3) throw new Error('spawn opencode ENOENT')+ return new FakeWorker(engine, attempts)+ }, 300_000, () => {})++ pool.setActiveCounts({ claude: 0, codex: 0, cursor: 0, opencode: 1, pi: 0, commandcode: 0, kilo: 0 })+ pool.setDeviceConnected(true)+ await settle()+ for (let i = 0; i < 5; i++) {+ await vi.advanceTimersByTimeAsync(2_000)+ await settle()+ }+ expect(pool.snapshot().opencode).toMatchObject({ ready: 1 })+ pool.shutdown()+ })+it('consumes a ready worker once and immediately replenishes that engine', async () => {let id = 0const created: FakeWorker[] = []diff --git a/cli/src/lib/disposableOneShotPool.ts b/cli/src/lib/disposableOneShotPool.tsindex 9abe6edf6..4ac6dd25a 100644--- a/cli/src/lib/disposableOneShotPool.ts+++ b/cli/src/lib/disposableOneShotPool.ts@@ -24,8 +24,24 @@ interface EngineState<Options, Result> {starting: numberready: Set<DisposableWorker<Options, Result>>retryTimer: NodeJS.Timeout | null+ /** Consecutive warm-spawn failures. Drives the backoff and silences the log. */+ failures: number}+/**+ * Warm-spawn retry backoff.+ *+ * A flat one-second retry is fine for a worker that crashed and will come back.+ * It is a disaster for a condition that never changes on its own — a missing or+ * unresolvable binary — because nothing in the loop ever notices it is losing.+ * MEASURED before this existed: one engine whose CLI could not be spawned+ * produced 69,341 log lines in ten hours, one process fork per second, forever.+ */+const RETRY_BASE_MS = 1_000+const RETRY_CAP_MS = 60_000+/** The attempt whose delay first reaches the cap — the last one that logs. */+const CAP_ANNOUNCE_AT = 7+export interface PoolSnapshot {enabled: booleanconnected: boolean@@ -41,13 +57,13 @@ export interface PoolSnapshot {/** Maintains disposable, pre-spawned workers. A checked-out worker is never returned to the pool. */export class DisposableOneShotPool<Options, Result> {private readonly states: Record<OneShotEngine, EngineState<Options, Result>> = {- claude: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- codex: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- cursor: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- opencode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- pi: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- commandcode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },- kilo: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null },+ claude: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ codex: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ cursor: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ opencode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ pi: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ commandcode: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },+ kilo: { active: 0, target: 0, starting: 0, ready: new Set(), retryTimer: null, failures: 0 },}private readonly all = new Set<DisposableWorker<Options, Result>>()private connected = false@@ -218,6 +234,10 @@ export class DisposableOneShotPool<Options, Result> {clearTimeout(state.retryTimer)state.retryTimer = null}+ // Going idle ends the streak. The next session of this engine deserves a+ // fast first attempt — the thing that was broken may well have been fixed+ // in between, and it is the only moment we get to find out cheaply.+ if (state.target === 0) state.failures = 0while (state.ready.size > state.target) {const worker = state.ready.values().next().value as DisposableWorker<Options, Result> | undefined@@ -242,13 +262,24 @@ export class DisposableOneShotPool<Options, Result> {if (!this.closed && this.enabled && state.ready.size < state.target) this.scheduleRetry(engine)} else {state.ready.add(worker)+ state.failures = 0this.log(`[recap-pool] ${engine} warm ready=${state.ready.size}/${state.target}`)this.reconcile(engine)}}).catch((err) => {state.starting--- this.log(`[recap-pool] ${engine} warm spawn failed: ${err instanceof Error ? err.message : String(err)}`)+ state.failures+++ const delay = this.retryDelay(state)+ // Every attempt while the backoff is still growing, then ONCE at the+ // ceiling. A failure that cannot fix itself should say so and go quiet,+ // not narrate the same sentence every second until the log is 9MB.+ if (delay < RETRY_CAP_MS || state.failures === CAP_ANNOUNCE_AT) {+ this.log(+ `[recap-pool] ${engine} warm spawn failed (${state.failures}): ${err instanceof Error ? err.message : String(err)}`+ + (delay >= RETRY_CAP_MS ? ` · backing off to ${RETRY_CAP_MS / 1000}s and staying quiet` : ` · retry in ${delay}ms`),+ )+ }this.scheduleRetry(engine)})}@@ -275,6 +306,12 @@ export class DisposableOneShotPool<Options, Result> {state.retryTimer = setTimeout(() => {state.retryTimer = nullthis.reconcile(engine)- }, 1000)+ }, this.retryDelay(state))+ }++ /** 1s, 2s, 4s … capped at a minute. Doubling is what makes a permanent failure cost nothing. */+ private retryDelay(state: EngineState<Options, Result>): number {+ const shift = Math.min(Math.max(state.failures - 1, 0), 6)+ return Math.min(RETRY_BASE_MS * 2 ** shift, RETRY_CAP_MS)}}diff --git a/cli/src/lib/engineBin.ts b/cli/src/lib/engineBin.tsindex bc0bdc20a..03db357d3 100644--- a/cli/src/lib/engineBin.ts+++ b/cli/src/lib/engineBin.ts@@ -115,9 +115,14 @@ function interactivePathEntries(): string[] {}}+/** Only what a spawned child inherits. A hit here is safe to return as a bare name. */+function processPathEntries(): string[] {+ return (process.env.PATH ?? '').split(delimiter).filter(Boolean)+}+function pathEntries(): string[] {return [...new Set([- ...(process.env.PATH ?? '').split(delimiter).filter(Boolean),+ ...processPathEntries(),...interactivePathEntries(),])]}@@ -334,12 +339,30 @@ export function cursorRuntimeBin(snapshot = agentCommandOwnershipSnapshot()): st*/export function opencodeBin(): string {if (env.OPENCODE_PATH) return env.OPENCODE_PATH- for (const entry of pathEntries()) {+ // THE CHILD'S PATH FIRST, and only that one may answer with a bare name.+ //+ // `pathEntries()` searches the interactive login shell's PATH as well as this+ // process's, and a spawned child inherits only the latter. A hit in a+ // shell-only directory therefore resolved here and then failed to spawn with+ // `ENOENT` — which is how the fallback below, added for exactly this bug, was+ // being skipped: the loop kept FINDING opencode and kept returning a name the+ // child could not resolve. Measured on a live daemon: 44 entries in the+ // process PATH, 48 in the interactive one, and `~/.opencode/bin` among the+ // four that only the shell knows about.+ for (const entry of processPathEntries()) {try {accessSync(join(entry, 'opencode'), constants.X_OK)return 'opencode'} catch { /* keep looking */ }}+ // Anywhere else it is reachable, it travels as an absolute path.+ for (const entry of pathEntries()) {+ const candidate = join(entry, 'opencode')+ try {+ accessSync(candidate, constants.X_OK)+ return candidate+ } catch { /* keep looking */ }+ }const installed = join(homedir(), '.opencode', 'bin', 'opencode')try {accessSync(installed, constants.X_OK)diff --git a/cli/src/lib/engineBinOpencode.spec.ts b/cli/src/lib/engineBinOpencode.spec.tsindex 589016a68..10bb48f51 100644--- a/cli/src/lib/engineBinOpencode.spec.ts+++ b/cli/src/lib/engineBinOpencode.spec.ts@@ -5,10 +5,12 @@ import { afterEach, describe, expect, it, vi } from 'vitest'const REAL_PATH = process.env.PATHconst REAL_HOME = process.env.HOME+const REAL_SHELL = process.env.SHELLafterEach(() => {process.env.PATH = REAL_PATHif (REAL_HOME) process.env.HOME = REAL_HOME+ if (REAL_SHELL) process.env.SHELL = REAL_SHELL; else delete process.env.SHELLvi.resetModules()})@@ -48,6 +50,39 @@ describe('opencodeBin', () => {expect(opencodeBin()).toBe('opencode')})+ it('returns an absolute path for an install only the login shell can see', async () => {+ // THE CASE THAT DEFEATED THE FALLBACK ABOVE. The resolver also searches the+ // interactive shell's PATH, and a child inherits only this process's — so a+ // hit there used to come back as the bare name and die on `spawn ENOENT`,+ // without the ~/.opencode fallback ever being reached. Measured on a live+ // daemon: 44 entries in the process PATH, 48 in the interactive one.+ const home = mkdtempSync(join(tmpdir(), 'oc-empty-'));+ const shellOnly = mkdtempSync(join(tmpdir(), 'oc-shell-'));+ const file = join(shellOnly, 'opencode');+ writeFileSync(file, '#!/bin/sh\nexit 0\n');+ chmodSync(file, 0o755);++ // A shell that answers the resolver's probe with a PATH this process lacks.+ const fakeShell = join(mkdtempSync(join(tmpdir(), 'oc-sh-')), 'shell');+ writeFileSync(+ fakeShell,+ `#!/bin/sh\nprintf '\\n__HARNESS_ENGINE_PATH__=%s\\n' "${shellOnly}"\n`,+ );+ chmodSync(fakeShell, 0o755);++ process.env.HOME = home;+ process.env.PATH = '/usr/bin:/bin';+ process.env.SHELL = fakeShell;+ const realNodeEnv = process.env.NODE_ENV;+ delete process.env.NODE_ENV; // the interactive probe is skipped under test+ try {+ const { opencodeBin } = await import('./engineBin.js');+ expect(opencodeBin()).toBe(file);+ } finally {+ if (realNodeEnv !== undefined) process.env.NODE_ENV = realNodeEnv;+ }+ });+it('still says `opencode` when nothing is installed, so the error is the usual one', async () => {process.env.HOME = mkdtempSync(join(tmpdir(), 'oc-none-'))process.env.PATH = '/nonexistent'