The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(cli): bound the liveness watcher's sleep forgiveness (#1029)
0xken15aed2ca989e51
diff --git a/cli/src/gateway/upstream.ts b/cli/src/gateway/upstream.tsindex 78c4e52dc..50f5d1348 100644--- a/cli/src/gateway/upstream.ts+++ b/cli/src/gateway/upstream.ts@@ -172,7 +172,11 @@ export class UpstreamLink {this.heartbeat = watchSocketLiveness(ws, {onIdle: (idleMs) => console.log(`[backend] no traffic for ${Math.round(idleMs / 1000)}s — terminating the link`),- onWake: (sleptMs, hungUp) => console.log(`[backend] woke after ${Math.round(sleptMs / 1000)}s asleep — ${hungUp ? 'the backend has hung up, redialing' : 're-probing the link'}`),+ // Two different rules end the socket on a wake, and the trace has to tell them apart: a sleep+ // the backend cannot have waited through (>= BACKEND_IDLE_DEADLINE_MS), or a second forgiven+ // wake with nothing heard back. GENUINE_SLEEP_MS (180s) is above this deadline, so the length+ // alone decides which fired.+ onWake: (sleptMs, givingUp) => console.log(`[backend] woke after ${Math.round(sleptMs / 1000)}s asleep — ${!givingUp ? 're-probing the link' : sleptMs >= BACKEND_IDLE_DEADLINE_MS ? 'the backend has hung up, redialing' : 'no answer since the last wake, redialing'}`),peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex 7cb983b38..4aa9a291b 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -726,6 +726,8 @@ export class RemoteRelayPool {})entry.heartbeat = watchSocketLiveness(ws, {onIdle: (idleMs) => console.log(`[relay] ${machineId.slice(0, 8)} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),+ // A wake can end the socket too, and never reaches onIdle (wsLiveness.ts, the wake branch).+ onWake: (sleptMs, givingUp) => console.log(`[relay] ${machineId.slice(0, 8)} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'giving up on the link' : 're-probing'}`),// The relay socket ends at the backend, which hung up on it while we slept.peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,// Piggybacked sweep for a migration that never completed (pane closed mid-flight, responder neverdiff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.tsindex 4ccd20eca..85e1b7588 100644--- a/cli/src/lib/terminalStreamManager.spec.ts+++ b/cli/src/lib/terminalStreamManager.spec.ts@@ -716,6 +716,28 @@ describe('TerminalStreamManager', () => {&& frame.payload.reason === 'heartbeat timeout')).toBe(true)})+ // A sweep read as a sleep carries each lease by `slept`, which `sleptFor` defines as the gap MINUS+ // one period — so every sweep still charges one EXPIRY_SWEEP_MS of silence however late it ran, and+ // a throttled machine cannot hold a dead client's lease open. Guards against "carry the whole gap",+ // which would never expire.+ it('still expires a lease when every sweep is late enough to read as a sleep', async () => {+ let asleepMs = 0+ await manager.stop()+ manager = newManager({ now: () => Date.now() + asleepMs })+ await manager.handleFrame('web-1', 'terminal_open', {+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,+ })+ // Twelve sweeps, each landing 15s late (gap 20s > 2 x 5s, so every one is read as a sleep) and no+ // `terminal_alive` in any of them. Six periods of charged silence is the 30s timeout.+ for (let i = 0; i < 12; i++) {+ asleepMs += 15_000+ await vi.advanceTimersByTimeAsync(5_000)+ }+ expect(stream.closed).toBe(true)+ expect(sent.some((frame) => frame.type === 'terminal_closed'+ && frame.payload.reason === 'heartbeat timeout')).toBe(true)+ })+// The incumbent's banner says WHO took over, so the close it gets carries what the winner// declared on open — verbatim through a relay, since the daemon never learns a peer's name.it('names the taker on the close when the winner introduced itself', async () => {diff --git a/cli/src/lib/wsLiveness.spec.ts b/cli/src/lib/wsLiveness.spec.tsindex a1e9c6e61..7d6dc9621 100644--- a/cli/src/lib/wsLiveness.spec.ts+++ b/cli/src/lib/wsLiveness.spec.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, it, vi } from 'vitest'import { EventEmitter } from 'node:events'import type { WebSocket } from 'ws'-import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'+import { BACKEND_IDLE_DEADLINE_MS, GENUINE_SLEEP_MS, watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'class FakeSocket extends EventEmitter {pings = 0@@ -148,4 +148,126 @@ describe('watchSocketLiveness', () => {await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)expect(ws.terminated).toBe(1)})++ // The 40s loopback deadline (localWsServer.ts) was unenforceable on a loaded machine: macOS+ // coalesces the 20s heartbeat so ticks land 44-80s late, every one read as a sleep, every one+ // forgiving the silence. Measured over one ~2-day harness.log: 968 such wakes reporting 24-60s+ // against 80 real sleeps (all >600s), and terminations logging `no traffic for 912-995s` — 23x+ // past the deadline, 276 times. One pass per proof of life is the bound.+ it('stops forgiving sleeps a peer never answers', async () => {+ vi.useFakeTimers()+ const ws = socket()+ ws.silent = true+ let asleepMs = 0+ const woke: Array<[number, boolean]> = []+ watchSocketLiveness(ws, {+ deadlineMs: 40_000, // the loopback shape+ now: () => performance.now() + asleepMs,+ onWake: (ms, givingUp) => woke.push([ms, givingUp]),+ })+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000 // a throttled tick, not a sleeping machine+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0) // the one free pass+ expect(woke).toEqual([[40_000, false]])+ asleepMs += 40_000 // and another, with nothing heard from the peer in between+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(1)+ expect(woke).toEqual([[40_000, false], [40_000, true]])+ })++ // The bound above must never cost a REAL sleep its forgiveness. A throttled tick can land just+ // before the lid closes, leaving the pass already spent, and terminating there would be the+ // 2026-09-28 incident again: a healthy loopback socket killed on waking.+ it('forgives a genuine long sleep even after a merely throttled tick', async () => {+ vi.useFakeTimers()+ const ws = socket()+ ws.silent = true+ let asleepMs = 0+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000 // throttled: spends the one pass+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0)+ asleepMs += 1_200_000 // a real sleep, unambiguously past GENUINE_SLEEP_MS+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0)+ expect(ws.pings).toBe(3) // re-probed on the wake, as before the bound existed+ // Still given up on afterwards, after a deadline of AWAKE time.+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0)+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(1)+ })++ // Guards the one ordering the bound depends on: `forgaveWithoutProof` is set BEFORE the re-probe,+ // so a peer that answers clears it and earns another pass. Set it after the ping and a healthy+ // socket dies on its second throttled tick — which no other test here would catch.+ it('forgives every sleep a peer answers', async () => {+ vi.useFakeTimers()+ const ws = socket()+ let asleepMs = 0+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })+ for (let i = 0; i < 5; i++) {+ asleepMs += 40_000+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ }+ expect(ws.terminated).toBe(0)+ })++ // A real sleep must hand back a WHOLE deadline of awake time, not a deadline conditional on the+ // next tick being punctual — and the tick right after a wake is the most throttled moment there is.+ it('restores the pass a genuine sleep was forgiven on', async () => {+ vi.useFakeTimers()+ const ws = socket()+ ws.silent = true+ let asleepMs = 0+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000 // throttled: spends the pass+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 1_200_000 // a real sleep: forgiven unconditionally, and the pass comes back with it+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000 // throttled again — on a restored pass this is forgiven, not fatal+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0)+ })++ // Pins GENUINE_SLEEP_MS itself: without this the constant could be set anywhere in (40s, 1200s]+ // — 50s, say, which forgives almost every throttled tick and is the original bug back — with every+ // other test still green.+ it('draws the genuine-sleep line at GENUINE_SLEEP_MS exactly', async () => {+ vi.useFakeTimers()+ for (const [slept, expected] of [[GENUINE_SLEEP_MS - 1, 1], [GENUINE_SLEEP_MS, 0]] as const) {+ const ws = socket()+ ws.silent = true+ let asleepMs = 0+ const watch = watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000 // spend the pass on a throttled tick+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += slept+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(expected)+ watch.stop()+ }+ })++ // The loopback peer's usual proof of life is a data frame, not a pong. markAlive is bound to all+ // three, but only the pong path was exercised with the pass spent.+ it('lets a data frame restore the pass, not only a pong', async () => {+ vi.useFakeTimers()+ const ws = socket()+ ws.silent = true+ let asleepMs = 0+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ asleepMs += 40_000+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS) // spends the pass+ ws.emit('message', Buffer.from('{}')) // the app is plainly there+ asleepMs += 40_000+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)+ expect(ws.terminated).toBe(0)+ })+})diff --git a/cli/src/lib/wsLiveness.ts b/cli/src/lib/wsLiveness.tsindex 49e795463..4b92e5776 100644--- a/cli/src/lib/wsLiveness.ts+++ b/cli/src/lib/wsLiveness.ts@@ -18,6 +18,12 @@* included, which had nothing wrong with it. A tick that arrives long after its period is the process* waking, not the peer failing: the clock restarts there, the peer is pinged at once, and it gets its* three pings after the wake before anyone gives up on it.+ *+ * That restart is forgiveness, and it needed a bound. A late tick is not necessarily a sleep —+ * `sleepAware.ts` says as much — and on a loaded Mac it usually is not. Forgiving every one of them+ * made the app's loopback deadline (40s) unenforceable, so a window that was already gone kept its+ * tile roster for a quarter of an hour; see GENUINE_SLEEP_MS and the wake branch for the measurement+ * and the rule.*/import type { WebSocket } from 'ws'@@ -25,6 +31,13 @@ import { sleptFor } from './sleepAware.js'export const WS_HEARTBEAT_MS = 20_000export const WS_IDLE_DEADLINE_MS = 60_000+/**+ * A detected "sleep" shorter than this is more likely a throttled timer than a sleeping machine, and+ * is forgiven only once (see the wake branch). Measured over one ~2-day harness.log: 968 detected+ * sleeps reported 24-60s against 80 real ones, every one of them over 600s. A macOS DarkWake (5-10s)+ * never reaches the branch at all, its gap staying under `SLEEP_GAP_FACTOR` periods.+ */+export const GENUINE_SLEEP_MS = 180_000/** The backend's `trackSocketLiveness` deadline: after this much silence from us it closes the link. */export const BACKEND_IDLE_DEADLINE_MS = 75_000@@ -42,8 +55,10 @@ export interface LivenessOptions {* from "the network did": both close as 1006, and only this one is said out loud first. */onIdle?: (idleMs: number) => void/** Announced when a tick finds the process has slept through [sleptMs]: the socket is re-probed on- * that tick, or — [hungUp], see [peerGivesUpAfterMs] — terminated at once. */- onWake?: (sleptMs: number, hungUp: boolean) => void+ * that tick, or — [givingUp] — terminated at once. Two things terminate on a wake: a sleep the far+ * end cannot have waited through ([peerGivesUpAfterMs]), and a second forgiven sleep with nothing+ * heard from the peer in between (see the wake branch). */+ onWake?: (sleptMs: number, givingUp: boolean) => void/** How long the FAR end puts up with our silence before it hangs up — the backend's own liveness* (75s). A sleep longer than that means the link is already gone at the other end, and waiting a* deadline to find out only delays the redial: such a wake terminates at once. Unset for a peer on@@ -60,7 +75,9 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):const now = opts.now ?? (() => performance.now())let lastAliveAt = now()let lastTickAt = lastAliveAt- const markAlive = (): void => { lastAliveAt = now() }+ /** A sleep was forgiven and the peer has not been heard from since. See the wake branch. */+ let forgaveWithoutProof = false+ const markAlive = (): void => { lastAliveAt = now(); forgaveWithoutProof = false }ws.on('pong', markAlive)ws.on('ping', markAlive) // the peer's own liveness ping — a proof that costs nothingws.on('message', markAlive) // data flowing is the strongest proof there is@@ -82,8 +99,32 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):// Back from sleep: nothing was said because nobody here was listening. Ask now, and give the// peer a whole deadline of awake time to answer.const hungUp = opts.peerGivesUpAfterMs !== undefined && slept >= opts.peerGivesUpAfterMs- opts.onWake?.(slept, hungUp)- if (hungUp) { giveUp(); return }+ // Forgiving EVERY detected sleep is what made the loopback socket's 40s deadline unenforceable:+ // on a throttled machine the detector fires constantly (see GENUINE_SLEEP_MS) and each firing+ // erased the silence, so terminations logged `no traffic for 912-995s`, 276 times in one log,+ // each closing every terminal stream on the connection. One pass is enough — a peer still there+ // answers the re-probe below, which clears the flag and earns it another. A sleep long enough to+ // be unambiguous is ALWAYS forgiven, so a real lid-close cannot be read as a throttled tick+ // (2026-09-28, the incident this whole file exists for).+ const forgive = !forgaveWithoutProof || slept >= GENUINE_SLEEP_MS+ opts.onWake?.(slept, hungUp || !forgive)+ if (hungUp || !forgive) { giveUp(); return }+ // Set BEFORE the ping so the spec's synchronous pong can clear it; on a real socket the pong+ // arrives a round trip later, through the poll phase, and clears it either way. An unambiguous+ // sleep hands the pass back with it, so the peer gets a whole deadline of AWAKE time however+ // late the following tick runs — the promise at the top of this file. Without that, a throttled+ // tick landing right after a wake (the most throttled moment there is) terminated a healthy+ // loopback socket with no grace at all, which is the 2026-09-28 class again.+ //+ // The alternative is `lastAliveAt += slept`: donate only the sleep and let the awake share of+ // every gap still count, which needs no flag and no constant. `terminalStreamManager.ts`+ // `expireLeases` does exactly that, and is right to — its timeout is six of its sweep periods,+ // so it keeps five periods of post-wake grace. Here `deadlineMs` is TWO heartbeats for the+ // loopback socket, so the same shape would leave a single ping after a wake instead of the three+ // this file promises. Hence the flag. What it costs: two genuine sleeps SHORTER than+ // GENUINE_SLEEP_MS, back to back, with the first one's pong never read, still terminate a+ // healthy socket — narrow, and the only hole left in the mechanism.+ forgaveWithoutProof = slept < GENUINE_SLEEP_MSlastAliveAt = attry { ws.ping() } catch { giveUp() }returndiff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.tsindex 43eb34926..b9ac0a051 100644--- a/cli/src/localWsServer.ts+++ b/cli/src/localWsServer.ts@@ -781,8 +781,10 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerOconst heartbeat = watchSocketLiveness(ws, {deadlineMs: LOCAL_IDLE_DEADLINE_MS,onIdle: (idleMs) => console.log(`[local-ws] ${connId} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),- // The app is on this computer and slept with us: a wake re-probes it, it never ends the socket.- onWake: (sleptMs) => console.log(`[local-ws] ${connId} woke after ${Math.round(sleptMs / 1000)}s asleep — re-probing`),+ // The app is on this computer and slept with us, so a wake re-probes it rather than ending the+ // socket — but only once per answer. Unbounded forgiveness is what let a window that was already+ // gone keep its tile roster for 912-995s against this 40s deadline (wsLiveness.ts GENUINE_SLEEP_MS).+ onWake: (sleptMs, givingUp) => console.log(`[local-ws] ${connId} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'the app has not answered since the last wake, terminating' : 're-probing'}`),})const cleanup = (): void => {diff --git a/cli/src/sharing/relay.spec.ts b/cli/src/sharing/relay.spec.tsindex 7bc0937b0..551fc8ecd 100644--- a/cli/src/sharing/relay.spec.ts+++ b/cli/src/sharing/relay.spec.ts@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'import type { AuthSessionManager } from '../lib/authSession.js'import { b64e, newIdentity } from '../lib/e2ee/core.js'import { ownerHandshake } from './crypto.js'-const state = vi.hoisted(() => ({ sockets: [] as any[], stop: vi.fn() }))+const state = vi.hoisted(() => ({ sockets: [] as any[], stop: vi.fn(), liveness: undefined as any }))vi.mock('ws', async () => {const { EventEmitter } = await import('node:events')class FakeWebSocket extends EventEmitter {@@ -14,7 +14,7 @@ vi.mock('ws', async () => {}return { WebSocket: FakeWebSocket }})-vi.mock('../lib/wsLiveness.js', () => ({ BACKEND_IDLE_DEADLINE_MS: 75_000, watchSocketLiveness: () => ({ stop: state.stop }) }))+vi.mock('../lib/wsLiveness.js', () => ({ BACKEND_IDLE_DEADLINE_MS: 75_000, watchSocketLiveness: (_ws: unknown, opts: unknown) => { state.liveness = opts; return { stop: state.stop } } }))import { HarnessShareRelay, SharingEndedError } from './relay.js'import { SHARE_REQUEST_TYPES, SHARE_RESULT_TYPES } from './protocol.js'describe('recipient relay', () => {@@ -141,4 +141,19 @@ describe('recipient relay', () => {const fourth = await begin(); const offline = expect(fourth.pending).rejects.toThrow('offline')fourth.ws.emit('close', 1006, Buffer.alloc(0)); await offline})++ it('says out loud when the liveness watcher is the one that gave up', async () => {+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})+ const { pending } = await begin()+ pending.catch(() => {}) // afterEach closes the relay, which rejects this handshake+ state.liveness.onIdle(80_000)+ expect(log).toHaveBeenCalledWith('[sharing] machine no traffic for 80s — terminating')+ // A wake can end the socket too, and never reaches onIdle (wsLiveness.ts, the wake branch).+ state.liveness.onWake(90_000, true)+ expect(log).toHaveBeenCalledWith('[sharing] machine woke after 90s asleep — giving up on the link')+ state.liveness.onWake(50_000, false)+ expect(log).toHaveBeenCalledWith('[sharing] machine woke after 50s asleep — re-probing')+ log.mockRestore()+ })+})diff --git a/cli/src/sharing/relay.ts b/cli/src/sharing/relay.tsindex 6507e95db..fe50f22e7 100644--- a/cli/src/sharing/relay.ts+++ b/cli/src/sharing/relay.ts@@ -49,7 +49,13 @@ export class HarnessShareRelay {if (!settled) reject(new Error(reason.toString() || 'The owner’s machine is offline.'))if (!detached) onClosed(code === 4403 ? 4403 : 1012, reason.toString() || 'Owner disconnected')})- ws.on('open', () => { heartbeat = watchSocketLiveness(ws, { peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS }) })+ // Without onIdle this closed the shared harness as a bare `1012 Owner disconnected` with+ // nothing in the log to say the watcher was the one that gave up.+ ws.on('open', () => { heartbeat = watchSocketLiveness(ws, {+ onIdle: (idleMs) => console.log(`[sharing] ${machineId.slice(0, 8)} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),+ onWake: (sleptMs, givingUp) => console.log(`[sharing] ${machineId.slice(0, 8)} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'giving up on the link' : 're-probing'}`),+ peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,+ }) })ws.on('message', raw => {try {const frame = JSON.parse(raw.toString()) as { type: string; payload: Record<string, unknown> }