Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(cli): bound the liveness watcher's sleep forgiveness (#1029)

0xken15aed2ca989e51
diff --git a/cli/src/gateway/upstream.ts b/cli/src/gateway/upstream.ts
index 78c4e52dc..50f5d1348 100644
--- a/cli/src/gateway/upstream.ts
+++ b/cli/src/gateway/upstream.ts
@@ -172,7 +172,11 @@ export class UpstreamLink {
this.heartbeat = watchSocketLiveness(ws, {
onIdle: (idleMs) => console.log(`[backend] no traffic for ${Math.round(idleMs / 1000)}s — terminating the link`),
- onWake: (sleptMs, hungUp) => console.log(`[backend] woke after ${Math.round(sleptMs / 1000)}s asleep — ${hungUp ? 'the backend has hung up, redialing' : 're-probing the link'}`),
+ // Two different rules end the socket on a wake, and the trace has to tell them apart: a sleep
+ // the backend cannot have waited through (>= BACKEND_IDLE_DEADLINE_MS), or a second forgiven
+ // wake with nothing heard back. GENUINE_SLEEP_MS (180s) is above this deadline, so the length
+ // alone decides which fired.
+ onWake: (sleptMs, givingUp) => console.log(`[backend] woke after ${Math.round(sleptMs / 1000)}s asleep — ${!givingUp ? 're-probing the link' : sleptMs >= BACKEND_IDLE_DEADLINE_MS ? 'the backend has hung up, redialing' : 'no answer since the last wake, redialing'}`),
peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 7cb983b38..4aa9a291b 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -726,6 +726,8 @@ export class RemoteRelayPool {
})
entry.heartbeat = watchSocketLiveness(ws, {
onIdle: (idleMs) => console.log(`[relay] ${machineId.slice(0, 8)} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),
+ // A wake can end the socket too, and never reaches onIdle (wsLiveness.ts, the wake branch).
+ onWake: (sleptMs, givingUp) => console.log(`[relay] ${machineId.slice(0, 8)} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'giving up on the link' : 're-probing'}`),
// The relay socket ends at the backend, which hung up on it while we slept.
peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
// Piggybacked sweep for a migration that never completed (pane closed mid-flight, responder never
diff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.ts
index 4ccd20eca..85e1b7588 100644
--- a/cli/src/lib/terminalStreamManager.spec.ts
+++ b/cli/src/lib/terminalStreamManager.spec.ts
@@ -716,6 +716,28 @@ describe('TerminalStreamManager', () => {
&& frame.payload.reason === 'heartbeat timeout')).toBe(true)
})
+ // A sweep read as a sleep carries each lease by `slept`, which `sleptFor` defines as the gap MINUS
+ // one period — so every sweep still charges one EXPIRY_SWEEP_MS of silence however late it ran, and
+ // a throttled machine cannot hold a dead client's lease open. Guards against "carry the whole gap",
+ // which would never expire.
+ it('still expires a lease when every sweep is late enough to read as a sleep', async () => {
+ let asleepMs = 0
+ await manager.stop()
+ manager = newManager({ now: () => Date.now() + asleepMs })
+ await manager.handleFrame('web-1', 'terminal_open', {
+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,
+ })
+ // Twelve sweeps, each landing 15s late (gap 20s > 2 x 5s, so every one is read as a sleep) and no
+ // `terminal_alive` in any of them. Six periods of charged silence is the 30s timeout.
+ for (let i = 0; i < 12; i++) {
+ asleepMs += 15_000
+ await vi.advanceTimersByTimeAsync(5_000)
+ }
+ expect(stream.closed).toBe(true)
+ expect(sent.some((frame) => frame.type === 'terminal_closed'
+ && frame.payload.reason === 'heartbeat timeout')).toBe(true)
+ })
+
// The incumbent's banner says WHO took over, so the close it gets carries what the winner
// declared on open — verbatim through a relay, since the daemon never learns a peer's name.
it('names the taker on the close when the winner introduced itself', async () => {
diff --git a/cli/src/lib/wsLiveness.spec.ts b/cli/src/lib/wsLiveness.spec.ts
index a1e9c6e61..7d6dc9621 100644
--- a/cli/src/lib/wsLiveness.spec.ts
+++ b/cli/src/lib/wsLiveness.spec.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { EventEmitter } from 'node:events'
import type { WebSocket } from 'ws'
-import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'
+import { BACKEND_IDLE_DEADLINE_MS, GENUINE_SLEEP_MS, watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'
class FakeSocket extends EventEmitter {
pings = 0
@@ -148,4 +148,126 @@ describe('watchSocketLiveness', () => {
await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
expect(ws.terminated).toBe(1)
})
+
+ // The 40s loopback deadline (localWsServer.ts) was unenforceable on a loaded machine: macOS
+ // coalesces the 20s heartbeat so ticks land 44-80s late, every one read as a sleep, every one
+ // forgiving the silence. Measured over one ~2-day harness.log: 968 such wakes reporting 24-60s
+ // against 80 real sleeps (all >600s), and terminations logging `no traffic for 912-995s` — 23x
+ // past the deadline, 276 times. One pass per proof of life is the bound.
+ it('stops forgiving sleeps a peer never answers', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ const woke: Array<[number, boolean]> = []
+ watchSocketLiveness(ws, {
+ deadlineMs: 40_000, // the loopback shape
+ now: () => performance.now() + asleepMs,
+ onWake: (ms, givingUp) => woke.push([ms, givingUp]),
+ })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000 // a throttled tick, not a sleeping machine
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0) // the one free pass
+ expect(woke).toEqual([[40_000, false]])
+ asleepMs += 40_000 // and another, with nothing heard from the peer in between
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(1)
+ expect(woke).toEqual([[40_000, false], [40_000, true]])
+ })
+
+ // The bound above must never cost a REAL sleep its forgiveness. A throttled tick can land just
+ // before the lid closes, leaving the pass already spent, and terminating there would be the
+ // 2026-09-28 incident again: a healthy loopback socket killed on waking.
+ it('forgives a genuine long sleep even after a merely throttled tick', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000 // throttled: spends the one pass
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ asleepMs += 1_200_000 // a real sleep, unambiguously past GENUINE_SLEEP_MS
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ expect(ws.pings).toBe(3) // re-probed on the wake, as before the bound existed
+ // Still given up on afterwards, after a deadline of AWAKE time.
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(1)
+ })
+
+ // Guards the one ordering the bound depends on: `forgaveWithoutProof` is set BEFORE the re-probe,
+ // so a peer that answers clears it and earns another pass. Set it after the ping and a healthy
+ // socket dies on its second throttled tick — which no other test here would catch.
+ it('forgives every sleep a peer answers', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ let asleepMs = 0
+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })
+ for (let i = 0; i < 5; i++) {
+ asleepMs += 40_000
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ }
+ expect(ws.terminated).toBe(0)
+ })
+
+ // A real sleep must hand back a WHOLE deadline of awake time, not a deadline conditional on the
+ // next tick being punctual — and the tick right after a wake is the most throttled moment there is.
+ it('restores the pass a genuine sleep was forgiven on', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000 // throttled: spends the pass
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 1_200_000 // a real sleep: forgiven unconditionally, and the pass comes back with it
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000 // throttled again — on a restored pass this is forgiven, not fatal
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ })
+
+ // Pins GENUINE_SLEEP_MS itself: without this the constant could be set anywhere in (40s, 1200s]
+ // — 50s, say, which forgives almost every throttled tick and is the original bug back — with every
+ // other test still green.
+ it('draws the genuine-sleep line at GENUINE_SLEEP_MS exactly', async () => {
+ vi.useFakeTimers()
+ for (const [slept, expected] of [[GENUINE_SLEEP_MS - 1, 1], [GENUINE_SLEEP_MS, 0]] as const) {
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ const watch = watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000 // spend the pass on a throttled tick
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += slept
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(expected)
+ watch.stop()
+ }
+ })
+
+ // The loopback peer's usual proof of life is a data frame, not a pong. markAlive is bound to all
+ // three, but only the pong path was exercised with the pass spent.
+ it('lets a data frame restore the pass, not only a pong', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ watchSocketLiveness(ws, { deadlineMs: 40_000, now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 40_000
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS) // spends the pass
+ ws.emit('message', Buffer.from('{}')) // the app is plainly there
+ asleepMs += 40_000
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ })
+
})
diff --git a/cli/src/lib/wsLiveness.ts b/cli/src/lib/wsLiveness.ts
index 49e795463..4b92e5776 100644
--- a/cli/src/lib/wsLiveness.ts
+++ b/cli/src/lib/wsLiveness.ts
@@ -18,6 +18,12 @@
* included, which had nothing wrong with it. A tick that arrives long after its period is the process
* waking, not the peer failing: the clock restarts there, the peer is pinged at once, and it gets its
* three pings after the wake before anyone gives up on it.
+ *
+ * That restart is forgiveness, and it needed a bound. A late tick is not necessarily a sleep —
+ * `sleepAware.ts` says as much — and on a loaded Mac it usually is not. Forgiving every one of them
+ * made the app's loopback deadline (40s) unenforceable, so a window that was already gone kept its
+ * tile roster for a quarter of an hour; see GENUINE_SLEEP_MS and the wake branch for the measurement
+ * and the rule.
*/
import type { WebSocket } from 'ws'
@@ -25,6 +31,13 @@ import { sleptFor } from './sleepAware.js'
export const WS_HEARTBEAT_MS = 20_000
export const WS_IDLE_DEADLINE_MS = 60_000
+/**
+ * A detected "sleep" shorter than this is more likely a throttled timer than a sleeping machine, and
+ * is forgiven only once (see the wake branch). Measured over one ~2-day harness.log: 968 detected
+ * sleeps reported 24-60s against 80 real ones, every one of them over 600s. A macOS DarkWake (5-10s)
+ * never reaches the branch at all, its gap staying under `SLEEP_GAP_FACTOR` periods.
+ */
+export const GENUINE_SLEEP_MS = 180_000
/** The backend's `trackSocketLiveness` deadline: after this much silence from us it closes the link. */
export const BACKEND_IDLE_DEADLINE_MS = 75_000
@@ -42,8 +55,10 @@ export interface LivenessOptions {
* from "the network did": both close as 1006, and only this one is said out loud first. */
onIdle?: (idleMs: number) => void
/** Announced when a tick finds the process has slept through [sleptMs]: the socket is re-probed on
- * that tick, or — [hungUp], see [peerGivesUpAfterMs] — terminated at once. */
- onWake?: (sleptMs: number, hungUp: boolean) => void
+ * that tick, or — [givingUp] — terminated at once. Two things terminate on a wake: a sleep the far
+ * end cannot have waited through ([peerGivesUpAfterMs]), and a second forgiven sleep with nothing
+ * heard from the peer in between (see the wake branch). */
+ onWake?: (sleptMs: number, givingUp: boolean) => void
/** How long the FAR end puts up with our silence before it hangs up — the backend's own liveness
* (75s). A sleep longer than that means the link is already gone at the other end, and waiting a
* deadline to find out only delays the redial: such a wake terminates at once. Unset for a peer on
@@ -60,7 +75,9 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):
const now = opts.now ?? (() => performance.now())
let lastAliveAt = now()
let lastTickAt = lastAliveAt
- const markAlive = (): void => { lastAliveAt = now() }
+ /** A sleep was forgiven and the peer has not been heard from since. See the wake branch. */
+ let forgaveWithoutProof = false
+ const markAlive = (): void => { lastAliveAt = now(); forgaveWithoutProof = false }
ws.on('pong', markAlive)
ws.on('ping', markAlive) // the peer's own liveness ping — a proof that costs nothing
ws.on('message', markAlive) // data flowing is the strongest proof there is
@@ -82,8 +99,32 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):
// Back from sleep: nothing was said because nobody here was listening. Ask now, and give the
// peer a whole deadline of awake time to answer.
const hungUp = opts.peerGivesUpAfterMs !== undefined && slept >= opts.peerGivesUpAfterMs
- opts.onWake?.(slept, hungUp)
- if (hungUp) { giveUp(); return }
+ // Forgiving EVERY detected sleep is what made the loopback socket's 40s deadline unenforceable:
+ // on a throttled machine the detector fires constantly (see GENUINE_SLEEP_MS) and each firing
+ // erased the silence, so terminations logged `no traffic for 912-995s`, 276 times in one log,
+ // each closing every terminal stream on the connection. One pass is enough — a peer still there
+ // answers the re-probe below, which clears the flag and earns it another. A sleep long enough to
+ // be unambiguous is ALWAYS forgiven, so a real lid-close cannot be read as a throttled tick
+ // (2026-09-28, the incident this whole file exists for).
+ const forgive = !forgaveWithoutProof || slept >= GENUINE_SLEEP_MS
+ opts.onWake?.(slept, hungUp || !forgive)
+ if (hungUp || !forgive) { giveUp(); return }
+ // Set BEFORE the ping so the spec's synchronous pong can clear it; on a real socket the pong
+ // arrives a round trip later, through the poll phase, and clears it either way. An unambiguous
+ // sleep hands the pass back with it, so the peer gets a whole deadline of AWAKE time however
+ // late the following tick runs — the promise at the top of this file. Without that, a throttled
+ // tick landing right after a wake (the most throttled moment there is) terminated a healthy
+ // loopback socket with no grace at all, which is the 2026-09-28 class again.
+ //
+ // The alternative is `lastAliveAt += slept`: donate only the sleep and let the awake share of
+ // every gap still count, which needs no flag and no constant. `terminalStreamManager.ts`
+ // `expireLeases` does exactly that, and is right to — its timeout is six of its sweep periods,
+ // so it keeps five periods of post-wake grace. Here `deadlineMs` is TWO heartbeats for the
+ // loopback socket, so the same shape would leave a single ping after a wake instead of the three
+ // this file promises. Hence the flag. What it costs: two genuine sleeps SHORTER than
+ // GENUINE_SLEEP_MS, back to back, with the first one's pong never read, still terminate a
+ // healthy socket — narrow, and the only hole left in the mechanism.
+ forgaveWithoutProof = slept < GENUINE_SLEEP_MS
lastAliveAt = at
try { ws.ping() } catch { giveUp() }
return
diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.ts
index 43eb34926..b9ac0a051 100644
--- a/cli/src/localWsServer.ts
+++ b/cli/src/localWsServer.ts
@@ -781,8 +781,10 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO
const heartbeat = watchSocketLiveness(ws, {
deadlineMs: LOCAL_IDLE_DEADLINE_MS,
onIdle: (idleMs) => console.log(`[local-ws] ${connId} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),
- // The app is on this computer and slept with us: a wake re-probes it, it never ends the socket.
- onWake: (sleptMs) => console.log(`[local-ws] ${connId} woke after ${Math.round(sleptMs / 1000)}s asleep — re-probing`),
+ // The app is on this computer and slept with us, so a wake re-probes it rather than ending the
+ // socket — but only once per answer. Unbounded forgiveness is what let a window that was already
+ // gone keep its tile roster for 912-995s against this 40s deadline (wsLiveness.ts GENUINE_SLEEP_MS).
+ onWake: (sleptMs, givingUp) => console.log(`[local-ws] ${connId} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'the app has not answered since the last wake, terminating' : 're-probing'}`),
})
const cleanup = (): void => {
diff --git a/cli/src/sharing/relay.spec.ts b/cli/src/sharing/relay.spec.ts
index 7bc0937b0..551fc8ecd 100644
--- a/cli/src/sharing/relay.spec.ts
+++ b/cli/src/sharing/relay.spec.ts
@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { AuthSessionManager } from '../lib/authSession.js'
import { b64e, newIdentity } from '../lib/e2ee/core.js'
import { ownerHandshake } from './crypto.js'
-const state = vi.hoisted(() => ({ sockets: [] as any[], stop: vi.fn() }))
+const state = vi.hoisted(() => ({ sockets: [] as any[], stop: vi.fn(), liveness: undefined as any }))
vi.mock('ws', async () => {
const { EventEmitter } = await import('node:events')
class FakeWebSocket extends EventEmitter {
@@ -14,7 +14,7 @@ vi.mock('ws', async () => {
}
return { WebSocket: FakeWebSocket }
})
-vi.mock('../lib/wsLiveness.js', () => ({ BACKEND_IDLE_DEADLINE_MS: 75_000, watchSocketLiveness: () => ({ stop: state.stop }) }))
+vi.mock('../lib/wsLiveness.js', () => ({ BACKEND_IDLE_DEADLINE_MS: 75_000, watchSocketLiveness: (_ws: unknown, opts: unknown) => { state.liveness = opts; return { stop: state.stop } } }))
import { HarnessShareRelay, SharingEndedError } from './relay.js'
import { SHARE_REQUEST_TYPES, SHARE_RESULT_TYPES } from './protocol.js'
describe('recipient relay', () => {
@@ -141,4 +141,19 @@ describe('recipient relay', () => {
const fourth = await begin(); const offline = expect(fourth.pending).rejects.toThrow('offline')
fourth.ws.emit('close', 1006, Buffer.alloc(0)); await offline
})
+
+ it('says out loud when the liveness watcher is the one that gave up', async () => {
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const { pending } = await begin()
+ pending.catch(() => {}) // afterEach closes the relay, which rejects this handshake
+ state.liveness.onIdle(80_000)
+ expect(log).toHaveBeenCalledWith('[sharing] machine no traffic for 80s — terminating')
+ // A wake can end the socket too, and never reaches onIdle (wsLiveness.ts, the wake branch).
+ state.liveness.onWake(90_000, true)
+ expect(log).toHaveBeenCalledWith('[sharing] machine woke after 90s asleep — giving up on the link')
+ state.liveness.onWake(50_000, false)
+ expect(log).toHaveBeenCalledWith('[sharing] machine woke after 50s asleep — re-probing')
+ log.mockRestore()
+ })
+
})
diff --git a/cli/src/sharing/relay.ts b/cli/src/sharing/relay.ts
index 6507e95db..fe50f22e7 100644
--- a/cli/src/sharing/relay.ts
+++ b/cli/src/sharing/relay.ts
@@ -49,7 +49,13 @@ export class HarnessShareRelay {
if (!settled) reject(new Error(reason.toString() || 'The owner’s machine is offline.'))
if (!detached) onClosed(code === 4403 ? 4403 : 1012, reason.toString() || 'Owner disconnected')
})
- ws.on('open', () => { heartbeat = watchSocketLiveness(ws, { peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS }) })
+ // Without onIdle this closed the shared harness as a bare `1012 Owner disconnected` with
+ // nothing in the log to say the watcher was the one that gave up.
+ ws.on('open', () => { heartbeat = watchSocketLiveness(ws, {
+ onIdle: (idleMs) => console.log(`[sharing] ${machineId.slice(0, 8)} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),
+ onWake: (sleptMs, givingUp) => console.log(`[sharing] ${machineId.slice(0, 8)} woke after ${Math.round(sleptMs / 1000)}s asleep — ${givingUp ? 'giving up on the link' : 're-probing'}`),
+ peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
+ }) })
ws.on('message', raw => {
try {
const frame = JSON.parse(raw.toString()) as { type: string; payload: Record<string, unknown> }