Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix: local terminals survive a Mac sleep/wake and the daemon respawns promptly

0xken15eca99fca4f6f
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index 120f01f2b..ea0569503 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -20,7 +20,7 @@ import type { AutonomousDeviceService, AutonomousDeviceFrame } from './lib/auton
*/
import { WebSocket } from 'ws'
-import { watchSocketLiveness, type LivenessWatch } from './lib/wsLiveness.js'
+import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, type LivenessWatch } from './lib/wsLiveness.js'
import { stat, readFile, readdir } from 'fs/promises'
import { existsSync } from 'node:fs'
import { dirname, isAbsolute, join } from 'path'
@@ -1058,6 +1058,8 @@ export class BackendSocket {
this.heartbeat = watchSocketLiveness(ws, {
onIdle: (idleMs) => console.log(`[backend] no traffic for ${Math.round(idleMs / 1000)}s — terminating the link`),
+ onWake: (sleptMs, hungUp) => console.log(`[backend] woke after ${Math.round(sleptMs / 1000)}s asleep — ${hungUp ? 'the backend has hung up, redialing' : 're-probing the link'}`),
+ peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
})
// App-level ping refreshes the backend's presence key (TTL 30s). The window's presence rides
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index db296f435..5f49b6d0b 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -54,6 +54,7 @@ import { engineSessionTitle } from './lib/sessionTitle.js'
import { installAmpPlugin, installCodexHooks, installCommandCodeHooks, installCursorHooks, installDevinHooks, installGrokHooks, installAgyHooks, installCopilotHooks, installHermesHooks, installKiloPlugin, installOpencodePlugin, installPiExtension, installSessionHooks } from './lib/hooks.js'
import { PID_FILE, daemonPort, isAlive, isDaemonRunning, readPid } from './lib/daemonState.js'
import { clearSafeModeMarker, readSafeModeMarker, runBootHandoff, safeModeDisposition, safeModeStatusBody, writeSafeModeMarker } from './lib/daemonSafeMode.js'
+import { awakeTimeout } from './lib/sleepAware.js'
import {
BIND_WAIT_MS, connectFailure, defaultLaunchDeps, removePidFileIf, waitForBind, waitForReady,
} from './lib/daemonLaunch.js'
@@ -7964,9 +7965,11 @@ const enterSafeMode = (err: unknown): void => {
// Bounded on purpose. A cause that has since cleared — tmux not yet on PATH after a reboot, a lock
// file, a port held for a moment — would otherwise leave the machine wedged in a state nobody
// respawns over, because not-ready is exactly what stops the app trying again.
+ // Counted in AWAKE time: a plain timer spent a closed lid on this clock and exited the daemon on the
+ // first loop turn after the wake, taking every local terminal with it (see lib/sleepAware.ts).
if (env.ADAPTER_SAFE_MODE_MS > 0) {
- setTimeout(() => leave(`no fix arrived within ${Math.round(env.ADAPTER_SAFE_MODE_MS / 60_000)}m — letting a clean start try`, 1),
- env.ADAPTER_SAFE_MODE_MS).unref?.()
+ awakeTimeout(() => leave(`no fix arrived within ${Math.round(env.ADAPTER_SAFE_MODE_MS / 60_000)}m — letting a clean start try`, 1),
+ env.ADAPTER_SAFE_MODE_MS)
}
}
diff --git a/cli/src/lib/authSession.spec.ts b/cli/src/lib/authSession.spec.ts
index f2de7cbda..28db16fdc 100644
--- a/cli/src/lib/authSession.spec.ts
+++ b/cli/src/lib/authSession.spec.ts
@@ -1,5 +1,5 @@
import { afterAll, afterEach, describe, expect, it, vi } from 'vitest'
-import { mkdtemp, readFile, rm } from 'fs/promises'
+import { mkdtemp, readFile, rm, stat, writeFile } from 'fs/promises'
import { tmpdir } from 'os'
const authDir = await mkdtemp(`${tmpdir()}/harness-auth-session-`)
@@ -11,6 +11,7 @@ const {
AuthSessionManager,
clearAuthSession,
readAuthSession,
+ releaseHeldAuthLock,
writeAuthSession,
} = await import('./authSession.js')
@@ -51,6 +52,27 @@ describe('AuthSessionManager', () => {
await expect(readFile(AUTH_SESSION_FILE, 'utf8')).resolves.toContain('new-access')
})
+ // `process.exit` skips `finally`: a daemon that exited mid-refresh left the lock for the next
+ // `harness auth status` to wait out (30s), which is how long the desktop app waits for it.
+ it('drops the refresh lock when the process exits while holding it, and only then', async () => {
+ const lock = `${authDir}/session.lock`
+ writeAuthSession(baseSession())
+ let answer!: (response: Response) => void
+ vi.stubGlobal('fetch', vi.fn<typeof fetch>().mockReturnValue(new Promise<Response>((resolve) => { answer = resolve })))
+ const refreshing = new AuthSessionManager('https://api.example.test').accessToken()
+ await vi.waitFor(() => stat(lock))
+ releaseHeldAuthLock() // what the `exit` handler runs
+ await expect(stat(lock)).rejects.toThrow()
+ answer(new Response(JSON.stringify({ success: true, data: { token: 'new-access', expiresIn: 3600 } })))
+ await expect(refreshing).resolves.toBe('new-access')
+
+ // Another process's lock is not ours to remove.
+ await writeFile(lock, '')
+ releaseHeldAuthLock()
+ await expect(stat(lock)).resolves.toBeTruthy()
+ await rm(lock, { force: true })
+ })
+
it('uses the newer persisted access token rather than refreshing a stale 401 again', async () => {
writeAuthSession({ ...baseSession(), accessToken: 'already-refreshed', expiresAt: Date.now() + 3_600_000 })
const fetchMock = vi.fn<typeof fetch>()
diff --git a/cli/src/lib/authSession.ts b/cli/src/lib/authSession.ts
index 82d0756e1..0da8a91d2 100644
--- a/cli/src/lib/authSession.ts
+++ b/cli/src/lib/authSession.ts
@@ -74,6 +74,20 @@ export function clearAuthSession(): void {
try { rmSync(AUTH_SESSION_FILE, { force: true }) } catch { /* ignore */ }
}
+// Whether THIS process holds the refresh lock right now. `process.exit` skips the `finally` below, and
+// a daemon that exits mid-refresh (safe mode's own deadline, a revoke) used to leave the lock behind:
+// the next `harness auth status` — the one the desktop app runs before respawning that daemon — then
+// sat out the whole LOCK_STALE_MS before it could answer, and the app gave up on it at exactly 30s.
+let holdingLock = false
+let releaseOnExitArmed = false
+
+/** Drop the refresh lock if this process holds it. Run on `exit`; exported for the spec. */
+export function releaseHeldAuthLock(): void {
+ if (!holdingLock) return
+ holdingLock = false
+ try { rmSync(LOCK_FILE, { force: true }) } catch { /* ignore */ }
+}
+
async function withLock<T>(action: () => Promise<T>): Promise<T> {
ensureDir()
let deadline = Date.now() + LOCK_STALE_MS
@@ -102,9 +116,15 @@ async function withLock<T>(action: () => Promise<T>): Promise<T> {
await new Promise<void>((resolve) => setTimeout(resolve, 40))
continue
}
+ holdingLock = true
+ if (!releaseOnExitArmed) {
+ releaseOnExitArmed = true
+ process.once('exit', releaseHeldAuthLock)
+ }
try {
return await action()
} finally {
+ holdingLock = false
closeSync(fd)
try { rmSync(LOCK_FILE, { force: true }) } catch { /* ignore */ }
}
diff --git a/cli/src/lib/autonomous-device/resultJournal.ts b/cli/src/lib/autonomous-device/resultJournal.ts
index 80856ed25..5885dd2d9 100644
--- a/cli/src/lib/autonomous-device/resultJournal.ts
+++ b/cli/src/lib/autonomous-device/resultJournal.ts
@@ -1,5 +1,5 @@
import { closeSync, fsyncSync, openSync, renameSync, rmSync, writeFileSync } from 'node:fs'
-import { dirname } from 'node:path'
+import { basename, dirname, join } from 'node:path'
import { randomUUID } from 'node:crypto'
import { hardenPrivateStateFileIfPresent, readPrivateStateFile, secureStateDirectory } from '../secureState.js'
const MAX_BYTES = 32 * 1024 * 1024
@@ -13,6 +13,14 @@ export class DeviceResultJournal {
if (!hardenPrivateStateFileIfPresent(this.path, MAX_BYTES)) return undefined
return JSON.parse(readPrivateStateFile(this.path, MAX_BYTES))
}
+ /** Set this journal aside under [owner] (the machine it was written for) so a fresh one can start,
+ * replacing an earlier copy set aside for the same owner. Returns the file it was moved to. */
+ archive(owner: string): string {
+ const tag = owner.replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 64) || 'unknown'
+ const target = join(dirname(this.path), `${basename(this.path).replace(/\.json$/, '')}.${tag}.json`)
+ renameSync(this.path, target)
+ return target
+ }
save(value: unknown): void {
secureStateDirectory(dirname(this.path))
hardenPrivateStateFileIfPresent(this.path, MAX_BYTES)
diff --git a/cli/src/lib/autonomous-device/results.spec.ts b/cli/src/lib/autonomous-device/results.spec.ts
index 8167fdfd1..0e0d3de38 100644
--- a/cli/src/lib/autonomous-device/results.spec.ts
+++ b/cli/src/lib/autonomous-device/results.spec.ts
@@ -175,6 +175,33 @@ describe('result persistence, opt-in and authorization', () => {
expect(setup('claude', { resultJournal: journal }).service.receipt('owner', 'A')).toBeNull()
})
+ // The same computer signs in as another machine (another account, another backend): the old
+ // journal's dedupe cannot apply to the new id, and refusing to start kept the whole daemon in safe
+ // mode (2026-09-28). It is kept aside, and a journal that is really broken still refuses.
+ it('sets aside a journal written for another machine instead of refusing to start', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'device-results-')); dirs.push(dir)
+ const path = join(dir, 'device-results.json')
+ const journal = new DeviceResultJournal(path)
+ journal.save({ version: 1, machineId: 'other-machine', entries: [], results: [] })
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const f = setup('claude', { resultJournal: journal })
+ log.mockRestore()
+ expect(() => statSync(path)).toThrow()
+ expect(JSON.parse(readFileSync(join(dir, 'device-results.other-machine.json'), 'utf8'))).toMatchObject({ machineId: 'other-machine' })
+ await f.send('A', claude.inputs[0])
+ expect(JSON.parse(readFileSync(path, 'utf8'))).toMatchObject({ machineId: 'machine' })
+
+ // Keeping the copy is best-effort: a journal that cannot be moved still does not stop the start.
+ const stuck = { load: () => ({ version: 1, machineId: 'third-machine', entries: [], results: [] }), save: () => {},
+ archive: () => { throw new Error('EACCES') } }
+ const quiet = vi.spyOn(console, 'log').mockImplementation(() => {})
+ expect(() => setup('claude', { resultJournal: stuck })).not.toThrow()
+ quiet.mockRestore()
+
+ journal.save({ version: 2, machineId: 'machine', entries: [], results: [] })
+ expect(() => setup('claude', { resultJournal: journal })).toThrow('Invalid Device result journal')
+ })
+
it('does not dispatch if durable reservation fails', async () => {
const f = setup('claude', { resultJournal: { load: () => undefined, save: () => { throw new Error('disk full') } } })
const response = await f.service.request('owner', { type: 'turn.send', requestId: randomUUID(), machineId: 'machine', agentId: 'agent', idempotencyKey: 'A', text: 'A' })
diff --git a/cli/src/lib/autonomous-device/service.ts b/cli/src/lib/autonomous-device/service.ts
index 22fa43e51..00ff3fa3d 100644
--- a/cli/src/lib/autonomous-device/service.ts
+++ b/cli/src/lib/autonomous-device/service.ts
@@ -21,7 +21,7 @@ export interface AutonomousDeviceAgent { agentId: string; name: string; engine:
const AGENT_RECAP_MAX_CHARS = 200
export interface AutonomousDeviceDelivery { deliveryId: string; sessionId: string; state: ReceiptState; reason?: string }
export interface AutonomousDeviceServiceOptions {
- resultJournal?: Pick<DeviceResultJournal, 'load' | 'save'>
+ resultJournal?: Pick<DeviceResultJournal, 'load' | 'save'> & Partial<Pick<DeviceResultJournal, 'archive'>>
inputConsumed?: (agentId: string, text: string) => void
store?: AutonomousDeviceStore
machineId: string; serverInstanceId?: string; now?: () => number
@@ -312,7 +312,22 @@ export class AutonomousDeviceService {
this.now = options.now ?? Date.now
this.streams = new AgentStreams({ machineId: options.machineId, serverInstanceId: this.serverInstanceId,
send: (deviceId, frame) => this.options.emit?.(frame, deviceId) })
- const snapshot = options.resultJournal?.load()
+ let snapshot = options.resultJournal?.load()
+ // Written on this computer under ANOTHER machine id: a sign-in to another account, a switch of
+ // backend, a machine re-registered. Its dedupe cannot apply here — every request naming that
+ // machine is refused as MACHINE_MISMATCH before a reservation is looked up — so it is set aside,
+ // not trusted and not treated as corruption. Throwing here kept the whole daemon in safe mode
+ // after a legitimate identity change, with every terminal on this computer down with it.
+ if (object(snapshot) && typeof snapshot.machineId === 'string' && snapshot.machineId
+ && snapshot.machineId !== options.machineId) {
+ const owner = snapshot.machineId
+ // Keeping a copy is a courtesy to a switch back; failing to keep one must not bring back the very
+ // start-up failure this avoids. The fresh journal's first save replaces the file either way.
+ let kept: string | undefined
+ try { kept = options.resultJournal?.archive?.(owner) } catch { /* not kept */ }
+ console.log(`[device] result journal belongs to machine ${owner.slice(0, 8)} — ${kept ? `kept as ${kept}` : 'discarded'}, starting fresh`)
+ snapshot = undefined
+ }
if (snapshot !== undefined) {
if (!object(snapshot) || snapshot.version !== 1 || snapshot.machineId !== options.machineId
|| !Array.isArray(snapshot.entries) || !Array.isArray(snapshot.results)) throw new Error('Invalid Device result journal; refusing to lose delivery dedupe')
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index bcc1281b7..be4d13c8c 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -14,7 +14,7 @@
*/
import { randomUUID } from 'node:crypto'
import { WebSocket, type RawData } from 'ws'
-import { watchSocketLiveness, type LivenessWatch } from './wsLiveness.js'
+import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, type LivenessWatch } from './wsLiveness.js'
import type { Frame, LocalClientSink } from '../backendSocket.js'
import type { AuthSessionManager } from './authSession.js'
import { b64d, type Identity } from './e2ee/core.js'
@@ -655,6 +655,8 @@ export class RemoteRelayPool {
})
entry.heartbeat = watchSocketLiveness(ws, {
onIdle: (idleMs) => console.log(`[relay] ${machineId.slice(0, 8)} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),
+ // The relay socket ends at the backend, which hung up on it while we slept.
+ peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
// Piggybacked sweep for a migration that never completed (pane closed mid-flight, responder never
// answered, etc.) — no dedicated timer needed, this tick is frequent enough (20s) against the 30s TTL.
onTick: () => {
diff --git a/cli/src/lib/sleepAware.spec.ts b/cli/src/lib/sleepAware.spec.ts
new file mode 100644
index 000000000..2df4bc466
--- /dev/null
+++ b/cli/src/lib/sleepAware.spec.ts
@@ -0,0 +1,49 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { awakeTimeout, sleptFor } from './sleepAware.js'
+
+afterEach(() => { vi.useRealTimers() })
+
+describe('sleptFor', () => {
+ it('reads an ordinary or merely late tick as awake, and a far-late one as the sleep it was', () => {
+ expect(sleptFor(20_000, 20_000)).toBe(0)
+ expect(sleptFor(39_000, 20_000)).toBe(0)
+ expect(sleptFor(2_300_000, 20_000)).toBe(2_280_000)
+ })
+})
+
+describe('awakeTimeout', () => {
+ it('fires after the awake time, like a timer that nothing interrupted', async () => {
+ vi.useFakeTimers()
+ const fired = vi.fn()
+ awakeTimeout(fired, 60_000, { tickMs: 10_000 })
+ await vi.advanceTimersByTimeAsync(50_000)
+ expect(fired).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(fired).toHaveBeenCalledOnce()
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fired).toHaveBeenCalledOnce()
+ })
+
+ // The safe-mode clock: 15 minutes that a lid closed for forty used to spend in one go, exiting
+ // the daemon on the first loop turn after the wake (2026-09-28 18:46:51).
+ it('does not spend a sleep, however long, on its deadline', async () => {
+ vi.useFakeTimers()
+ let asleepMs = 0
+ const fired = vi.fn()
+ awakeTimeout(fired, 900_000, { tickMs: 10_000, now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(300_000)
+ asleepMs += 2_280_000
+ await vi.advanceTimersByTimeAsync(10_000)
+ expect(fired).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(590_000)
+ expect(fired).toHaveBeenCalledOnce()
+ })
+
+ it('can be called off', async () => {
+ vi.useFakeTimers()
+ const fired = vi.fn()
+ awakeTimeout(fired, 20_000, { tickMs: 10_000 }).cancel()
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fired).not.toHaveBeenCalled()
+ })
+})
diff --git a/cli/src/lib/sleepAware.ts b/cli/src/lib/sleepAware.ts
new file mode 100644
index 000000000..69ea3f899
--- /dev/null
+++ b/cli/src/lib/sleepAware.ts
@@ -0,0 +1,59 @@
+/**
+ * Telling a machine that SLEPT apart from a peer that went silent.
+ *
+ * Node's clocks do not stop while a Mac sleeps: `performance.now()`, `process.hrtime` and the event
+ * loop's own timer clock all run on the clock that keeps counting (measured on Node 22.23 and 23.11
+ * against CLOCK_UPTIME_RAW / CLOCK_MONOTONIC_RAW — they match the second). A lid closed for forty
+ * minutes therefore arrives, on the first loop turn after the wake, as forty minutes of "silence" on
+ * every deadline at once: every socket judged dead, every terminal lease expired, the safe-mode clock
+ * run out — measured 2026-09-28, `[local-ws] no traffic for 2287s — terminating` against a loopback
+ * peer that was perfectly healthy, and the daemon exiting 200ms later.
+ *
+ * Nothing here asks the OS about power state. A periodic tick already knows how long it expects to
+ * wait; a tick that arrives far later than that means the process was not running in between —
+ * asleep, or a loop blocked for that long, and neither is the peer's silence.
+ */
+
+/** A tick later than this many periods is read as the process having slept through it. */
+export const SLEEP_GAP_FACTOR = 2
+
+/**
+ * How much of [gapMs] — the time since the previous tick of a timer that fires every [periodMs] —
+ * the process spent asleep. Zero for an ordinary tick, however late a busy loop made it.
+ */
+export function sleptFor(gapMs: number, periodMs: number): number {
+ return gapMs > periodMs * SLEEP_GAP_FACTOR ? gapMs - periodMs : 0
+}
+
+export interface AwakeTimeout {
+ cancel(): void
+}
+
+export interface AwakeTimeoutOptions {
+ /** How often awake time is counted. Coarse on purpose: a long deadline only needs to be about right. */
+ tickMs?: number
+ now?: () => number
+}
+
+/**
+ * `setTimeout` that counts only the time the process was awake: [fn] runs once [ms] of it has passed.
+ * For the long deadlines a sleep must not spend — a plain timer set before the lid closed fires on
+ * the first loop turn after the wake. Unref'd, like the timers it replaces: it never holds the loop.
+ */
+export function awakeTimeout(fn: () => void, ms: number, opts: AwakeTimeoutOptions = {}): AwakeTimeout {
+ const tickMs = opts.tickMs ?? 10_000
+ const now = opts.now ?? (() => performance.now())
+ let awake = 0
+ let last = now()
+ const timer = setInterval(() => {
+ const at = now()
+ const gap = at - last
+ last = at
+ awake += gap - sleptFor(gap, tickMs)
+ if (awake < ms) return
+ clearInterval(timer)
+ fn()
+ }, tickMs)
+ timer.unref?.()
+ return { cancel: () => clearInterval(timer) }
+}
diff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.ts
index ccc287839..ce781e0db 100644
--- a/cli/src/lib/terminalStreamManager.spec.ts
+++ b/cli/src/lib/terminalStreamManager.spec.ts
@@ -619,6 +619,29 @@ describe('TerminalStreamManager', () => {
&& frame.payload.reason === 'heartbeat timeout')).toBe(true)
})
+ // Node's clock keeps running while a Mac sleeps, so a lid closed for forty minutes reached the next
+ // sweep as forty minutes without `terminal_alive` and closed every stream at the wake — measured
+ // 2026-09-28 as `[terminal-stream] closed` on every local terminal the moment the lid opened.
+ it('carries a lease over the time the computer slept, and still expires it on awake silence', async () => {
+ let asleepMs = 0
+ await manager.stop()
+ manager = newManager({ now: () => Date.now() + asleepMs })
+ await manager.handleFrame('web-1', 'terminal_open', {
+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,
+ })
+ await vi.advanceTimersByTimeAsync(20_000)
+ asleepMs += 2_280_000 // the lid closes; no timer runs until it opens again
+ await vi.advanceTimersByTimeAsync(5_000)
+ expect(stream.closed).toBe(false)
+ expect(sent.some((frame) => frame.type === 'terminal_closed')).toBe(false)
+
+ // Awake and still silent: the lease runs out on awake time, as before.
+ await vi.advanceTimersByTimeAsync(30_000)
+ expect(stream.closed).toBe(true)
+ expect(sent.some((frame) => frame.type === 'terminal_closed'
+ && frame.payload.reason === 'heartbeat timeout')).toBe(true)
+ })
+
// The incumbent's banner says WHO took over, so the close it gets carries what the winner
// declared on open — verbatim through a relay, since the daemon never learns a peer's name.
it('names the taker on the close when the winner introduced itself', async () => {
diff --git a/cli/src/lib/terminalStreamManager.ts b/cli/src/lib/terminalStreamManager.ts
index 3fbf2fa5c..3fa4ac2b7 100644
--- a/cli/src/lib/terminalStreamManager.ts
+++ b/cli/src/lib/terminalStreamManager.ts
@@ -4,6 +4,7 @@ import { deflateSync } from 'node:zlib'
import { ENGINES } from '../engines/types.js'
import type { RegisteredSession } from './registry.js'
import { writeImageToOsClipboard } from './osClipboard.js'
+import { sleptFor } from './sleepAware.js'
import { writePasteDropFile, writePasteImageFile } from './pasteDropFiles.js'
import type { TerminalBackendCoordinator } from './terminalBackendCoordinator.js'
import { terminalPlacementKey, terminalRouteKey } from './terminalRuntime.js'
@@ -77,6 +78,7 @@ export function clientDescriptorFrom(raw: unknown): TerminalClientDescriptor | u
const PROTOCOL_VERSION = 3
const HEARTBEAT_TIMEOUT_MS = 30_000
const SYNC_INTERVAL_MS = 5_000
+const EXPIRY_SWEEP_MS = 5_000
const OUTPUT_FLUSH_MS = 8
// The terminal the person is typing into on the loopback desktop gains nothing from the 8ms window:
// a frame across 127.0.0.1 costs tens of microseconds, and the app already folds every write between
@@ -213,10 +215,12 @@ export class TerminalStreamManager {
private readonly focusByConn = new Map<string, string | null>()
private readonly now: () => number
private readonly expiryTimer: ReturnType<typeof setInterval>
+ private lastSweepAt: number
constructor(private readonly deps: TerminalStreamManagerDeps) {
this.now = deps.now ?? (() => Date.now())
- this.expiryTimer = setInterval(() => this.expireLeases(), 5_000)
+ this.lastSweepAt = this.now()
+ this.expiryTimer = setInterval(() => this.expireLeases(), EXPIRY_SWEEP_MS)
this.expiryTimer.unref?.()
}
@@ -1200,6 +1204,15 @@ export class TerminalStreamManager {
private expireLeases(): void {
const now = this.now()
+ // A sweep that arrives long after its period is the computer waking up, and the time asleep is
+ // nobody's silence: the window slept too, and could not have sent `terminal_alive`. Every lease is
+ // carried over the gap, so the window's first heartbeat after the wake finds its stream still
+ // there. Closing them all here is what put every local terminal on "reconnecting" at each wake.
+ const slept = sleptFor(now - this.lastSweepAt, EXPIRY_SWEEP_MS)
+ this.lastSweepAt = now
+ if (slept > 0) {
+ for (const state of this.streams.values()) state.expiresAt += slept
+ }
for (const state of [...this.streams.values()]) {
if (!state.closing && state.expiresAt <= now) void this.closeStream(state, 'heartbeat timeout', true)
else if (!state.closing && now - state.lastSyncAt >= SYNC_INTERVAL_MS) this.sendSync(state, now)
diff --git a/cli/src/lib/wsLiveness.spec.ts b/cli/src/lib/wsLiveness.spec.ts
index 77bbe5290..a1e9c6e61 100644
--- a/cli/src/lib/wsLiveness.spec.ts
+++ b/cli/src/lib/wsLiveness.spec.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { EventEmitter } from 'node:events'
import type { WebSocket } from 'ws'
-import { watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'
+import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness, WS_HEARTBEAT_MS, WS_IDLE_DEADLINE_MS } from './wsLiveness.js'
class FakeSocket extends EventEmitter {
pings = 0
@@ -88,6 +88,58 @@ describe('watchSocketLiveness', () => {
expect(ws.pings).toBe(1)
})
+ // Node's clock runs on while a Mac sleeps: the first tick after a long sleep saw the whole sleep as
+ // silence and terminated every socket, the app's loopback one included (2026-09-28, `no traffic for
+ // 2287s`). Sleep is simulated by moving the clock without running any timer.
+ it('re-probes instead of terminating on the first tick after the computer slept', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ ws.silent = true
+ let asleepMs = 0
+ const woke: number[] = []
+ watchSocketLiveness(ws, { now: () => performance.now() + asleepMs, onWake: (ms) => woke.push(ms) })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 2_280_000
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ expect(ws.pings).toBe(2) // the wake tick asks at once
+ expect(woke).toEqual([2_280_000])
+ // A peer that stays silent is still given up on, after a deadline of AWAKE time.
+ await vi.advanceTimersByTimeAsync(WS_IDLE_DEADLINE_MS - WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(1)
+ })
+
+ it('keeps a live peer across a sleep without a single missed beat', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ let asleepMs = 0
+ watchSocketLiveness(ws, { now: () => performance.now() + asleepMs })
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 600_000
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS * 10)
+ expect(ws.terminated).toBe(0)
+ })
+
+ it('hangs up at once after a sleep the far end could not have waited through', async () => {
+ vi.useFakeTimers()
+ const ws = socket()
+ let asleepMs = 0
+ const verdicts: boolean[] = []
+ const opts = { now: () => performance.now() + asleepMs, peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS,
+ onWake: (_: number, hungUp: boolean) => { verdicts.push(hungUp) } }
+ watchSocketLiveness(ws, opts)
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ asleepMs += 30_000 // shorter than the backend waits: the link may well still be there
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(0)
+ asleepMs += BACKEND_IDLE_DEADLINE_MS
+ await vi.advanceTimersByTimeAsync(WS_HEARTBEAT_MS)
+ expect(ws.terminated).toBe(1)
+ expect(verdicts).toEqual([false, true])
+ })
+
it('terminates a socket it cannot even ping', async () => {
vi.useFakeTimers()
const ws = socket()
diff --git a/cli/src/lib/wsLiveness.ts b/cli/src/lib/wsLiveness.ts
index 09c796c65..49e795463 100644
--- a/cli/src/lib/wsLiveness.ts
+++ b/cli/src/lib/wsLiveness.ts
@@ -10,16 +10,23 @@
*
* Here ANY inbound frame proves the peer — data, its own ping, a pong — and the socket is only given
* up after `WS_IDLE_DEADLINE_MS` of nothing at all: three pings' worth, the same shape the backend's
- * `trackSocketLiveness` applies from its side (75s there). Silence is measured on the MONOTONIC clock
- * (`performance.now()` does not advance while the machine sleeps), so the deadline is sixty seconds of
- * silence while awake — a lid closed for an hour is not an hour of silence, and the link gets its
+ * `trackSocketLiveness` applies from its side (75s there).
+ *
+ * The deadline is sixty seconds of silence WHILE AWAKE. `performance.now()` does not stop while a Mac
+ * sleeps (see `sleepAware.ts`), so a lid closed for an hour used to read as an hour of silence and the
+ * first tick after the wake terminated every socket this daemon held — the loopback one to the app
+ * included, which had nothing wrong with it. A tick that arrives long after its period is the process
+ * waking, not the peer failing: the clock restarts there, the peer is pinged at once, and it gets its
* three pings after the wake before anyone gives up on it.
*/
import type { WebSocket } from 'ws'
+import { sleptFor } from './sleepAware.js'
export const WS_HEARTBEAT_MS = 20_000
export const WS_IDLE_DEADLINE_MS = 60_000
+/** The backend's `trackSocketLiveness` deadline: after this much silence from us it closes the link. */
+export const BACKEND_IDLE_DEADLINE_MS = 75_000
export interface LivenessWatch {
/** Stop pinging and judging. Idempotent; the socket itself is left alone. */
@@ -34,13 +41,26 @@ export interface LivenessOptions {
/** Announced right before a silent socket is terminated, so a later trace can tell "we gave up"
* from "the network did": both close as 1006, and only this one is said out loud first. */
onIdle?: (idleMs: number) => void
+ /** Announced when a tick finds the process has slept through [sleptMs]: the socket is re-probed on
+ * that tick, or — [hungUp], see [peerGivesUpAfterMs] — terminated at once. */
+ onWake?: (sleptMs: number, hungUp: boolean) => void
+ /** How long the FAR end puts up with our silence before it hangs up — the backend's own liveness
+ * (75s). A sleep longer than that means the link is already gone at the other end, and waiting a
+ * deadline to find out only delays the redial: such a wake terminates at once. Unset for a peer on
+ * this machine (the app's loopback socket), which slept when we did and is still there. */
+ peerGivesUpAfterMs?: number
+ /** The clock silence is measured on. Tests only. */
+ now?: () => number
}
/** Start watching an OPEN socket. Call once the handshake is done — the clock starts now. */
export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}): LivenessWatch {
const deadlineMs = opts.deadlineMs ?? WS_IDLE_DEADLINE_MS
- let lastAliveAt = performance.now()
- const markAlive = (): void => { lastAliveAt = performance.now() }
+ const heartbeatMs = opts.heartbeatMs ?? WS_HEARTBEAT_MS
+ const now = opts.now ?? (() => performance.now())
+ let lastAliveAt = now()
+ let lastTickAt = lastAliveAt
+ const markAlive = (): void => { lastAliveAt = now() }
ws.on('pong', markAlive)
ws.on('ping', markAlive) // the peer's own liveness ping — a proof that costs nothing
ws.on('message', markAlive) // data flowing is the strongest proof there is
@@ -55,7 +75,20 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):
// (it is silently dropped), so an unstopped watch would tick against a corpse forever.
ws.once('close', stop)
const timer = setInterval(() => {
- const idleMs = performance.now() - lastAliveAt
+ const at = now()
+ const slept = sleptFor(at - lastTickAt, heartbeatMs)
+ lastTickAt = at
+ if (slept > 0) {
+ // Back from sleep: nothing was said because nobody here was listening. Ask now, and give the
+ // peer a whole deadline of awake time to answer.
+ const hungUp = opts.peerGivesUpAfterMs !== undefined && slept >= opts.peerGivesUpAfterMs
+ opts.onWake?.(slept, hungUp)
+ if (hungUp) { giveUp(); return }
+ lastAliveAt = at
+ try { ws.ping() } catch { giveUp() }
+ return
+ }
+ const idleMs = at - lastAliveAt
if (idleMs >= deadlineMs) {
opts.onIdle?.(idleMs)
giveUp()
@@ -64,7 +97,7 @@ export function watchSocketLiveness(ws: WebSocket, opts: LivenessOptions = {}):
// A ping that cannot even be written means the socket is already gone under us.
try { ws.ping() } catch { giveUp(); return }
opts.onTick?.()
- }, opts.heartbeatMs ?? WS_HEARTBEAT_MS)
+ }, heartbeatMs)
timer.unref?.()
return { stop }
}
diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.ts
index a9183d629..815d6adcd 100644
--- a/cli/src/localWsServer.ts
+++ b/cli/src/localWsServer.ts
@@ -722,6 +722,8 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO
const heartbeat = watchSocketLiveness(ws, {
deadlineMs: LOCAL_IDLE_DEADLINE_MS,
onIdle: (idleMs) => console.log(`[local-ws] ${connId} no traffic for ${Math.round(idleMs / 1000)}s — terminating`),
+ // The app is on this computer and slept with us: a wake re-probes it, it never ends the socket.
+ onWake: (sleptMs) => console.log(`[local-ws] ${connId} woke after ${Math.round(sleptMs / 1000)}s asleep — re-probing`),
})
const cleanup = (): void => {
diff --git a/cli/src/sharing/relay.spec.ts b/cli/src/sharing/relay.spec.ts
index 057d2415b..64a2e3134 100644
--- a/cli/src/sharing/relay.spec.ts
+++ b/cli/src/sharing/relay.spec.ts
@@ -14,7 +14,7 @@ vi.mock('ws', async () => {
}
return { WebSocket: FakeWebSocket }
})
-vi.mock('../lib/wsLiveness.js', () => ({ watchSocketLiveness: () => ({ stop: state.stop }) }))
+vi.mock('../lib/wsLiveness.js', () => ({ BACKEND_IDLE_DEADLINE_MS: 75_000, watchSocketLiveness: () => ({ stop: state.stop }) }))
import { HarnessShareRelay, SharingEndedError } from './relay.js'
import { SHARE_REQUEST_TYPES, SHARE_RESULT_TYPES } from './protocol.js'
describe('recipient relay', () => {
diff --git a/cli/src/sharing/relay.ts b/cli/src/sharing/relay.ts
index d3ab1e4d3..668a21016 100644
--- a/cli/src/sharing/relay.ts
+++ b/cli/src/sharing/relay.ts
@@ -4,7 +4,7 @@ import type { AuthSessionManager } from '../lib/authSession.js'
import type { Frame, LocalClientSink } from '../backendSocket.js'
import type { RelaySession } from '../lib/remoteRelay.js'
import { recipientHandshake, type ObserverCipher } from './crypto.js'
-import { watchSocketLiveness } from '../lib/wsLiveness.js'
+import { BACKEND_IDLE_DEADLINE_MS, watchSocketLiveness } from '../lib/wsLiveness.js'
export interface SharedHarnessReference {
id: string; agentId: string; name: string; engine: string | null; ownerPublicKey: string; expiresAt: string
@@ -48,7 +48,7 @@ export class HarnessShareRelay {
if (!settled) reject(new Error(reason.toString() || 'The owner’s machine is offline.'))
if (!detached) onClosed(code === 4403 ? 4403 : 1012, reason.toString() || 'Owner disconnected')
})
- ws.on('open', () => { heartbeat = watchSocketLiveness(ws) })
+ ws.on('open', () => { heartbeat = watchSocketLiveness(ws, { peerGivesUpAfterMs: BACKEND_IDLE_DEADLINE_MS }) })
ws.on('message', raw => {
try {
const frame = JSON.parse(raw.toString()) as { type: string; payload: Record<string, unknown> }
diff --git a/desktop/lib/ws/local_cli_discovery.dart b/desktop/lib/ws/local_cli_discovery.dart
index 760911507..b2c514b4f 100644
--- a/desktop/lib/ws/local_cli_discovery.dart
+++ b/desktop/lib/ws/local_cli_discovery.dart
@@ -392,8 +392,26 @@ class LocalCliDiscovery {
// Asked here and not on every tick because it costs a `harness auth status` process, and
// the respawn point is already rate-limited by the backoff above — so this runs once per
// spawn attempt rather than once every [checkInterval].
- if (stillSignedIn != null && !await stillSignedIn()) {
- onSignedOut?.call();
+ //
+ // Asked BESIDE the spawn, never before it: the spawn goes ahead whatever the answer, so
+ // waiting for it only delays the one thing that brings the terminals back. It used to be
+ // awaited first, and a check that threw — `auth status` sitting out the refresh lock a
+ // dying daemon left behind, 30s, the app's own timeout for it — skipped the spawn and left
+ // the terminals dark until something else happened to start the daemon (measured
+ // 2026-09-28 18:47: a minute and eight seconds). A slow answer still arrives and is still
+ // told; a failed one is not a sign-out.
+ final signedInCheck = stillSignedIn?.call();
+ if (signedInCheck != null) {
+ unawaited(
+ signedInCheck.then(
+ (signedIn) {
+ if (!signedIn) onSignedOut?.call();
+ },
+ onError: (Object error) => debugPrint(
+ 'LocalCliDiscovery.startSupervising: auth check failed: $error',
+ ),
+ ),
+ );
}
try {
await _spawnCommand();
diff --git a/desktop/test/local_cli_discovery_test.dart b/desktop/test/local_cli_discovery_test.dart
index 577397703..df4c7becf 100644
--- a/desktop/test/local_cli_discovery_test.dart
+++ b/desktop/test/local_cli_discovery_test.dart
@@ -861,4 +861,102 @@ void main() {
expect(timer.isActive, isTrue);
},
);
+
+ // `harness auth status` sat out a refresh lock a dying daemon had left (30s) and threw on the app's
+ // own timeout; the throw skipped the spawn and the local terminals stayed dark (2026-09-28 18:47).
+ for (final (name, check) in <(String, Future<bool> Function())>[
+ (
+ 'fails',
+ () async => throw ProcessException(
+ 'harness',
+ const [],
+ 'did not finish within 30s',
+ ),
+ ),
+ ('never answers', () => Completer<bool>().future),
+ ]) {
+ test('startSupervising still respawns when the auth check $name', () async {
+ const computerId = '0123456789abcdef0123456789abcdef';
+ final identityFile = File('${scratch.path}/computer-id')
+ ..writeAsStringSync(computerId);
+ final probe = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
+ final closedPort = probe.port;
+ await probe.close(force: true);
+
+ var spawnCount = 0;
+ var signedOutCalls = 0;
+ final discovery = LocalCliDiscovery(
+ config: AppConfig(
+ apiBaseUrl: 'https://harness-api.autonomous.ai',
+ localCliBaseUrl: 'http://127.0.0.1:$closedPort',
+ ),
+ identity: LocalMachineIdentity(computerIdFile: identityFile),
+ spawnCommand: () async {
+ spawnCount++;
+ },
+ );
+
+ final timer = discovery.startSupervising(
+ checkInterval: const Duration(milliseconds: 20),
+ graceStep: const Duration(milliseconds: 10),
+ graceWindow: const Duration(milliseconds: 50),
+ initialBackoff: const Duration(milliseconds: 20),
+ maxBackoff: const Duration(milliseconds: 20),
+ stillSignedIn: check,
+ onSignedOut: () => signedOutCalls++,
+ );
+ addTearDown(timer.cancel);
+
+ await Future.delayed(const Duration(milliseconds: 300));
+
+ expect(spawnCount, greaterThan(0));
+ expect(
+ signedOutCalls,
+ 0,
+ reason: 'an auth check with no answer is not a sign-out',
+ );
+ });
+ }
+
+ test('startSupervising spawns without waiting for a slow auth check, and still reports its sign-out', () async {
+ const computerId = '0123456789abcdef0123456789abcdef';
+ final identityFile = File('${scratch.path}/computer-id')
+ ..writeAsStringSync(computerId);
+ final probe = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
+ final closedPort = probe.port;
+ await probe.close(force: true);
+
+ var spawnCount = 0;
+ var signedOutCalls = 0;
+ final answer = Completer<bool>();
+ final discovery = LocalCliDiscovery(
+ config: AppConfig(
+ apiBaseUrl: 'https://harness-api.autonomous.ai',
+ localCliBaseUrl: 'http://127.0.0.1:$closedPort',
+ ),
+ identity: LocalMachineIdentity(computerIdFile: identityFile),
+ spawnCommand: () async {
+ spawnCount++;
+ },
+ );
+
+ final timer = discovery.startSupervising(
+ checkInterval: const Duration(milliseconds: 20),
+ graceStep: const Duration(milliseconds: 10),
+ graceWindow: const Duration(milliseconds: 50),
+ initialBackoff: const Duration(seconds: 10),
+ maxBackoff: const Duration(seconds: 10),
+ stillSignedIn: () => answer.future,
+ onSignedOut: () => signedOutCalls++,
+ );
+ addTearDown(timer.cancel);
+
+ await Future.delayed(const Duration(milliseconds: 150));
+ expect(spawnCount, 1, reason: 'the spawn does not wait on the auth check');
+ expect(signedOutCalls, 0);
+
+ answer.complete(false);
+ await Future.delayed(Duration.zero);
+ expect(signedOutCalls, 1, reason: 'a late answer is still told');
+ });
}