Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: implement websocket heartbeat and relay pool invalidation to handle stale machine-node sessions

0xken15340cd9cb5330
diff --git a/cli/src/lib/e2ee/relayLink.spec.ts b/cli/src/lib/e2ee/relayLink.spec.ts
index db4ad9c11..cf3a6070b 100644
--- a/cli/src/lib/e2ee/relayLink.spec.ts
+++ b/cli/src/lib/e2ee/relayLink.spec.ts
@@ -267,4 +267,89 @@ describe('RemoteRelayPool drops a peer the responder no longer trusts', () => {
await new Promise<void>((resolve) => wss.close(() => resolve()))
}
})
+
+ it('invalidate() drops a pooled entry so the next acquire() dials fresh, without firing its old onClosed', async () => {
+ let selects = 0
+ const wss = new WebSocketServer({ port: 0 })
+ const manager = new E2eeManagerCtor({
+ machineId: MACHINE_ID,
+ sendTo: (_connId, frame) => {
+ // Unlike the other fixtures in this file, this test leaves a terminated (invalidated) client
+ // behind mid-test — only broadcast to sockets still actually open.
+ for (const client of wss.clients) {
+ if (client.readyState === client.OPEN) client.send(JSON.stringify(frame))
+ }
+ },
+ isConnected: () => true,
+ })
+ wss.on('connection', (ws) => {
+ let selected = false
+ ws.on('message', (raw) => {
+ let frame: Frame
+ try { frame = JSON.parse(raw.toString()) as Frame } catch { return }
+ if (!selected) {
+ if (frame.type === 'machine_select') {
+ selected = true
+ selects++
+ ws.send(JSON.stringify({ type: 'connected', payload: { machineId: MACHINE_ID } }))
+ }
+ return
+ }
+ const type = frame.type as string
+ if (type.startsWith('e2e_')) manager.handleFrame(`fake-conn-${selects}`, frame)
+ })
+ })
+
+ try {
+ const port = (wss.address() as AddressInfo).port
+ const clientIdentity = C.newIdentity()
+ const token = manager.createSetupToken(MACHINE_ID).token
+ const claim = await claimSetupToken({
+ token,
+ targetMachineId: MACHINE_ID,
+ selfIdentity: clientIdentity,
+ accessToken: 'unused-in-this-fake',
+ backendWsBase: `ws://127.0.0.1:${port}`,
+ autonomousEnv: 'prod',
+ timeoutMs: 2_000,
+ })
+ expect(claim.ok).toBe(true)
+ if (!claim.ok) return
+
+ const peers = new MachinePeerStore()
+ peers.pin(MACHINE_ID, C.b64e(claim.peerPub), 'harness link')
+ const pool = new RemoteRelayPool(fakeAuth, `ws://127.0.0.1:${port}`, clientIdentity, peers)
+
+ const staleSink = { sendFrame: () => true, sendBinary: () => true }
+ let staleOnClosedFired = false
+ await pool.acquire(
+ MACHINE_ID,
+ 'prod',
+ { type: 'machine_select', payload: { machineId: MACHINE_ID } },
+ staleSink,
+ () => { staleOnClosedFired = true },
+ )
+ expect(selects).toBe(2) // 1 for claimSetupToken's own connection, 1 for the acquire() dial
+
+ pool.invalidate(MACHINE_ID)
+ // onClosed is a stand-in for the LOCAL app socket's own close() — invalidate() must never call
+ // it, since a real caller invalidates right before reusing that same local connection for a
+ // fresh acquire(); firing it here would self-destruct the very connection driving the retry.
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ expect(staleOnClosedFired).toBe(false)
+
+ const freshSink = { sendFrame: () => true, sendBinary: () => true }
+ await pool.acquire(
+ MACHINE_ID,
+ 'prod',
+ { type: 'machine_select', payload: { machineId: MACHINE_ID } },
+ freshSink,
+ () => {},
+ )
+ expect(selects).toBe(3) // invalidate() forced a brand new dial instead of reusing the old entry
+ pool.invalidate(MACHINE_ID) // drop the still-open final entry so wss.close() below can settle
+ } finally {
+ await new Promise<void>((resolve) => wss.close(() => resolve()))
+ }
+ })
})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index b4b06012f..782f27f50 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -21,6 +21,12 @@ import { encodeTerminalLocal, type TerminalBinaryClear } from './terminalBinary.
const CONNECT_TIMEOUT_MS = 15_000
const LINGER_MS = 30_000
+// Same convention/value as localWsServer.ts's app<->daemon heartbeat. Without this, a machine-node
+// cycling (e.g. `harness start` on the OTHER end after a crash/restart) can leave this daemon holding
+// an upstream socket the backend silently dropped with no close frame — every RPC sent through it then
+// times out client-side forever, since nothing ever removes the dead entry to let the next select
+// redial. `ws.terminate()` on a missed pong forces the existing `close` handler to run cleanup.
+const HEARTBEAT_MS = 20_000
export class RelayConnectError extends Error {
constructor(message: string, readonly closeCode?: number) {
@@ -42,6 +48,8 @@ interface Entry {
sink: LocalClientSink | null
onClosed: ((code: number, reason: string) => void) | null
lingerTimer: ReturnType<typeof setTimeout> | null
+ alive: boolean
+ heartbeatTimer: ReturnType<typeof setInterval> | null
}
export interface RelaySession {
@@ -61,6 +69,26 @@ export class RemoteRelayPool {
private readonly peers: MachinePeerStore,
) {}
+ /** Force-drops a pooled entry so the next `acquire()` dials fresh instead of reusing it. For when
+ * the transport itself never closed but the app-level session behind it is known dead anyway — e.g.
+ * the relayed machine's own Harness process restarted, dropping its in-memory E2EE session state
+ * without ever touching this socket (nothing else — not even the heartbeat, since backend itself
+ * keeps answering pings fine — would ever notice on its own). Signalled by the local client sending
+ * `forceReconnect: true` on a fresh `machine_select` after observing a live RPC time out. */
+ invalidate(machineId: string): void {
+ const entry = this.entries.get(machineId)
+ if (!entry) return
+ this.entries.delete(machineId)
+ if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
+ if (entry.lingerTimer) clearTimeout(entry.lingerTimer)
+ // The caller is invalidating so it can immediately acquire() a fresh entry on the SAME local
+ // connection (it just got a forceReconnect select) — null this out first so the generic
+ // `ws.on('close', ...)` cleanup below doesn't turn around and close that same local socket via a
+ // now-stale onClosed callback.
+ entry.onClosed = null
+ try { entry.ws.terminate() } catch { /* already gone */ }
+ }
+
/** Attach `sink` to the (possibly newly-created, possibly reused) upstream connection for
* `machineId`. `selectFrame` is the local client's own `machine_select` frame, forwarded upstream
* verbatim on a fresh connect — backend only reads its `.machineId`, so the local protocol's extra
@@ -118,7 +146,15 @@ export class RemoteRelayPool {
const url = `${this.backendWsBase}/api/web-ws?autonomousEnv=${encodeURIComponent(autonomousEnv)}`
const ws = new WebSocket(url, [token])
const crypto = new RelaySessionCrypto({ machineId, selfIdentity: this.selfIdentity, peerPub: b64d(peer.pub) })
- const entry: Entry = { ws, crypto, sink: null, onClosed: null, lingerTimer: null }
+ const entry: Entry = {
+ ws,
+ crypto,
+ sink: null,
+ onClosed: null,
+ lingerTimer: null,
+ alive: true,
+ heartbeatTimer: null,
+ }
// Two phases before this connection is usable: (1) machine_select ack, (2) this daemon's own
// e2e_hello/e2e_welcome as the "client" role — see lib/e2ee/relayClient.ts. Only once BOTH are done
// does the app's onOutgoing/sink start receiving anything, so it never sees a half-encrypted stream.
@@ -214,9 +250,17 @@ export class RemoteRelayPool {
})
// Handshake done — from here on, a close is the entry's real end-of-life, not a handshake failure.
ws.on('close', (code, reasonBuf) => {
+ if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)
this.entries.delete(machineId)
entry.onClosed?.(code, reasonBuf?.toString() ?? '')
})
+ ws.on('pong', () => { entry.alive = true })
+ entry.heartbeatTimer = setInterval(() => {
+ if (!entry.alive) { ws.terminate(); return }
+ entry.alive = false
+ try { ws.ping() } catch { ws.terminate() }
+ }, HEARTBEAT_MS)
+ entry.heartbeatTimer.unref?.()
this.entries.set(machineId, entry)
return entry
}
diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.ts
index b269831cc..e02dda023 100644
--- a/cli/src/localWsServer.ts
+++ b/cli/src/localWsServer.ts
@@ -161,6 +161,11 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerO
close(4403, 'machine mismatch')
return
}
+ // The local client observed a live RPC time out against an otherwise-"connected" machine —
+ // its pooled entry is suspect (most commonly the relayed machine's own Harness process
+ // restarted, dropping its E2EE session without the transport itself ever closing). Drop it
+ // so this select dials fresh instead of handing back the same dead session again.
+ if (payload?.forceReconnect === true) options.relayPool.invalidate(requestedMachineId)
try {
relay = await options.relayPool.acquire(requestedMachineId, options.autonomousEnv, frame, sink, close)
selected = true