The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: implement websocket heartbeat and relay pool invalidation to handle stale machine-node sessions
0xken15340cd9cb5330
diff --git a/cli/src/lib/e2ee/relayLink.spec.ts b/cli/src/lib/e2ee/relayLink.spec.tsindex db4ad9c11..cf3a6070b 100644--- a/cli/src/lib/e2ee/relayLink.spec.ts+++ b/cli/src/lib/e2ee/relayLink.spec.ts@@ -267,4 +267,89 @@ describe('RemoteRelayPool drops a peer the responder no longer trusts', () => {await new Promise<void>((resolve) => wss.close(() => resolve()))}})++ it('invalidate() drops a pooled entry so the next acquire() dials fresh, without firing its old onClosed', async () => {+ let selects = 0+ const wss = new WebSocketServer({ port: 0 })+ const manager = new E2eeManagerCtor({+ machineId: MACHINE_ID,+ sendTo: (_connId, frame) => {+ // Unlike the other fixtures in this file, this test leaves a terminated (invalidated) client+ // behind mid-test — only broadcast to sockets still actually open.+ for (const client of wss.clients) {+ if (client.readyState === client.OPEN) client.send(JSON.stringify(frame))+ }+ },+ isConnected: () => true,+ })+ wss.on('connection', (ws) => {+ let selected = false+ ws.on('message', (raw) => {+ let frame: Frame+ try { frame = JSON.parse(raw.toString()) as Frame } catch { return }+ if (!selected) {+ if (frame.type === 'machine_select') {+ selected = true+ selects+++ ws.send(JSON.stringify({ type: 'connected', payload: { machineId: MACHINE_ID } }))+ }+ return+ }+ const type = frame.type as string+ if (type.startsWith('e2e_')) manager.handleFrame(`fake-conn-${selects}`, frame)+ })+ })++ try {+ const port = (wss.address() as AddressInfo).port+ const clientIdentity = C.newIdentity()+ const token = manager.createSetupToken(MACHINE_ID).token+ const claim = await claimSetupToken({+ token,+ targetMachineId: MACHINE_ID,+ selfIdentity: clientIdentity,+ accessToken: 'unused-in-this-fake',+ backendWsBase: `ws://127.0.0.1:${port}`,+ autonomousEnv: 'prod',+ timeoutMs: 2_000,+ })+ expect(claim.ok).toBe(true)+ if (!claim.ok) return++ const peers = new MachinePeerStore()+ peers.pin(MACHINE_ID, C.b64e(claim.peerPub), 'harness link')+ const pool = new RemoteRelayPool(fakeAuth, `ws://127.0.0.1:${port}`, clientIdentity, peers)++ const staleSink = { sendFrame: () => true, sendBinary: () => true }+ let staleOnClosedFired = false+ await pool.acquire(+ MACHINE_ID,+ 'prod',+ { type: 'machine_select', payload: { machineId: MACHINE_ID } },+ staleSink,+ () => { staleOnClosedFired = true },+ )+ expect(selects).toBe(2) // 1 for claimSetupToken's own connection, 1 for the acquire() dial++ pool.invalidate(MACHINE_ID)+ // onClosed is a stand-in for the LOCAL app socket's own close() — invalidate() must never call+ // it, since a real caller invalidates right before reusing that same local connection for a+ // fresh acquire(); firing it here would self-destruct the very connection driving the retry.+ await new Promise((resolve) => setTimeout(resolve, 50))+ expect(staleOnClosedFired).toBe(false)++ const freshSink = { sendFrame: () => true, sendBinary: () => true }+ await pool.acquire(+ MACHINE_ID,+ 'prod',+ { type: 'machine_select', payload: { machineId: MACHINE_ID } },+ freshSink,+ () => {},+ )+ expect(selects).toBe(3) // invalidate() forced a brand new dial instead of reusing the old entry+ pool.invalidate(MACHINE_ID) // drop the still-open final entry so wss.close() below can settle+ } finally {+ await new Promise<void>((resolve) => wss.close(() => resolve()))+ }+ })})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex b4b06012f..782f27f50 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -21,6 +21,12 @@ import { encodeTerminalLocal, type TerminalBinaryClear } from './terminalBinary.const CONNECT_TIMEOUT_MS = 15_000const LINGER_MS = 30_000+// Same convention/value as localWsServer.ts's app<->daemon heartbeat. Without this, a machine-node+// cycling (e.g. `harness start` on the OTHER end after a crash/restart) can leave this daemon holding+// an upstream socket the backend silently dropped with no close frame — every RPC sent through it then+// times out client-side forever, since nothing ever removes the dead entry to let the next select+// redial. `ws.terminate()` on a missed pong forces the existing `close` handler to run cleanup.+const HEARTBEAT_MS = 20_000export class RelayConnectError extends Error {constructor(message: string, readonly closeCode?: number) {@@ -42,6 +48,8 @@ interface Entry {sink: LocalClientSink | nullonClosed: ((code: number, reason: string) => void) | nulllingerTimer: ReturnType<typeof setTimeout> | null+ alive: boolean+ heartbeatTimer: ReturnType<typeof setInterval> | null}export interface RelaySession {@@ -61,6 +69,26 @@ export class RemoteRelayPool {private readonly peers: MachinePeerStore,) {}+ /** Force-drops a pooled entry so the next `acquire()` dials fresh instead of reusing it. For when+ * the transport itself never closed but the app-level session behind it is known dead anyway — e.g.+ * the relayed machine's own Harness process restarted, dropping its in-memory E2EE session state+ * without ever touching this socket (nothing else — not even the heartbeat, since backend itself+ * keeps answering pings fine — would ever notice on its own). Signalled by the local client sending+ * `forceReconnect: true` on a fresh `machine_select` after observing a live RPC time out. */+ invalidate(machineId: string): void {+ const entry = this.entries.get(machineId)+ if (!entry) return+ this.entries.delete(machineId)+ if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)+ if (entry.lingerTimer) clearTimeout(entry.lingerTimer)+ // The caller is invalidating so it can immediately acquire() a fresh entry on the SAME local+ // connection (it just got a forceReconnect select) — null this out first so the generic+ // `ws.on('close', ...)` cleanup below doesn't turn around and close that same local socket via a+ // now-stale onClosed callback.+ entry.onClosed = null+ try { entry.ws.terminate() } catch { /* already gone */ }+ }+/** Attach `sink` to the (possibly newly-created, possibly reused) upstream connection for* `machineId`. `selectFrame` is the local client's own `machine_select` frame, forwarded upstream* verbatim on a fresh connect — backend only reads its `.machineId`, so the local protocol's extra@@ -118,7 +146,15 @@ export class RemoteRelayPool {const url = `${this.backendWsBase}/api/web-ws?autonomousEnv=${encodeURIComponent(autonomousEnv)}`const ws = new WebSocket(url, [token])const crypto = new RelaySessionCrypto({ machineId, selfIdentity: this.selfIdentity, peerPub: b64d(peer.pub) })- const entry: Entry = { ws, crypto, sink: null, onClosed: null, lingerTimer: null }+ const entry: Entry = {+ ws,+ crypto,+ sink: null,+ onClosed: null,+ lingerTimer: null,+ alive: true,+ heartbeatTimer: null,+ }// Two phases before this connection is usable: (1) machine_select ack, (2) this daemon's own// e2e_hello/e2e_welcome as the "client" role — see lib/e2ee/relayClient.ts. Only once BOTH are done// does the app's onOutgoing/sink start receiving anything, so it never sees a half-encrypted stream.@@ -214,9 +250,17 @@ export class RemoteRelayPool {})// Handshake done — from here on, a close is the entry's real end-of-life, not a handshake failure.ws.on('close', (code, reasonBuf) => {+ if (entry.heartbeatTimer) clearInterval(entry.heartbeatTimer)this.entries.delete(machineId)entry.onClosed?.(code, reasonBuf?.toString() ?? '')})+ ws.on('pong', () => { entry.alive = true })+ entry.heartbeatTimer = setInterval(() => {+ if (!entry.alive) { ws.terminate(); return }+ entry.alive = false+ try { ws.ping() } catch { ws.terminate() }+ }, HEARTBEAT_MS)+ entry.heartbeatTimer.unref?.()this.entries.set(machineId, entry)return entry}diff --git a/cli/src/localWsServer.ts b/cli/src/localWsServer.tsindex b269831cc..e02dda023 100644--- a/cli/src/localWsServer.ts+++ b/cli/src/localWsServer.ts@@ -161,6 +161,11 @@ export function attachLocalWsServer(server: http.Server, options: LocalWsServerOclose(4403, 'machine mismatch')return}+ // The local client observed a live RPC time out against an otherwise-"connected" machine —+ // its pooled entry is suspect (most commonly the relayed machine's own Harness process+ // restarted, dropping its E2EE session without the transport itself ever closing). Drop it+ // so this select dials fresh instead of handing back the same dead session again.+ if (payload?.forceReconnect === true) options.relayPool.invalidate(requestedMachineId)try {relay = await options.relayPool.acquire(requestedMachineId, options.autonomousEnv, frame, sink, close)selected = true