The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(device): a machine that stops answering gets a new session, not eight minutes of silence
0xdiego254836eea6292c
diff --git a/cli/src/device/deviceFleet.spec.ts b/cli/src/device/deviceFleet.spec.tsindex 8fd226f44..6818348f0 100644--- a/cli/src/device/deviceFleet.spec.ts+++ b/cli/src/device/deviceFleet.spec.ts@@ -21,6 +21,7 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: strconst rows = listOf(over.rows ?? [{}])let frameCb: ((f: DeviceFrame) => void) | null = nullconst sent: DeviceFrame[] = []+ const resets: string[] = []const link = {selectedMachine: 'm1',onFrame: (cb: (f: DeviceFrame) => void) => { frameCb = cb; return () => {} },@@ -30,6 +31,7 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: strrelease: vi.fn(),send: (f: DeviceFrame) => { sent.push(f) },rpc: over.rpc ?? (async () => ({})),+ resetSession: vi.fn((machineId: string) => { resets.push(machineId); return true }),}const fleet = new DeviceFleet({list: {@@ -40,10 +42,54 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: strhasPeerLink: (id: string) => (over.linked ?? ['m1']).includes(id),log: () => {},})- return { fleet, link, sent, say: (f: DeviceFrame) => frameCb?.(f) }+ return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f) }}describe('DeviceFleet', () => {+ it('throws away the E2EE session after two missed round trips, and keeps the last verdict', async () => {+ // THE FAILURE THIS EXISTS FOR IS SILENT AND PERMANENT. A session outlives the machine that agreed to+ // it: establish() returns instantly for anything in the map, so once the far end stops answering,+ // every later call is encrypted, sent, and waits out its timeout — forever. Measured on the desk as+ // eight minutes of `agents_list timed out` twenty seconds apart, healed only by a daemon restart that+ // happened for an unrelated reason.+ let calls = 0+ const { fleet, resets } = make({+ rpc: async () => { calls++; throw new Error('timed out') },+ })++ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')+ // One miss is a dropped request, not a dead machine. Nothing is torn down for it.+ expect(resets).toEqual([])+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })++ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')+ expect(resets).toEqual(['m1'])+ expect(calls).toBe(2)+ })++ it('a round trip that comes back clears the count, so misses have to be consecutive', async () => {+ let fail = true+ const { fleet, resets } = make({+ rpc: async () => { if (fail) throw new Error('timed out'); return {} },+ })+ await expect(fleet.listAgents('m1')).rejects.toThrow()+ fail = false+ await fleet.listAgents('m1')+ expect(fleet.reachable('m1')).toMatchObject({ ok: true })+ fail = true+ await expect(fleet.listAgents('m1')).rejects.toThrow()+ // Two failures, but a success between them: a flaky link is not a gone machine.+ expect(resets).toEqual([])+ })++ it('says nothing about a machine nobody has asked', async () => {+ // Callers refuse to deliver on a NEGATIVE verdict. Answering "unreachable" for a machine that has+ // simply never been called would turn every cold start into a refusal.+ const { fleet } = make()+ expect(fleet.reachable('m1')).toBeNull()+ })++it('hides this computer from the fleet list — the host adds it back with real facts', async () => {const { fleet } = make({ rows: [{ machineId: 'mine', local: true }, { machineId: 'other' }], linked: ['other'] })const { machines } = await fleet.list()diff --git a/cli/src/device/deviceFleet.ts b/cli/src/device/deviceFleet.tsindex 9983ca990..282c6af8b 100644--- a/cli/src/device/deviceFleet.ts+++ b/cli/src/device/deviceFleet.ts@@ -14,6 +14,9 @@ import type { DeviceFrame, DeviceLink } from './deviceLink.js'* not pay a full dial each way. Mirrors RemoteRelayPool's own linger for the same reason. */const LINGER_MS = 60_000+/** Consecutive failed round trips before a machine's E2EE session is thrown away — see `rpc()`. */+const RESET_AFTER_FAILURES = 2+/** Split `runtime-v1:<sid>:<engine>:<model>@<effort>` into the two words the dial's chips show. */function chipsFromProfile(profile: unknown): { model?: string; effort?: string } {if (typeof profile !== 'string' || !profile.startsWith('runtime-v1:')) return {}@@ -46,6 +49,21 @@ export class DeviceFleet implements MachineFleet {private lingerTimer: NodeJS.Timeout | null = nullprivate readonly recapCache = new Map<string, RecentTurn[]>()private readonly listeners = new Set<(e: FleetEvent) => void>()+ /**+ * How the last round trip to each machine went.+ *+ * THE ONLY HONEST PROOF WE HAVE. A machine can look fine from every other angle and still swallow+ * everything sent to it: the backend's list still calls it online, and the E2EE session still reports+ * `ready` because it was established while the machine was alive — `establish()` returns instantly for+ * a session that handshook an hour ago and has been deaf since. What actually knows is a request that+ * came back, or did not.+ *+ * It costs no traffic of its own: the refresh loop already asks these machines for their agents every+ * few minutes, so the answer is a by-product of work that was happening anyway.+ */+ private readonly health = new Map<string, { ok: boolean; at: number }>()+ /** Consecutive failed round trips per machine — the counter behind RESET_AFTER_FAILURES. */+ private readonly misses = new Map<string, number>()constructor(private readonly opts: DeviceFleetOpts) {this.opts.link.onFrame((frame) => this.onFrame(frame))@@ -136,11 +154,56 @@ export class DeviceFleet implements MachineFleet {// Dropping the tag would not fail loudly — it would quietly ask the wrong computer.async listAgents(machineId: string): Promise<CableAgent[]> {- const res = await this.opts.link.rpc('agents_list', {}, machineId)+ const res = await this.rpc(machineId, 'agents_list', {})const raw = Array.isArray(res.agents) ? res.agents : []return raw.map(toCableAgent).filter((a) => a.id)}+ /**+ * One round trip, and one record of how it went.+ *+ * Every RPC goes through here so `reachable()` reads a fact rather than a guess. It rethrows+ * untouched — this is a witness, not a handler.+ */+ private async rpc(machineId: string, type: string, payload: Record<string, unknown>): Promise<Record<string, unknown>> {+ try {+ const res = await this.opts.link.rpc(type, payload, machineId)+ this.health.set(machineId, { ok: true, at: Date.now() })+ this.misses.delete(machineId)+ return res+ } catch (err) {+ this.health.set(machineId, { ok: false, at: Date.now() })+ const missed = (this.misses.get(machineId) ?? 0) + 1+ this.misses.set(machineId, missed)+ // TWO IN A ROW AND THE SESSION GOES. A live machine drops the odd request; a machine that has gone+ // away answers nothing, and the E2EE session it agreed to keeps every later call believing there is+ // somebody at the other end — establish() returns instantly for a session in the map, so nothing+ // ever rebuilds it. Measured: eight minutes of timeouts, twenty seconds apart, healed only by a+ // daemon restart that happened for another reason.+ //+ // The counter is the pacing: two more failures cost two more timeouts, so a machine that is simply+ // gone is retried on the order of half a minute rather than in a loop.+ if (missed >= RESET_AFTER_FAILURES) {+ this.misses.set(machineId, 0)+ if (this.opts.link.resetSession(machineId)) {+ this.opts.log(`device: ${machineId.slice(0, 8)} missed ${missed} in a row — dropped its E2EE session, next call rebuilds it`)+ }+ }+ throw err+ }+ }++ /**+ * Whether the last thing we asked this machine came back.+ *+ * `null` for a machine nobody has asked yet — which is NOT the same as unreachable, and callers must+ * not treat it as one: refusing to deliver to a machine we have simply never spoken to would turn a+ * cold start into a failure.+ */+ reachable(machineId: string): { ok: boolean; at: number } | null {+ return this.health.get(machineId) ?? null+ }+sendTurn(machineId: string, agentId: string, text: string): void {// `resumeLatest` matches what the backend's own voice path dispatches, so a dial turn and a spoken// turn land on the same session rather than one of them starting a new one.@@ -159,14 +222,14 @@ export class DeviceFleet implements MachineFleet {if (!model) return// The far end owns what the profile means; this only reassembles the opaque string it round-trips.const selectedModel = `runtime-v1:${agentId}:claude:${model}@${effort || 'auto'}`- void this.opts.link.rpc('agent_update', { agentId, selectedModel }, machineId)+ void this.rpc(machineId, 'agent_update', { agentId, selectedModel }).catch((err) => this.opts.log(`device: agent_update failed (${(err as Error).message})`))}async listModels(machineId: string, agentId: string): Promise<string[]> {// `compact` is what keeps the catalog inside the dial's picker; the backend trims to ≤24 entries.- const res = await this.opts.link.rpc('models_list', { agentId, compact: true }, machineId)+ const res = await this.rpc(machineId, 'models_list', { agentId, compact: true })const raw = Array.isArray(res.models) ? res.models : []return raw.map((m) => (typeof m === 'string' ? m : String((m as Record<string, unknown>)?.id ?? ''))).filter(Boolean)@@ -177,7 +240,7 @@ export class DeviceFleet implements MachineFleet {const cached = this.recapCache.get(key)if (cached) return cachedtry {- const res = await this.opts.link.rpc('agent_recent', { agentId, n: 3 }, machineId)+ const res = await this.rpc(machineId, 'agent_recent', { agentId, n: 3 })const events = Array.isArray(res.events) ? res.events : []const turns = events.map((e) => {diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.tsindex fc4d7ebc5..78e27a31a 100644--- a/cli/src/device/deviceLink.ts+++ b/cli/src/device/deviceLink.ts@@ -189,6 +189,25 @@ export class DeviceLink {if (this.ws?.readyState === WebSocket.OPEN) this.send({ type: 'machine_deselect' })}+ /**+ * Throw away a machine's E2EE session so the next call handshakes a new one.+ *+ * A SESSION OUTLIVES THE MACHINE THAT AGREED TO IT. `establish()` returns instantly for any session in+ * this map, and nothing in the map notices that the far end stopped answering — so once a machine goes+ * away mid-session, every request to it is encrypted, sent, and waits out its timeout, forever. Measured+ * on the desk: eight minutes of `agents_list timed out` every twenty seconds, with no attempt to rebuild+ * anything, until an unrelated daemon restart cleared the map and the very first call afterwards worked.+ *+ * Deliberately NOT `failCrypto`: that rejects an in-flight handshake, and a handshake in flight is the+ * repair already happening. Returns whether there was anything to drop, so the caller can say so.+ */+ resetSession(machineId: string): boolean {+ if (!machineId || this.handshakes.has(machineId)) return false+ if (!this.sessions.has(machineId)) return false+ this.sessions.delete(machineId)+ return true+ }+release(): void {this.wanted = ''this.held = false