Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(device): a machine that stops answering gets a new session, not eight minutes of silence

0xdiego254836eea6292c
diff --git a/cli/src/device/deviceFleet.spec.ts b/cli/src/device/deviceFleet.spec.ts
index 8fd226f44..6818348f0 100644
--- a/cli/src/device/deviceFleet.spec.ts
+++ b/cli/src/device/deviceFleet.spec.ts
@@ -21,6 +21,7 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: str
const rows = listOf(over.rows ?? [{}])
let frameCb: ((f: DeviceFrame) => void) | null = null
const sent: DeviceFrame[] = []
+ const resets: string[] = []
const link = {
selectedMachine: 'm1',
onFrame: (cb: (f: DeviceFrame) => void) => { frameCb = cb; return () => {} },
@@ -30,6 +31,7 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: str
release: vi.fn(),
send: (f: DeviceFrame) => { sent.push(f) },
rpc: over.rpc ?? (async () => ({})),
+ resetSession: vi.fn((machineId: string) => { resets.push(machineId); return true }),
}
const fleet = new DeviceFleet({
list: {
@@ -40,10 +42,54 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: str
hasPeerLink: (id: string) => (over.linked ?? ['m1']).includes(id),
log: () => {},
})
- return { fleet, link, sent, say: (f: DeviceFrame) => frameCb?.(f) }
+ return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f) }
}
describe('DeviceFleet', () => {
+ it('throws away the E2EE session after two missed round trips, and keeps the last verdict', async () => {
+ // THE FAILURE THIS EXISTS FOR IS SILENT AND PERMANENT. A session outlives the machine that agreed to
+ // it: establish() returns instantly for anything in the map, so once the far end stops answering,
+ // every later call is encrypted, sent, and waits out its timeout — forever. Measured on the desk as
+ // eight minutes of `agents_list timed out` twenty seconds apart, healed only by a daemon restart that
+ // happened for an unrelated reason.
+ let calls = 0
+ const { fleet, resets } = make({
+ rpc: async () => { calls++; throw new Error('timed out') },
+ })
+
+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
+ // One miss is a dropped request, not a dead machine. Nothing is torn down for it.
+ expect(resets).toEqual([])
+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })
+
+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
+ expect(resets).toEqual(['m1'])
+ expect(calls).toBe(2)
+ })
+
+ it('a round trip that comes back clears the count, so misses have to be consecutive', async () => {
+ let fail = true
+ const { fleet, resets } = make({
+ rpc: async () => { if (fail) throw new Error('timed out'); return {} },
+ })
+ await expect(fleet.listAgents('m1')).rejects.toThrow()
+ fail = false
+ await fleet.listAgents('m1')
+ expect(fleet.reachable('m1')).toMatchObject({ ok: true })
+ fail = true
+ await expect(fleet.listAgents('m1')).rejects.toThrow()
+ // Two failures, but a success between them: a flaky link is not a gone machine.
+ expect(resets).toEqual([])
+ })
+
+ it('says nothing about a machine nobody has asked', async () => {
+ // Callers refuse to deliver on a NEGATIVE verdict. Answering "unreachable" for a machine that has
+ // simply never been called would turn every cold start into a refusal.
+ const { fleet } = make()
+ expect(fleet.reachable('m1')).toBeNull()
+ })
+
+
it('hides this computer from the fleet list — the host adds it back with real facts', async () => {
const { fleet } = make({ rows: [{ machineId: 'mine', local: true }, { machineId: 'other' }], linked: ['other'] })
const { machines } = await fleet.list()
diff --git a/cli/src/device/deviceFleet.ts b/cli/src/device/deviceFleet.ts
index 9983ca990..282c6af8b 100644
--- a/cli/src/device/deviceFleet.ts
+++ b/cli/src/device/deviceFleet.ts
@@ -14,6 +14,9 @@ import type { DeviceFrame, DeviceLink } from './deviceLink.js'
* not pay a full dial each way. Mirrors RemoteRelayPool's own linger for the same reason. */
const LINGER_MS = 60_000
+/** Consecutive failed round trips before a machine's E2EE session is thrown away — see `rpc()`. */
+const RESET_AFTER_FAILURES = 2
+
/** Split `runtime-v1:<sid>:<engine>:<model>@<effort>` into the two words the dial's chips show. */
function chipsFromProfile(profile: unknown): { model?: string; effort?: string } {
if (typeof profile !== 'string' || !profile.startsWith('runtime-v1:')) return {}
@@ -46,6 +49,21 @@ export class DeviceFleet implements MachineFleet {
private lingerTimer: NodeJS.Timeout | null = null
private readonly recapCache = new Map<string, RecentTurn[]>()
private readonly listeners = new Set<(e: FleetEvent) => void>()
+ /**
+ * How the last round trip to each machine went.
+ *
+ * THE ONLY HONEST PROOF WE HAVE. A machine can look fine from every other angle and still swallow
+ * everything sent to it: the backend's list still calls it online, and the E2EE session still reports
+ * `ready` because it was established while the machine was alive — `establish()` returns instantly for
+ * a session that handshook an hour ago and has been deaf since. What actually knows is a request that
+ * came back, or did not.
+ *
+ * It costs no traffic of its own: the refresh loop already asks these machines for their agents every
+ * few minutes, so the answer is a by-product of work that was happening anyway.
+ */
+ private readonly health = new Map<string, { ok: boolean; at: number }>()
+ /** Consecutive failed round trips per machine — the counter behind RESET_AFTER_FAILURES. */
+ private readonly misses = new Map<string, number>()
constructor(private readonly opts: DeviceFleetOpts) {
this.opts.link.onFrame((frame) => this.onFrame(frame))
@@ -136,11 +154,56 @@ export class DeviceFleet implements MachineFleet {
// Dropping the tag would not fail loudly — it would quietly ask the wrong computer.
async listAgents(machineId: string): Promise<CableAgent[]> {
- const res = await this.opts.link.rpc('agents_list', {}, machineId)
+ const res = await this.rpc(machineId, 'agents_list', {})
const raw = Array.isArray(res.agents) ? res.agents : []
return raw.map(toCableAgent).filter((a) => a.id)
}
+ /**
+ * One round trip, and one record of how it went.
+ *
+ * Every RPC goes through here so `reachable()` reads a fact rather than a guess. It rethrows
+ * untouched — this is a witness, not a handler.
+ */
+ private async rpc(machineId: string, type: string, payload: Record<string, unknown>): Promise<Record<string, unknown>> {
+ try {
+ const res = await this.opts.link.rpc(type, payload, machineId)
+ this.health.set(machineId, { ok: true, at: Date.now() })
+ this.misses.delete(machineId)
+ return res
+ } catch (err) {
+ this.health.set(machineId, { ok: false, at: Date.now() })
+ const missed = (this.misses.get(machineId) ?? 0) + 1
+ this.misses.set(machineId, missed)
+ // TWO IN A ROW AND THE SESSION GOES. A live machine drops the odd request; a machine that has gone
+ // away answers nothing, and the E2EE session it agreed to keeps every later call believing there is
+ // somebody at the other end — establish() returns instantly for a session in the map, so nothing
+ // ever rebuilds it. Measured: eight minutes of timeouts, twenty seconds apart, healed only by a
+ // daemon restart that happened for another reason.
+ //
+ // The counter is the pacing: two more failures cost two more timeouts, so a machine that is simply
+ // gone is retried on the order of half a minute rather than in a loop.
+ if (missed >= RESET_AFTER_FAILURES) {
+ this.misses.set(machineId, 0)
+ if (this.opts.link.resetSession(machineId)) {
+ this.opts.log(`device: ${machineId.slice(0, 8)} missed ${missed} in a row — dropped its E2EE session, next call rebuilds it`)
+ }
+ }
+ throw err
+ }
+ }
+
+ /**
+ * Whether the last thing we asked this machine came back.
+ *
+ * `null` for a machine nobody has asked yet — which is NOT the same as unreachable, and callers must
+ * not treat it as one: refusing to deliver to a machine we have simply never spoken to would turn a
+ * cold start into a failure.
+ */
+ reachable(machineId: string): { ok: boolean; at: number } | null {
+ return this.health.get(machineId) ?? null
+ }
+
sendTurn(machineId: string, agentId: string, text: string): void {
// `resumeLatest` matches what the backend's own voice path dispatches, so a dial turn and a spoken
// turn land on the same session rather than one of them starting a new one.
@@ -159,14 +222,14 @@ export class DeviceFleet implements MachineFleet {
if (!model) return
// The far end owns what the profile means; this only reassembles the opaque string it round-trips.
const selectedModel = `runtime-v1:${agentId}:claude:${model}@${effort || 'auto'}`
- void this.opts.link.rpc('agent_update', { agentId, selectedModel }, machineId)
+ void this.rpc(machineId, 'agent_update', { agentId, selectedModel })
.catch((err) => this.opts.log(`device: agent_update failed (${(err as Error).message})`))
}
async listModels(machineId: string, agentId: string): Promise<string[]> {
// `compact` is what keeps the catalog inside the dial's picker; the backend trims to ≤24 entries.
- const res = await this.opts.link.rpc('models_list', { agentId, compact: true }, machineId)
+ const res = await this.rpc(machineId, 'models_list', { agentId, compact: true })
const raw = Array.isArray(res.models) ? res.models : []
return raw.map((m) => (typeof m === 'string' ? m : String((m as Record<string, unknown>)?.id ?? ''))).filter(Boolean)
@@ -177,7 +240,7 @@ export class DeviceFleet implements MachineFleet {
const cached = this.recapCache.get(key)
if (cached) return cached
try {
- const res = await this.opts.link.rpc('agent_recent', { agentId, n: 3 }, machineId)
+ const res = await this.rpc(machineId, 'agent_recent', { agentId, n: 3 })
const events = Array.isArray(res.events) ? res.events : []
const turns = events.map((e) => {
diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.ts
index fc4d7ebc5..78e27a31a 100644
--- a/cli/src/device/deviceLink.ts
+++ b/cli/src/device/deviceLink.ts
@@ -189,6 +189,25 @@ export class DeviceLink {
if (this.ws?.readyState === WebSocket.OPEN) this.send({ type: 'machine_deselect' })
}
+ /**
+ * Throw away a machine's E2EE session so the next call handshakes a new one.
+ *
+ * A SESSION OUTLIVES THE MACHINE THAT AGREED TO IT. `establish()` returns instantly for any session in
+ * this map, and nothing in the map notices that the far end stopped answering — so once a machine goes
+ * away mid-session, every request to it is encrypted, sent, and waits out its timeout, forever. Measured
+ * on the desk: eight minutes of `agents_list timed out` every twenty seconds, with no attempt to rebuild
+ * anything, until an unrelated daemon restart cleared the map and the very first call afterwards worked.
+ *
+ * Deliberately NOT `failCrypto`: that rejects an in-flight handshake, and a handshake in flight is the
+ * repair already happening. Returns whether there was anything to drop, so the caller can say so.
+ */
+ resetSession(machineId: string): boolean {
+ if (!machineId || this.handshakes.has(machineId)) return false
+ if (!this.sessions.has(machineId)) return false
+ this.sessions.delete(machineId)
+ return true
+ }
+
release(): void {
this.wanted = ''
this.held = false