The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(cli): the dial lane keeps a new socket past an old one's close, takes select answers only from the backend, and reconnects where the dial is now (#2402)
0xken15b7c4f3934354
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.tsindex a2ea2ad36..d884db126 100644--- a/cli/src/cable/cableHost.spec.ts+++ b/cli/src/cable/cableHost.spec.ts@@ -1204,8 +1204,7 @@ describe('announcing over the real lane', () => {await soon('the first pick', () => selects.length === 1)answerNewest()expect(await first).toEqual({ ok: true })- host.onDialGone() // unplugged: the lane goes with it- await soon('the old socket to close', () => lines.some((line) => line.startsWith('device: closed')))+ host.onDialGone() // unplugged: the lane goes with it, and comes back at oncevi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })host.onDialAttached() // and back: announces cloud-1, where the dial wasawait soon('the announce', () => selects.length === 2)diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.tsindex afef22d79..da15d13a6 100644--- a/cli/src/device/deviceLink.spec.ts+++ b/cli/src/device/deviceLink.spec.ts@@ -39,8 +39,10 @@ async function fakeDeviceWs(opts: { refuseToken?: string } = {}) {})await new Promise<void>((resolve) => wss.once('listening', () => resolve()))const listeners = new Set<(frame: Frame) => void>()+ const sockets: WebSocket[] = []wss.on('connection', (ws) => {socket = ws+ sockets.push(ws)ws.on('message', (raw) => {const frame = JSON.parse(raw.toString()) as Framereceived.push(frame)@@ -55,6 +57,10 @@ async function fakeDeviceWs(opts: { refuseToken?: string } = {}) {/** Frames for machines, as the lane sent them: its own control frames left out. */forMachines: () => received.filter((frame) => frame.type !== 'device_hello' && frame.type !== 'machines_watch' && frame.type !== 'ping'),send: (frame: Frame) => socket?.send(JSON.stringify(frame)),+ /** Every socket the lane opened, oldest first. */+ sockets,+ /** The backend closes the newest socket, as a restart or a network drop does. */+ closeCurrent: (code = 1012) => socket?.close(code),/** The socket to this lane closing under it, as a relay restart or a network change does. */drop: () => socket?.terminate(),on: (listener: (frame: Frame) => void) => { listeners.add(listener) },@@ -1095,15 +1101,153 @@ describe('overlapping machine selects', () => {})it('fails a select in flight when the lane is revoked, and starts the next one fresh', async () => {- const { link, server, selects, lines } = await selecting(60_000)+ const { link, server, selects } = await selecting(60_000)const pending = settled(link.attach(CLOUD))await soon('the select', () => selects().length === 1)server.send({ type: 'device_revoked' })expect(await pending).toBe('revoked')- await soon('the revoked socket to close', () => lines.some((line) => line.startsWith('device: closed')))const next = settled(link.attach(CLOUD))await soon('a select on a new socket', () => selects().length === 2)server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })expect(await next).toBe('ok')})})++/**+ * The lane's own lifecycle around a select, found while fixing overlapping selects (#2400) and in its two security+ * reviews: a socket closed on purpose is not the lane's any more, a select answer is the backend's own, the attached+ * machine is the one an answer settled, and a reconnect goes where the dial is now.+ */+describe('the lane around a select', () => {+ const CLOUD_2 = 'second-cloud-machine'+ const settled = (promise: Promise<void>) => promise.then(() => 'ok', (error: Error) => error.message)+ const selectsOf = (server: { received: Frame[] }) => server.received.filter((frame) => frame.type === 'machine_select')+ .map((frame) => (frame.payload as { machineId: string }).machineId)+ /** A lane whose fake backend answers every select after a moment, unless told to stay quiet. */+ async function answering(opts: { quiet?: boolean } = {}) {+ const up = await lane()+ up.server.on((frame) => {+ if (frame.type !== 'machine_select' || opts.quiet) return+ const machineId = (frame.payload as { machineId: string }).machineId+ setTimeout(() => up.server.send({ type: 'machine_selected', payload: { machineId } }), 20)+ })+ await up.link.online()+ return up+ }+ const settle = () => new Promise((resolve) => setTimeout(resolve, 300))++ it('keeps the new socket of a fast replug: the old one closing later is not the lane any more', async () => {+ const { link, server, lines } = await answering()+ link.release()+ expect(await link.online().then(() => 'ok', (error: Error) => error.message)).toBe('ok')+ await settle() // the first socket's close lands now+ expect(server.sockets).toHaveLength(2)+ expect(server.sockets[1]!.readyState).toBe(1) // OPEN+ expect(lines.filter((line) => line.startsWith('device: closed'))).toEqual([])+ expect(await settled(link.attach(CLOUD))).toBe('ok')+ })++ it('reselects at once after a revoke, without waiting for the revoked socket to close', async () => {+ const { link, server } = await answering()+ await link.attach(CLOUD)+ server.send({ type: 'device_revoked' })+ // The dial asks again the moment the revoke is read, before the revoked socket has finished closing.+ for (let turn = 0; turn < 50_000 && link.status() !== 'idle'; turn++) await new Promise((resolve) => setImmediate(resolve))+ expect(await settled(link.attach(CLOUD))).toBe('ok')+ await settle()+ expect(await settled(link.attach(CLOUD))).toBe('ok')+ // Still attached: the old socket's close tore nothing down, so the second attach had nothing to send.+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD])+ expect(server.sockets.at(-1)!.readyState).toBe(1)+ })++ it('still comes back after the backend closes the current socket, and after a revoke while plugged in', async () => {+ const { link, server, lines } = await answering()+ server.closeCurrent(1012)+ await until('the reconnect', () => server.sockets.length === 2 && link.status() === 'ready', 4_000)+ expect(lines).toContain('device: closed (1012)')+ // A revoke, with the dial still plugged in, comes back as before: the held socket is opened again.+ server.send({ type: 'device_revoked' })+ await until('the reconnect after the revoke', () => server.sockets.length === 3 && link.status() === 'ready', 4_000)+ })++ it('settles nothing with a select answer relayed from a machine, or one opened from a sealed frame', async () => {+ const { link, server, seal, lines, heard } = await answering({ quiet: true })+ await link.establish(LINKED)+ let how = ''+ void settled(link.attach(CLOUD)).then((result) => { how = result })+ await until('the select', () => selectsOf(server).length === 1)+ // Tagged with a machine: what another machine on the account (or the relay on its behalf) can put on this lane.+ server.send({ type: 'machine_selected', machineId: CLOUD_2, payload: {} })+ server.send({ type: 'machine_select_error', machineId: CLOUD_2, payload: { error: 'NOT_YOUR_MACHINE' } })+ // Sealed by a linked machine, which opens to a select answer.+ seal.open.mockResolvedValueOnce({ frame: { type: 'machine_selected', machineId: LINKED, payload: {} } })+ server.send({ type: 'machine_selected', machineId: LINKED, payload: { __e2e: { n: 0 } } })+ await settle()+ expect(how).toBe('')+ expect(lines.filter((line) => line.includes('not from the backend'))).toHaveLength(3)+ expect(heard.filter((frame) => String(frame.type).startsWith('machine_select'))).toEqual([])+ // The backend's own answer settles it.+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })+ await until('the answer', () => how !== '')+ expect(how).toBe('ok')+ })++ it('marks attached only the machine whose select an answer settled, so a later select of a replaced one is sent', async () => {+ const { link, server } = await answering({ quiet: true })+ const older = settled(link.attach(CLOUD))+ await until('the first select', () => selectsOf(server).length === 1)+ const newer = settled(link.attach(CLOUD_2))+ await until('the second select', () => selectsOf(server).length === 2)+ expect(await older).toBe('SELECT_SUPERSEDED')+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } }) // late, for the replaced one+ server.send({ type: 'machine_select_error', payload: { machineId: CLOUD_2, error: 'MACHINE_START_FAILED' } })+ expect(await newer).toBe('MACHINE_START_FAILED')+ let how = ''+ void settled(link.attach(CLOUD)).then((result) => { how = result })+ await settle()+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2, CLOUD])+ expect(how).toBe('') // waits for its own answer, not skipped+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })+ await until('its answer', () => how !== '')+ expect(how).toBe('ok')+ })++ it('tears down a socket still closing before it opens the next, so the next starts clean and the old cleanup runs once', async () => {+ const { link, server, seal, lines } = await answering()+ await link.establish(LINKED)+ expect(seal.hello).toHaveBeenCalledTimes(1)+ let asked = ''+ void link.rpc('models_list', {}, LINKED).then(() => { asked = 'answered' }, (error: Error) => { asked = error.message })+ await until('the request', () => server.forMachines().some((frame) => frame.type === 'models_list'))+ // The socket starts closing (a close handshake under way, its close event not yet fired) when the next call needs one.+ const old = (link as unknown as { ws: { close(): void; readyState: number } }).ws+ old.close()+ expect(old.readyState).toBe(2) // CLOSING+ await link.online()+ await new Promise((resolve) => setTimeout(resolve, 50))+ // The old socket's lane went with it: its request failed, its session ended at the gateway, once.+ expect(asked).toMatch(/socket closed/)+ expect(seal.drop).toHaveBeenCalledWith(LINKED)+ await until('the old close to land', () => lines.some((line) => line.startsWith('device: an earlier socket closed')))+ expect(seal.drop.mock.calls.filter(([machineId]) => machineId === LINKED)).toHaveLength(1)+ // And the new socket holds no session of the old one: the next call to the machine says hello again.+ await link.sendSealed({ type: 'message', machineId: LINKED, payload: { content: 'on the new socket' } })+ await until('a new hello', () => seal.hello.mock.calls.length === 2)+ expect(server.sockets).toHaveLength(2)+ expect(server.sockets[1]!.readyState).toBe(1)+ })++ it('reconnects to the machine chosen during the wait, not the one before the drop', async () => {+ const { link, server } = await answering()+ await link.attach(CLOUD)+ server.closeCurrent(1012)+ await until('the drop', () => link.status() === 'idle')+ // The person picks another machine before the reconnect timer fires.+ expect(await settled(link.attach(CLOUD_2))).toBe('ok')+ await new Promise((resolve) => setTimeout(resolve, 1_500)) // past the reconnect timer+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2])+ expect(await settled(link.attach(CLOUD_2))).toBe('ok')+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2]) // still there: nothing to send+ })+})diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.tsindex 927635b07..c206b49cf 100644--- a/cli/src/device/deviceLink.ts+++ b/cli/src/device/deviceLink.ts@@ -622,6 +622,13 @@ export class DeviceLink {private async connect(): Promise<void> {if (this.ws?.readyState === WebSocket.OPEN) returnif (this.ws?.readyState === WebSocket.CONNECTING) return this.waitOpen()+ // A socket still closing (its close event not yet fired) is torn down here, before the next one replaces it: its+ // close is ignored once it is not the lane's socket (open()), so nothing else would end its sessions, fail what+ // waits on it, or stop its ping, and the next socket would start with all of it (review 2 on #2402).+ if (this.ws) {+ this.opts.log(`device: the socket is still closing — tearing it down before the next one`)+ this.teardown('socket closed while closing')+ }this.state = 'connecting'const base = this.opts.backendWsBase.replace(/\/$/, '')const url = `${base}/api/device-ws?computer=${encodeURIComponent(this.opts.computerId)}`@@ -682,7 +689,12 @@ export class DeviceLink {this.opts.log(`device: a frame could not be read (${err instanceof Error ? err.message : String(err)})`)})})- ws.on('close', (code) => this.onClose(code))+ // Only while it is still the lane's socket. One closed on purpose (teardown: an unplug, a revoke) closes a moment+ // later, when a new socket may already be up; read then, it tore the new one down (a fast replug failed once).+ ws.on('close', (code) => {+ if (this.ws === ws) this.onClose(code)+ else this.opts.log(`device: an earlier socket closed (${code}) — not the lane's any more`)+ })})}@@ -889,10 +901,18 @@ export class DeviceLink {this.opts.log('device: not authorized — other machines are unavailable until the next sign-in')return}+ this.reconnectLater()+ }++ private reconnectLater(): void {const delay = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** Math.min(this.attempts++, 5))this.reconnectTimer = setTimeout(() => {- // Re-select only if a machine was selected. A socket held open purely for presence comes back the- // same way it went up — connected and attached to nothing.+ this.reconnectTimer = null+ // Re-select only if a machine is wanted NOW: one chosen during the wait wins over the one before the drop (a+ // select of the old one would end the backend there while the dial shows the new one). A socket held open purely+ // for presence comes back the same way it went up — connected and attached to nothing.+ const wanted = this.wanted+ if (!wanted && !this.heldOpen) returnconst retry = wanted ? this.attach(wanted) : this.online()// Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.void retry.catch((err) => { if ((err as Error).message !== SELECT_SUPERSEDED) this.opts.log(`device: reconnect failed (${(err as Error).message})`) })@@ -968,7 +988,7 @@ export class DeviceLink {this.route(opened.frame as DeviceFrame, this.answered(from, session, frame))return}- this.route(frame)+ this.route(frame, '', !from && !wrapped)}/**@@ -1056,16 +1076,26 @@ export class DeviceLink {/** A frame read (opened, or never sealed): to its waiter, or to the listeners. [answered] is the machine it proves* up (`answered`), or ''. */- private route(frame: DeviceFrame, answered = ''): void {+ private route(frame: DeviceFrame, answered = '', backend = false): void {const from = typeof frame.machineId === 'string' ? frame.machineId : ''const type = frame.type ?? ''const payload = (frame.payload ?? {}) as Record<string, unknown>+ // A select is answered by the backend itself, in the clear and tagged with no machine (deviceWs.ts bindMachine). One+ // tagged with a machine was relayed from that machine's side, and one opened from a sealed frame was written by a+ // machine: neither says where the backend put this dial, and one naming no machine would settle the select in flight.+ if ((type === 'machine_selected' || type === 'machine_select_error') && !backend) {+ this.opts.log(`device: ignored a ${type} not from the backend${from ? ` (tagged ${from})` : ''}`)+ return+ }if (type === 'machine_selected') {const named = typeof payload.machineId === 'string' ? payload.machineId : ''- this.attached = named || this.wanted- this.opts.log(`device: machine_selected ${this.attached}`)- if (this.answersSelect(type, named)) this.settleSelect()+ this.opts.log(`device: machine_selected ${named || this.selecting?.machineId || '?'}`)+ if (!this.answersSelect(type, named)) return+ // Only the answer that settles the select in flight says what is attached: a late one for a machine already+ // replaced would make the next select of that machine look done and be skipped (attach).+ this.attached = this.selecting!.machineId+ this.settleSelect()return}if (type === 'machine_select_error') {@@ -1079,6 +1109,9 @@ export class DeviceLink {this.opts.log('device: revoked — this dial was removed from the account')this.wanted = ''this.teardown('revoked')+ // As before: a lane held open for a plugged-in dial comes back. This used to happen only through the revoked+ // socket's own close, which now no longer reaches the lane (see open()).+ if (this.heldOpen) this.reconnectLater()return}// A machine the backend says went offline took its session with it: its daemon is gone or going, and the