Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(cli): the dial lane keeps a new socket past an old one's close, takes select answers only from the backend, and reconnects where the dial is now (#2402)

0xken15b7c4f3934354
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.ts
index a2ea2ad36..d884db126 100644
--- a/cli/src/cable/cableHost.spec.ts
+++ b/cli/src/cable/cableHost.spec.ts
@@ -1204,8 +1204,7 @@ describe('announcing over the real lane', () => {
await soon('the first pick', () => selects.length === 1)
answerNewest()
expect(await first).toEqual({ ok: true })
- host.onDialGone() // unplugged: the lane goes with it
- await soon('the old socket to close', () => lines.some((line) => line.startsWith('device: closed')))
+ host.onDialGone() // unplugged: the lane goes with it, and comes back at once
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
host.onDialAttached() // and back: announces cloud-1, where the dial was
await soon('the announce', () => selects.length === 2)
diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.ts
index afef22d79..da15d13a6 100644
--- a/cli/src/device/deviceLink.spec.ts
+++ b/cli/src/device/deviceLink.spec.ts
@@ -39,8 +39,10 @@ async function fakeDeviceWs(opts: { refuseToken?: string } = {}) {
})
await new Promise<void>((resolve) => wss.once('listening', () => resolve()))
const listeners = new Set<(frame: Frame) => void>()
+ const sockets: WebSocket[] = []
wss.on('connection', (ws) => {
socket = ws
+ sockets.push(ws)
ws.on('message', (raw) => {
const frame = JSON.parse(raw.toString()) as Frame
received.push(frame)
@@ -55,6 +57,10 @@ async function fakeDeviceWs(opts: { refuseToken?: string } = {}) {
/** Frames for machines, as the lane sent them: its own control frames left out. */
forMachines: () => received.filter((frame) => frame.type !== 'device_hello' && frame.type !== 'machines_watch' && frame.type !== 'ping'),
send: (frame: Frame) => socket?.send(JSON.stringify(frame)),
+ /** Every socket the lane opened, oldest first. */
+ sockets,
+ /** The backend closes the newest socket, as a restart or a network drop does. */
+ closeCurrent: (code = 1012) => socket?.close(code),
/** The socket to this lane closing under it, as a relay restart or a network change does. */
drop: () => socket?.terminate(),
on: (listener: (frame: Frame) => void) => { listeners.add(listener) },
@@ -1095,15 +1101,153 @@ describe('overlapping machine selects', () => {
})
it('fails a select in flight when the lane is revoked, and starts the next one fresh', async () => {
- const { link, server, selects, lines } = await selecting(60_000)
+ const { link, server, selects } = await selecting(60_000)
const pending = settled(link.attach(CLOUD))
await soon('the select', () => selects().length === 1)
server.send({ type: 'device_revoked' })
expect(await pending).toBe('revoked')
- await soon('the revoked socket to close', () => lines.some((line) => line.startsWith('device: closed')))
const next = settled(link.attach(CLOUD))
await soon('a select on a new socket', () => selects().length === 2)
server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })
expect(await next).toBe('ok')
})
})
+
+/**
+ * The lane's own lifecycle around a select, found while fixing overlapping selects (#2400) and in its two security
+ * reviews: a socket closed on purpose is not the lane's any more, a select answer is the backend's own, the attached
+ * machine is the one an answer settled, and a reconnect goes where the dial is now.
+ */
+describe('the lane around a select', () => {
+ const CLOUD_2 = 'second-cloud-machine'
+ const settled = (promise: Promise<void>) => promise.then(() => 'ok', (error: Error) => error.message)
+ const selectsOf = (server: { received: Frame[] }) => server.received.filter((frame) => frame.type === 'machine_select')
+ .map((frame) => (frame.payload as { machineId: string }).machineId)
+ /** A lane whose fake backend answers every select after a moment, unless told to stay quiet. */
+ async function answering(opts: { quiet?: boolean } = {}) {
+ const up = await lane()
+ up.server.on((frame) => {
+ if (frame.type !== 'machine_select' || opts.quiet) return
+ const machineId = (frame.payload as { machineId: string }).machineId
+ setTimeout(() => up.server.send({ type: 'machine_selected', payload: { machineId } }), 20)
+ })
+ await up.link.online()
+ return up
+ }
+ const settle = () => new Promise((resolve) => setTimeout(resolve, 300))
+
+ it('keeps the new socket of a fast replug: the old one closing later is not the lane any more', async () => {
+ const { link, server, lines } = await answering()
+ link.release()
+ expect(await link.online().then(() => 'ok', (error: Error) => error.message)).toBe('ok')
+ await settle() // the first socket's close lands now
+ expect(server.sockets).toHaveLength(2)
+ expect(server.sockets[1]!.readyState).toBe(1) // OPEN
+ expect(lines.filter((line) => line.startsWith('device: closed'))).toEqual([])
+ expect(await settled(link.attach(CLOUD))).toBe('ok')
+ })
+
+ it('reselects at once after a revoke, without waiting for the revoked socket to close', async () => {
+ const { link, server } = await answering()
+ await link.attach(CLOUD)
+ server.send({ type: 'device_revoked' })
+ // The dial asks again the moment the revoke is read, before the revoked socket has finished closing.
+ for (let turn = 0; turn < 50_000 && link.status() !== 'idle'; turn++) await new Promise((resolve) => setImmediate(resolve))
+ expect(await settled(link.attach(CLOUD))).toBe('ok')
+ await settle()
+ expect(await settled(link.attach(CLOUD))).toBe('ok')
+ // Still attached: the old socket's close tore nothing down, so the second attach had nothing to send.
+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD])
+ expect(server.sockets.at(-1)!.readyState).toBe(1)
+ })
+
+ it('still comes back after the backend closes the current socket, and after a revoke while plugged in', async () => {
+ const { link, server, lines } = await answering()
+ server.closeCurrent(1012)
+ await until('the reconnect', () => server.sockets.length === 2 && link.status() === 'ready', 4_000)
+ expect(lines).toContain('device: closed (1012)')
+ // A revoke, with the dial still plugged in, comes back as before: the held socket is opened again.
+ server.send({ type: 'device_revoked' })
+ await until('the reconnect after the revoke', () => server.sockets.length === 3 && link.status() === 'ready', 4_000)
+ })
+
+ it('settles nothing with a select answer relayed from a machine, or one opened from a sealed frame', async () => {
+ const { link, server, seal, lines, heard } = await answering({ quiet: true })
+ await link.establish(LINKED)
+ let how = ''
+ void settled(link.attach(CLOUD)).then((result) => { how = result })
+ await until('the select', () => selectsOf(server).length === 1)
+ // Tagged with a machine: what another machine on the account (or the relay on its behalf) can put on this lane.
+ server.send({ type: 'machine_selected', machineId: CLOUD_2, payload: {} })
+ server.send({ type: 'machine_select_error', machineId: CLOUD_2, payload: { error: 'NOT_YOUR_MACHINE' } })
+ // Sealed by a linked machine, which opens to a select answer.
+ seal.open.mockResolvedValueOnce({ frame: { type: 'machine_selected', machineId: LINKED, payload: {} } })
+ server.send({ type: 'machine_selected', machineId: LINKED, payload: { __e2e: { n: 0 } } })
+ await settle()
+ expect(how).toBe('')
+ expect(lines.filter((line) => line.includes('not from the backend'))).toHaveLength(3)
+ expect(heard.filter((frame) => String(frame.type).startsWith('machine_select'))).toEqual([])
+ // The backend's own answer settles it.
+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })
+ await until('the answer', () => how !== '')
+ expect(how).toBe('ok')
+ })
+
+ it('marks attached only the machine whose select an answer settled, so a later select of a replaced one is sent', async () => {
+ const { link, server } = await answering({ quiet: true })
+ const older = settled(link.attach(CLOUD))
+ await until('the first select', () => selectsOf(server).length === 1)
+ const newer = settled(link.attach(CLOUD_2))
+ await until('the second select', () => selectsOf(server).length === 2)
+ expect(await older).toBe('SELECT_SUPERSEDED')
+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } }) // late, for the replaced one
+ server.send({ type: 'machine_select_error', payload: { machineId: CLOUD_2, error: 'MACHINE_START_FAILED' } })
+ expect(await newer).toBe('MACHINE_START_FAILED')
+ let how = ''
+ void settled(link.attach(CLOUD)).then((result) => { how = result })
+ await settle()
+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2, CLOUD])
+ expect(how).toBe('') // waits for its own answer, not skipped
+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })
+ await until('its answer', () => how !== '')
+ expect(how).toBe('ok')
+ })
+
+ it('tears down a socket still closing before it opens the next, so the next starts clean and the old cleanup runs once', async () => {
+ const { link, server, seal, lines } = await answering()
+ await link.establish(LINKED)
+ expect(seal.hello).toHaveBeenCalledTimes(1)
+ let asked = ''
+ void link.rpc('models_list', {}, LINKED).then(() => { asked = 'answered' }, (error: Error) => { asked = error.message })
+ await until('the request', () => server.forMachines().some((frame) => frame.type === 'models_list'))
+ // The socket starts closing (a close handshake under way, its close event not yet fired) when the next call needs one.
+ const old = (link as unknown as { ws: { close(): void; readyState: number } }).ws
+ old.close()
+ expect(old.readyState).toBe(2) // CLOSING
+ await link.online()
+ await new Promise((resolve) => setTimeout(resolve, 50))
+ // The old socket's lane went with it: its request failed, its session ended at the gateway, once.
+ expect(asked).toMatch(/socket closed/)
+ expect(seal.drop).toHaveBeenCalledWith(LINKED)
+ await until('the old close to land', () => lines.some((line) => line.startsWith('device: an earlier socket closed')))
+ expect(seal.drop.mock.calls.filter(([machineId]) => machineId === LINKED)).toHaveLength(1)
+ // And the new socket holds no session of the old one: the next call to the machine says hello again.
+ await link.sendSealed({ type: 'message', machineId: LINKED, payload: { content: 'on the new socket' } })
+ await until('a new hello', () => seal.hello.mock.calls.length === 2)
+ expect(server.sockets).toHaveLength(2)
+ expect(server.sockets[1]!.readyState).toBe(1)
+ })
+
+ it('reconnects to the machine chosen during the wait, not the one before the drop', async () => {
+ const { link, server } = await answering()
+ await link.attach(CLOUD)
+ server.closeCurrent(1012)
+ await until('the drop', () => link.status() === 'idle')
+ // The person picks another machine before the reconnect timer fires.
+ expect(await settled(link.attach(CLOUD_2))).toBe('ok')
+ await new Promise((resolve) => setTimeout(resolve, 1_500)) // past the reconnect timer
+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2])
+ expect(await settled(link.attach(CLOUD_2))).toBe('ok')
+ expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2]) // still there: nothing to send
+ })
+})
diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.ts
index 927635b07..c206b49cf 100644
--- a/cli/src/device/deviceLink.ts
+++ b/cli/src/device/deviceLink.ts
@@ -622,6 +622,13 @@ export class DeviceLink {
private async connect(): Promise<void> {
if (this.ws?.readyState === WebSocket.OPEN) return
if (this.ws?.readyState === WebSocket.CONNECTING) return this.waitOpen()
+ // A socket still closing (its close event not yet fired) is torn down here, before the next one replaces it: its
+ // close is ignored once it is not the lane's socket (open()), so nothing else would end its sessions, fail what
+ // waits on it, or stop its ping, and the next socket would start with all of it (review 2 on #2402).
+ if (this.ws) {
+ this.opts.log(`device: the socket is still closing — tearing it down before the next one`)
+ this.teardown('socket closed while closing')
+ }
this.state = 'connecting'
const base = this.opts.backendWsBase.replace(/\/$/, '')
const url = `${base}/api/device-ws?computer=${encodeURIComponent(this.opts.computerId)}`
@@ -682,7 +689,12 @@ export class DeviceLink {
this.opts.log(`device: a frame could not be read (${err instanceof Error ? err.message : String(err)})`)
})
})
- ws.on('close', (code) => this.onClose(code))
+ // Only while it is still the lane's socket. One closed on purpose (teardown: an unplug, a revoke) closes a moment
+ // later, when a new socket may already be up; read then, it tore the new one down (a fast replug failed once).
+ ws.on('close', (code) => {
+ if (this.ws === ws) this.onClose(code)
+ else this.opts.log(`device: an earlier socket closed (${code}) — not the lane's any more`)
+ })
})
}
@@ -889,10 +901,18 @@ export class DeviceLink {
this.opts.log('device: not authorized — other machines are unavailable until the next sign-in')
return
}
+ this.reconnectLater()
+ }
+
+ private reconnectLater(): void {
const delay = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** Math.min(this.attempts++, 5))
this.reconnectTimer = setTimeout(() => {
- // Re-select only if a machine was selected. A socket held open purely for presence comes back the
- // same way it went up — connected and attached to nothing.
+ this.reconnectTimer = null
+ // Re-select only if a machine is wanted NOW: one chosen during the wait wins over the one before the drop (a
+ // select of the old one would end the backend there while the dial shows the new one). A socket held open purely
+ // for presence comes back the same way it went up — connected and attached to nothing.
+ const wanted = this.wanted
+ if (!wanted && !this.heldOpen) return
const retry = wanted ? this.attach(wanted) : this.online()
// Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.
void retry.catch((err) => { if ((err as Error).message !== SELECT_SUPERSEDED) this.opts.log(`device: reconnect failed (${(err as Error).message})`) })
@@ -968,7 +988,7 @@ export class DeviceLink {
this.route(opened.frame as DeviceFrame, this.answered(from, session, frame))
return
}
- this.route(frame)
+ this.route(frame, '', !from && !wrapped)
}
/**
@@ -1056,16 +1076,26 @@ export class DeviceLink {
/** A frame read (opened, or never sealed): to its waiter, or to the listeners. [answered] is the machine it proves
* up (`answered`), or ''. */
- private route(frame: DeviceFrame, answered = ''): void {
+ private route(frame: DeviceFrame, answered = '', backend = false): void {
const from = typeof frame.machineId === 'string' ? frame.machineId : ''
const type = frame.type ?? ''
const payload = (frame.payload ?? {}) as Record<string, unknown>
+ // A select is answered by the backend itself, in the clear and tagged with no machine (deviceWs.ts bindMachine). One
+ // tagged with a machine was relayed from that machine's side, and one opened from a sealed frame was written by a
+ // machine: neither says where the backend put this dial, and one naming no machine would settle the select in flight.
+ if ((type === 'machine_selected' || type === 'machine_select_error') && !backend) {
+ this.opts.log(`device: ignored a ${type} not from the backend${from ? ` (tagged ${from})` : ''}`)
+ return
+ }
if (type === 'machine_selected') {
const named = typeof payload.machineId === 'string' ? payload.machineId : ''
- this.attached = named || this.wanted
- this.opts.log(`device: machine_selected ${this.attached}`)
- if (this.answersSelect(type, named)) this.settleSelect()
+ this.opts.log(`device: machine_selected ${named || this.selecting?.machineId || '?'}`)
+ if (!this.answersSelect(type, named)) return
+ // Only the answer that settles the select in flight says what is attached: a late one for a machine already
+ // replaced would make the next select of that machine look done and be skipped (attach).
+ this.attached = this.selecting!.machineId
+ this.settleSelect()
return
}
if (type === 'machine_select_error') {
@@ -1079,6 +1109,9 @@ export class DeviceLink {
this.opts.log('device: revoked — this dial was removed from the account')
this.wanted = ''
this.teardown('revoked')
+ // As before: a lane held open for a plugged-in dial comes back. This used to happen only through the revoked
+ // socket's own close, which now no longer reaches the lane (see open()).
+ if (this.heldOpen) this.reconnectLater()
return
}
// A machine the backend says went offline took its session with it: its daemon is gone or going, and the