Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Device lane: a sign-in refresh that fails during reconnect tries again instead of giving up (Diego) (#2483)

0xken15b4289d771c5c
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.ts
index d884db126..46b837e80 100644
--- a/cli/src/cable/cableHost.spec.ts
+++ b/cli/src/cable/cableHost.spec.ts
@@ -1219,3 +1219,132 @@ describe('announcing over the real lane', () => {
expect(host.selectedMachine()).toBe('mine')
})
})
+
+/**
+ * A plug-in whose first lane open fails — Diego, 11-01: the sign-in refresh failed in a network blip, nothing tried
+ * again, and the wheel lost every other Mac until the dial was replugged.
+ */
+describe('a plug-in whose first lane open fails', () => {
+ const stops: Array<() => Promise<void> | void> = []
+ afterEach(async () => {
+ vi.useRealTimers()
+ for (const stop of stops.splice(0)) await stop()
+ })
+ /** A bare fleet whose online() throws `fails` times, as a token read lost to a blip does, then opens. */
+ const failing = (fails: number) => {
+ let left = fails
+ const fleet = fleetOf([REMOTE])
+ fleet.online = vi.fn(async () => { if (left-- > 0) throw new Error('UNAVAILABLE: the sign-in could not be refreshed') })
+ fleet.select = vi.fn(async () => {})
+ fleet.release = vi.fn()
+ return fleet
+ }
+ const turns = async () => { for (let turn = 0; turn < 20; turn++) await new Promise((resolve) => setImmediate(resolve)) }
+
+ it('opens the lane later, and then announces where the dial is', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(2)
+ const log = vi.fn()
+ const host = new DaemonCableHost(wiring({ log }), fleet)
+ host.onDialAttached()
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(1)
+ expect(log).toHaveBeenCalledWith('cable: could not open the lane (UNAVAILABLE: the sign-in could not be refreshed)')
+ expect(fleet.select).not.toHaveBeenCalled()
+ await vi.advanceTimersByTimeAsync(1_000)
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(2) // failed again: the next try waits longer
+ await vi.advanceTimersByTimeAsync(1_999)
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(1)
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(3)
+ expect(fleet.select).toHaveBeenCalledWith('mine')
+ // Open now: nothing more is scheduled.
+ await vi.advanceTimersByTimeAsync(60_000)
+ expect(fleet.online).toHaveBeenCalledTimes(3)
+ })
+
+ it('stops trying once the dial is unplugged', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(Infinity)
+ const host = new DaemonCableHost(wiring(), fleet)
+ host.onDialAttached()
+ await turns()
+ host.onDialGone()
+ await vi.advanceTimersByTimeAsync(120_000)
+ expect(fleet.online).toHaveBeenCalledTimes(1)
+ })
+
+ it('stops trying once this computer is signed out', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(Infinity)
+ let signedIn = true
+ const host = new DaemonCableHost(wiring({ signedIn: () => signedIn }), fleet)
+ host.onDialAttached()
+ await turns()
+ signedIn = false
+ await vi.advanceTimersByTimeAsync(120_000)
+ expect(fleet.online).toHaveBeenCalledTimes(1)
+ })
+
+ it('announces nothing for an open that lands after the dial was unplugged', async () => {
+ let finish: () => void = () => {}
+ const fleet = failing(0)
+ fleet.online = vi.fn(() => new Promise<void>((resolve) => { finish = resolve }))
+ const host = new DaemonCableHost(wiring(), fleet)
+ host.onDialAttached()
+ await turns()
+ host.onDialGone()
+ finish()
+ await turns()
+ expect(fleet.select).not.toHaveBeenCalled()
+ })
+
+ it('keeps one retry for a replug while one is waiting, not two', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(Infinity)
+ const host = new DaemonCableHost(wiring(), fleet)
+ host.onDialAttached()
+ await turns()
+ host.onDialAttached() // the session re-reports the dial without an unplug in between
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(2)
+ await vi.advanceTimersByTimeAsync(1_000)
+ await turns()
+ expect(fleet.online).toHaveBeenCalledTimes(3)
+ })
+
+ it('opens the real lane later when its first token read throws', async () => {
+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1', handleProtocols: (protocols) => [...protocols][0] ?? false })
+ await new Promise<void>((resolve) => wss.once('listening', () => resolve()))
+ const frames: Array<{ type?: string; payload?: { machineId?: string } }> = []
+ wss.on('connection', (ws) => { ws.on('message', (raw) => { frames.push(JSON.parse(raw.toString())) }) })
+ stops.push(() => new Promise<void>((resolve) => { for (const client of wss.clients) client.terminate(); wss.close(() => resolve()) }))
+ let fails = 1
+ const lines: string[] = []
+ const link = new DeviceLink({
+ auth: { accessToken: async () => { if (fails-- > 0) throw new Error('UNAVAILABLE'); return 'token' } },
+ backendWsBase: `ws://127.0.0.1:${(wss.address() as { port: number }).port}`,
+ computerId: 'this-computer', autonomousEnv: 'test',
+ seal: { hello: vi.fn(), welcome: vi.fn(), rekey: vi.fn(), seal: vi.fn(), open: vi.fn(), drop: vi.fn() } as never,
+ peer: () => null, localMachineId: () => 'mine', log: (line) => { lines.push(line) },
+ })
+ stops.push(() => link.stop())
+ const rows = [{ machineId: 'mine', name: 'MacbookPro.local', state: 'ready', authMode: 'remote', local: true }]
+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,
+ link, hasPeerLink: () => false, log: (line) => { lines.push(line) } })
+ const host = new DaemonCableHost(wiring({ log: (line: string) => { lines.push(line) } }), fleet)
+ host.onDialAttached()
+ const deadline = Date.now() + 4_000
+ while (!frames.some((frame) => frame.type === 'machine_select')) {
+ if (Date.now() > deadline) throw new Error(`the lane never opened: ${lines.join(' | ')}`)
+ await new Promise((resolve) => setTimeout(resolve, 20))
+ }
+ expect(lines).toContain('cable: could not open the lane (UNAVAILABLE)')
+ expect(frames.filter((frame) => frame.type === 'machines_watch')).toHaveLength(1)
+ expect(frames.find((frame) => frame.type === 'machine_select')?.payload?.machineId).toBe('mine')
+ expect(link.status()).toBe('ready')
+ }, 10_000)
+})
diff --git a/cli/src/cable/cableHost.ts b/cli/src/cable/cableHost.ts
index e34723798..74740aae1 100644
--- a/cli/src/cable/cableHost.ts
+++ b/cli/src/cable/cableHost.ts
@@ -150,6 +150,13 @@ function isPlaceholder(id: string): boolean {
/** How long a keepalive may be answered from the last manifest read — see firmwareFor. */
const RELEASE_KEEP_MS = 30 * 60_000
+/**
+ * The wait before trying the plug-in's lane again, doubling to the cap: the same backoff as the lane's own reconnect
+ * (deviceLink.ts reconnectLater). Diego, 11-01: a first open lost to a refresh that failed in a network blip was
+ * never tried again, and the wheel had no other Mac until the dial was replugged.
+ */
+const LANE_RETRY_MS = 1_000
+const LANE_RETRY_MAX_MS = 30_000
export class DaemonCableHost implements CableHost {
/**
@@ -173,6 +180,12 @@ export class DaemonCableHost implements CableHost {
*/
private selecting = ''
+ /** Counts plug-ins: a lane open that lands after an unplug or a replug belongs to an older one and does nothing. */
+ private plugIn = 0
+ /** The next try at the lane for this plug-in, after an open that failed; cleared by an unplug or a replug. */
+ private laneRetry: NodeJS.Timeout | null = null
+ private laneTries = 0
+
/** The last good manifest read per OTA key, for keepalives — see firmwareFor. */
private readonly releases = new Map<string, { release: FirmwareRelease; at: number }>()
@@ -271,24 +284,59 @@ export class DaemonCableHost implements CableHost {
*/
onDialAttached(): void {
this.wiring.watching?.(true)
+ this.stopLaneRetry()
+ this.laneTries = 0
+ this.openLane(++this.plugIn, 'cable: dial on the wire — opening the cloud lane')
+ }
+
+ private openLane(plugIn: number, line: string): void {
if (!this.viaFleet((r) => r.hasLane())) return
// Signed out there is no lane to open: the socket is authenticated, so dialling it would fail once
// per plug-in and log a failure for something nobody asked for. The dial still works — it is on the
- // cable, and everything it shows on this computer is served in-process.
+ // cable, and everything it shows on this computer is served in-process. A retry stops here too.
if (this.wiring.signedIn?.() === false) {
this.wiring.log('cable: dial on the wire — this computer only (not signed in)')
return
}
- this.wiring.log('cable: dial on the wire — opening the cloud lane')
+ this.wiring.log(line)
void (async () => {
// The socket first, and unconditionally: it is what makes the machine wheel's dots live, and it is
// held for as long as the dial is plugged in whether or not anything is selected.
- const online = await this.viaFleet((r) => r.online())
- if (!online.ok) throw new Error(online.message)
+ let online: { ok: true } | { ok: false; message: string }
+ try {
+ online = await this.viaFleet((r) => r.online())
+ } catch (err) {
+ online = { ok: false, message: (err as Error).message }
+ }
+ if (!online.ok) {
+ this.wiring.log(`cable: could not open the lane (${online.message})`)
+ // Tried again with a backoff while this dial stays plugged in: a token read that throws leaves the lane no
+ // socket, so nothing else would (deviceLink.ts readToken).
+ this.retryLane(plugIn)
+ return
+ }
+ // Unplugged or replugged while it opened: the newer plug-in announces for itself, and an unplugged dial is
+ // nowhere to announce.
+ if (plugIn !== this.plugIn) return
+ this.laneTries = 0
await this.announceSelection()
})().catch((err) => this.wiring.log(`cable: could not open the lane (${(err as Error).message})`))
}
+ private retryLane(plugIn: number): void {
+ if (plugIn !== this.plugIn) return
+ this.stopLaneRetry()
+ const delay = Math.min(LANE_RETRY_MAX_MS, LANE_RETRY_MS * 2 ** Math.min(this.laneTries++, 5))
+ this.laneRetry = setTimeout(() => {
+ this.laneRetry = null
+ if (plugIn === this.plugIn) this.openLane(plugIn, 'cable: trying the cloud lane again')
+ }, delay)
+ }
+
+ private stopLaneRetry(): void {
+ if (this.laneRetry) { clearTimeout(this.laneRetry); this.laneRetry = null }
+ }
+
/**
* Tell the backend which machine the dial is on.
*
@@ -318,6 +366,8 @@ export class DaemonCableHost implements CableHost {
* a screen that is not there.
*/
onDialGone(): void {
+ this.plugIn++
+ this.stopLaneRetry()
this.wiring.watching?.(false)
this.wiring.clearSelection?.()
this.wiring.clearVisit?.()
diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.ts
index da15d13a6..c5f168758 100644
--- a/cli/src/device/deviceLink.spec.ts
+++ b/cli/src/device/deviceLink.spec.ts
@@ -1251,3 +1251,120 @@ describe('the lane around a select', () => {
expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2]) // still there: nothing to send
})
})
+
+describe('a sign-in refresh that fails while the lane dials', () => {
+ /** A token read that throws `fails` times (a refresh lost to a network blip, as UNAVAILABLE), then reads. */
+ const flaky = (fails: number) => {
+ let left = fails
+ return { accessToken: vi.fn(async () => {
+ if (left > 0) { left -= 1; throw new Error('UNAVAILABLE: the sign-in could not be refreshed') }
+ return 'token-1'
+ }) }
+ }
+ /** A token read held open until the test lets it go. */
+ const held = () => {
+ const waiting: Array<(token: string) => void> = []
+ return { waiting, auth: { accessToken: vi.fn(() => new Promise<string>((resolve) => { waiting.push(resolve) })) } }
+ }
+
+ it('goes back to idle, not connecting forever, when the first token read throws', async () => {
+ const { link, server } = await lane({ auth: flaky(1) })
+ expect(await link.online().then(() => 'ok', (error: Error) => error.message)).toMatch(/UNAVAILABLE/)
+ expect(link.status()).toBe('idle')
+ expect(server.sockets).toHaveLength(0)
+ })
+
+ it('tries again after a reconnect whose token read fails once: a second socket within the backoff', async () => {
+ const auth = { accessToken: vi.fn(async () => 'token-1') }
+ const { link, server, lines } = await lane({ auth })
+ await link.online()
+ // The network blips: the backend drops the socket, and the refresh the reconnect needs fails once.
+ auth.accessToken.mockRejectedValueOnce(new Error('UNAVAILABLE: the sign-in could not be refreshed'))
+ server.closeCurrent(1012)
+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)
+ expect(link.status()).toBe('idle')
+ // 1 s for the first try, 2 s for the next: well inside 5 s.
+ await until('the second socket', () => server.sockets.length === 2 && link.status() === 'ready', 5_000)
+ expect(server.received.filter((frame) => frame.type === 'machines_watch')).toHaveLength(2)
+ }, 12_000)
+
+ it('stops trying once the dial is unplugged during the wait', async () => {
+ const auth = { accessToken: vi.fn(async () => 'token-1') }
+ const { link, server, lines } = await lane({ auth })
+ await link.online()
+ auth.accessToken.mockRejectedValue(new Error('UNAVAILABLE'))
+ server.closeCurrent(1012)
+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)
+ link.release()
+ const reads = auth.accessToken.mock.calls.length
+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past the next try's backoff
+ expect(auth.accessToken.mock.calls.length).toBe(reads)
+ expect(server.sockets).toHaveLength(1)
+ expect(link.status()).toBe('idle')
+ }, 12_000)
+
+ it('opens no socket when the dial is unplugged while the token is still being read', async () => {
+ const { waiting, auth } = held()
+ const { link, server } = await lane({ auth })
+ const opened = link.online().then(() => 'ok', (error: Error) => error.message)
+ await until('the token read', () => waiting.length === 1)
+ link.release()
+ waiting[0]!('token-1')
+ expect(await opened).not.toBe('ok')
+ await new Promise((resolve) => setTimeout(resolve, 100))
+ expect(server.sockets).toHaveLength(0)
+ expect(link.status()).toBe('idle')
+ })
+
+ it('reads one token and opens one socket for two callers that dial at once', async () => {
+ const { waiting, auth } = held()
+ const { link, server } = await lane({ auth })
+ const first = link.online()
+ const second = link.online()
+ await until('the token read', () => waiting.length >= 1)
+ await new Promise((resolve) => setTimeout(resolve, 20))
+ expect(waiting).toHaveLength(1)
+ waiting[0]!('token-1')
+ await Promise.all([first, second])
+ await new Promise((resolve) => setTimeout(resolve, 100))
+ expect(server.sockets).toHaveLength(1)
+ expect(link.status()).toBe('ready')
+ })
+
+ it('stops, as not authorized, when the token read says the session is over rather than unavailable', async () => {
+ const over = (code: string) => Object.assign(new Error('sign in again'), { code })
+ const auth = { accessToken: vi.fn(async () => 'token-1') }
+ const { link, server, lines } = await lane({ auth })
+ await link.online()
+ auth.accessToken.mockRejectedValue(over('INVALID_REFRESH'))
+ server.closeCurrent(1012)
+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)
+ expect(link.status()).toBe('error')
+ const reads = auth.accessToken.mock.calls.length
+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past the next try's backoff
+ expect(auth.accessToken.mock.calls.length).toBe(reads)
+ expect(server.sockets).toHaveLength(1)
+ // Signed out on a first dial reads the same: not authorized, not connecting.
+ const fresh = await lane({ auth: { accessToken: vi.fn(async () => { throw over('MISSING') }) } })
+ expect(await fresh.link.online().then(() => 'ok', (error: Error) => error.message)).toBe('sign in again')
+ expect(fresh.link.status()).toBe('error')
+ }, 12_000)
+
+ it('does not keep dialling after the backend says not authorized during a reconnect', async () => {
+ const { link, server } = await lane()
+ // The backend reads selects and says nothing back, so the reconnect's attach is still waiting when the 4401 lands.
+ await link.online()
+ const selects = () => server.received.filter((frame) => frame.type === 'machine_select').length
+ const attached = link.attach(CLOUD).catch(() => {})
+ await until('the select', () => selects() === 1)
+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })
+ await attached
+ server.closeCurrent(1012)
+ await until('the reconnect select', () => selects() === 2, 4_000)
+ server.closeCurrent(4401)
+ await until('not authorized', () => link.status() === 'error', 2_000)
+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past any backoff
+ expect(server.sockets).toHaveLength(2)
+ expect(link.status()).toBe('error')
+ }, 12_000)
+})
diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.ts
index c206b49cf..0ad96a472 100644
--- a/cli/src/device/deviceLink.ts
+++ b/cli/src/device/deviceLink.ts
@@ -51,6 +51,13 @@ const RPC_TIMEOUT_MS = 15_000
* a machine nobody could reach. Callers say nothing of it as a failure (deviceFleet.ts, cableHost.ts).
*/
export const SELECT_SUPERSEDED = 'SELECT_SUPERSEDED'
+/** What a dial fails with when the lane was let go while its token was read (an unplug, the daemon stopping). */
+const LET_GO = 'the lane was let go while it dialled'
+/**
+ * The token read's codes for a session that is over (authSession.ts AuthSessionError): signed out, a refresh the backend
+ * refused, SSO asking for a new sign-in. Trying again cannot fix them; only UNAVAILABLE (a refresh lost to a blip) can.
+ */
+const SESSION_OVER = new Set(['MISSING', 'INVALID_REFRESH', 'SIGN_IN_AGAIN'])
/**
* How long an agent list may take: a background read nobody is watching a spinner for, refreshed every few seconds
* (fleetRouter.ts). Diego, 10-30: a slow but working Mac answered its lists 5 to 25 s after the 15 s every request had,
@@ -158,6 +165,11 @@ export class DeviceLink {
private wanted = ''
private attempts = 0
private reconnectTimer: NodeJS.Timeout | null = null
+ /**
+ * The dial in progress, from its token read to its socket opening. A caller meanwhile joins it: two dials at once
+ * would read two tokens and open two sockets, and the one replaced would stay open with nobody to close it.
+ */
+ private dialing: Promise<void> | null = null
private pingTimer: NodeJS.Timeout | null = null
private state: DeviceLinkStatus = 'idle'
/** The dial is plugged in, so this socket should exist even with no machine selected. */
@@ -622,6 +634,14 @@ export class DeviceLink {
private async connect(): Promise<void> {
if (this.ws?.readyState === WebSocket.OPEN) return
if (this.ws?.readyState === WebSocket.CONNECTING) return this.waitOpen()
+ if (!this.dialing) {
+ const dialing: Promise<void> = this.dial().finally(() => { if (this.dialing === dialing) this.dialing = null })
+ this.dialing = dialing
+ }
+ return this.dialing
+ }
+
+ private async dial(): Promise<void> {
// A socket still closing (its close event not yet fired) is torn down here, before the next one replaces it: its
// close is ignored once it is not the lane's socket (open()), so nothing else would end its sessions, fail what
// waits on it, or stop its ping, and the next socket would start with all of it (review 2 on #2402).
@@ -635,7 +655,7 @@ export class DeviceLink {
+ `&label=${encodeURIComponent(hostname())}`
+ `&autonomousEnv=${encodeURIComponent(this.opts.autonomousEnv)}`
- let token = await this.opts.auth.accessToken()
+ let token = await this.readToken()
this.opts.log(`device: connecting → ${base}/api/device-ws`)
try {
await this.open(url, token)
@@ -643,11 +663,43 @@ export class DeviceLink {
// Exactly ONE forced refresh, and only for a 401. Anything else is not a token problem, and a
// refresh loop against a backend that is simply down burns the refresh token for nothing.
if (!/401/.test(String((err as Error).message))) throw err
- token = await this.opts.auth.accessToken({ force: true, failedToken: token })
+ token = await this.readToken({ force: true, failedToken: token })
await this.open(url, token)
}
}
+ /**
+ * The token for the next socket, read before that socket exists. A read that throws (a refresh lost to the same
+ * network blip, as UNAVAILABLE) leaves no socket, so no close event follows to reset the lane or schedule the next
+ * try: the lane goes back to idle here, and the caller retries (reconnectLater, cableHost). A lane let go while the
+ * read was out (the dial unplugged, the daemon stopping) opens nothing with the token it got.
+ */
+ private async readToken(options?: { force: true; failedToken: string }): Promise<string> {
+ let token: string
+ try {
+ token = await (options ? this.opts.auth.accessToken(options) : this.opts.auth.accessToken())
+ } catch (err) {
+ this.noSocket()
+ // As a 4401 after the one refresh (onClose): stop, and let the cabled machine carry on. Retrying a session that
+ // is over would read for nothing every 30 s until someone signs in again.
+ if (!this.ws && SESSION_OVER.has(String((err as { code?: unknown }).code))) {
+ this.state = 'error'
+ this.opts.log('device: not signed in — other machines are unavailable until the next sign-in')
+ }
+ throw err
+ }
+ if (!this.wanted && !this.heldOpen) {
+ this.noSocket()
+ throw new Error(LET_GO)
+ }
+ return token
+ }
+
+ /** No socket came of this dial: idle, not connecting forever. One that did exist reset it already (teardown). */
+ private noSocket(): void {
+ if (!this.ws && this.state === 'connecting') this.state = 'idle'
+ }
+
private open(url: string, token: string): Promise<void> {
return new Promise<void>((resolve, reject) => {
const ws = new WebSocket(url, [token])
@@ -914,8 +966,17 @@ export class DeviceLink {
const wanted = this.wanted
if (!wanted && !this.heldOpen) return
const retry = wanted ? this.attach(wanted) : this.online()
- // Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.
- void retry.catch((err) => { if ((err as Error).message !== SELECT_SUPERSEDED) this.opts.log(`device: reconnect failed (${(err as Error).message})`) })
+ void retry.catch((err) => {
+ // Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.
+ if ((err as Error).message === SELECT_SUPERSEDED) return
+ this.opts.log(`device: reconnect failed (${(err as Error).message})`)
+ // No socket came of this try (its token read threw), so no close will schedule the next one: it is scheduled
+ // here, with the backoff, while the lane is still wanted. A socket that failed already scheduled its own
+ // (onClose), another dial may be under way, and a lane the backend refused (4401/4403) stays refused.
+ if (!this.wanted && !this.heldOpen) return
+ if (this.ws || this.reconnectTimer || this.dialing || this.state === 'error') return
+ this.reconnectLater()
+ })
}, delay)
}