The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Device lane: a sign-in refresh that fails during reconnect tries again instead of giving up (Diego) (#2483)
0xken15b4289d771c5c
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.tsindex d884db126..46b837e80 100644--- a/cli/src/cable/cableHost.spec.ts+++ b/cli/src/cable/cableHost.spec.ts@@ -1219,3 +1219,132 @@ describe('announcing over the real lane', () => {expect(host.selectedMachine()).toBe('mine')})})++/**+ * A plug-in whose first lane open fails — Diego, 11-01: the sign-in refresh failed in a network blip, nothing tried+ * again, and the wheel lost every other Mac until the dial was replugged.+ */+describe('a plug-in whose first lane open fails', () => {+ const stops: Array<() => Promise<void> | void> = []+ afterEach(async () => {+ vi.useRealTimers()+ for (const stop of stops.splice(0)) await stop()+ })+ /** A bare fleet whose online() throws `fails` times, as a token read lost to a blip does, then opens. */+ const failing = (fails: number) => {+ let left = fails+ const fleet = fleetOf([REMOTE])+ fleet.online = vi.fn(async () => { if (left-- > 0) throw new Error('UNAVAILABLE: the sign-in could not be refreshed') })+ fleet.select = vi.fn(async () => {})+ fleet.release = vi.fn()+ return fleet+ }+ const turns = async () => { for (let turn = 0; turn < 20; turn++) await new Promise((resolve) => setImmediate(resolve)) }++ it('opens the lane later, and then announces where the dial is', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ const fleet = failing(2)+ const log = vi.fn()+ const host = new DaemonCableHost(wiring({ log }), fleet)+ host.onDialAttached()+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(1)+ expect(log).toHaveBeenCalledWith('cable: could not open the lane (UNAVAILABLE: the sign-in could not be refreshed)')+ expect(fleet.select).not.toHaveBeenCalled()+ await vi.advanceTimersByTimeAsync(1_000)+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(2) // failed again: the next try waits longer+ await vi.advanceTimersByTimeAsync(1_999)+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(1)+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(3)+ expect(fleet.select).toHaveBeenCalledWith('mine')+ // Open now: nothing more is scheduled.+ await vi.advanceTimersByTimeAsync(60_000)+ expect(fleet.online).toHaveBeenCalledTimes(3)+ })++ it('stops trying once the dial is unplugged', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ const fleet = failing(Infinity)+ const host = new DaemonCableHost(wiring(), fleet)+ host.onDialAttached()+ await turns()+ host.onDialGone()+ await vi.advanceTimersByTimeAsync(120_000)+ expect(fleet.online).toHaveBeenCalledTimes(1)+ })++ it('stops trying once this computer is signed out', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ const fleet = failing(Infinity)+ let signedIn = true+ const host = new DaemonCableHost(wiring({ signedIn: () => signedIn }), fleet)+ host.onDialAttached()+ await turns()+ signedIn = false+ await vi.advanceTimersByTimeAsync(120_000)+ expect(fleet.online).toHaveBeenCalledTimes(1)+ })++ it('announces nothing for an open that lands after the dial was unplugged', async () => {+ let finish: () => void = () => {}+ const fleet = failing(0)+ fleet.online = vi.fn(() => new Promise<void>((resolve) => { finish = resolve }))+ const host = new DaemonCableHost(wiring(), fleet)+ host.onDialAttached()+ await turns()+ host.onDialGone()+ finish()+ await turns()+ expect(fleet.select).not.toHaveBeenCalled()+ })++ it('keeps one retry for a replug while one is waiting, not two', async () => {+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ const fleet = failing(Infinity)+ const host = new DaemonCableHost(wiring(), fleet)+ host.onDialAttached()+ await turns()+ host.onDialAttached() // the session re-reports the dial without an unplug in between+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(2)+ await vi.advanceTimersByTimeAsync(1_000)+ await turns()+ expect(fleet.online).toHaveBeenCalledTimes(3)+ })++ it('opens the real lane later when its first token read throws', async () => {+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1', handleProtocols: (protocols) => [...protocols][0] ?? false })+ await new Promise<void>((resolve) => wss.once('listening', () => resolve()))+ const frames: Array<{ type?: string; payload?: { machineId?: string } }> = []+ wss.on('connection', (ws) => { ws.on('message', (raw) => { frames.push(JSON.parse(raw.toString())) }) })+ stops.push(() => new Promise<void>((resolve) => { for (const client of wss.clients) client.terminate(); wss.close(() => resolve()) }))+ let fails = 1+ const lines: string[] = []+ const link = new DeviceLink({+ auth: { accessToken: async () => { if (fails-- > 0) throw new Error('UNAVAILABLE'); return 'token' } },+ backendWsBase: `ws://127.0.0.1:${(wss.address() as { port: number }).port}`,+ computerId: 'this-computer', autonomousEnv: 'test',+ seal: { hello: vi.fn(), welcome: vi.fn(), rekey: vi.fn(), seal: vi.fn(), open: vi.fn(), drop: vi.fn() } as never,+ peer: () => null, localMachineId: () => 'mine', log: (line) => { lines.push(line) },+ })+ stops.push(() => link.stop())+ const rows = [{ machineId: 'mine', name: 'MacbookPro.local', state: 'ready', authMode: 'remote', local: true }]+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,+ link, hasPeerLink: () => false, log: (line) => { lines.push(line) } })+ const host = new DaemonCableHost(wiring({ log: (line: string) => { lines.push(line) } }), fleet)+ host.onDialAttached()+ const deadline = Date.now() + 4_000+ while (!frames.some((frame) => frame.type === 'machine_select')) {+ if (Date.now() > deadline) throw new Error(`the lane never opened: ${lines.join(' | ')}`)+ await new Promise((resolve) => setTimeout(resolve, 20))+ }+ expect(lines).toContain('cable: could not open the lane (UNAVAILABLE)')+ expect(frames.filter((frame) => frame.type === 'machines_watch')).toHaveLength(1)+ expect(frames.find((frame) => frame.type === 'machine_select')?.payload?.machineId).toBe('mine')+ expect(link.status()).toBe('ready')+ }, 10_000)+})diff --git a/cli/src/cable/cableHost.ts b/cli/src/cable/cableHost.tsindex e34723798..74740aae1 100644--- a/cli/src/cable/cableHost.ts+++ b/cli/src/cable/cableHost.ts@@ -150,6 +150,13 @@ function isPlaceholder(id: string): boolean {/** How long a keepalive may be answered from the last manifest read — see firmwareFor. */const RELEASE_KEEP_MS = 30 * 60_000+/**+ * The wait before trying the plug-in's lane again, doubling to the cap: the same backoff as the lane's own reconnect+ * (deviceLink.ts reconnectLater). Diego, 11-01: a first open lost to a refresh that failed in a network blip was+ * never tried again, and the wheel had no other Mac until the dial was replugged.+ */+const LANE_RETRY_MS = 1_000+const LANE_RETRY_MAX_MS = 30_000export class DaemonCableHost implements CableHost {/**@@ -173,6 +180,12 @@ export class DaemonCableHost implements CableHost {*/private selecting = ''+ /** Counts plug-ins: a lane open that lands after an unplug or a replug belongs to an older one and does nothing. */+ private plugIn = 0+ /** The next try at the lane for this plug-in, after an open that failed; cleared by an unplug or a replug. */+ private laneRetry: NodeJS.Timeout | null = null+ private laneTries = 0+/** The last good manifest read per OTA key, for keepalives — see firmwareFor. */private readonly releases = new Map<string, { release: FirmwareRelease; at: number }>()@@ -271,24 +284,59 @@ export class DaemonCableHost implements CableHost {*/onDialAttached(): void {this.wiring.watching?.(true)+ this.stopLaneRetry()+ this.laneTries = 0+ this.openLane(++this.plugIn, 'cable: dial on the wire — opening the cloud lane')+ }++ private openLane(plugIn: number, line: string): void {if (!this.viaFleet((r) => r.hasLane())) return// Signed out there is no lane to open: the socket is authenticated, so dialling it would fail once// per plug-in and log a failure for something nobody asked for. The dial still works — it is on the- // cable, and everything it shows on this computer is served in-process.+ // cable, and everything it shows on this computer is served in-process. A retry stops here too.if (this.wiring.signedIn?.() === false) {this.wiring.log('cable: dial on the wire — this computer only (not signed in)')return}- this.wiring.log('cable: dial on the wire — opening the cloud lane')+ this.wiring.log(line)void (async () => {// The socket first, and unconditionally: it is what makes the machine wheel's dots live, and it is// held for as long as the dial is plugged in whether or not anything is selected.- const online = await this.viaFleet((r) => r.online())- if (!online.ok) throw new Error(online.message)+ let online: { ok: true } | { ok: false; message: string }+ try {+ online = await this.viaFleet((r) => r.online())+ } catch (err) {+ online = { ok: false, message: (err as Error).message }+ }+ if (!online.ok) {+ this.wiring.log(`cable: could not open the lane (${online.message})`)+ // Tried again with a backoff while this dial stays plugged in: a token read that throws leaves the lane no+ // socket, so nothing else would (deviceLink.ts readToken).+ this.retryLane(plugIn)+ return+ }+ // Unplugged or replugged while it opened: the newer plug-in announces for itself, and an unplugged dial is+ // nowhere to announce.+ if (plugIn !== this.plugIn) return+ this.laneTries = 0await this.announceSelection()})().catch((err) => this.wiring.log(`cable: could not open the lane (${(err as Error).message})`))}+ private retryLane(plugIn: number): void {+ if (plugIn !== this.plugIn) return+ this.stopLaneRetry()+ const delay = Math.min(LANE_RETRY_MAX_MS, LANE_RETRY_MS * 2 ** Math.min(this.laneTries++, 5))+ this.laneRetry = setTimeout(() => {+ this.laneRetry = null+ if (plugIn === this.plugIn) this.openLane(plugIn, 'cable: trying the cloud lane again')+ }, delay)+ }++ private stopLaneRetry(): void {+ if (this.laneRetry) { clearTimeout(this.laneRetry); this.laneRetry = null }+ }+/*** Tell the backend which machine the dial is on.*@@ -318,6 +366,8 @@ export class DaemonCableHost implements CableHost {* a screen that is not there.*/onDialGone(): void {+ this.plugIn+++ this.stopLaneRetry()this.wiring.watching?.(false)this.wiring.clearSelection?.()this.wiring.clearVisit?.()diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.tsindex da15d13a6..c5f168758 100644--- a/cli/src/device/deviceLink.spec.ts+++ b/cli/src/device/deviceLink.spec.ts@@ -1251,3 +1251,120 @@ describe('the lane around a select', () => {expect(selectsOf(server)).toEqual([CLOUD, CLOUD_2]) // still there: nothing to send})})++describe('a sign-in refresh that fails while the lane dials', () => {+ /** A token read that throws `fails` times (a refresh lost to a network blip, as UNAVAILABLE), then reads. */+ const flaky = (fails: number) => {+ let left = fails+ return { accessToken: vi.fn(async () => {+ if (left > 0) { left -= 1; throw new Error('UNAVAILABLE: the sign-in could not be refreshed') }+ return 'token-1'+ }) }+ }+ /** A token read held open until the test lets it go. */+ const held = () => {+ const waiting: Array<(token: string) => void> = []+ return { waiting, auth: { accessToken: vi.fn(() => new Promise<string>((resolve) => { waiting.push(resolve) })) } }+ }++ it('goes back to idle, not connecting forever, when the first token read throws', async () => {+ const { link, server } = await lane({ auth: flaky(1) })+ expect(await link.online().then(() => 'ok', (error: Error) => error.message)).toMatch(/UNAVAILABLE/)+ expect(link.status()).toBe('idle')+ expect(server.sockets).toHaveLength(0)+ })++ it('tries again after a reconnect whose token read fails once: a second socket within the backoff', async () => {+ const auth = { accessToken: vi.fn(async () => 'token-1') }+ const { link, server, lines } = await lane({ auth })+ await link.online()+ // The network blips: the backend drops the socket, and the refresh the reconnect needs fails once.+ auth.accessToken.mockRejectedValueOnce(new Error('UNAVAILABLE: the sign-in could not be refreshed'))+ server.closeCurrent(1012)+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)+ expect(link.status()).toBe('idle')+ // 1 s for the first try, 2 s for the next: well inside 5 s.+ await until('the second socket', () => server.sockets.length === 2 && link.status() === 'ready', 5_000)+ expect(server.received.filter((frame) => frame.type === 'machines_watch')).toHaveLength(2)+ }, 12_000)++ it('stops trying once the dial is unplugged during the wait', async () => {+ const auth = { accessToken: vi.fn(async () => 'token-1') }+ const { link, server, lines } = await lane({ auth })+ await link.online()+ auth.accessToken.mockRejectedValue(new Error('UNAVAILABLE'))+ server.closeCurrent(1012)+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)+ link.release()+ const reads = auth.accessToken.mock.calls.length+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past the next try's backoff+ expect(auth.accessToken.mock.calls.length).toBe(reads)+ expect(server.sockets).toHaveLength(1)+ expect(link.status()).toBe('idle')+ }, 12_000)++ it('opens no socket when the dial is unplugged while the token is still being read', async () => {+ const { waiting, auth } = held()+ const { link, server } = await lane({ auth })+ const opened = link.online().then(() => 'ok', (error: Error) => error.message)+ await until('the token read', () => waiting.length === 1)+ link.release()+ waiting[0]!('token-1')+ expect(await opened).not.toBe('ok')+ await new Promise((resolve) => setTimeout(resolve, 100))+ expect(server.sockets).toHaveLength(0)+ expect(link.status()).toBe('idle')+ })++ it('reads one token and opens one socket for two callers that dial at once', async () => {+ const { waiting, auth } = held()+ const { link, server } = await lane({ auth })+ const first = link.online()+ const second = link.online()+ await until('the token read', () => waiting.length >= 1)+ await new Promise((resolve) => setTimeout(resolve, 20))+ expect(waiting).toHaveLength(1)+ waiting[0]!('token-1')+ await Promise.all([first, second])+ await new Promise((resolve) => setTimeout(resolve, 100))+ expect(server.sockets).toHaveLength(1)+ expect(link.status()).toBe('ready')+ })++ it('stops, as not authorized, when the token read says the session is over rather than unavailable', async () => {+ const over = (code: string) => Object.assign(new Error('sign in again'), { code })+ const auth = { accessToken: vi.fn(async () => 'token-1') }+ const { link, server, lines } = await lane({ auth })+ await link.online()+ auth.accessToken.mockRejectedValue(over('INVALID_REFRESH'))+ server.closeCurrent(1012)+ await until('the failed reconnect', () => lines.some((line) => line.startsWith('device: reconnect failed')), 3_000)+ expect(link.status()).toBe('error')+ const reads = auth.accessToken.mock.calls.length+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past the next try's backoff+ expect(auth.accessToken.mock.calls.length).toBe(reads)+ expect(server.sockets).toHaveLength(1)+ // Signed out on a first dial reads the same: not authorized, not connecting.+ const fresh = await lane({ auth: { accessToken: vi.fn(async () => { throw over('MISSING') }) } })+ expect(await fresh.link.online().then(() => 'ok', (error: Error) => error.message)).toBe('sign in again')+ expect(fresh.link.status()).toBe('error')+ }, 12_000)++ it('does not keep dialling after the backend says not authorized during a reconnect', async () => {+ const { link, server } = await lane()+ // The backend reads selects and says nothing back, so the reconnect's attach is still waiting when the 4401 lands.+ await link.online()+ const selects = () => server.received.filter((frame) => frame.type === 'machine_select').length+ const attached = link.attach(CLOUD).catch(() => {})+ await until('the select', () => selects() === 1)+ server.send({ type: 'machine_selected', payload: { machineId: CLOUD } })+ await attached+ server.closeCurrent(1012)+ await until('the reconnect select', () => selects() === 2, 4_000)+ server.closeCurrent(4401)+ await until('not authorized', () => link.status() === 'error', 2_000)+ await new Promise((resolve) => setTimeout(resolve, 2_500)) // past any backoff+ expect(server.sockets).toHaveLength(2)+ expect(link.status()).toBe('error')+ }, 12_000)+})diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.tsindex c206b49cf..0ad96a472 100644--- a/cli/src/device/deviceLink.ts+++ b/cli/src/device/deviceLink.ts@@ -51,6 +51,13 @@ const RPC_TIMEOUT_MS = 15_000* a machine nobody could reach. Callers say nothing of it as a failure (deviceFleet.ts, cableHost.ts).*/export const SELECT_SUPERSEDED = 'SELECT_SUPERSEDED'+/** What a dial fails with when the lane was let go while its token was read (an unplug, the daemon stopping). */+const LET_GO = 'the lane was let go while it dialled'+/**+ * The token read's codes for a session that is over (authSession.ts AuthSessionError): signed out, a refresh the backend+ * refused, SSO asking for a new sign-in. Trying again cannot fix them; only UNAVAILABLE (a refresh lost to a blip) can.+ */+const SESSION_OVER = new Set(['MISSING', 'INVALID_REFRESH', 'SIGN_IN_AGAIN'])/*** How long an agent list may take: a background read nobody is watching a spinner for, refreshed every few seconds* (fleetRouter.ts). Diego, 10-30: a slow but working Mac answered its lists 5 to 25 s after the 15 s every request had,@@ -158,6 +165,11 @@ export class DeviceLink {private wanted = ''private attempts = 0private reconnectTimer: NodeJS.Timeout | null = null+ /**+ * The dial in progress, from its token read to its socket opening. A caller meanwhile joins it: two dials at once+ * would read two tokens and open two sockets, and the one replaced would stay open with nobody to close it.+ */+ private dialing: Promise<void> | null = nullprivate pingTimer: NodeJS.Timeout | null = nullprivate state: DeviceLinkStatus = 'idle'/** The dial is plugged in, so this socket should exist even with no machine selected. */@@ -622,6 +634,14 @@ export class DeviceLink {private async connect(): Promise<void> {if (this.ws?.readyState === WebSocket.OPEN) returnif (this.ws?.readyState === WebSocket.CONNECTING) return this.waitOpen()+ if (!this.dialing) {+ const dialing: Promise<void> = this.dial().finally(() => { if (this.dialing === dialing) this.dialing = null })+ this.dialing = dialing+ }+ return this.dialing+ }++ private async dial(): Promise<void> {// A socket still closing (its close event not yet fired) is torn down here, before the next one replaces it: its// close is ignored once it is not the lane's socket (open()), so nothing else would end its sessions, fail what// waits on it, or stop its ping, and the next socket would start with all of it (review 2 on #2402).@@ -635,7 +655,7 @@ export class DeviceLink {+ `&label=${encodeURIComponent(hostname())}`+ `&autonomousEnv=${encodeURIComponent(this.opts.autonomousEnv)}`- let token = await this.opts.auth.accessToken()+ let token = await this.readToken()this.opts.log(`device: connecting → ${base}/api/device-ws`)try {await this.open(url, token)@@ -643,11 +663,43 @@ export class DeviceLink {// Exactly ONE forced refresh, and only for a 401. Anything else is not a token problem, and a// refresh loop against a backend that is simply down burns the refresh token for nothing.if (!/401/.test(String((err as Error).message))) throw err- token = await this.opts.auth.accessToken({ force: true, failedToken: token })+ token = await this.readToken({ force: true, failedToken: token })await this.open(url, token)}}+ /**+ * The token for the next socket, read before that socket exists. A read that throws (a refresh lost to the same+ * network blip, as UNAVAILABLE) leaves no socket, so no close event follows to reset the lane or schedule the next+ * try: the lane goes back to idle here, and the caller retries (reconnectLater, cableHost). A lane let go while the+ * read was out (the dial unplugged, the daemon stopping) opens nothing with the token it got.+ */+ private async readToken(options?: { force: true; failedToken: string }): Promise<string> {+ let token: string+ try {+ token = await (options ? this.opts.auth.accessToken(options) : this.opts.auth.accessToken())+ } catch (err) {+ this.noSocket()+ // As a 4401 after the one refresh (onClose): stop, and let the cabled machine carry on. Retrying a session that+ // is over would read for nothing every 30 s until someone signs in again.+ if (!this.ws && SESSION_OVER.has(String((err as { code?: unknown }).code))) {+ this.state = 'error'+ this.opts.log('device: not signed in — other machines are unavailable until the next sign-in')+ }+ throw err+ }+ if (!this.wanted && !this.heldOpen) {+ this.noSocket()+ throw new Error(LET_GO)+ }+ return token+ }++ /** No socket came of this dial: idle, not connecting forever. One that did exist reset it already (teardown). */+ private noSocket(): void {+ if (!this.ws && this.state === 'connecting') this.state = 'idle'+ }+private open(url: string, token: string): Promise<void> {return new Promise<void>((resolve, reject) => {const ws = new WebSocket(url, [token])@@ -914,8 +966,17 @@ export class DeviceLink {const wanted = this.wantedif (!wanted && !this.heldOpen) returnconst retry = wanted ? this.attach(wanted) : this.online()- // Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.- void retry.catch((err) => { if ((err as Error).message !== SELECT_SUPERSEDED) this.opts.log(`device: reconnect failed (${(err as Error).message})`) })+ void retry.catch((err) => {+ // Replaced by a newer select meanwhile is not a failed reconnect: the newer one carries on.+ if ((err as Error).message === SELECT_SUPERSEDED) return+ this.opts.log(`device: reconnect failed (${(err as Error).message})`)+ // No socket came of this try (its token read threw), so no close will schedule the next one: it is scheduled+ // here, with the backoff, while the lane is still wanted. A socket that failed already scheduled its own+ // (onClose), another dial may be under way, and a lane the backend refused (4401/4403) stays refused.+ if (!this.wanted && !this.heldOpen) return+ if (this.ws || this.reconnectTimer || this.dialing || this.state === 'error') return+ this.reconnectLater()+ })}, delay)}