Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Device lane: the plug-in retry stops when the session is over and runs one backoff (Diego, #2483 follow-up) (#2489)

0xken15d1089a82cd23
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.ts
index 46b837e80..f5422d2b8 100644
--- a/cli/src/cable/cableHost.spec.ts
+++ b/cli/src/cable/cableHost.spec.ts
@@ -1234,7 +1234,7 @@ describe('a plug-in whose first lane open fails', () => {
const failing = (fails: number) => {
let left = fails
const fleet = fleetOf([REMOTE])
- fleet.online = vi.fn(async () => { if (left-- > 0) throw new Error('UNAVAILABLE: the sign-in could not be refreshed') })
+ fleet.online = vi.fn(async () => { if (left-- > 0) throw Object.assign(new Error('UNAVAILABLE: the sign-in could not be refreshed'), { code: 'TOKEN_UNAVAILABLE' }) })
fleet.select = vi.fn(async () => {})
fleet.release = vi.fn()
return fleet
@@ -1316,6 +1316,59 @@ describe('a plug-in whose first lane open fails', () => {
expect(fleet.online).toHaveBeenCalledTimes(3)
})
+ it('stops at once when the session is over, and says why', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(0)
+ fleet.online = vi.fn(async () => { throw Object.assign(new Error('sign in again'), { code: 'SESSION_OVER' }) })
+ const log = vi.fn()
+ const host = new DaemonCableHost(wiring({ log }), fleet)
+ host.onDialAttached()
+ await turns()
+ await vi.advanceTimersByTimeAsync(120_000)
+ expect(fleet.online).toHaveBeenCalledTimes(1)
+ expect(log).toHaveBeenCalledWith('cable: could not open the lane (sign in again) — other machines are unavailable until the next sign-in')
+ expect(log).not.toHaveBeenCalledWith('cable: trying the cloud lane again')
+ })
+
+ it('leaves a socket that failed to the lane\'s own reconnect: one backoff, not two', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const fleet = failing(0)
+ fleet.online = vi.fn(async () => { throw Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:1'), { code: 'ECONNREFUSED' }) })
+ const log = vi.fn()
+ const host = new DaemonCableHost(wiring({ log }), fleet)
+ host.onDialAttached()
+ await turns()
+ await vi.advanceTimersByTimeAsync(120_000)
+ expect(fleet.online).toHaveBeenCalledTimes(1)
+ expect(log).toHaveBeenCalledWith('cable: could not open the lane (connect ECONNREFUSED 127.0.0.1:1)')
+ })
+
+ it('runs one backoff over the real lane when the backend is down at the first plug-in', async () => {
+ // A port nothing listens on: every dial is refused at once, as with the backend down.
+ const probe = new WebSocketServer({ port: 0, host: '127.0.0.1' })
+ await new Promise<void>((resolve) => probe.once('listening', () => resolve()))
+ const port = (probe.address() as { port: number }).port
+ await new Promise<void>((resolve) => probe.close(() => resolve()))
+ const lines: string[] = []
+ const link = new DeviceLink({
+ auth: { accessToken: async () => 'token' }, backendWsBase: `ws://127.0.0.1:${port}`,
+ computerId: 'this-computer', autonomousEnv: 'test',
+ seal: { hello: vi.fn(), welcome: vi.fn(), rekey: vi.fn(), seal: vi.fn(), open: vi.fn(), drop: vi.fn() } as never,
+ peer: () => null, localMachineId: () => 'mine', log: (line) => { lines.push(line) },
+ })
+ stops.push(() => link.stop())
+ const rows = [{ machineId: 'mine', name: 'MacbookPro.local', state: 'ready', authMode: 'remote', local: true }]
+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,
+ link, hasPeerLink: () => false, log: (line) => { lines.push(line) } })
+ const host = new DaemonCableHost(wiring({ log: (line: string) => { lines.push(line) } }), fleet)
+ host.onDialAttached()
+ await new Promise((resolve) => setTimeout(resolve, 3_500)) // the first dial, then the lane's own at 1 s and 3 s
+ const dials = lines.filter((line) => line.startsWith('device: connecting'))
+ expect(dials.length).toBeGreaterThanOrEqual(2) // the lane did try again by itself
+ expect(dials.length).toBeLessThanOrEqual(3)
+ expect(lines).not.toContain('cable: trying the cloud lane again')
+ }, 10_000)
+
it('opens the real lane later when its first token read throws', async () => {
const wss = new WebSocketServer({ port: 0, host: '127.0.0.1', handleProtocols: (protocols) => [...protocols][0] ?? false })
await new Promise<void>((resolve) => wss.once('listening', () => resolve()))
diff --git a/cli/src/cable/cableHost.ts b/cli/src/cable/cableHost.ts
index 74740aae1..700ce1c33 100644
--- a/cli/src/cable/cableHost.ts
+++ b/cli/src/cable/cableHost.ts
@@ -35,7 +35,7 @@ import type { SelectionCommand, SelectionResult } from './windowSelection.js'
import type { VisitCommand, VisitResult } from './windowVisit.js'
import type { FormCommand, FormResult } from './windowForm.js'
import type { PetStore } from './pets/store.js'
-import type { MachineFleet } from './machineFleet.js'
+import { LANE_SESSION_OVER, LANE_TOKEN_UNAVAILABLE, type MachineFleet } from './machineFleet.js'
import type { ReviewedAnswer, AnswerReceipt } from './questionInbox.js'
import { isTier, type Tier } from '../lib/notifyTier.js'
@@ -302,17 +302,24 @@ export class DaemonCableHost implements CableHost {
void (async () => {
// The socket first, and unconditionally: it is what makes the machine wheel's dots live, and it is
// held for as long as the dial is plugged in whether or not anything is selected.
- let online: { ok: true } | { ok: false; message: string }
+ let online: { ok: true } | { ok: false; message: string; code?: string }
try {
online = await this.viaFleet((r) => r.online())
} catch (err) {
online = { ok: false, message: (err as Error).message }
}
if (!online.ok) {
+ // A session that is over is not tried again: each try would be a refused refresh and two log lines every 30 s
+ // until someone signs in (security review 2 on #2483). The next sign-in and plug-in open it.
+ if (online.code === LANE_SESSION_OVER) {
+ this.wiring.log(`cable: could not open the lane (${online.message}) — other machines are unavailable until the next sign-in`)
+ return
+ }
this.wiring.log(`cable: could not open the lane (${online.message})`)
- // Tried again with a backoff while this dial stays plugged in: a token read that throws leaves the lane no
- // socket, so nothing else would (deviceLink.ts readToken).
- this.retryLane(plugIn)
+ // Only a token that could not be read is tried again from here: it left the lane no socket, so nothing else
+ // would (deviceLink.ts readToken). A socket that failed is the lane's own reconnect's, and trying it here too
+ // ran two backoffs against a backend that is down (security review 2 on #2483).
+ if (online.code === LANE_TOKEN_UNAVAILABLE) this.retryLane(plugIn)
return
}
// Unplugged or replugged while it opened: the newer plug-in announces for itself, and an unplugged dial is
diff --git a/cli/src/cable/machineFleet.ts b/cli/src/cable/machineFleet.ts
index b2ec062c1..fb358d688 100644
--- a/cli/src/cable/machineFleet.ts
+++ b/cli/src/cable/machineFleet.ts
@@ -12,6 +12,18 @@ import type { CableAgent } from './cableSession.js'
import type { RecentTurn } from './cableHost.js'
/** One of the owner's machines, as the fleet knows it. The local row is added by the host, not here. */
+/**
+ * Why `online()` failed, as the `code` on what it rejects with, so the dial's plug-in retry (cableHost.ts) tries again
+ * only what nothing else will (follow-up to #2483, both security reviews):
+ * - LANE_TOKEN_UNAVAILABLE: the token could not be read just now (a refresh lost to a network blip). No socket was
+ * made, so the lane has no close to retry from: only the caller can try again.
+ * - LANE_SESSION_OVER: the session is over (signed out, a refresh refused, SSO asking for a new sign-in). Trying again
+ * cannot fix it; the next sign-in and plug-in do.
+ * Any other failure was the socket's, and the lane's own reconnect retries it (deviceLink.ts onClose).
+ */
+export const LANE_TOKEN_UNAVAILABLE = 'TOKEN_UNAVAILABLE'
+export const LANE_SESSION_OVER = 'SESSION_OVER'
+
export interface FleetMachine {
machineId: string
name: string
diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.ts
index c5f168758..ee21654b4 100644
--- a/cli/src/device/deviceLink.spec.ts
+++ b/cli/src/device/deviceLink.spec.ts
@@ -1350,6 +1350,24 @@ describe('a sign-in refresh that fails while the lane dials', () => {
expect(fresh.link.status()).toBe('error')
}, 12_000)
+ it('says why a dial failed, so the plug-in retries only what nothing else will', async () => {
+ const codeOf = (promise: Promise<void>) => promise.then(() => 'ok', (error: Error & { code?: unknown }) => error.code)
+ // A token that could not be read just now: no socket, so only the caller can try again.
+ const blip = await lane({ auth: flaky(1) })
+ expect(await codeOf(blip.link.online())).toBe('TOKEN_UNAVAILABLE')
+ // A session that is over: trying again cannot fix it.
+ const over = await lane({ auth: { accessToken: vi.fn(async () => { throw Object.assign(new Error('sign in again'), { code: 'SIGN_IN_AGAIN' }) }) } })
+ const failed = await over.link.online().then(() => null, (error: Error & { code?: unknown }) => error)
+ expect(failed?.code).toBe('SESSION_OVER')
+ expect(failed?.message).toBe('sign in again')
+ // A backend that is not there: the socket failed, and the lane's own reconnect has that one.
+ const down = await lane()
+ await down.server.close()
+ const refused = await codeOf(down.link.online())
+ expect(['TOKEN_UNAVAILABLE', 'SESSION_OVER']).not.toContain(refused)
+ await until('the lane\'s own reconnect', () => (down.link as unknown as { reconnectTimer: unknown }).reconnectTimer !== null, 2_000)
+ })
+
it('does not keep dialling after the backend says not authorized during a reconnect', async () => {
const { link, server } = await lane()
// The backend reads selects and says nothing back, so the reconnect's attach is still waiting when the 4401 lands.
diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.ts
index 0ad96a472..02d264f56 100644
--- a/cli/src/device/deviceLink.ts
+++ b/cli/src/device/deviceLink.ts
@@ -34,6 +34,7 @@ import type { LaneSeal } from '../core/api.js'
import type { MachinePeer } from '../lib/e2ee/machinePeers.js'
import { isWrapped } from '../lib/relayFrames.js'
import { VERSION } from '../version.js'
+import { LANE_SESSION_OVER, LANE_TOKEN_UNAVAILABLE } from '../cable/machineFleet.js'
/** One frame on the wire. The backend's own vocabulary, unchanged. */
export interface DeviceFrame {
@@ -680,13 +681,17 @@ export class DeviceLink {
token = await (options ? this.opts.auth.accessToken(options) : this.opts.auth.accessToken())
} catch (err) {
this.noSocket()
+ const message = err instanceof Error ? err.message : String(err)
// As a 4401 after the one refresh (onClose): stop, and let the cabled machine carry on. Retrying a session that
// is over would read for nothing every 30 s until someone signs in again.
if (!this.ws && SESSION_OVER.has(String((err as { code?: unknown }).code))) {
this.state = 'error'
this.opts.log('device: not signed in — other machines are unavailable until the next sign-in')
+ throw Object.assign(new Error(message), { code: LANE_SESSION_OVER })
}
- throw err
+ // The caller retries this one (machineFleet.ts), unless the lane's own reconnect is already set: after a 401 the
+ // socket that failed scheduled it (onClose), and a second loop would dial twice per backoff.
+ throw Object.assign(new Error(message), this.reconnectTimer ? {} : { code: LANE_TOKEN_UNAVAILABLE })
}
if (!this.wanted && !this.heldOpen) {
this.noSocket()
diff --git a/cli/src/services/fleet.ts b/cli/src/services/fleet.ts
index 7e44ad0bb..22e569a87 100644
--- a/cli/src/services/fleet.ts
+++ b/cli/src/services/fleet.ts
@@ -61,8 +61,9 @@ export interface FleetRouting {
forkAgent(agentId: string): Promise<ForkOutcome>
/** Whether a lane to the other machines exists: signed in, with the fleet's own fleet. */
hasLane(): boolean
- /** Hold the lane for a dial, attached to no machine. Resolves with how it went; never rejects. */
- online(): Promise<{ ok: true } | { ok: false; message: string }>
+ /** Hold the lane for a dial, attached to no machine. Resolves with how it went, and why when the lane said
+ * (cable/machineFleet.ts LANE_*); never rejects. */
+ online(): Promise<{ ok: true } | { ok: false; message: string; code?: string }>
select(machineId: string): Promise<SelectResult>
release(immediate?: boolean): void
}
diff --git a/cli/src/services/fleetRouter.spec.ts b/cli/src/services/fleetRouter.spec.ts
index d535aa0c2..bd5ce1250 100644
--- a/cli/src/services/fleetRouter.spec.ts
+++ b/cli/src/services/fleetRouter.spec.ts
@@ -592,6 +592,18 @@ describe('the lane, held while a dial is plugged in', () => {
expect(await router.select('other')).toEqual({ ok: false, code: 'NEEDS_LINK', message: 'Link office-imac to this computer first' })
expect(await router.select('third')).toEqual({ ok: false, code: 'UNREACHABLE', message: 'socket closed' })
})
+
+ it('passes on why the lane would not open, so the dial knows whether to try again', async () => {
+ const fleet = fleetOf([REMOTE])
+ const router = new FleetRouter(localOf(), fleet)
+ fleet.online = vi.fn(async () => { throw Object.assign(new Error('the sign-in could not be refreshed'), { code: 'TOKEN_UNAVAILABLE' }) })
+ expect(await router.online()).toEqual({ ok: false, message: 'the sign-in could not be refreshed', code: 'TOKEN_UNAVAILABLE' })
+ fleet.online = vi.fn(async () => { throw Object.assign(new Error('sign in again'), { code: 'SESSION_OVER' }) })
+ expect(await router.online()).toEqual({ ok: false, message: 'sign in again', code: 'SESSION_OVER' })
+ // Only a word: anything else on the error stays behind.
+ fleet.online = vi.fn(async () => { throw Object.assign(new Error('odd'), { code: 42 }) })
+ expect(await router.online()).toEqual({ ok: false, message: 'odd' })
+ })
})
describe('an answer reviewed on a device', () => {
diff --git a/cli/src/services/fleetRouter.ts b/cli/src/services/fleetRouter.ts
index 4f0444eb8..3fb50e360 100644
--- a/cli/src/services/fleetRouter.ts
+++ b/cli/src/services/fleetRouter.ts
@@ -367,12 +367,14 @@ export class FleetRouter implements FleetRouting {
/** Come online for a dial — see MachineFleet.online. A lane that will not open is an answer: the dial
* says so in its log and carries on with this computer. */
- async online(): Promise<{ ok: true } | { ok: false; message: string }> {
+ async online(): Promise<{ ok: true } | { ok: false; message: string; code?: string }> {
try {
await this.fleet!.online()
return { ok: true }
} catch (err) {
- return { ok: false, message: (err as Error).message }
+ // Why it failed, when the lane says (machineFleet.ts LANE_*): whether the dial tries again turns on it.
+ const code = (err as { code?: unknown }).code
+ return { ok: false, message: (err as Error).message, ...(typeof code === 'string' && code ? { code } : {}) }
}
}