Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(backend): refuse a relay's select of an offline machine at once, with a retry-after (#1798)

0xken1594818fdcdb0a
diff --git a/backend/src/lib/webWs.ts b/backend/src/lib/webWs.ts
index c9ab991d9..8707ad313 100644
--- a/backend/src/lib/webWs.ts
+++ b/backend/src/lib/webWs.ts
@@ -73,6 +73,10 @@ const wss = createWss(WS_LIMITS.web, { echoFirstProtocol: true })
const DEVICE_RESEED_MS = 30_000
// Trailing debounce for list re-derivations poked by machines_watch (see scheduleWatchAgents).
const WATCH_DEBOUNCE_MS = 250
+/** How long a relay refused for an offline machine waits before it selects that machine again: a daemon's relay
+ * re-dialled offline machines every few seconds, 68% of 1,884 selects in 10 minutes (prod, 2026-10-27, #314). The
+ * Desktop's own longest reconnect backoff, so a machine coming back is seen no later than it was. */
+export const OFFLINE_RETRY_AFTER_MS = 30_000
/** Who is on the other end of a per-user socket, as the client declares it: the surface (`?client=`), and
* for the phone the release (`?release=`, the public x.y.z) and whether the build is a debug or simulator
@@ -327,7 +331,7 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'we
send({ type: 'node_status', payload: { online, status: online ? 'running' : lifecycle === 'unknown' ? 'offline' : lifecycle } })
})(), 'web-ws seed node_status', { userId: user.sub, machineId })
}
- const bindAgent = async (machineIdRaw: unknown): Promise<void> => {
+ const bindAgent = async (machineIdRaw: unknown, refuseOffline: boolean): Promise<void> => {
if (typeof machineIdRaw !== 'string' || !machineIdRaw) return
const machineId = machineIdRaw
const gen = ++selectGen
@@ -369,6 +373,17 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'we
// Idempotent same-agent re-select keeps the connId, but still passed through ensureMachineReady so
// selecting a deliberately stopped machine wakes it before the readiness ack.
if (client && machineId === currentAgentId) { currentBinding = binding; ack(machineId); return }
+ // A daemon's relay (a socket that declares no phone or web client) cannot use an offline machine: it hung up on the
+ // ack half a second later and dialled again, every daemon already out there included. Refused here instead, before
+ // a hub client and an ack, in the machine_select_error every daemon already turns into a failed dial. A phone or
+ // the web viewer waits attached for the machine to come back, as before.
+ if ((refuseOffline || !surface) && !await machineOnline(machineId, !!await getAgentPresence(machineId))) {
+ if (!closed && gen === selectGen) {
+ send({ type: 'machine_select_error', payload: { machineId, error: 'MACHINE_OFFLINE', retryAfterMs: OFFLINE_RETRY_AFTER_MS } })
+ }
+ return
+ }
+ if (closed || gen !== selectGen) return
client?.detach()
// A guest never hears the machine's broadcasts, only what the machine addresses to it (lib/hub.ts).
client = attachHubClient(ws, machineId, 'web', { guest: !!shared })
@@ -481,8 +496,8 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'we
}
}
if (type === 'machine_select') {
- const machineId = (frame.payload as { machineId?: unknown } | undefined)?.machineId
- await bindAgent(machineId)
+ const payload = frame.payload as { machineId?: unknown; offline?: unknown } | undefined
+ await bindAgent(payload?.machineId, payload?.offline === 'refuse')
return
}
if (type === 'machine_deselect') {
diff --git a/backend/src/lib/webWsOffline.test.ts b/backend/src/lib/webWsOffline.test.ts
new file mode 100644
index 000000000..73b3a4f0c
--- /dev/null
+++ b/backend/src/lib/webWsOffline.test.ts
@@ -0,0 +1,87 @@
+import { EventEmitter } from 'node:events'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+// A daemon's relay selecting a machine with no socket to the backend (#314). Production saw 1,884 selects in 10 minutes,
+// 68% of them to offline machines: each one set up the whole relay (hub client, ack, presence reads) only for the
+// daemon to hang up half a second later. A relay (any socket that declares no phone or web client, so every daemon
+// already out there) is refused at once, with how long to wait; a phone or the web viewer still waits attached.
+const m = vi.hoisted(() => ({
+ auth: vi.fn(), upgrade: vi.fn(), presence: vi.fn(), online: vi.fn(), attach: vi.fn(),
+ machineFindUnique: vi.fn(),
+}))
+vi.mock('./ssoAuth.js', async original => ({ ...await original<typeof import('./ssoAuth.js')>(), authenticateAccessToken: m.auth }))
+vi.mock('./wsServer.js', async original => ({ ...await original<typeof import('./wsServer.js')>(), createWss: () => ({ handleUpgrade: m.upgrade }) }))
+vi.mock('./prisma.js', () => ({
+ prisma: { machine: { findUnique: m.machineFindUnique, findMany: vi.fn(async () => []) }, deviceBinding: { findMany: vi.fn(async () => []) } },
+ machineAlive: { OR: [{ deletedAt: null }, { deletedAt: { isSet: false } }] },
+}))
+vi.mock('./bus.js', () => ({
+ getAgentPresence: m.presence, getAgentPresenceMany: vi.fn(async () => new Map()), subscribeStatus: vi.fn(async () => () => {}),
+ getDevicePresence: vi.fn(async () => false), subscribeDeviceStatus: vi.fn(async () => () => {}),
+ subscribeDeviceMachineListChanged: vi.fn(async () => () => {}), subscribeDeviceE2eePair: vi.fn(async () => () => {}),
+ subscribeMachineShareChanged: vi.fn(async () => () => {}),
+}))
+vi.mock('./providerLink.js', () => ({ machineOnline: m.online }))
+vi.mock('./machineLifecycle.js', () => ({ ensureMachineReady: vi.fn(async () => {}), getMachineLifecycleStatus: vi.fn(async () => 'offline') }))
+vi.mock('./webAccountPushes.js', () => ({ relayWebDocumentPushes: vi.fn(async () => () => {}) }))
+vi.mock('./hub.js', () => ({ attachHubClient: m.attach, trackSocketLiveness: () => () => {} }))
+
+import { handleWebUpgrade, OFFLINE_RETRY_AFTER_MS } from './webWs.js'
+
+const owner = { sub: 'owner', email: '[email protected]', role: 'user', autonomousEnv: 'prod' }
+const mac = { machineId: 'owner-mac', userId: 'owner', autonomousEnv: 'prod', billingStatus: 'not_required', authMode: 'remote', deletedAt: null }
+const flush = async () => { for (let i = 0; i < 400; i++) await Promise.resolve() }
+
+class Socket extends EventEmitter {
+ readyState = 1; bufferedAmount = 0
+ send = vi.fn()
+ close = vi.fn(() => { this.readyState = 3; this.emit('close') })
+ terminate = vi.fn(() => this.close())
+ frame(body: unknown) { this.emit('message', Buffer.from(JSON.stringify(body)), false) }
+ sent() { return this.send.mock.calls.map(c => JSON.parse(c[0] as string)) }
+}
+const raw = () => ({ write: vi.fn(), destroy: vi.fn(), destroyed: false, on: vi.fn(), once: vi.fn() })
+
+async function selects(payload: Record<string, unknown>, query = ''): Promise<Socket> {
+ const ws = new Socket()
+ m.upgrade.mockImplementation((_r, _s, _h, done) => done(ws))
+ handleWebUpgrade({ url: `/api/web-ws?autonomousEnv=prod${query}`, headers: { 'sec-websocket-protocol': 'access-token' } } as never, raw() as never, Buffer.alloc(0))
+ await flush()
+ ws.frame({ type: 'machine_select', payload: { machineId: 'owner-mac', ...payload } })
+ await flush()
+ return ws
+}
+
+beforeEach(() => {
+ vi.clearAllMocks()
+ m.auth.mockResolvedValue(owner)
+ m.machineFindUnique.mockResolvedValue(mac)
+ m.attach.mockReturnValue({ connId: 'conn', detach: vi.fn() })
+ m.presence.mockResolvedValue(null)
+ m.online.mockResolvedValue(false)
+})
+
+describe('a relay selecting an offline machine', () => {
+ it.each([['an older daemon', {}], ['a daemon that asks', { offline: 'refuse' }]])('%s is refused at once with how long to wait, and no relay is set up', async (_who, payload) => {
+ const ws = await selects(payload)
+ expect(ws.sent()).toContainEqual({ type: 'machine_select_error',
+ payload: { machineId: 'owner-mac', error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 } })
+ expect(OFFLINE_RETRY_AFTER_MS).toBe(30_000)
+ expect(m.attach).not.toHaveBeenCalled()
+ expect(ws.sent().some(f => f.type === 'connected' && f.payload?.machineId)).toBe(false)
+ })
+
+ it('goes through when the machine is online', async () => {
+ m.presence.mockResolvedValue('manager')
+ m.online.mockResolvedValue(true)
+ const ws = await selects({})
+ expect(m.attach).toHaveBeenCalledOnce()
+ expect(ws.sent()).toContainEqual(expect.objectContaining({ type: 'connected', payload: expect.objectContaining({ machineId: 'owner-mac' }) }))
+ })
+})
+
+it.each(['mobile', 'web'])('a %s client still attaches to an offline machine and waits for it', async client => {
+ const ws = await selects({}, `&client=${client}`)
+ expect(m.attach).toHaveBeenCalledOnce()
+ expect(ws.sent()).toContainEqual(expect.objectContaining({ type: 'connected', payload: expect.objectContaining({ machineId: 'owner-mac' }) }))
+})