The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(backend): refuse a relay's select of an offline machine at once, with a retry-after (#1798)
0xken1594818fdcdb0a
diff --git a/backend/src/lib/webWs.ts b/backend/src/lib/webWs.tsindex c9ab991d9..8707ad313 100644--- a/backend/src/lib/webWs.ts+++ b/backend/src/lib/webWs.ts@@ -73,6 +73,10 @@ const wss = createWss(WS_LIMITS.web, { echoFirstProtocol: true })const DEVICE_RESEED_MS = 30_000// Trailing debounce for list re-derivations poked by machines_watch (see scheduleWatchAgents).const WATCH_DEBOUNCE_MS = 250+/** How long a relay refused for an offline machine waits before it selects that machine again: a daemon's relay+ * re-dialled offline machines every few seconds, 68% of 1,884 selects in 10 minutes (prod, 2026-10-27, #314). The+ * Desktop's own longest reconnect backoff, so a machine coming back is seen no later than it was. */+export const OFFLINE_RETRY_AFTER_MS = 30_000/** Who is on the other end of a per-user socket, as the client declares it: the surface (`?client=`), and* for the phone the release (`?release=`, the public x.y.z) and whether the build is a debug or simulator@@ -327,7 +331,7 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'wesend({ type: 'node_status', payload: { online, status: online ? 'running' : lifecycle === 'unknown' ? 'offline' : lifecycle } })})(), 'web-ws seed node_status', { userId: user.sub, machineId })}- const bindAgent = async (machineIdRaw: unknown): Promise<void> => {+ const bindAgent = async (machineIdRaw: unknown, refuseOffline: boolean): Promise<void> => {if (typeof machineIdRaw !== 'string' || !machineIdRaw) returnconst machineId = machineIdRawconst gen = ++selectGen@@ -369,6 +373,17 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'we// Idempotent same-agent re-select keeps the connId, but still passed through ensureMachineReady so// selecting a deliberately stopped machine wakes it before the readiness ack.if (client && machineId === currentAgentId) { currentBinding = binding; ack(machineId); return }+ // A daemon's relay (a socket that declares no phone or web client) cannot use an offline machine: it hung up on the+ // ack half a second later and dialled again, every daemon already out there included. Refused here instead, before+ // a hub client and an ack, in the machine_select_error every daemon already turns into a failed dial. A phone or+ // the web viewer waits attached for the machine to come back, as before.+ if ((refuseOffline || !surface) && !await machineOnline(machineId, !!await getAgentPresence(machineId))) {+ if (!closed && gen === selectGen) {+ send({ type: 'machine_select_error', payload: { machineId, error: 'MACHINE_OFFLINE', retryAfterMs: OFFLINE_RETRY_AFTER_MS } })+ }+ return+ }+ if (closed || gen !== selectGen) returnclient?.detach()// A guest never hears the machine's broadcasts, only what the machine addresses to it (lib/hub.ts).client = attachHubClient(ws, machineId, 'web', { guest: !!shared })@@ -481,8 +496,8 @@ function attachUserClient(ws: WebSocket, user: AuthUser, surface: 'mobile' | 'we}}if (type === 'machine_select') {- const machineId = (frame.payload as { machineId?: unknown } | undefined)?.machineId- await bindAgent(machineId)+ const payload = frame.payload as { machineId?: unknown; offline?: unknown } | undefined+ await bindAgent(payload?.machineId, payload?.offline === 'refuse')return}if (type === 'machine_deselect') {diff --git a/backend/src/lib/webWsOffline.test.ts b/backend/src/lib/webWsOffline.test.tsnew file mode 100644index 000000000..73b3a4f0c--- /dev/null+++ b/backend/src/lib/webWsOffline.test.ts@@ -0,0 +1,87 @@+import { EventEmitter } from 'node:events'+import { beforeEach, describe, expect, it, vi } from 'vitest'++// A daemon's relay selecting a machine with no socket to the backend (#314). Production saw 1,884 selects in 10 minutes,+// 68% of them to offline machines: each one set up the whole relay (hub client, ack, presence reads) only for the+// daemon to hang up half a second later. A relay (any socket that declares no phone or web client, so every daemon+// already out there) is refused at once, with how long to wait; a phone or the web viewer still waits attached.+const m = vi.hoisted(() => ({+ auth: vi.fn(), upgrade: vi.fn(), presence: vi.fn(), online: vi.fn(), attach: vi.fn(),+ machineFindUnique: vi.fn(),+}))+vi.mock('./ssoAuth.js', async original => ({ ...await original<typeof import('./ssoAuth.js')>(), authenticateAccessToken: m.auth }))+vi.mock('./wsServer.js', async original => ({ ...await original<typeof import('./wsServer.js')>(), createWss: () => ({ handleUpgrade: m.upgrade }) }))+vi.mock('./prisma.js', () => ({+ prisma: { machine: { findUnique: m.machineFindUnique, findMany: vi.fn(async () => []) }, deviceBinding: { findMany: vi.fn(async () => []) } },+ machineAlive: { OR: [{ deletedAt: null }, { deletedAt: { isSet: false } }] },+}))+vi.mock('./bus.js', () => ({+ getAgentPresence: m.presence, getAgentPresenceMany: vi.fn(async () => new Map()), subscribeStatus: vi.fn(async () => () => {}),+ getDevicePresence: vi.fn(async () => false), subscribeDeviceStatus: vi.fn(async () => () => {}),+ subscribeDeviceMachineListChanged: vi.fn(async () => () => {}), subscribeDeviceE2eePair: vi.fn(async () => () => {}),+ subscribeMachineShareChanged: vi.fn(async () => () => {}),+}))+vi.mock('./providerLink.js', () => ({ machineOnline: m.online }))+vi.mock('./machineLifecycle.js', () => ({ ensureMachineReady: vi.fn(async () => {}), getMachineLifecycleStatus: vi.fn(async () => 'offline') }))+vi.mock('./webAccountPushes.js', () => ({ relayWebDocumentPushes: vi.fn(async () => () => {}) }))+vi.mock('./hub.js', () => ({ attachHubClient: m.attach, trackSocketLiveness: () => () => {} }))++import { handleWebUpgrade, OFFLINE_RETRY_AFTER_MS } from './webWs.js'++const owner = { sub: 'owner', email: '[email protected]', role: 'user', autonomousEnv: 'prod' }+const mac = { machineId: 'owner-mac', userId: 'owner', autonomousEnv: 'prod', billingStatus: 'not_required', authMode: 'remote', deletedAt: null }+const flush = async () => { for (let i = 0; i < 400; i++) await Promise.resolve() }++class Socket extends EventEmitter {+ readyState = 1; bufferedAmount = 0+ send = vi.fn()+ close = vi.fn(() => { this.readyState = 3; this.emit('close') })+ terminate = vi.fn(() => this.close())+ frame(body: unknown) { this.emit('message', Buffer.from(JSON.stringify(body)), false) }+ sent() { return this.send.mock.calls.map(c => JSON.parse(c[0] as string)) }+}+const raw = () => ({ write: vi.fn(), destroy: vi.fn(), destroyed: false, on: vi.fn(), once: vi.fn() })++async function selects(payload: Record<string, unknown>, query = ''): Promise<Socket> {+ const ws = new Socket()+ m.upgrade.mockImplementation((_r, _s, _h, done) => done(ws))+ handleWebUpgrade({ url: `/api/web-ws?autonomousEnv=prod${query}`, headers: { 'sec-websocket-protocol': 'access-token' } } as never, raw() as never, Buffer.alloc(0))+ await flush()+ ws.frame({ type: 'machine_select', payload: { machineId: 'owner-mac', ...payload } })+ await flush()+ return ws+}++beforeEach(() => {+ vi.clearAllMocks()+ m.auth.mockResolvedValue(owner)+ m.machineFindUnique.mockResolvedValue(mac)+ m.attach.mockReturnValue({ connId: 'conn', detach: vi.fn() })+ m.presence.mockResolvedValue(null)+ m.online.mockResolvedValue(false)+})++describe('a relay selecting an offline machine', () => {+ it.each([['an older daemon', {}], ['a daemon that asks', { offline: 'refuse' }]])('%s is refused at once with how long to wait, and no relay is set up', async (_who, payload) => {+ const ws = await selects(payload)+ expect(ws.sent()).toContainEqual({ type: 'machine_select_error',+ payload: { machineId: 'owner-mac', error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 } })+ expect(OFFLINE_RETRY_AFTER_MS).toBe(30_000)+ expect(m.attach).not.toHaveBeenCalled()+ expect(ws.sent().some(f => f.type === 'connected' && f.payload?.machineId)).toBe(false)+ })++ it('goes through when the machine is online', async () => {+ m.presence.mockResolvedValue('manager')+ m.online.mockResolvedValue(true)+ const ws = await selects({})+ expect(m.attach).toHaveBeenCalledOnce()+ expect(ws.sent()).toContainEqual(expect.objectContaining({ type: 'connected', payload: expect.objectContaining({ machineId: 'owner-mac' }) }))+ })+})++it.each(['mobile', 'web'])('a %s client still attaches to an offline machine and waits for it', async client => {+ const ws = await selects({}, `&client=${client}`)+ expect(m.attach).toHaveBeenCalledOnce()+ expect(ws.sent()).toContainEqual(expect.objectContaining({ type: 'connected', payload: expect.objectContaining({ machineId: 'owner-mac' }) }))+})