The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(cli): a slow linked Mac that answers late stays reachable, with its session (#2381)
harness-for-autonomous[bot]14fff78142ac
diff --git a/cli/e2e/fleet.e2e.ts b/cli/e2e/fleet.e2e.tsindex e7fff10f5..64f23c9cf 100644--- a/cli/e2e/fleet.e2e.ts+++ b/cli/e2e/fleet.e2e.ts@@ -178,6 +178,30 @@ describe('two machines in one fleet', () => {onA.close()})+ // Diego, 10-30: a slow but working Mac was treated as dead. Its replies came 5 to 25 s after the 15 s every request+ // waited, two timeouts threw its session away, its agents left the carousel and turns to them were refused.+ it('B slow to answer stays on A\'s carousel with its session, and its turns go through', async () => {+ const f = await fresh({ dialOnA: true })+ const { a, b, backend } = f+ const { onA, onB, agentId } = await linked(f)+ const sessions = () => a.daemon.log().split(`[device] device: e2e session ready ${b.machineId}`).length - 1+ const before = sessions()+ // From here B answers 20 s late: past the 15 s a request used to wait, inside the 45 s an agent list waits now.+ backend.slowUp.set(b.machineId, 20_000)+ await new Promise((resolve) => setTimeout(resolve, 75_000))+ const log = a.daemon.log()+ expect(log).not.toContain('in a row — dropped its E2EE session')+ expect(log).not.toContain('dropped off the carousel')+ expect(sessions()).toBe(before)+ // Still on the carousel, still reachable: ⌘K picks it, and the turn reaches B.+ expect((await routeTask(onA, 'fix the parser')).agentId).toBe(agentId)+ const started = onB.next(isTurn('turn_started', agentId), 30_000, 'turn_started on B')+ expect(await routeSend(onA, agentId, 'to a slow machine b')).toMatchObject({ ok: true })+ expect((await started).payload?.userMessage).toBe('to a slow machine b')+ onA.close()+ onB.close()+ }, 300_000)+it('the dial\'s routing failing through the port leaves the dial serving A, and ⌘K serving both machines', async () => {const f = await fresh({ dialOnA: true, envA: { HARNESSD_TEST_FAULTS: 'fleet.sendTurn' } })const { a, b } = fdiff --git a/cli/e2e/harness/fakeBackend.ts b/cli/e2e/harness/fakeBackend.tsindex 299cd8b51..856950ce2 100644--- a/cli/e2e/harness/fakeBackend.ts+++ b/cli/e2e/harness/fakeBackend.ts@@ -135,6 +135,9 @@ export class FakeBackend {private down = false/** How long everything relayed between a machine and its web clients is held, ms. */latencyMs = 0+ /** How long what each machine sends up to the devices' lanes is held, ms, by machine id: a machine slow to answer+ * (Diego's, 10-30), not a slow relay. Its handshakes and replies come late alike; what goes down to it does not. */+ readonly slowUp = new Map<string, number>()/** What each web client sent towards its machine, as sent (sealed), by connection id. */readonly webSent = new Map<string, Frame[]>()/** What reached each web client from its machine, as relayed (sealed), by connection id. */@@ -617,13 +620,13 @@ export class FakeBackend {const tagged = { ...envelope.frame, machineId }if (envelope.targetConnId) {const owner = this.connOwners.get(envelope.targetConnId)- if (owner?.machineId === machineId) this.sendDevice(owner.device, tagged)+ if (owner?.machineId === machineId) this.upToDevice(owner.device, machineId, tagged)const web = [...this.webClients].find((client) => client.connId === envelope.targetConnId && client.machineId === machineId)if (web) this.toWeb(web, envelope.frame)if (!owner && !web) this.targeted.set(envelope.targetConnId, [...(this.targeted.get(envelope.targetConnId) ?? []), envelope.frame])return}- if (envelope.commanderEligible) for (const device of this.devices) this.sendDevice(device, tagged)+ if (envelope.commanderEligible) for (const device of this.devices) this.upToDevice(device, machineId, tagged)// The real hub: web clients get every broadcast that is not device-only.if (envelope.webEligible !== false) this.toWebClients(machineId, envelope.frame)}@@ -788,6 +791,13 @@ export class FakeBackend {if (connId) this.sendNode(machineId, connId, frame)}+ /** What a machine sends up to a device, later by `slowUp` if the machine is slow: equal delays fire in order. */+ private upToDevice(device: Device, machineId: string, frame: Frame): void {+ const late = this.slowUp.get(machineId) ?? 0+ if (late > 0) setTimeout(() => this.sendDevice(device, frame), late)+ else this.sendDevice(device, frame)+ }+private sendDevice(device: Device, frame: Frame): void {if (device.ws.readyState === WebSocket.OPEN) device.ws.send(json(frame))}diff --git a/cli/src/device/deviceFleet.spec.ts b/cli/src/device/deviceFleet.spec.tsindex 419999d80..24d50b1f0 100644--- a/cli/src/device/deviceFleet.spec.ts+++ b/cli/src/device/deviceFleet.spec.ts@@ -21,12 +21,14 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: strconst rows = listOf(over.rows ?? [{}])let frameCb: ((f: DeviceFrame) => void) | null = nulllet undeliveredCb: ((what: Undelivered) => void) | null = null+ let aliveCb: ((machineId: string) => void) | null = nullconst sent: DeviceFrame[] = []const resets: string[] = []const link = {selectedMachine: 'm1',onFrame: (cb: (f: DeviceFrame) => void) => { frameCb = cb; return () => {} },onUndelivered: (cb: (what: Undelivered) => void) => { undeliveredCb = cb; return () => {} },+ onAlive: (cb: (machineId: string) => void) => { aliveCb = cb; return () => {} },attach: vi.fn(async () => {}),online: vi.fn(async () => {}),detach: vi.fn(),@@ -45,7 +47,8 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: strhasPeerLink: (id: string) => (over.linked ?? ['m1']).includes(id),log: () => {},})- return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f), undelivered: (what: Undelivered) => undeliveredCb?.(what) }+ return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f), undelivered: (what: Undelivered) => undeliveredCb?.(what),+ alive: (machineId: string) => aliveCb?.(machineId) }}describe('DeviceFleet', () => {@@ -108,6 +111,49 @@ describe('DeviceFleet', () => {expect(resets).toEqual([])})+ // Diego, 10-30: a slow but working Mac was treated as dead. Each session drop was followed within 5-25 s by the late+ // reply it had given up on, from the same machine (42 drops, 20 unmatched agents_list_result in a day).+ it('keeps a slow machine reachable, with its session, once a late reply of its own comes back', async () => {+ const { fleet, resets, alive } = make({ rpc: async () => { throw new Error('agents_list timed out') } })+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })+ // The reply it timed out on arrives late, opened under its session (deviceLink.ts `onAlive`).+ alive('m1')+ expect(fleet.reachable('m1')).toMatchObject({ ok: true })+ // The count starts again: the next timeout is the first, not the second.+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')+ expect(resets).toEqual([])+ })++ it('counts no miss for a request its machine proved alive during, however it ended', async () => {+ let fail!: (error: Error) => void+ const { fleet, resets, alive } = make({ rpc: () => new Promise((_resolve, reject) => { fail = reject }) })+ for (const round of [1, 2, 3]) {+ const asked = fleet.listAgents('m1')+ alive('m1')+ fail(new Error(round === 3 ? 'AGENT_NOT_FOUND' : 'agents_list timed out'))+ await expect(asked).rejects.toThrow()+ expect(fleet.reachable('m1'), `round ${round}`).toMatchObject({ ok: true })+ }+ expect(resets).toEqual([])+ })++ it('still throws away the session of a machine that proves nothing, whoever else answers', async () => {+ let fail!: (error: Error) => void+ const { fleet, resets, alive } = make({ rows: [{}, { machineId: 'm2', name: 'studio' }], linked: ['m1', 'm2'],+ rpc: () => new Promise((_resolve, reject) => { fail = reject }) })+ for (const round of [1, 2]) {+ const asked = fleet.listAgents('m1')+ // Another machine answering while m1's request is out says nothing about m1.+ alive('m2')+ fail(new Error('agents_list timed out'))+ await expect(asked, `round ${round}`).rejects.toThrow()+ }+ expect(resets).toEqual(['m1'])+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })+ expect(fleet.reachable('m2')).toMatchObject({ ok: true })+ })+it('shows a turn, a stop or an answer that may not have reached its machine on that agent\'s tile, never inviting a blind resend', async () => {// Diego, 10-30: after the far daemon restarted, a dial turn vanished with nothing on the dial to say so.const { fleet, undelivered } = make()@@ -291,7 +337,7 @@ describe('DeviceFleet', () => {find: (id: string) => rows.find((r) => r.machineId === id),applyLive: (r: unknown) => { applied.push(r); return true },} as never,- link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {} } as never,+ link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {} } as never,hasPeerLink: () => true,log: () => {},})@@ -308,7 +354,7 @@ describe('DeviceFleet', () => {let frameCb!: (f: { type?: string; machineId?: string; payload?: Record<string, unknown> }) => voidconst fleet = new DeviceFleet({list: { list: () => ({ machines: [], source: 'backend' as const }), find: () => undefined, applyLive: () => true } as never,- link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {} } as never,+ link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {} } as never,hasPeerLink: () => true,log: () => {},})diff --git a/cli/src/device/deviceFleet.ts b/cli/src/device/deviceFleet.tsindex ada45f9a4..203379cf4 100644--- a/cli/src/device/deviceFleet.ts+++ b/cli/src/device/deviceFleet.ts@@ -89,10 +89,17 @@ export class DeviceFleet implements MachineFleet {private readonly health = new Map<string, { ok: boolean; at: number }>()/** Consecutive failed round trips per machine — the counter behind RESET_AFTER_FAILURES. */private readonly misses = new Map<string, number>()+ /**+ * How many times each machine has answered under its own session (DeviceLink.onAlive): a request it answered+ * anything during is no miss, however it ended. Diego, 10-30: a slow Mac's answers came 5 to 25 s after their+ * requests timed out, so every pair of timeouts threw away a session that was answering (42 drops in a day).+ */+ private readonly answers = new Map<string, number>()constructor(private readonly opts: DeviceFleetOpts) {this.opts.link.onFrame((frame) => this.onFrame(frame))this.opts.link.onUndelivered((what) => this.onUndelivered(what))+ this.opts.link.onAlive((machineId) => this.alive(machineId))}async list(): Promise<{ machines: FleetMachine[]; source: 'backend' | 'local' | 'signed-out' }> {@@ -198,6 +205,7 @@ export class DeviceFleet implements MachineFleet {* untouched — this is a witness, not a handler.*/private async rpc(machineId: string, type: string, payload: Record<string, unknown>, again = false): Promise<Record<string, unknown>> {+ const heard = this.answers.get(machineId) ?? 0try {const res = await this.opts.link.rpc(type, payload, machineId)this.health.set(machineId, { ok: true, at: Date.now() })@@ -207,6 +215,9 @@ export class DeviceFleet implements MachineFleet {// Its session went before the answer came (the far daemon restarted): a read is asked once more, on// the new session, and is no miss — nothing about the machine is known yet.if (!again && (err as Error).message === SESSION_GONE && READS.has(type)) return this.rpc(machineId, type, payload, true)+ // The machine answered something of its own while this was out (a late reply to an earlier request, or this+ // one's error): it is up, and slow or refusing, not gone. No miss, and its session stays.+ if ((this.answers.get(machineId) ?? 0) !== heard) throw errthis.health.set(machineId, { ok: false, at: Date.now() })const missed = (this.misses.get(machineId) ?? 0) + 1this.misses.set(machineId, missed)@@ -228,6 +239,13 @@ export class DeviceFleet implements MachineFleet {}}+ /** A machine answered under its own session (DeviceLink.onAlive): it is up, and its misses start again. */+ private alive(machineId: string): void {+ this.answers.set(machineId, (this.answers.get(machineId) ?? 0) + 1)+ this.health.set(machineId, { ok: true, at: Date.now() })+ this.misses.delete(machineId)+ }+/*** Whether the last thing we asked this machine came back.*diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.tsindex 9477ca923..66d05f45f 100644--- a/cli/src/device/deviceLink.spec.ts+++ b/cli/src/device/deviceLink.spec.ts@@ -778,3 +778,84 @@ describe('the lane\'s own bounds', () => {expect(heard.find((frame) => frame.agentId === 'a2')).toMatchObject({ type: 'commander_question' })})})++// Diego, 10-30: a slow but working Mac was treated as dead. Its late replies had no waiter and were only logged, and two+// timeouts in a row threw away a session that had just answered: 20 unmatched agents_list_result, 42 session drops.+describe('a late reply from a slow machine', () => {+ /** A lane with LINKED's session up, and what its proof-of-life hook heard. */+ async function upWithAlive() {+ const lane$ = await lane()+ await lane$.link.online()+ await lane$.link.establish(LINKED)+ const alive: string[] = []+ lane$.link.onAlive((machineId) => { alive.push(machineId) })+ return { ...lane$, alive }+ }++ it('is proof of life when it opens under the machine\'s own session, matched or not, once each', async () => {+ const { server, seal, link, lines, alive } = await upWithAlive()+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'timed-out-long-ago', agents: [] } } })+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 7 } } })+ await until('the late reply', () => lines.some((line) => line.includes('unmatched agents_list_result')))+ expect(alive).toEqual([LINKED])+ // An answer that came in time is proof too.+ const answer = link.rpc('models_list', {}, LINKED)+ await until('the request', () => seal.seal.mock.calls.some(([, frame]) => frame.type === 'models_list'))+ seal.open.mockResolvedValueOnce({ frame: { type: 'models_list_result', machineId: LINKED, payload: { requestId: askedId(seal, LINKED, 'models_list'), models: [] } } })+ server.send({ type: 'models_list_result', machineId: LINKED, payload: { __e2e: { n: 8 } } })+ await answer+ expect(alive).toEqual([LINKED, LINKED])+ })++ it('is no proof when the relay could have made it: in the clear, under the group key, unreadable, or for another session', async () => {+ const { server, seal, link, lines, alive, heard } = await upWithAlive()+ // In the clear: the relay can write any of these.+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'forged', agents: [] } })+ // Under the group key, which every commander of the machine holds.+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'group', agents: [] } } })+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { k: 'g', n: 1, epoch: 'e' } } })+ // Sealed, but not under this session's keys.+ seal.open.mockResolvedValueOnce({ unreadable: true })+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 9 } } })+ // From a machine with no session at all.+ server.send({ type: 'agents_list_result', machineId: CLOUD, payload: { requestId: 'cloud', agents: [] } })+ await until('every frame read', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 3+ && lines.some((line) => line.includes('could not decrypt agents_list_result')))+ // Opened under a session that was thrown away while the gateway read it.+ let release!: (opened: { frame: Frame }) => void+ seal.open.mockImplementationOnce(() => new Promise((resolve) => { release = resolve }))+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 10 } } })+ await until('the slow open', () => seal.open.mock.calls.length === 3)+ expect(link.resetSession(LINKED)).toBe(true)+ release({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'stale', agents: [] } } })+ await until('the stale reply', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 4)+ expect(alive).toEqual([])+ expect(heard.filter((frame) => frame.type === 'agents_list_result')).toEqual([])+ })++ it('is proof once a frame held for a renewed session opens under it', async () => {+ const { server, seal, link, alive } = await upWithAlive()+ expect(link.resetSession(LINKED)).toBe(true)+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'held', agents: [] } } })+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 0 } } })+ await until('the held reply, under the new session', () => alive.length === 1)+ expect(alive).toEqual([LINKED])+ expect(seal.hello).toHaveBeenCalledTimes(2)+ })++ it('has longer to answer an agent list, a background read, than anything a person waits on', async () => {+ const { link, lines } = await upWithAlive()+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ try {+ const list = link.rpc('agents_list', {}, LINKED).catch((error: Error) => error)+ const models = link.rpc('models_list', {}, LINKED).catch((error: Error) => error)+ await vi.advanceTimersByTimeAsync(15_000)+ expect(await models).toEqual(new Error('models_list timed out'))+ await vi.advanceTimersByTimeAsync(29_999)+ expect(lines.some((line) => line.includes('agents_list timed out'))).toBe(false)+ await vi.advanceTimersByTimeAsync(1)+ expect(await list).toEqual(new Error('agents_list timed out'))+ expect(lines).toContain('device: agents_list timed out after 45000 ms')+ } finally { vi.useRealTimers() }+ })+})diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.tsindex 770efc1bf..5d3f0d730 100644--- a/cli/src/device/deviceLink.ts+++ b/cli/src/device/deviceLink.ts@@ -45,6 +45,13 @@ export interface DeviceFrame {}const RPC_TIMEOUT_MS = 15_000+/**+ * How long an agent list may take: a background read nobody is watching a spinner for, refreshed every few seconds+ * (fleetRouter.ts). Diego, 10-30: a slow but working Mac answered its lists 5 to 25 s after the 15 s every request had,+ * so each answer arrived with nobody waiting and the machine was counted dead (20 such answers in a day). A dead machine+ * is still found: two lists it never answers throw its session away (deviceFleet.ts).+ */+const LIST_TIMEOUT_MS = 45_000const PING_EVERY_MS = 15_000const BASE_DELAY_MS = 1_000const MAX_DELAY_MS = 30_000@@ -139,6 +146,7 @@ export class DeviceLink {private readonly pending = new Map<string, { resolve: (v: Record<string, unknown>) => void; reject: (e: Error) => void; timer: NodeJS.Timeout; machineId: string }>()private readonly listeners = new Set<(frame: DeviceFrame) => void>()private readonly undeliveredListeners = new Set<(what: Undelivered) => void>()+ private readonly aliveListeners = new Set<(machineId: string) => void>()/** Resolves when `machine_selected` lands, so callers can await the attach rather than poll for it. */private selectWaiter: { resolve: () => void; reject: (e: Error) => void; timer: NodeJS.Timeout } | null = null/**@@ -204,6 +212,17 @@ export class DeviceLink {return () => this.listeners.delete(cb)}+ /**+ * Hear each machine that answered: a reply, waited for or not, that opened under that machine's current session with+ * the key it shares with this daemon alone. Proof the machine is up, however late it came (deviceFleet.ts). Never a+ * frame in the clear or under the machine's group key: the relay can write the one, and every commander of the+ * machine can seal the other. Returns how to stop hearing.+ */+ onAlive(cb: (machineId: string) => void): () => void {+ this.aliveListeners.add(cb)+ return () => this.aliveListeners.delete(cb)+ }+/** Hear each turn, stop or answer that did not reach its machine. Returns how to stop hearing. */onUndelivered(cb: (what: Undelivered) => void): () => void {this.undeliveredListeners.add(cb)@@ -528,12 +547,13 @@ export class DeviceLink {if (machineId && this.opts.peer(machineId) && !this.sessions.get(machineId)?.ready) throw new Error(SESSION_GONE)const requestId = randomUUID()+ const waitMs = type === 'agents_list' ? LIST_TIMEOUT_MS : RPC_TIMEOUT_MSreturn new Promise<Record<string, unknown>>((resolve, reject) => {const timer = setTimeout(() => {this.pending.delete(requestId)- this.opts.log(`device: ${type} timed out after ${RPC_TIMEOUT_MS} ms`)+ this.opts.log(`device: ${type} timed out after ${waitMs} ms`)reject(new Error(`${type} timed out`))- }, RPC_TIMEOUT_MS)+ }, waitMs)this.pending.set(requestId, { resolve, reject, timer, machineId: machineId || this.attached })this.send(machineId ? { type, machineId, payload: { ...payload, requestId } } : { type, payload: { ...payload, requestId } })@@ -711,7 +731,7 @@ export class DeviceLink {this.opts.log(`device: a held ${frame.type ?? '?'} from ${machineId} is not from the session it renewed — dropped`)continue}- this.route(opened.frame as DeviceFrame)+ this.route(opened.frame as DeviceFrame, this.answered(machineId, session, frame))}}@@ -848,11 +868,23 @@ export class DeviceLink {this.opts.log(`device: could not decrypt ${frame.type ?? '?'} — dropped`)return}- frame = opened.frame as DeviceFrame+ this.route(opened.frame as DeviceFrame, this.answered(from, session, frame))+ return}this.route(frame)}+ /**+ * Whether a frame the gateway just opened proves its machine answered: sealed with the key this daemon shares with+ * that machine alone (not its group key), and opened under the session that is still the machine's (one thrown away+ * while the gateway read it proves nothing about the next). The relay cannot make such a frame, and the session's+ * counter refuses one it replays. Returns the machine, or '' for no proof.+ */+ private answered(machineId: string, session: LaneSession, sealed: DeviceFrame): string {+ const envelope = (sealed.payload as { __e2e?: { k?: unknown } } | undefined)?.__e2e+ return envelope?.k !== 'g' && this.sessions.get(machineId) === session && session.ready ? machineId : ''+ }+/*** The far daemon says it holds no session for what this one sealed: it restarted, and opened none of it.*@@ -925,8 +957,9 @@ export class DeviceLink {return linked}- /** A frame read (opened, or never sealed): to its waiter, or to the listeners. */- private route(frame: DeviceFrame): void {+ /** A frame read (opened, or never sealed): to its waiter, or to the listeners. [answered] is the machine it proves+ * up (`answered`), or ''. */+ private route(frame: DeviceFrame, answered = ''): void {const from = typeof frame.machineId === 'string' ? frame.machineId : ''const type = frame.type ?? ''const payload = (frame.payload ?? {}) as Record<string, unknown>@@ -961,6 +994,10 @@ export class DeviceLink {}if (type.endsWith('_result')) {+ // Said before its waiter hears the answer, so a request that ends now in an error, or ended already in a timeout,+ // is not counted against a machine that just answered. Only the machine that sealed it: the tag names whose keys+ // opened it.+ if (answered && answered === from) for (const cb of this.aliveListeners) cb(answered)const requestId = typeof payload.requestId === 'string' ? payload.requestId : ''const waiter = requestId ? this.pending.get(requestId) : undefinedif (!waiter) {diff --git a/cli/src/device/slowMachine.spec.ts b/cli/src/device/slowMachine.spec.tsnew file mode 100644index 000000000..8499b214a--- /dev/null+++ b/cli/src/device/slowMachine.spec.ts@@ -0,0 +1,151 @@+/**+ * A slow linked machine, with real keys end to end: the fleet (deviceFleet.ts) over its lane (deviceLink.ts), sealed by+ * the gateway's sessions (gateway/lane.ts), against a far machine running its own E2eeManager, through a fake device+ * socket the test plays the relay on. Diego, 10-30: a slow but working Mac's replies came after their requests had+ * timed out, and two timeouts threw away a session that was answering. Here a late reply the far machine sealed keeps+ * it reachable with its session; what the relay could make (a reply in the clear, an old one replayed) or any commander+ * of the machine could seal (under its group key) proves nothing; and a machine that proves nothing is still reset.+ */+import { mkdtempSync, rmSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterAll, afterEach, beforeAll, expect, it, vi } from 'vitest'+import { WebSocketServer, type WebSocket } from 'ws'+import type * as CoreCrypto from '../lib/e2ee/core.js'+import type { E2eeManager as Manager } from '../lib/e2ee/manager.js'+import { LaneSessions } from '../gateway/lane.js'+import { DeviceFleet } from './deviceFleet.js'+import { DeviceLink } from './deviceLink.js'++type Frame = Record<string, unknown>+const MACHINE = 'b1b2c3d4e5f60718293a4b5c6d7e8f90'+const CONN = 'device:slow-lane'++// The far machine's store reads its data folder when its module loads: a throwaway one, set first.+const dataDir = mkdtempSync(join(tmpdir(), 'slow-machine-'))+let C: typeof CoreCrypto+let E2eeManager: typeof Manager+beforeAll(async () => {+ process.env.ADAPTER_DATA_DIR = dataDir+ C = await import('../lib/e2ee/core.js')+ E2eeManager = (await import('../lib/e2ee/manager.js')).E2eeManager+})+afterAll(() => { rmSync(dataDir, { recursive: true, force: true }) })++const stops: Array<() => Promise<void>> = []+afterEach(async () => {+ vi.useRealTimers()+ for (const stop of stops.splice(0)) await stop()+})++/** Wait on the lane without a timer (they are faked): it reads each frame through promises alone. */+async function until(what: string, check: () => boolean): Promise<void> {+ for (let turn = 0; turn < 20_000; turn++) {+ if (check()) return+ await new Promise((resolve) => setImmediate(resolve))+ }+ throw new Error(`never: ${what}`)+}++async function slowMachine() {+ const identity = C.newIdentity()+ const lane = new LaneSessions(() => identity)+ // The far machine, which trusts this one's key (as `harness link connect` leaves two machines).+ const out: Frame[] = []+ const far = new E2eeManager({ machineId: MACHINE, sendTo: (_conn, frame) => { out.push(frame) }, isConnected: () => true })+ far.trustPeer({ pub: C.b64e(identity.pub), machineId: 'this-machine', label: 'this machine' })+ const farPub = C.b64e((far as unknown as { store: { getIdentity(): CoreCrypto.Identity } }).store.getIdentity().pub)++ // The relay: hellos go to the far machine and its welcome comes back; requests are opened there and kept unanswered.+ const asked: Array<{ type: string; requestId: string }> = []+ const turns: Frame[] = []+ let socket: WebSocket | null = null+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1', handleProtocols: (protocols) => [...protocols][0] ?? false })+ await new Promise<void>((resolve) => wss.once('listening', () => resolve()))+ const toLane = (frame: Frame) => socket?.send(JSON.stringify({ ...frame, machineId: MACHINE }))+ wss.on('connection', (ws) => {+ socket = ws+ ws.on('message', (raw) => {+ const frame = JSON.parse(raw.toString()) as Frame+ if (frame.machineId !== MACHINE) return+ if (frame.type === 'e2e_hello') {+ const before = out.length+ far.handleFrame(CONN, frame)+ for (const reply of out.slice(before)) toLane(reply)+ return+ }+ const plain = far.unwrapDown(CONN, frame)+ if (!plain) return+ if (plain.type === 'message') { turns.push(plain); return }+ asked.push({ type: String(plain.type), requestId: String((plain.payload as Frame).requestId) })+ })+ })+ stops.push(() => new Promise<void>((resolve) => { for (const client of wss.clients) client.terminate(); wss.close(() => resolve()) }))++ const lines: string[] = []+ const link = new DeviceLink({+ auth: { accessToken: async () => 'token' }, backendWsBase: `ws://127.0.0.1:${(wss.address() as { port: number }).port}`,+ computerId: 'this-computer', autonomousEnv: 'test', seal: lane,+ peer: (machineId) => machineId === MACHINE ? { machineId, pub: farPub, label: 'far', linkedAt: 1 } as never : null,+ localMachineId: () => 'this-machine', log: (line) => { lines.push(line) },+ })+ stops.push(async () => link.stop())+ const rows = [{ machineId: MACHINE, name: 'slow-mac', state: 'ready', authMode: 'remote', local: false }]+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,+ link, hasPeerLink: () => true, log: (line) => { lines.push(line) } })+ await link.online()+ const hellos = () => out.filter((frame) => frame.type === 'e2e_welcome').length+ /** One agent list that the far machine reads and does not answer in time. */+ const timesOut = async () => {+ const before = asked.length+ const listed = fleet.listAgents(MACHINE).catch((error: Error) => error)+ await until('the far machine to read the request', () => asked.length === before + 1)+ await vi.advanceTimersByTimeAsync(45_000)+ expect(await listed).toEqual(new Error('agents_list timed out'))+ return asked[before]!+ }+ const read = (what: string) => until(what, () => lines.some((line) => line.includes(what)))+ return { far, fleet, link, lines, toLane, timesOut, hellos, read, turns }+}++it('keeps a slow machine reachable with its session on its own late reply, and on nothing the relay or a commander could make', async () => {+ const { far, fleet, lines, toLane, timesOut, hellos, read, turns } = await slowMachine()+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })+ const first = await timesOut()+ expect(hellos()).toBe(1)+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })++ // What the relay can write: a reply in the clear.+ toLane({ type: 'agents_list_result', payload: { requestId: first.requestId, agents: [] } })+ // What any commander of the machine can seal: a reply under its group key, which opens.+ const group = far as unknown as { groupKey: Uint8Array; groupCounter: number; epoch: string }+ toLane({ type: 'agents_list_result', payload: C.wrapPayload(group.groupKey, 'g', group.groupCounter++, 'agents_list_result', undefined,+ { requestId: first.requestId, agents: [] }, group.epoch) })+ await until('both read', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 2)+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })++ // Its own late reply, sealed for this session alone.+ const late = far.wrapRpcReply(CONN, 'agents_list_result', first.requestId, { agents: [] })!+ toLane(late)+ await until('the late reply to count', () => fleet.reachable(MACHINE)?.ok === true)++ // Its misses start again, so the next timeout is the first: the session stays, and a turn is sealed for it.+ const second = await timesOut()+ expect(lines.some((line) => line.includes('in a row — dropped its E2EE session'))).toBe(false)+ fleet.sendTurn(MACHINE, 'agent-1', 'still there?')+ await until('the turn to reach the far machine', () => turns.length === 1)+ expect(turns[0]).toMatchObject({ type: 'message', payload: { content: 'still there?', agentId: 'agent-1' } })+ expect(hellos()).toBe(1)++ // The same late reply again, as a relay replays it: the session's counter refuses it.+ toLane(late)+ await read('could not decrypt agents_list_result')+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })+ void second++ // Nothing of its own comes back now: the second miss in a row throws the session away, and the next call says hello.+ await timesOut()+ await read('missed 2 in a row — dropped its E2EE session')+ await timesOut()+ expect(hellos()).toBe(2)+})diff --git a/cli/src/services/fleet.spec.ts b/cli/src/services/fleet.spec.tsindex 144a18bc1..566a43960 100644--- a/cli/src/services/fleet.spec.ts+++ b/cli/src/services/fleet.spec.ts@@ -30,6 +30,7 @@ vi.mock('../device/deviceLink.js', () => ({constructor(opts: DeviceLinkOpts) { lane.opts = opts }onFrame(cb: (frame: Record<string, unknown>) => void) { lane.frames = cb; return () => {} }onUndelivered() { return () => {} }+ onAlive() { return () => {} }stop() { lane.stop() }rpc(type: string) { return lane.rpc(type) }sendSealed(...args: unknown[]) { return lane.sendSealed(...(args as [])) }diff --git a/cli/src/services/fleetRouter.spec.ts b/cli/src/services/fleetRouter.spec.tsindex a73557230..f03c1678a 100644--- a/cli/src/services/fleetRouter.spec.ts+++ b/cli/src/services/fleetRouter.spec.ts@@ -7,6 +7,7 @@ import type { ReviewedAnswer } from '../cable/questionInbox.js'import type { RegisteredSession } from '../lib/registry.js'import { transcriptActivityAt } from '../lib/transcriptActivity.js'import { FleetRouter, type FleetLocal } from './fleetRouter.js'+import { DeviceFleet } from '../device/deviceFleet.js'vi.mock('../lib/transcriptActivity.js', () => ({ transcriptActivityAt: vi.fn(async () => null) }))@@ -167,6 +168,45 @@ describe('which machine an agent is on', () => {expect(log).not.toHaveBeenCalled()})+ // Diego, 10-30: a slow but working Mac's agents left the carousel and turns to them were refused, though its replies+ // were arriving, late (15 carousel drops in a day). The real fleet, over a lane whose list times out each time.+ it('keeps a slow machine on the carousel, with its session and its turns, while late replies of its own prove it alive', async () => {+ const rows = [{ ...REMOTE, local: false, agentCount: 1 }]+ let alive: (machineId: string) => void = () => {}+ const link = {+ selectedMachine: '', onFrame: () => () => {}, onUndelivered: () => () => {},+ onAlive: (cb: (machineId: string) => void) => { alive = cb; return () => {} },+ rpc: vi.fn(async (): Promise<Record<string, unknown>> => ({ agents: [{ id: 'r1', name: 'api', engine: 'claude' }] })),+ resetSession: vi.fn(() => true), sendSealed: vi.fn(async () => {}),+ }+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,+ link: link as never, hasPeerLink: () => true, log: () => {} })+ const log = vi.fn()+ const router = new FleetRouter(localOf({ log }), fleet)+ expect((await settled(router)).map((a) => a.id)).toEqual(['r1'])+ // Slow from here: every list times out, and the reply to the one before arrives while the next is out.+ link.rpc.mockImplementation(async () => { alive('other'); throw new Error('agents_list timed out') })+ const now = Date.now()+ for (const later of [60_000, 120_000, 180_000]) {+ vi.spyOn(Date, 'now').mockReturnValue(now + later)+ expect((await settled(router)).map((a) => a.id)).toEqual(['r1'])+ }+ expect(log).not.toHaveBeenCalledWith(expect.stringContaining('dropped off the carousel'))+ expect(link.resetSession).not.toHaveBeenCalled()+ expect(router.sendTurn('r1', 'still there?')).toEqual({ ok: true })+ expect(link.sendSealed).toHaveBeenCalledWith(expect.objectContaining({ type: 'message', machineId: 'other' }))+ // Gone for real: nothing of its own comes back, so its tiles go, its session is thrown away, and the verdict turns+ // refuse on is that it did not answer.+ link.rpc.mockImplementation(async () => { throw new Error('agents_list timed out') })+ for (const later of [240_000, 300_000]) {+ vi.spyOn(Date, 'now').mockReturnValue(now + later)+ await settled(router)+ }+ expect(log).toHaveBeenCalledWith('cable: office-imac dropped off the carousel (agents_list timed out)')+ expect(link.resetSession).toHaveBeenCalledWith('other')+ expect(fleet.reachable('other')).toMatchObject({ ok: false })+ })+it('keeps a machine’s agents while the backend cannot be asked, and drops them when the machine says offline', async () => {const machines: FleetMachine[] = [{ ...REMOTE }]const fleet = fleetOf(machines, { other: [{ id: 'r1', name: 'api' }] })diff --git a/cli/src/services/fleetRouter.ts b/cli/src/services/fleetRouter.tsindex 299be2589..073ed3ec6 100644--- a/cli/src/services/fleetRouter.ts+++ b/cli/src/services/fleetRouter.ts@@ -440,7 +440,9 @@ export class FleetRouter implements FleetRouting {}).catch((err) => {const before = this.remoteAgents.get(m.id)- const stale = before && Date.now() - before.at > REMOTE_GRACE_MS+ // A machine that answered under its own session while this list was out is slow, not gone (Diego, 10-30:+ // 15 carousel drops in a day of a Mac whose late replies kept arriving): its last good list stays.+ const stale = before && Date.now() - before.at > REMOTE_GRACE_MS && fleet.reachable?.(m.id)?.ok !== trueif (stale) this.remoteAgents.set(m.id, { agents: [], at: Date.now(), asked: Date.now() })if (before?.agents.length && stale) {this.local.log(`cable: ${m.name} dropped off the carousel (${(err as Error).message})`)