Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(cli): a slow linked Mac that answers late stays reachable, with its session (#2381)

harness-for-autonomous[bot]14fff78142ac
diff --git a/cli/e2e/fleet.e2e.ts b/cli/e2e/fleet.e2e.ts
index e7fff10f5..64f23c9cf 100644
--- a/cli/e2e/fleet.e2e.ts
+++ b/cli/e2e/fleet.e2e.ts
@@ -178,6 +178,30 @@ describe('two machines in one fleet', () => {
onA.close()
})
+ // Diego, 10-30: a slow but working Mac was treated as dead. Its replies came 5 to 25 s after the 15 s every request
+ // waited, two timeouts threw its session away, its agents left the carousel and turns to them were refused.
+ it('B slow to answer stays on A\'s carousel with its session, and its turns go through', async () => {
+ const f = await fresh({ dialOnA: true })
+ const { a, b, backend } = f
+ const { onA, onB, agentId } = await linked(f)
+ const sessions = () => a.daemon.log().split(`[device] device: e2e session ready ${b.machineId}`).length - 1
+ const before = sessions()
+ // From here B answers 20 s late: past the 15 s a request used to wait, inside the 45 s an agent list waits now.
+ backend.slowUp.set(b.machineId, 20_000)
+ await new Promise((resolve) => setTimeout(resolve, 75_000))
+ const log = a.daemon.log()
+ expect(log).not.toContain('in a row — dropped its E2EE session')
+ expect(log).not.toContain('dropped off the carousel')
+ expect(sessions()).toBe(before)
+ // Still on the carousel, still reachable: ⌘K picks it, and the turn reaches B.
+ expect((await routeTask(onA, 'fix the parser')).agentId).toBe(agentId)
+ const started = onB.next(isTurn('turn_started', agentId), 30_000, 'turn_started on B')
+ expect(await routeSend(onA, agentId, 'to a slow machine b')).toMatchObject({ ok: true })
+ expect((await started).payload?.userMessage).toBe('to a slow machine b')
+ onA.close()
+ onB.close()
+ }, 300_000)
+
it('the dial\'s routing failing through the port leaves the dial serving A, and ⌘K serving both machines', async () => {
const f = await fresh({ dialOnA: true, envA: { HARNESSD_TEST_FAULTS: 'fleet.sendTurn' } })
const { a, b } = f
diff --git a/cli/e2e/harness/fakeBackend.ts b/cli/e2e/harness/fakeBackend.ts
index 299cd8b51..856950ce2 100644
--- a/cli/e2e/harness/fakeBackend.ts
+++ b/cli/e2e/harness/fakeBackend.ts
@@ -135,6 +135,9 @@ export class FakeBackend {
private down = false
/** How long everything relayed between a machine and its web clients is held, ms. */
latencyMs = 0
+ /** How long what each machine sends up to the devices' lanes is held, ms, by machine id: a machine slow to answer
+ * (Diego's, 10-30), not a slow relay. Its handshakes and replies come late alike; what goes down to it does not. */
+ readonly slowUp = new Map<string, number>()
/** What each web client sent towards its machine, as sent (sealed), by connection id. */
readonly webSent = new Map<string, Frame[]>()
/** What reached each web client from its machine, as relayed (sealed), by connection id. */
@@ -617,13 +620,13 @@ export class FakeBackend {
const tagged = { ...envelope.frame, machineId }
if (envelope.targetConnId) {
const owner = this.connOwners.get(envelope.targetConnId)
- if (owner?.machineId === machineId) this.sendDevice(owner.device, tagged)
+ if (owner?.machineId === machineId) this.upToDevice(owner.device, machineId, tagged)
const web = [...this.webClients].find((client) => client.connId === envelope.targetConnId && client.machineId === machineId)
if (web) this.toWeb(web, envelope.frame)
if (!owner && !web) this.targeted.set(envelope.targetConnId, [...(this.targeted.get(envelope.targetConnId) ?? []), envelope.frame])
return
}
- if (envelope.commanderEligible) for (const device of this.devices) this.sendDevice(device, tagged)
+ if (envelope.commanderEligible) for (const device of this.devices) this.upToDevice(device, machineId, tagged)
// The real hub: web clients get every broadcast that is not device-only.
if (envelope.webEligible !== false) this.toWebClients(machineId, envelope.frame)
}
@@ -788,6 +791,13 @@ export class FakeBackend {
if (connId) this.sendNode(machineId, connId, frame)
}
+ /** What a machine sends up to a device, later by `slowUp` if the machine is slow: equal delays fire in order. */
+ private upToDevice(device: Device, machineId: string, frame: Frame): void {
+ const late = this.slowUp.get(machineId) ?? 0
+ if (late > 0) setTimeout(() => this.sendDevice(device, frame), late)
+ else this.sendDevice(device, frame)
+ }
+
private sendDevice(device: Device, frame: Frame): void {
if (device.ws.readyState === WebSocket.OPEN) device.ws.send(json(frame))
}
diff --git a/cli/src/device/deviceFleet.spec.ts b/cli/src/device/deviceFleet.spec.ts
index 419999d80..24d50b1f0 100644
--- a/cli/src/device/deviceFleet.spec.ts
+++ b/cli/src/device/deviceFleet.spec.ts
@@ -21,12 +21,14 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: str
const rows = listOf(over.rows ?? [{}])
let frameCb: ((f: DeviceFrame) => void) | null = null
let undeliveredCb: ((what: Undelivered) => void) | null = null
+ let aliveCb: ((machineId: string) => void) | null = null
const sent: DeviceFrame[] = []
const resets: string[] = []
const link = {
selectedMachine: 'm1',
onFrame: (cb: (f: DeviceFrame) => void) => { frameCb = cb; return () => {} },
onUndelivered: (cb: (what: Undelivered) => void) => { undeliveredCb = cb; return () => {} },
+ onAlive: (cb: (machineId: string) => void) => { aliveCb = cb; return () => {} },
attach: vi.fn(async () => {}),
online: vi.fn(async () => {}),
detach: vi.fn(),
@@ -45,7 +47,8 @@ function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: str
hasPeerLink: (id: string) => (over.linked ?? ['m1']).includes(id),
log: () => {},
})
- return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f), undelivered: (what: Undelivered) => undeliveredCb?.(what) }
+ return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f), undelivered: (what: Undelivered) => undeliveredCb?.(what),
+ alive: (machineId: string) => aliveCb?.(machineId) }
}
describe('DeviceFleet', () => {
@@ -108,6 +111,49 @@ describe('DeviceFleet', () => {
expect(resets).toEqual([])
})
+ // Diego, 10-30: a slow but working Mac was treated as dead. Each session drop was followed within 5-25 s by the late
+ // reply it had given up on, from the same machine (42 drops, 20 unmatched agents_list_result in a day).
+ it('keeps a slow machine reachable, with its session, once a late reply of its own comes back', async () => {
+ const { fleet, resets, alive } = make({ rpc: async () => { throw new Error('agents_list timed out') } })
+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })
+ // The reply it timed out on arrives late, opened under its session (deviceLink.ts `onAlive`).
+ alive('m1')
+ expect(fleet.reachable('m1')).toMatchObject({ ok: true })
+ // The count starts again: the next timeout is the first, not the second.
+ await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
+ expect(resets).toEqual([])
+ })
+
+ it('counts no miss for a request its machine proved alive during, however it ended', async () => {
+ let fail!: (error: Error) => void
+ const { fleet, resets, alive } = make({ rpc: () => new Promise((_resolve, reject) => { fail = reject }) })
+ for (const round of [1, 2, 3]) {
+ const asked = fleet.listAgents('m1')
+ alive('m1')
+ fail(new Error(round === 3 ? 'AGENT_NOT_FOUND' : 'agents_list timed out'))
+ await expect(asked).rejects.toThrow()
+ expect(fleet.reachable('m1'), `round ${round}`).toMatchObject({ ok: true })
+ }
+ expect(resets).toEqual([])
+ })
+
+ it('still throws away the session of a machine that proves nothing, whoever else answers', async () => {
+ let fail!: (error: Error) => void
+ const { fleet, resets, alive } = make({ rows: [{}, { machineId: 'm2', name: 'studio' }], linked: ['m1', 'm2'],
+ rpc: () => new Promise((_resolve, reject) => { fail = reject }) })
+ for (const round of [1, 2]) {
+ const asked = fleet.listAgents('m1')
+ // Another machine answering while m1's request is out says nothing about m1.
+ alive('m2')
+ fail(new Error('agents_list timed out'))
+ await expect(asked, `round ${round}`).rejects.toThrow()
+ }
+ expect(resets).toEqual(['m1'])
+ expect(fleet.reachable('m1')).toMatchObject({ ok: false })
+ expect(fleet.reachable('m2')).toMatchObject({ ok: true })
+ })
+
it('shows a turn, a stop or an answer that may not have reached its machine on that agent\'s tile, never inviting a blind resend', async () => {
// Diego, 10-30: after the far daemon restarted, a dial turn vanished with nothing on the dial to say so.
const { fleet, undelivered } = make()
@@ -291,7 +337,7 @@ describe('DeviceFleet', () => {
find: (id: string) => rows.find((r) => r.machineId === id),
applyLive: (r: unknown) => { applied.push(r); return true },
} as never,
- link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {} } as never,
+ link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {} } as never,
hasPeerLink: () => true,
log: () => {},
})
@@ -308,7 +354,7 @@ describe('DeviceFleet', () => {
let frameCb!: (f: { type?: string; machineId?: string; payload?: Record<string, unknown> }) => void
const fleet = new DeviceFleet({
list: { list: () => ({ machines: [], source: 'backend' as const }), find: () => undefined, applyLive: () => true } as never,
- link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {} } as never,
+ link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {} } as never,
hasPeerLink: () => true,
log: () => {},
})
diff --git a/cli/src/device/deviceFleet.ts b/cli/src/device/deviceFleet.ts
index ada45f9a4..203379cf4 100644
--- a/cli/src/device/deviceFleet.ts
+++ b/cli/src/device/deviceFleet.ts
@@ -89,10 +89,17 @@ export class DeviceFleet implements MachineFleet {
private readonly health = new Map<string, { ok: boolean; at: number }>()
/** Consecutive failed round trips per machine — the counter behind RESET_AFTER_FAILURES. */
private readonly misses = new Map<string, number>()
+ /**
+ * How many times each machine has answered under its own session (DeviceLink.onAlive): a request it answered
+ * anything during is no miss, however it ended. Diego, 10-30: a slow Mac's answers came 5 to 25 s after their
+ * requests timed out, so every pair of timeouts threw away a session that was answering (42 drops in a day).
+ */
+ private readonly answers = new Map<string, number>()
constructor(private readonly opts: DeviceFleetOpts) {
this.opts.link.onFrame((frame) => this.onFrame(frame))
this.opts.link.onUndelivered((what) => this.onUndelivered(what))
+ this.opts.link.onAlive((machineId) => this.alive(machineId))
}
async list(): Promise<{ machines: FleetMachine[]; source: 'backend' | 'local' | 'signed-out' }> {
@@ -198,6 +205,7 @@ export class DeviceFleet implements MachineFleet {
* untouched — this is a witness, not a handler.
*/
private async rpc(machineId: string, type: string, payload: Record<string, unknown>, again = false): Promise<Record<string, unknown>> {
+ const heard = this.answers.get(machineId) ?? 0
try {
const res = await this.opts.link.rpc(type, payload, machineId)
this.health.set(machineId, { ok: true, at: Date.now() })
@@ -207,6 +215,9 @@ export class DeviceFleet implements MachineFleet {
// Its session went before the answer came (the far daemon restarted): a read is asked once more, on
// the new session, and is no miss — nothing about the machine is known yet.
if (!again && (err as Error).message === SESSION_GONE && READS.has(type)) return this.rpc(machineId, type, payload, true)
+ // The machine answered something of its own while this was out (a late reply to an earlier request, or this
+ // one's error): it is up, and slow or refusing, not gone. No miss, and its session stays.
+ if ((this.answers.get(machineId) ?? 0) !== heard) throw err
this.health.set(machineId, { ok: false, at: Date.now() })
const missed = (this.misses.get(machineId) ?? 0) + 1
this.misses.set(machineId, missed)
@@ -228,6 +239,13 @@ export class DeviceFleet implements MachineFleet {
}
}
+ /** A machine answered under its own session (DeviceLink.onAlive): it is up, and its misses start again. */
+ private alive(machineId: string): void {
+ this.answers.set(machineId, (this.answers.get(machineId) ?? 0) + 1)
+ this.health.set(machineId, { ok: true, at: Date.now() })
+ this.misses.delete(machineId)
+ }
+
/**
* Whether the last thing we asked this machine came back.
*
diff --git a/cli/src/device/deviceLink.spec.ts b/cli/src/device/deviceLink.spec.ts
index 9477ca923..66d05f45f 100644
--- a/cli/src/device/deviceLink.spec.ts
+++ b/cli/src/device/deviceLink.spec.ts
@@ -778,3 +778,84 @@ describe('the lane\'s own bounds', () => {
expect(heard.find((frame) => frame.agentId === 'a2')).toMatchObject({ type: 'commander_question' })
})
})
+
+// Diego, 10-30: a slow but working Mac was treated as dead. Its late replies had no waiter and were only logged, and two
+// timeouts in a row threw away a session that had just answered: 20 unmatched agents_list_result, 42 session drops.
+describe('a late reply from a slow machine', () => {
+ /** A lane with LINKED's session up, and what its proof-of-life hook heard. */
+ async function upWithAlive() {
+ const lane$ = await lane()
+ await lane$.link.online()
+ await lane$.link.establish(LINKED)
+ const alive: string[] = []
+ lane$.link.onAlive((machineId) => { alive.push(machineId) })
+ return { ...lane$, alive }
+ }
+
+ it('is proof of life when it opens under the machine\'s own session, matched or not, once each', async () => {
+ const { server, seal, link, lines, alive } = await upWithAlive()
+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'timed-out-long-ago', agents: [] } } })
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 7 } } })
+ await until('the late reply', () => lines.some((line) => line.includes('unmatched agents_list_result')))
+ expect(alive).toEqual([LINKED])
+ // An answer that came in time is proof too.
+ const answer = link.rpc('models_list', {}, LINKED)
+ await until('the request', () => seal.seal.mock.calls.some(([, frame]) => frame.type === 'models_list'))
+ seal.open.mockResolvedValueOnce({ frame: { type: 'models_list_result', machineId: LINKED, payload: { requestId: askedId(seal, LINKED, 'models_list'), models: [] } } })
+ server.send({ type: 'models_list_result', machineId: LINKED, payload: { __e2e: { n: 8 } } })
+ await answer
+ expect(alive).toEqual([LINKED, LINKED])
+ })
+
+ it('is no proof when the relay could have made it: in the clear, under the group key, unreadable, or for another session', async () => {
+ const { server, seal, link, lines, alive, heard } = await upWithAlive()
+ // In the clear: the relay can write any of these.
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'forged', agents: [] } })
+ // Under the group key, which every commander of the machine holds.
+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'group', agents: [] } } })
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { k: 'g', n: 1, epoch: 'e' } } })
+ // Sealed, but not under this session's keys.
+ seal.open.mockResolvedValueOnce({ unreadable: true })
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 9 } } })
+ // From a machine with no session at all.
+ server.send({ type: 'agents_list_result', machineId: CLOUD, payload: { requestId: 'cloud', agents: [] } })
+ await until('every frame read', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 3
+ && lines.some((line) => line.includes('could not decrypt agents_list_result')))
+ // Opened under a session that was thrown away while the gateway read it.
+ let release!: (opened: { frame: Frame }) => void
+ seal.open.mockImplementationOnce(() => new Promise((resolve) => { release = resolve }))
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 10 } } })
+ await until('the slow open', () => seal.open.mock.calls.length === 3)
+ expect(link.resetSession(LINKED)).toBe(true)
+ release({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'stale', agents: [] } } })
+ await until('the stale reply', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 4)
+ expect(alive).toEqual([])
+ expect(heard.filter((frame) => frame.type === 'agents_list_result')).toEqual([])
+ })
+
+ it('is proof once a frame held for a renewed session opens under it', async () => {
+ const { server, seal, link, alive } = await upWithAlive()
+ expect(link.resetSession(LINKED)).toBe(true)
+ seal.open.mockResolvedValueOnce({ frame: { type: 'agents_list_result', machineId: LINKED, payload: { requestId: 'held', agents: [] } } })
+ server.send({ type: 'agents_list_result', machineId: LINKED, payload: { __e2e: { n: 0 } } })
+ await until('the held reply, under the new session', () => alive.length === 1)
+ expect(alive).toEqual([LINKED])
+ expect(seal.hello).toHaveBeenCalledTimes(2)
+ })
+
+ it('has longer to answer an agent list, a background read, than anything a person waits on', async () => {
+ const { link, lines } = await upWithAlive()
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ try {
+ const list = link.rpc('agents_list', {}, LINKED).catch((error: Error) => error)
+ const models = link.rpc('models_list', {}, LINKED).catch((error: Error) => error)
+ await vi.advanceTimersByTimeAsync(15_000)
+ expect(await models).toEqual(new Error('models_list timed out'))
+ await vi.advanceTimersByTimeAsync(29_999)
+ expect(lines.some((line) => line.includes('agents_list timed out'))).toBe(false)
+ await vi.advanceTimersByTimeAsync(1)
+ expect(await list).toEqual(new Error('agents_list timed out'))
+ expect(lines).toContain('device: agents_list timed out after 45000 ms')
+ } finally { vi.useRealTimers() }
+ })
+})
diff --git a/cli/src/device/deviceLink.ts b/cli/src/device/deviceLink.ts
index 770efc1bf..5d3f0d730 100644
--- a/cli/src/device/deviceLink.ts
+++ b/cli/src/device/deviceLink.ts
@@ -45,6 +45,13 @@ export interface DeviceFrame {
}
const RPC_TIMEOUT_MS = 15_000
+/**
+ * How long an agent list may take: a background read nobody is watching a spinner for, refreshed every few seconds
+ * (fleetRouter.ts). Diego, 10-30: a slow but working Mac answered its lists 5 to 25 s after the 15 s every request had,
+ * so each answer arrived with nobody waiting and the machine was counted dead (20 such answers in a day). A dead machine
+ * is still found: two lists it never answers throw its session away (deviceFleet.ts).
+ */
+const LIST_TIMEOUT_MS = 45_000
const PING_EVERY_MS = 15_000
const BASE_DELAY_MS = 1_000
const MAX_DELAY_MS = 30_000
@@ -139,6 +146,7 @@ export class DeviceLink {
private readonly pending = new Map<string, { resolve: (v: Record<string, unknown>) => void; reject: (e: Error) => void; timer: NodeJS.Timeout; machineId: string }>()
private readonly listeners = new Set<(frame: DeviceFrame) => void>()
private readonly undeliveredListeners = new Set<(what: Undelivered) => void>()
+ private readonly aliveListeners = new Set<(machineId: string) => void>()
/** Resolves when `machine_selected` lands, so callers can await the attach rather than poll for it. */
private selectWaiter: { resolve: () => void; reject: (e: Error) => void; timer: NodeJS.Timeout } | null = null
/**
@@ -204,6 +212,17 @@ export class DeviceLink {
return () => this.listeners.delete(cb)
}
+ /**
+ * Hear each machine that answered: a reply, waited for or not, that opened under that machine's current session with
+ * the key it shares with this daemon alone. Proof the machine is up, however late it came (deviceFleet.ts). Never a
+ * frame in the clear or under the machine's group key: the relay can write the one, and every commander of the
+ * machine can seal the other. Returns how to stop hearing.
+ */
+ onAlive(cb: (machineId: string) => void): () => void {
+ this.aliveListeners.add(cb)
+ return () => this.aliveListeners.delete(cb)
+ }
+
/** Hear each turn, stop or answer that did not reach its machine. Returns how to stop hearing. */
onUndelivered(cb: (what: Undelivered) => void): () => void {
this.undeliveredListeners.add(cb)
@@ -528,12 +547,13 @@ export class DeviceLink {
if (machineId && this.opts.peer(machineId) && !this.sessions.get(machineId)?.ready) throw new Error(SESSION_GONE)
const requestId = randomUUID()
+ const waitMs = type === 'agents_list' ? LIST_TIMEOUT_MS : RPC_TIMEOUT_MS
return new Promise<Record<string, unknown>>((resolve, reject) => {
const timer = setTimeout(() => {
this.pending.delete(requestId)
- this.opts.log(`device: ${type} timed out after ${RPC_TIMEOUT_MS} ms`)
+ this.opts.log(`device: ${type} timed out after ${waitMs} ms`)
reject(new Error(`${type} timed out`))
- }, RPC_TIMEOUT_MS)
+ }, waitMs)
this.pending.set(requestId, { resolve, reject, timer, machineId: machineId || this.attached })
this.send(machineId ? { type, machineId, payload: { ...payload, requestId } } : { type, payload: { ...payload, requestId } })
@@ -711,7 +731,7 @@ export class DeviceLink {
this.opts.log(`device: a held ${frame.type ?? '?'} from ${machineId} is not from the session it renewed — dropped`)
continue
}
- this.route(opened.frame as DeviceFrame)
+ this.route(opened.frame as DeviceFrame, this.answered(machineId, session, frame))
}
}
@@ -848,11 +868,23 @@ export class DeviceLink {
this.opts.log(`device: could not decrypt ${frame.type ?? '?'} — dropped`)
return
}
- frame = opened.frame as DeviceFrame
+ this.route(opened.frame as DeviceFrame, this.answered(from, session, frame))
+ return
}
this.route(frame)
}
+ /**
+ * Whether a frame the gateway just opened proves its machine answered: sealed with the key this daemon shares with
+ * that machine alone (not its group key), and opened under the session that is still the machine's (one thrown away
+ * while the gateway read it proves nothing about the next). The relay cannot make such a frame, and the session's
+ * counter refuses one it replays. Returns the machine, or '' for no proof.
+ */
+ private answered(machineId: string, session: LaneSession, sealed: DeviceFrame): string {
+ const envelope = (sealed.payload as { __e2e?: { k?: unknown } } | undefined)?.__e2e
+ return envelope?.k !== 'g' && this.sessions.get(machineId) === session && session.ready ? machineId : ''
+ }
+
/**
* The far daemon says it holds no session for what this one sealed: it restarted, and opened none of it.
*
@@ -925,8 +957,9 @@ export class DeviceLink {
return linked
}
- /** A frame read (opened, or never sealed): to its waiter, or to the listeners. */
- private route(frame: DeviceFrame): void {
+ /** A frame read (opened, or never sealed): to its waiter, or to the listeners. [answered] is the machine it proves
+ * up (`answered`), or ''. */
+ private route(frame: DeviceFrame, answered = ''): void {
const from = typeof frame.machineId === 'string' ? frame.machineId : ''
const type = frame.type ?? ''
const payload = (frame.payload ?? {}) as Record<string, unknown>
@@ -961,6 +994,10 @@ export class DeviceLink {
}
if (type.endsWith('_result')) {
+ // Said before its waiter hears the answer, so a request that ends now in an error, or ended already in a timeout,
+ // is not counted against a machine that just answered. Only the machine that sealed it: the tag names whose keys
+ // opened it.
+ if (answered && answered === from) for (const cb of this.aliveListeners) cb(answered)
const requestId = typeof payload.requestId === 'string' ? payload.requestId : ''
const waiter = requestId ? this.pending.get(requestId) : undefined
if (!waiter) {
diff --git a/cli/src/device/slowMachine.spec.ts b/cli/src/device/slowMachine.spec.ts
new file mode 100644
index 000000000..8499b214a
--- /dev/null
+++ b/cli/src/device/slowMachine.spec.ts
@@ -0,0 +1,151 @@
+/**
+ * A slow linked machine, with real keys end to end: the fleet (deviceFleet.ts) over its lane (deviceLink.ts), sealed by
+ * the gateway's sessions (gateway/lane.ts), against a far machine running its own E2eeManager, through a fake device
+ * socket the test plays the relay on. Diego, 10-30: a slow but working Mac's replies came after their requests had
+ * timed out, and two timeouts threw away a session that was answering. Here a late reply the far machine sealed keeps
+ * it reachable with its session; what the relay could make (a reply in the clear, an old one replayed) or any commander
+ * of the machine could seal (under its group key) proves nothing; and a machine that proves nothing is still reset.
+ */
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterAll, afterEach, beforeAll, expect, it, vi } from 'vitest'
+import { WebSocketServer, type WebSocket } from 'ws'
+import type * as CoreCrypto from '../lib/e2ee/core.js'
+import type { E2eeManager as Manager } from '../lib/e2ee/manager.js'
+import { LaneSessions } from '../gateway/lane.js'
+import { DeviceFleet } from './deviceFleet.js'
+import { DeviceLink } from './deviceLink.js'
+
+type Frame = Record<string, unknown>
+const MACHINE = 'b1b2c3d4e5f60718293a4b5c6d7e8f90'
+const CONN = 'device:slow-lane'
+
+// The far machine's store reads its data folder when its module loads: a throwaway one, set first.
+const dataDir = mkdtempSync(join(tmpdir(), 'slow-machine-'))
+let C: typeof CoreCrypto
+let E2eeManager: typeof Manager
+beforeAll(async () => {
+ process.env.ADAPTER_DATA_DIR = dataDir
+ C = await import('../lib/e2ee/core.js')
+ E2eeManager = (await import('../lib/e2ee/manager.js')).E2eeManager
+})
+afterAll(() => { rmSync(dataDir, { recursive: true, force: true }) })
+
+const stops: Array<() => Promise<void>> = []
+afterEach(async () => {
+ vi.useRealTimers()
+ for (const stop of stops.splice(0)) await stop()
+})
+
+/** Wait on the lane without a timer (they are faked): it reads each frame through promises alone. */
+async function until(what: string, check: () => boolean): Promise<void> {
+ for (let turn = 0; turn < 20_000; turn++) {
+ if (check()) return
+ await new Promise((resolve) => setImmediate(resolve))
+ }
+ throw new Error(`never: ${what}`)
+}
+
+async function slowMachine() {
+ const identity = C.newIdentity()
+ const lane = new LaneSessions(() => identity)
+ // The far machine, which trusts this one's key (as `harness link connect` leaves two machines).
+ const out: Frame[] = []
+ const far = new E2eeManager({ machineId: MACHINE, sendTo: (_conn, frame) => { out.push(frame) }, isConnected: () => true })
+ far.trustPeer({ pub: C.b64e(identity.pub), machineId: 'this-machine', label: 'this machine' })
+ const farPub = C.b64e((far as unknown as { store: { getIdentity(): CoreCrypto.Identity } }).store.getIdentity().pub)
+
+ // The relay: hellos go to the far machine and its welcome comes back; requests are opened there and kept unanswered.
+ const asked: Array<{ type: string; requestId: string }> = []
+ const turns: Frame[] = []
+ let socket: WebSocket | null = null
+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1', handleProtocols: (protocols) => [...protocols][0] ?? false })
+ await new Promise<void>((resolve) => wss.once('listening', () => resolve()))
+ const toLane = (frame: Frame) => socket?.send(JSON.stringify({ ...frame, machineId: MACHINE }))
+ wss.on('connection', (ws) => {
+ socket = ws
+ ws.on('message', (raw) => {
+ const frame = JSON.parse(raw.toString()) as Frame
+ if (frame.machineId !== MACHINE) return
+ if (frame.type === 'e2e_hello') {
+ const before = out.length
+ far.handleFrame(CONN, frame)
+ for (const reply of out.slice(before)) toLane(reply)
+ return
+ }
+ const plain = far.unwrapDown(CONN, frame)
+ if (!plain) return
+ if (plain.type === 'message') { turns.push(plain); return }
+ asked.push({ type: String(plain.type), requestId: String((plain.payload as Frame).requestId) })
+ })
+ })
+ stops.push(() => new Promise<void>((resolve) => { for (const client of wss.clients) client.terminate(); wss.close(() => resolve()) }))
+
+ const lines: string[] = []
+ const link = new DeviceLink({
+ auth: { accessToken: async () => 'token' }, backendWsBase: `ws://127.0.0.1:${(wss.address() as { port: number }).port}`,
+ computerId: 'this-computer', autonomousEnv: 'test', seal: lane,
+ peer: (machineId) => machineId === MACHINE ? { machineId, pub: farPub, label: 'far', linkedAt: 1 } as never : null,
+ localMachineId: () => 'this-machine', log: (line) => { lines.push(line) },
+ })
+ stops.push(async () => link.stop())
+ const rows = [{ machineId: MACHINE, name: 'slow-mac', state: 'ready', authMode: 'remote', local: false }]
+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,
+ link, hasPeerLink: () => true, log: (line) => { lines.push(line) } })
+ await link.online()
+ const hellos = () => out.filter((frame) => frame.type === 'e2e_welcome').length
+ /** One agent list that the far machine reads and does not answer in time. */
+ const timesOut = async () => {
+ const before = asked.length
+ const listed = fleet.listAgents(MACHINE).catch((error: Error) => error)
+ await until('the far machine to read the request', () => asked.length === before + 1)
+ await vi.advanceTimersByTimeAsync(45_000)
+ expect(await listed).toEqual(new Error('agents_list timed out'))
+ return asked[before]!
+ }
+ const read = (what: string) => until(what, () => lines.some((line) => line.includes(what)))
+ return { far, fleet, link, lines, toLane, timesOut, hellos, read, turns }
+}
+
+it('keeps a slow machine reachable with its session on its own late reply, and on nothing the relay or a commander could make', async () => {
+ const { far, fleet, lines, toLane, timesOut, hellos, read, turns } = await slowMachine()
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
+ const first = await timesOut()
+ expect(hellos()).toBe(1)
+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })
+
+ // What the relay can write: a reply in the clear.
+ toLane({ type: 'agents_list_result', payload: { requestId: first.requestId, agents: [] } })
+ // What any commander of the machine can seal: a reply under its group key, which opens.
+ const group = far as unknown as { groupKey: Uint8Array; groupCounter: number; epoch: string }
+ toLane({ type: 'agents_list_result', payload: C.wrapPayload(group.groupKey, 'g', group.groupCounter++, 'agents_list_result', undefined,
+ { requestId: first.requestId, agents: [] }, group.epoch) })
+ await until('both read', () => lines.filter((line) => line.includes('unmatched agents_list_result')).length === 2)
+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })
+
+ // Its own late reply, sealed for this session alone.
+ const late = far.wrapRpcReply(CONN, 'agents_list_result', first.requestId, { agents: [] })!
+ toLane(late)
+ await until('the late reply to count', () => fleet.reachable(MACHINE)?.ok === true)
+
+ // Its misses start again, so the next timeout is the first: the session stays, and a turn is sealed for it.
+ const second = await timesOut()
+ expect(lines.some((line) => line.includes('in a row — dropped its E2EE session'))).toBe(false)
+ fleet.sendTurn(MACHINE, 'agent-1', 'still there?')
+ await until('the turn to reach the far machine', () => turns.length === 1)
+ expect(turns[0]).toMatchObject({ type: 'message', payload: { content: 'still there?', agentId: 'agent-1' } })
+ expect(hellos()).toBe(1)
+
+ // The same late reply again, as a relay replays it: the session's counter refuses it.
+ toLane(late)
+ await read('could not decrypt agents_list_result')
+ expect(fleet.reachable(MACHINE)).toMatchObject({ ok: false })
+ void second
+
+ // Nothing of its own comes back now: the second miss in a row throws the session away, and the next call says hello.
+ await timesOut()
+ await read('missed 2 in a row — dropped its E2EE session')
+ await timesOut()
+ expect(hellos()).toBe(2)
+})
diff --git a/cli/src/services/fleet.spec.ts b/cli/src/services/fleet.spec.ts
index 144a18bc1..566a43960 100644
--- a/cli/src/services/fleet.spec.ts
+++ b/cli/src/services/fleet.spec.ts
@@ -30,6 +30,7 @@ vi.mock('../device/deviceLink.js', () => ({
constructor(opts: DeviceLinkOpts) { lane.opts = opts }
onFrame(cb: (frame: Record<string, unknown>) => void) { lane.frames = cb; return () => {} }
onUndelivered() { return () => {} }
+ onAlive() { return () => {} }
stop() { lane.stop() }
rpc(type: string) { return lane.rpc(type) }
sendSealed(...args: unknown[]) { return lane.sendSealed(...(args as [])) }
diff --git a/cli/src/services/fleetRouter.spec.ts b/cli/src/services/fleetRouter.spec.ts
index a73557230..f03c1678a 100644
--- a/cli/src/services/fleetRouter.spec.ts
+++ b/cli/src/services/fleetRouter.spec.ts
@@ -7,6 +7,7 @@ import type { ReviewedAnswer } from '../cable/questionInbox.js'
import type { RegisteredSession } from '../lib/registry.js'
import { transcriptActivityAt } from '../lib/transcriptActivity.js'
import { FleetRouter, type FleetLocal } from './fleetRouter.js'
+import { DeviceFleet } from '../device/deviceFleet.js'
vi.mock('../lib/transcriptActivity.js', () => ({ transcriptActivityAt: vi.fn(async () => null) }))
@@ -167,6 +168,45 @@ describe('which machine an agent is on', () => {
expect(log).not.toHaveBeenCalled()
})
+ // Diego, 10-30: a slow but working Mac's agents left the carousel and turns to them were refused, though its replies
+ // were arriving, late (15 carousel drops in a day). The real fleet, over a lane whose list times out each time.
+ it('keeps a slow machine on the carousel, with its session and its turns, while late replies of its own prove it alive', async () => {
+ const rows = [{ ...REMOTE, local: false, agentCount: 1 }]
+ let alive: (machineId: string) => void = () => {}
+ const link = {
+ selectedMachine: '', onFrame: () => () => {}, onUndelivered: () => () => {},
+ onAlive: (cb: (machineId: string) => void) => { alive = cb; return () => {} },
+ rpc: vi.fn(async (): Promise<Record<string, unknown>> => ({ agents: [{ id: 'r1', name: 'api', engine: 'claude' }] })),
+ resetSession: vi.fn(() => true), sendSealed: vi.fn(async () => {}),
+ }
+ const fleet = new DeviceFleet({ list: { list: () => ({ machines: rows, source: 'backend' as const }), find: (id: string) => rows.find((r) => r.machineId === id) } as never,
+ link: link as never, hasPeerLink: () => true, log: () => {} })
+ const log = vi.fn()
+ const router = new FleetRouter(localOf({ log }), fleet)
+ expect((await settled(router)).map((a) => a.id)).toEqual(['r1'])
+ // Slow from here: every list times out, and the reply to the one before arrives while the next is out.
+ link.rpc.mockImplementation(async () => { alive('other'); throw new Error('agents_list timed out') })
+ const now = Date.now()
+ for (const later of [60_000, 120_000, 180_000]) {
+ vi.spyOn(Date, 'now').mockReturnValue(now + later)
+ expect((await settled(router)).map((a) => a.id)).toEqual(['r1'])
+ }
+ expect(log).not.toHaveBeenCalledWith(expect.stringContaining('dropped off the carousel'))
+ expect(link.resetSession).not.toHaveBeenCalled()
+ expect(router.sendTurn('r1', 'still there?')).toEqual({ ok: true })
+ expect(link.sendSealed).toHaveBeenCalledWith(expect.objectContaining({ type: 'message', machineId: 'other' }))
+ // Gone for real: nothing of its own comes back, so its tiles go, its session is thrown away, and the verdict turns
+ // refuse on is that it did not answer.
+ link.rpc.mockImplementation(async () => { throw new Error('agents_list timed out') })
+ for (const later of [240_000, 300_000]) {
+ vi.spyOn(Date, 'now').mockReturnValue(now + later)
+ await settled(router)
+ }
+ expect(log).toHaveBeenCalledWith('cable: office-imac dropped off the carousel (agents_list timed out)')
+ expect(link.resetSession).toHaveBeenCalledWith('other')
+ expect(fleet.reachable('other')).toMatchObject({ ok: false })
+ })
+
it('keeps a machine’s agents while the backend cannot be asked, and drops them when the machine says offline', async () => {
const machines: FleetMachine[] = [{ ...REMOTE }]
const fleet = fleetOf(machines, { other: [{ id: 'r1', name: 'api' }] })
diff --git a/cli/src/services/fleetRouter.ts b/cli/src/services/fleetRouter.ts
index 299be2589..073ed3ec6 100644
--- a/cli/src/services/fleetRouter.ts
+++ b/cli/src/services/fleetRouter.ts
@@ -440,7 +440,9 @@ export class FleetRouter implements FleetRouting {
})
.catch((err) => {
const before = this.remoteAgents.get(m.id)
- const stale = before && Date.now() - before.at > REMOTE_GRACE_MS
+ // A machine that answered under its own session while this list was out is slow, not gone (Diego, 10-30:
+ // 15 carousel drops in a day of a Mac whose late replies kept arriving): its last good list stays.
+ const stale = before && Date.now() - before.at > REMOTE_GRACE_MS && fleet.reachable?.(m.id)?.ok !== true
if (stale) this.remoteAgents.set(m.id, { agents: [], at: Date.now(), asked: Date.now() })
if (before?.agents.length && stale) {
this.local.log(`cable: ${m.name} dropped off the carousel (${(err as Error).message})`)