Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

22.9 KB
Raw Blame
// The fan-out between the machine list and the one device socket.
//
// The cases here are the ones that fail SILENTLY: a machine with no pinned key answering an empty
// carousel instead of an instruction, and another machine's turn card painting a tile on the machine the
// dial is actually showing.
import { describe, expect, it, vi } from 'vitest'

import { DeviceFleet } from './deviceFleet.js'
import { FleetRouter } from '../services/fleetRouter.js'
import { FleetError, type FleetEvent } from '../cable/machineFleet.js'
import { SESSION_GONE, type DeviceFrame, type Undelivered } from './deviceLink.js'

type Listed = { machineId: string; name: string; state: string; agentCount: number; authMode: string; local: boolean }

function listOf(rows: Partial<Listed>[]): Listed[] {
  return rows.map((r) => ({
    machineId: 'm1', name: 'office-imac', state: 'ready', authMode: 'remote', local: false, ...r,
  })) as Listed[]
}

function make(over: { rows?: Partial<Listed>[]; linked?: string[]; rpc?: (t: string, p: Record<string, unknown>) => Promise<Record<string, unknown>> } = {}) {
  const rows = listOf(over.rows ?? [{}])
  let frameCb: ((f: DeviceFrame) => void) | null = null
  let undeliveredCb: ((what: Undelivered) => void) | null = null
  let aliveCb: ((machineId: string) => void) | null = null
  let goneCb: ((machineId: string) => void) | null = null
  const sent: DeviceFrame[] = []
  const resets: string[] = []
  const link = {
    selectedMachine: 'm1',
    onFrame: (cb: (f: DeviceFrame) => void) => { frameCb = cb; return () => {} },
    onUndelivered: (cb: (what: Undelivered) => void) => { undeliveredCb = cb; return () => {} },
    onAlive: (cb: (machineId: string) => void) => { aliveCb = cb; return () => {} },
    onGone: (cb: (machineId: string) => void) => { goneCb = cb; return () => {} },
    attach: vi.fn(async () => {}),
    online: vi.fn(async () => {}),
    detach: vi.fn(),
    release: vi.fn(),
    send: (f: DeviceFrame) => { sent.push(f) },
    sendSealed: async (f: DeviceFrame) => { sent.push(f) },
    rpc: over.rpc ?? (async () => ({})),
    resetSession: vi.fn((machineId: string) => { resets.push(machineId); return true }),
  }
  const fleet = new DeviceFleet({
    list: {
      list: () => ({ machines: rows, source: 'backend' as const }),
      find: (id: string) => rows.find((r) => r.machineId === id),
    } as never,
    link: link as never,
    hasPeerLink: (id: string) => (over.linked ?? ['m1']).includes(id),
    log: () => {},
  })
  return { fleet, link, sent, resets, say: (f: DeviceFrame) => frameCb?.(f), undelivered: (what: Undelivered) => undeliveredCb?.(what),
    alive: (machineId: string) => aliveCb?.(machineId), gone: (machineId: string) => goneCb?.(machineId) }
}

describe('DeviceFleet', () => {
  it('preserves reviewed question contents on the remote lane and forwards its matching close', async () => {
    const { fleet, say, sent } = make()
    const questions = [{ key: 'scope', q: 'Which scope?', options: ['File'], multi: false }]
    fleet.answerReviewed('m1', { agentId: 'a1', requestId: 'r1', questions,
      answers: { scope: 'File' }, selections: { scope: ['File'] } })
    await vi.waitFor(() => expect(sent).toHaveLength(1))
    expect(sent[0]).toMatchObject({ type: 'question_response', machineId: 'm1', payload: {
      requestId: 'r1', expectedQuestions: questions, selectedLabels: { scope: ['File'] } } })
    const seen: FleetEvent[] = []; fleet.onEvent(event => seen.push(event))
    say({ type: 'commander_question_close', machineId: 'm1', agentId: 'a1', payload: { requestId: 'r1' } })
    expect(seen).toEqual([{ machineId: 'm1', kind: 'questionClosed', agentId: 'a1', requestId: 'r1' }])
  })

  it('never lists another machine\'s terminal to the dial — the far daemon answers as to an app', async () => {
    const { fleet } = make({
      rpc: async () => ({ agents: [
        { id: 't1', name: 'Terminal 1', engine: 'terminal' },
        { id: 'c1', name: 'Claude 1', engine: 'claude' },
      ] }),
    })
    expect((await fleet.listAgents('m1')).map((a) => a.id)).toEqual(['c1'])
  })

  it('throws away the E2EE session after two missed round trips, and keeps the last verdict', async () => {
    // THE FAILURE THIS EXISTS FOR IS SILENT AND PERMANENT. A session outlives the machine that agreed to
    // it: establish() returns instantly for anything in the map, so once the far end stops answering,
    // every later call is encrypted, sent, and waits out its timeout — forever. Measured on the desk as
    // eight minutes of `agents_list timed out` twenty seconds apart, healed only by a daemon restart that
    // happened for an unrelated reason.
    let calls = 0
    const { fleet, resets } = make({
      rpc: async () => { calls++; throw new Error('timed out') },
    })

    await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
    // One miss is a dropped request, not a dead machine. Nothing is torn down for it.
    expect(resets).toEqual([])
    expect(fleet.reachable('m1')).toMatchObject({ ok: false })

    await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
    expect(resets).toEqual(['m1'])
    expect(calls).toBe(2)
  })

  it('a round trip that comes back clears the count, so misses have to be consecutive', async () => {
    let fail = true
    const { fleet, resets } = make({
      rpc: async () => { if (fail) throw new Error('timed out'); return {} },
    })
    await expect(fleet.listAgents('m1')).rejects.toThrow()
    fail = false
    await fleet.listAgents('m1')
    expect(fleet.reachable('m1')).toMatchObject({ ok: true })
    fail = true
    await expect(fleet.listAgents('m1')).rejects.toThrow()
    // Two failures, but a success between them: a flaky link is not a gone machine.
    expect(resets).toEqual([])
  })

  // Diego, 10-30: a slow but working Mac was treated as dead. Each session drop was followed within 5-25 s by the late
  // reply it had given up on, from the same machine (42 drops, 20 unmatched agents_list_result in a day).
  it('keeps a slow machine reachable, with its session, once a late reply of its own comes back', async () => {
    const { fleet, resets, alive } = make({ rpc: async () => { throw new Error('agents_list timed out') } })
    await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
    expect(fleet.reachable('m1')).toMatchObject({ ok: false })
    // The reply it timed out on arrives late, opened under its session (deviceLink.ts `onAlive`).
    alive('m1')
    expect(fleet.reachable('m1')).toMatchObject({ ok: true })
    // The count starts again: the next timeout is the first, not the second.
    await expect(fleet.listAgents('m1')).rejects.toThrow('timed out')
    expect(resets).toEqual([])
  })

  // #2387's review: B's daemon gone, a turn to its agent was sent to be lost for up to 50 s, while a list to it waited out
  // its time. Told it went away (DeviceLink.onGone), the router refuses the turn at once, and sends again once it answers.
  it('refuses a turn to a machine the lane says went away at once, and sends to it again once it answers', async () => {
    const { fleet, sent, resets, alive, gone } = make({ rpc: async () => ({ agents: [{ id: 'r1', name: 'api', engine: 'claude' }] }) })
    const log = vi.fn()
    const router = new FleetRouter({ machineName: () => 'here', machineId: () => 'this-one', computerId: () => 'c', sessions: () => [],
      displayName: () => '', desk: () => [], sendTurn: vi.fn(), stopTurn: vi.fn(), answer: vi.fn(), recent: () => [], recentAsks: () => [], log }, fleet)
    await router.listAgentsFlat()
    await vi.waitFor(async () => expect((await router.listAgentsFlat()).map((a) => a.id)).toEqual(['r1']))
    expect(router.sendTurn('r1', 'before')).toEqual({ ok: true })
    gone('m1')
    expect(fleet.reachable('m1')).toMatchObject({ ok: false })
    expect(router.sendTurn('r1', 'while it is away')).toMatchObject({ ok: false, machine: 'office-imac', reason: 'the last request to it did not come back' })
    expect(sent.filter((frame) => frame.type === 'message').map((frame) => (frame.payload as { content?: string }).content)).toEqual(['before'])
    // The word may be the relay's: it throws no session away and counts no miss.
    expect(resets).toEqual([])
    alive('m1')
    expect(router.sendTurn('r1', 'back')).toEqual({ ok: true })
  })

  it('counts no miss for a request its machine proved alive during, however it ended', async () => {
    let fail!: (error: Error) => void
    const { fleet, resets, alive } = make({ rpc: () => new Promise((_resolve, reject) => { fail = reject }) })
    for (const round of [1, 2, 3]) {
      const asked = fleet.listAgents('m1')
      alive('m1')
      fail(new Error(round === 3 ? 'AGENT_NOT_FOUND' : 'agents_list timed out'))
      await expect(asked).rejects.toThrow()
      expect(fleet.reachable('m1'), `round ${round}`).toMatchObject({ ok: true })
    }
    expect(resets).toEqual([])
  })

  it('still throws away the session of a machine that proves nothing, whoever else answers', async () => {
    let fail!: (error: Error) => void
    const { fleet, resets, alive } = make({ rows: [{}, { machineId: 'm2', name: 'studio' }], linked: ['m1', 'm2'],
      rpc: () => new Promise((_resolve, reject) => { fail = reject }) })
    for (const round of [1, 2]) {
      const asked = fleet.listAgents('m1')
      // Another machine answering while m1's request is out says nothing about m1.
      alive('m2')
      fail(new Error('agents_list timed out'))
      await expect(asked, `round ${round}`).rejects.toThrow()
    }
    expect(resets).toEqual(['m1'])
    expect(fleet.reachable('m1')).toMatchObject({ ok: false })
    expect(fleet.reachable('m2')).toMatchObject({ ok: true })
  })

  it('shows a turn, a stop or an answer that may not have reached its machine on that agent\'s tile, never inviting a blind resend', async () => {
    // Diego, 10-30: after the far daemon restarted, a dial turn vanished with nothing on the dial to say so.
    const { fleet, undelivered } = make()
    const seen: FleetEvent[] = []
    fleet.onEvent((e) => seen.push(e))
    undelivered({ machineId: 'm1', type: 'message', agentId: 'a1' })
    undelivered({ machineId: 'm1', type: 'cancel', agentId: 'a2' })
    undelivered({ machineId: 'm1', type: 'question_response', agentId: 'a3', requestId: 'q1' })
    // With no agent to put it on there is no tile to tell.
    undelivered({ machineId: 'm1', type: 'message' })
    undelivered({ machineId: 'gone', type: 'message', agentId: 'a4' })
    expect(seen).toEqual([
      { machineId: 'm1', kind: 'error', agentId: 'a1', text: 'May not have reached office-imac. Check before sending again.', recap: 'May not have arrived' },
      { machineId: 'm1', kind: 'error', agentId: 'a2', text: 'Stop may not have reached office-imac. Check before stopping again.', recap: 'May not have arrived' },
      { machineId: 'm1', kind: 'error', agentId: 'a3', text: 'Answer may not have reached office-imac. Check before answering again.', recap: 'May not have arrived' },
      { machineId: 'gone', kind: 'error', agentId: 'a4', text: 'May not have reached that computer. Check before sending again.', recap: 'May not have arrived' },
    ])
  })

  it('asks a read again once when its machine\'s session went before the answer came, and counts no miss for it', async () => {
    // The far daemon restarted mid-request: it never saw the request, so asking again is safe for a read.
    const asked: string[] = []
    let fail = 1
    const { fleet, resets } = make({
      rpc: async (type) => { asked.push(type); if (fail-- > 0) throw new Error(SESSION_GONE); return { agents: [{ id: 'c1', engine: 'claude' }] } },
    })
    expect((await fleet.listAgents('m1')).map((a) => a.id)).toEqual(['c1'])
    expect(asked).toEqual(['agents_list', 'agents_list'])
    expect(fleet.reachable('m1')).toMatchObject({ ok: true })
    // Only once: a second loss is a failure like any other.
    fail = 2
    await expect(fleet.listAgents('m1')).rejects.toThrow(SESSION_GONE)
    expect(asked).toHaveLength(4)
    expect(resets).toEqual([])
    // A write is never asked twice: the far side may have run it.
    const writes: string[] = []
    const w = make({ rpc: async (type) => { writes.push(type); throw new Error(SESSION_GONE) } })
    await expect(w.fleet.forkAgent('m1', 'a1')).rejects.toThrow(SESSION_GONE)
    expect(writes).toEqual(['agent_fork'])
  })

  it('says nothing about a machine nobody has asked', async () => {
    // Callers refuse to deliver on a NEGATIVE verdict. Answering "unreachable" for a machine that has
    // simply never been called would turn every cold start into a refusal.
    const { fleet } = make()
    expect(fleet.reachable('m1')).toBeNull()
  })


  it('hides this computer from the fleet list — the host adds it back with real facts', async () => {
    const { fleet } = make({ rows: [{ machineId: 'mine', local: true }, { machineId: 'other' }], linked: ['other'] })
    const { machines } = await fleet.list()
    expect(machines.map((m) => m.machineId)).toEqual(['other'])
  })

  it('marks an unlinked remote machine needs-link instead of leaving it looking ready', async () => {
    // Reachable but unreadable: everything on it is encrypted to a key this daemon does not hold. Saying
    // "ready" would send the user into an empty carousel with nothing explaining it.
    const { fleet } = make({ rows: [{ machineId: 'm1', authMode: 'remote' }], linked: [] })
    const { machines } = await fleet.list()
    expect(machines[0].state).toBe('needs-link')
  })

  it('leaves a cloud machine alone — E2EE is a remote-machine rule, not a universal one', async () => {
    const { fleet } = make({ rows: [{ machineId: 'm1', authMode: 'managed' }], linked: [] })
    const { machines } = await fleet.list()
    expect(machines[0].state).toBe('ready')
  })

  it('refuses an unlinked select LOCALLY, without dialling anything', async () => {
    const { fleet, link } = make({ rows: [{ machineId: 'm1', name: 'imac', authMode: 'remote' }], linked: [] })
    await expect(fleet.select('m1')).rejects.toMatchObject({ code: 'NEEDS_LINK' })
    expect(link.attach).not.toHaveBeenCalled()   // the guide appears instantly, not after a doomed connect
  })

  it('refuses a machine that is not on the account', async () => {
    const { fleet } = make()
    await expect(fleet.select('nope')).rejects.toBeInstanceOf(FleetError)
  })

  it('maps the far end’s refusal onto a code and words a person can act on', async () => {
    const { fleet, link } = make()
    link.attach = vi.fn(async () => { throw new Error('NOT_YOUR_MACHINE') })
    await expect(fleet.select('m1')).rejects.toMatchObject({ code: 'NOT_YOURS' })
  })

  it('sends a turn in the shape the backend dispatches', async () => {
    const { fleet, sent } = make()
    fleet.sendTurn('m1', 'a1', 'ship it')
    expect(sent[0]).toMatchObject({ type: 'message', payload: { content: 'ship it', agentId: 'a1', resumeLatest: true } })
  })

  it('forwards a question answer keyed by the question keys', async () => {
    const { fleet, sent } = make()
    fleet.answer('m1', 'a1', 'req-1', { scope: 'wide' })
    expect(sent[0]).toMatchObject({ type: 'question_response', payload: { agentId: 'a1', requestId: 'req-1', answers: { scope: 'wide' } } })
  })

  it('turns an agents_list_result into tiles, chips and all', async () => {
    const { fleet } = make({
      rpc: async () => ({ agents: [{ id: 'a1', name: 'api refactor', engine: 'codex', selectedModel: 'runtime-v1:s1:codex:opus@high' }] }),
    })
    const agents = await fleet.listAgents('m1')
    expect(agents[0]).toMatchObject({ id: 'a1', name: 'api refactor', engine: 'codex', model: 'opus', effort: 'high' })
  })

  it('delivers a card from a machine the wheel is not pointed at, tagged with its own machine', () => {
    // The carousel spans machines, so a card from a background machine belongs to a tile that is on
    // screen. This used to be DROPPED — which, once every machine's agents share one carousel, is what a
    // tile stuck on "Working…" for a turn that finished minutes ago looks like from the outside.
    const { fleet, say } = make()
    const seen: FleetEvent[] = []
    fleet.onEvent((e) => seen.push(e))
    say({ type: 'commander_event', machineId: 'other', agentId: 'a1', payload: { kind: 'summary', recap: 'nope', text: 'b' } })
    expect(seen).toEqual([{ machineId: 'other', kind: 'summary', agentId: 'a1', text: 'b', recap: 'nope' }])

    say({ type: 'commander_event', machineId: 'm1', agentId: 'a1', payload: { kind: 'summary', recap: 'yes', text: 'body' } })
    expect(seen[1]).toEqual({ machineId: 'm1', kind: 'summary', agentId: 'a1', text: 'body', recap: 'yes' })
  })


  // #2648's review (Orion): the emit dropped the tier the other machine's recaps decided, so the dial decided again here
  // with no agent list for that machine, and a Swarm agent on another of the person's Macs alerted.
  it('carries the tier the other machine decided on its summary card, and only a real one', () => {
    const { fleet, say } = make()
    const seen: FleetEvent[] = []
    fleet.onEvent((e) => seen.push(e))
    say({ type: 'commander_event', machineId: 'm1', agentId: 'a1', payload: { kind: 'summary', recap: 'Built.', text: 'done', tier: 'silent' } })
    say({ type: 'commander_event', machineId: 'm1', agentId: 'a1', payload: { kind: 'summary', recap: 'Built.', text: 'done', tier: 'loud' } })
    say({ type: 'commander_event', machineId: 'm1', agentId: 'a1', payload: { kind: 'processing', text: 'go', tier: 'alert' } })
    expect(seen.map((e) => (e as { tier?: string }).tier)).toEqual(['silent', undefined, undefined])
  })

  it('ignores card kinds the dial has no use for', () => {
    const { fleet, say } = make()
    const seen: FleetEvent[] = []
    fleet.onEvent((e) => seen.push(e))
    say({ type: 'commander_event', machineId: 'm1', agentId: 'a1', payload: { kind: 'tool', text: 'grep' } })
    expect(seen).toHaveLength(0)
  })

  it('surfaces a question and a presence flip', () => {
    const { fleet, say } = make()
    const seen: FleetEvent[] = []
    fleet.onEvent((e) => seen.push(e))
    say({ type: 'commander_question', machineId: 'm1', agentId: 'a1', payload: { requestId: 'r1', questions: [{ key: 'k' }] } })
    say({ type: 'node_status', machineId: 'm1', payload: { online: false } })
    expect(seen.map((e) => e.kind)).toEqual(['question', 'state'])
  })

  it('caches recaps so a redraw does not re-ask the far machine for every tile', async () => {
    const rpc = vi.fn(async () => ({ events: [{ kind: 'summary', recap: 'r', text: 't' }] }))
    const { fleet } = make({ rpc })
    await fleet.recentSummaries('m1', 'a1')
    await fleet.recentSummaries('m1', 'a1')
    expect(rpc).toHaveBeenCalledTimes(1)
  })

  it('answers an empty history rather than throwing when a recap RPC fails', async () => {
    const { fleet } = make({ rpc: async () => { throw new Error('timed out') } })
    await expect(fleet.recentSummaries('m1', 'a1')).resolves.toEqual([])
  })

  it('selects THIS computer without asking for a pinned key', async () => {
    // Our own machine is `authMode: 'remote'` like any other computer-backed one and has no pin for
    // itself — nor does it need one: its agents are read in-process, never over the relay.
    const { fleet, link } = make({ rows: [{ machineId: 'mine', local: true, authMode: 'remote' }], linked: [] })
    await expect(fleet.select('mine')).resolves.toBeUndefined()
    expect(link.attach).toHaveBeenCalledWith('mine')
  })

  it('keeps the socket when the dial goes back to local, and drops it when the dial goes', () => {
    // Two different events. "Looked away" detaches from the machine but the wheel's dots must stay live;
    // "unplugged" takes the socket with it, because it was only ever held for the dial.
    const { fleet, link } = make()
    // Lingering: nothing happens yet, and when it does it DETACHES rather than closing.
    fleet.release(false)
    expect(link.release).not.toHaveBeenCalled()

    // Unplugged: the socket goes now.
    fleet.release(true)
    expect(link.release).toHaveBeenCalled()
    expect(link.detach).not.toHaveBeenCalled()
  })

  it('folds a live machines_status into the list and reports the change', () => {
    const rows = [{ machineId: 'm1', name: 'a', state: 'offline', authMode: 'remote', local: false }]
    const applied: unknown[] = []
    let frameCb!: (f: { type?: string; payload?: Record<string, unknown> }) => void
    const fleet = new DeviceFleet({
      list: {
        list: () => ({ machines: rows, source: 'backend' as const }),
        find: (id: string) => rows.find((r) => r.machineId === id),
        applyLive: (r: unknown) => { applied.push(r); return true },
      } as never,
      link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {}, onGone: () => () => {} } as never,
      hasPeerLink: () => true,
      log: () => {},
    })
    const seen: string[] = []
    fleet.onEvent((e) => seen.push(e.kind))

    frameCb({ type: 'machines_status', payload: { statuses: [{ machineId: 'm1', online: true }] } })
    expect(applied).toHaveLength(1)
    expect(seen).toEqual(['state'])
  })

  it('does not filter machines_status by the selected machine', () => {
    // It is about the whole account, and it arrives while nothing at all is selected.
    let frameCb!: (f: { type?: string; machineId?: string; payload?: Record<string, unknown> }) => void
    const fleet = new DeviceFleet({
      list: { list: () => ({ machines: [], source: 'backend' as const }), find: () => undefined, applyLive: () => true } as never,
      link: { selectedMachine: 'm1', onFrame: (cb: never) => { frameCb = cb; return () => {} }, onUndelivered: () => () => {}, onAlive: () => () => {}, onGone: () => () => {} } as never,
      hasPeerLink: () => true,
      log: () => {},
    })
    const seen: string[] = []
    fleet.onEvent((e) => seen.push(e.kind))
    frameCb({ type: 'machines_status', machineId: 'someone-else', payload: { statuses: [] } })
    expect(seen).toEqual(['state'])
  })

  it('reads nothing over the lane for THIS computer', async () => {
    // Selecting the local machine is an announcement, not an attach-to-read: its agents are in-process.
    // Asking the cloud round-trips to ourselves and is refused with E2EE_REQUIRED, because a
    // computer-backed machine is `authMode: 'remote'` — which is exactly what happened on hardware.
    const rpc = vi.fn(async () => ({ agents: [] }))
    const { fleet } = make({ rows: [{ machineId: 'mine', local: true, authMode: 'remote' }], linked: [], rpc })
    await fleet.select('mine')
    await new Promise((r) => setTimeout(r, 0))
    expect(rpc).not.toHaveBeenCalled()
  })

  it('drops our own cards when they come back the long way', () => {
    // Selecting the local machine attaches a commander to it, so everything this daemon emits is fanned
    // back down the socket — after the dial already had it in-process.
    const { fleet, say } = make({ rows: [{ machineId: 'mine', local: true }, { machineId: 'm1' }] })
    const seen: string[] = []
    fleet.onEvent((e) => seen.push(e.kind))
    say({ type: 'commander_event', machineId: 'mine', agentId: 'a1', payload: { kind: 'summary', recap: 'echo' } })
    expect(seen).toHaveLength(0)
  })
})