Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

20.1 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import { applyCommand } from './commands.js'
import { crewKeep, crewNeedsMachine, type CrewDeps } from './crewKeep.js'
import { recordAgentPresence } from './machines.js'
import { config, fixture, GB, machine, NOW, person } from './crewKeep.test-support.js'
import * as crewRules from './crewRules.js'
import type { MachineRule } from './policy.js'
import { FOLDER_ASLEEP_MS, MOVE_LOW_DISK_BYTES, MOVE_OFFLINE_MS, MOVE_QUOTA_OUT_MS, SLEEP_IDLE_MS, DISK_FLOOR_BYTES } from './crewRules.js'
import type { Agent, SuperTabState } from './model.js'

const everywhere: CrewDeps = { admits: () => true }
/** The rules live in the project rules record, never in source (River, 11-05). */
const MACHINE_RULES_FROM_SOURCE = (crewRules as Record<string, unknown>).MACHINE_RULES
const kinds = (state: SuperTabState, deps = everywhere) => crewKeep(state, NOW, deps)

describe('crewKeep: sleep', () => {
  it('sleeps a crew member idle over an hour, not one idle under it, and never one with active work', () => {
    expect(kinds(fixture(undefined, SLEEP_IDLE_MS + 60_000).state)).toEqual([{ kind: 'sleep', agentId: expect.any(String) }])
    expect(kinds(fixture(undefined, SLEEP_IDLE_MS - 60_000).state)).toEqual([])
    const busy = fixture()
    busy.state = applyCommand(busy.state, person, { type: 'conversation.create', title: 'Fix it', brief: 'b', checks: [] } as never,
      { now: NOW - 2 * SLEEP_IDLE_MS, operationId: 'g1' }).state
    const group = busy.state.conversations.find(c => c.title === 'Fix it')!
    group.agentIds.push(busy.ada.id)
    group.assignment = { id: 'a1', agentId: busy.ada.id, status: 'working', assignedAt: NOW - 2 * SLEEP_IDLE_MS } as never
    expect(kinds(busy.state)).toEqual([])
  })

  it('never touches a PM, a member who is not crew, or one already asleep', () => {
    const f = fixture()
    ;(f.lead as Agent & { crew?: unknown }).crew = { role: 'PM', at: 1, by: 'owner' }
    delete (f.ada as Agent & { crew?: unknown }).crew
    expect(kinds(f.state)).toEqual([])
    const asleep = fixture()
    asleep.ada.asleep = { since: NOW - 60_000 }
    expect(kinds(asleep.state)).toEqual([])
  })
})

describe('crewKeep: move when its machine fails', () => {
  const moveOf = (state: SuperTabState, deps = everywhere) => crewKeep(state, NOW, deps).find(a => a.kind === 'move')
  it('moves a member holding work after the machine is offline over 5 minutes, not before', () => {
    // Holding work: the task its machine took from it, as liveness does at 2 minutes offline.
    const holding = (pools: ReturnType<typeof machine>[]) => {
      const f = fixture(pools)
      f.state = applyCommand(f.state, person, { type: 'conversation.create', title: 'Fix it', brief: 'b', checks: [] } as never, { now: NOW - 7_200_000, operationId: 'held' }).state
      const group = f.state.conversations.find(c => c.title === 'Fix it')!
      group.assignment = { id: 'a1', agentId: f.ada.id, status: 'lost', lost: { reason: 'machine offline', at: NOW - 180_000 } } as never
      f.state.agents.find(a => a.id === f.ada.id)!.lostTask = { conversationId: group.id, assignmentId: 'a1', reason: 'machine offline', at: NOW - 180_000 }
      return f.state
    }
    expect(moveOf(holding([machine('mac', { receivedAt: NOW - MOVE_OFFLINE_MS + 30_000 }), machine('mini')]))).toBeUndefined()
    expect(moveOf(holding([machine('mac', { receivedAt: NOW - MOVE_OFFLINE_MS - 1 }), machine('mini')])))
      .toEqual({ kind: 'move', agentId: expect.any(String), toMachineId: 'mini', reason: 'offline' })
  })
  it('moves at once when its engine is signed out', () => {
    expect(moveOf(fixture([machine('mac', { signedIn: false }), machine('mini')]).state)).toMatchObject({ toMachineId: 'mini', reason: 'signed-out' })
  })
  it('moves when out of quota for over an hour; not when it resets within the hour, nor when quota is unknown', () => {
    expect(moveOf(fixture([machine('mac', { usedPercent: 100, resetIn: MOVE_QUOTA_OUT_MS + 60_000 }), machine('mini')]).state))
      .toMatchObject({ toMachineId: 'mini', reason: 'quota' })
    expect(moveOf(fixture([machine('mac', { usedPercent: 100, resetIn: MOVE_QUOTA_OUT_MS - 60_000 }), machine('mini')]).state)).toBeUndefined()
    expect(moveOf(fixture([machine('mac', { usedPercent: 100, known: false }), machine('mini')]).state)).toBeUndefined()
  })
  it('moves when low on disk', () => {
    expect(moveOf(fixture([machine('mac', { diskFree: MOVE_LOW_DISK_BYTES - 1 }), machine('mini')]).state)).toMatchObject({ reason: 'disk' })
    expect(moveOf(fixture([machine('mac', { diskFree: MOVE_LOW_DISK_BYTES + GB }), machine('mini')]).state)).toBeUndefined()
  })
  it('picks the fallback in the route\'s order: same account kind, then the account resetting soonest, then most left, then fewer working', () => {
    const failing = machine('mac', { signedIn: false })
    // Soonest reset first among usable machines of the same kind.
    expect(moveOf(fixture([failing, machine('late', { resetIn: 5 * 86_400_000 }), machine('soon', { resetIn: 86_400_000 })]).state))
      .toMatchObject({ toMachineId: 'soon' })
    // Same reset: most left.
    expect(moveOf(fixture([failing, machine('half', { usedPercent: 50 }), machine('fresh', { usedPercent: 10 })]).state))
      .toMatchObject({ toMachineId: 'fresh' })
    // A machine that is failing itself, or that does not admit the start, is never a fallback.
    expect(moveOf(fixture([failing, machine('down', { receivedAt: NOW - MOVE_OFFLINE_MS - 1 }), machine('ok')]).state)).toMatchObject({ toMachineId: 'ok' })
    expect(moveOf(fixture([failing, machine('refuses'), machine('ok')]).state, { admits: id => id !== 'refuses' })).toMatchObject({ toMachineId: 'ok' })
  })
  it('with no fallback says it once, and moves no member twice', () => {
    const lonely = fixture([machine('mac', { signedIn: false })])
    expect(kinds(lonely.state)).toEqual([{ kind: 'stuck', agentId: lonely.ada.id, reason: 'signed-out' }])
    lonely.ada.crewStuck = 'signed-out'
    expect(kinds(lonely.state)).toEqual([])
    const moving = fixture([machine('mac', { signedIn: false }), machine('mini')])
    moving.ada.movedTo = { machineId: 'mini', startId: 's1', reason: 'signed-out', at: NOW - 1000 }
    expect(kinds(moving.state)).toEqual([])
  })
})

describe('crewKeep: folders', () => {
  it('drops the folder of a member asleep 7 days, once', () => {
    const f = fixture()
    f.ada.asleep = { since: NOW - FOLDER_ASLEEP_MS - 1 }
    expect(kinds(f.state)).toEqual([{ kind: 'dropFolder', agentId: f.ada.id, why: 'asleep-7d' }])
    f.ada.asleep.folderDropped = true
    expect(kinds(f.state)).toEqual([])
    f.ada.asleep = { since: NOW - FOLDER_ASLEEP_MS + 60_000 }
    expect(kinds(f.state)).toEqual([])
  })
  it('below 10 GB free, drops the folder of the member asleep longest on that machine first, one a pass', () => {
    const f = fixture([machine('mac', { diskFree: DISK_FLOOR_BYTES - GB }), machine('mini')])
    f.ada.asleep = { since: NOW - 3_600_000 }
    f.state = applyCommand(f.state, person, { type: 'agent.add', conversationId: 'lead', machineId: 'mac', harnessId: 'bo', name: 'bo', lead: false, configuration: config },
      { now: NOW - 2 * SLEEP_IDLE_MS, operationId: 'add-bo', contribution: { machineOwnerId: 'owner', contributorId: 'owner' } }).state
    const bo = f.state.agents.find(a => a.harnessId === 'bo')!
    ;(bo as Agent & { crew?: unknown }).crew = { role: 'Engineer', at: 1, by: 'owner' }
    bo.asleep = { since: NOW - 2 * 3_600_000 }
    expect(kinds(f.state)).toEqual([{ kind: 'dropFolder', agentId: bo.id, why: 'low-disk' }])
    bo.asleep.folderDropped = true
    expect(kinds(f.state)).toEqual([{ kind: 'dropFolder', agentId: f.ada.id, why: 'low-disk' }])
  })
})

describe('crewKeep: Team-written work stays in Team accounts (security review of #3229)', () => {
  it('a Team-account member moves only to a Team-account machine, and is stuck with none', () => {
    const team = (pool: ReturnType<typeof machine>) => ({ ...pool, hostMachineId: `${pool.machineId}-mac` }) as typeof pool
    const onlyMain = fixture([team(machine('mac', { signedIn: false })), machine('mini')])
    expect(kinds(onlyMain.state)).toEqual([{ kind: 'stuck', agentId: onlyMain.ada.id, reason: 'signed-out' }])
    const withTeam = fixture([team(machine('mac', { signedIn: false })), machine('mini', { resetIn: 86_400_000 }), team(machine('studio'))])
    expect(kinds(withTeam.state)).toEqual([{ kind: 'move', agentId: withTeam.ada.id, toMachineId: 'studio', reason: 'signed-out' }])
  })
  it('a main-account member in a Team-written conversation moves only to a Team-account machine', () => {
    const f = fixture([machine('mac', { signedIn: false }), machine('mini')])
    f.state.conversations.find(c => c.id === 'lead')!.teamWritten = true
    // The project's chat is its coordinator's; a task the member works in is what counts.
    f.state = applyCommand(f.state, person, { type: 'conversation.create', title: 'Team task', brief: 'b', checks: [] } as never, { now: NOW - 7_200_000, operationId: 'tw' }).state
    const task = f.state.conversations.find(c => c.title === 'Team task')!
    task.agentIds.push(f.ada.id)
    task.teamWritten = true
    expect(kinds(f.state)).toEqual([{ kind: 'stuck', agentId: f.ada.id, reason: 'signed-out' }])
  })
})

describe('crewKeep: account kinds and Team sessions (security review 2 of #3229)', () => {
  const team = (pool: ReturnType<typeof machine>) => ({ ...pool, hostMachineId: `${pool.machineId}-mac` }) as typeof pool
  it('a main-account member never moves into a Team account', () => {
    const f = fixture([machine('mac', { signedIn: false }), team(machine('studio'))])
    expect(kinds(f.state)).toEqual([{ kind: 'stuck', agentId: f.ada.id, reason: 'signed-out' }])
  })
  it('a member in a Team account never sleeps: its close there deletes what a wake would resume', () => {
    const f = fixture([team(machine('mac')), machine('mini')])
    expect(kinds(f.state)).toEqual([])
  })
  it('a machine that refused the member\'s move start is not picked again for an hour', () => {
    const f = fixture([machine('mac', { signedIn: false }), machine('mini', { resetIn: 86_400_000 }), machine('late')])
    f.ada.crewMoveRefused = { machineId: 'mini', at: NOW - 30 * 60_000 }
    expect(kinds(f.state)).toEqual([{ kind: 'move', agentId: f.ada.id, toMachineId: 'late', reason: 'signed-out' }])
    f.ada.crewMoveRefused = { machineId: 'mini', at: NOW - 61 * 60_000 }
    expect(kinds(f.state)).toEqual([{ kind: 'move', agentId: f.ada.id, toMachineId: 'mini', reason: 'signed-out' }])
  })
})

describe('crewKeep: an asleep member stays where it sleeps (review of #3229)', () => {
  it('is not moved off a machine that went offline: it has no work there, and moves once woken if it must', () => {
    const f = fixture([machine('mac', { receivedAt: NOW - MOVE_OFFLINE_MS - 1 }), machine('mini')])
    f.ada.asleep = { since: NOW - 3_600_000 }
    expect(kinds(f.state)).toEqual([])
  })
  it('says whether any member needs a machine now, so the sweep reads the Team facts only then', () => {
    const calm = fixture()
    expect(crewNeedsMachine(calm.state, NOW)).toBe(false)
    expect(crewNeedsMachine(fixture([machine('mac', { signedIn: false }), machine('mini')]).state, NOW)).toBe(true)
  })
})

describe('crewKeep: machine rules apply to moves, read from the project rules record (review of #3229, River)', () => {
  const at = (iso: string) => Date.parse(iso)
  // Generic rules as the record would hold them: never a person's real machines or hours in source (River, 11-05).
  const rules: MachineRule[] = [
    { machineIds: ['no-codex'], why: 'no Codex there', noEngines: ['codex'] },
    { machineIds: ['travels'], why: 'away', away: { days: [1, 2, 3, 4, 5], windows: [['07:30', '09:30'], ['18:00', '21:00']], timeZone: 'Asia/Ho_Chi_Minh' } },
  ]
  // Each project's own rules (#3964): the member's project here is the Swarm's first.
  const withRules: CrewDeps = { admits: () => true, machineRules: projectId => projectId === 'lead' ? rules : [] }
  it('never moves a member to a machine whose rule refuses its engine; another engine may go there', () => {
    const codex = fixture([machine('mac', { signedIn: false }), machine('no-codex'), machine('other', { resetIn: 6 * 86_400_000 })])
    codex.ada.configuration = { ...codex.ada.configuration, engine: 'codex' }
    for (const pool of codex.state.machines!) {
      pool.engines = ['claude', 'codex']
      ;(pool.observation as { readiness: { engines: unknown[] } }).readiness.engines = [{ engine: 'codex', installed: true, signedIn: pool.machineId !== 'mac' }]
      ;(pool.observation as { quotas: Array<{ engine: string }> }).quotas[0]!.engine = 'codex'
    }
    expect(crewKeep(codex.state, NOW, withRules)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'other' })])
    const claude = fixture([machine('mac', { signedIn: false }), machine('no-codex'), machine('other', { resetIn: 6 * 86_400_000 })])
    expect(crewKeep(claude.state, NOW, withRules)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'no-codex' })])
  })
  it('never moves to a machine while its rule says it is away, in its own time zone', () => {
    const pools = () => [machine('mac', { signedIn: false }), machine('travels'), machine('other', { resetIn: 6 * 86_400_000 })]
    // Monday 2026-11-09 08:00 in Vietnam (UTC+7) is 01:00 UTC; 10:00 there is 03:00 UTC.
    const away = at('2026-11-09T01:00:00Z'), back = at('2026-11-09T03:00:00Z')
    const onAway = fixture(pools()); onAway.state.machines!.forEach(p => { p.observation!.receivedAt = away - 5000 })
    expect(crewKeep(onAway.state, away, withRules)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'other' })])
    const onBack = fixture(pools()); onBack.state.machines!.forEach(p => { p.observation!.receivedAt = back - 5000 })
    expect(crewKeep(onBack.state, back, withRules)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'travels' })])
  })
  // Review of PR #3436 (SecAlder): a crew Reviewer is never moved onto a machine where no Reviewer runs.
  it('never moves a Reviewer to a machine where no Reviewer runs; an Engineer may go there', () => {
    const noReviews: CrewDeps = { admits: () => true, machineRules: () => [{ machineIds: ['local-codex'], why: 'Its Codex defaults to a local model', noReviews: true }] }
    const pools = () => [machine('mac', { signedIn: false }), machine('local-codex'), machine('other', { resetIn: 6 * 86_400_000 })]
    const reviewer = fixture(pools())
    ;(reviewer.ada as Agent & { crew?: unknown }).crew = { role: 'Reviewer', at: 1, by: 'owner' }
    expect(crewKeep(reviewer.state, NOW, noReviews)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'other' })])
    // Any role the review steps take as a Reviewer's (reviewerPick.ts), not only one starting with the word.
    const security = fixture(pools())
    ;(security.ada as Agent & { crew?: unknown }).crew = { role: 'Security reviewer', at: 1, by: 'owner' }
    expect(crewKeep(security.state, NOW, noReviews)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'other' })])
    const engineer = fixture(pools())
    expect(crewKeep(engineer.state, NOW, noReviews)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'local-codex' })])
  })
  it('reads the member\'s own project\'s rules, never another project\'s', () => {
    const codex = fixture([machine('mac', { signedIn: false }), machine('no-codex'), machine('other', { resetIn: 6 * 86_400_000 })])
    codex.ada.configuration = { ...codex.ada.configuration, engine: 'codex' }
    for (const pool of codex.state.machines!) {
      pool.engines = ['claude', 'codex']
      ;(pool.observation as { readiness: { engines: unknown[] } }).readiness.engines = [{ engine: 'codex', installed: true, signedIn: pool.machineId !== 'mac' }]
      ;(pool.observation as { quotas: Array<{ engine: string }> }).quotas[0]!.engine = 'codex'
    }
    const elsewhere: CrewDeps = { admits: () => true, machineRules: projectId => projectId === 'lead' ? [] : rules }
    expect(crewKeep(codex.state, NOW, elsewhere)).toEqual([expect.objectContaining({ kind: 'move', toMachineId: 'no-codex' })])
  })
  it('with no rules record, no machine rule applies', () => {
    const f = fixture([machine('mac', { signedIn: false }), machine('travels'), machine('other', { resetIn: 6 * 86_400_000 })])
    expect(crewKeep(f.state, at('2026-11-09T01:00:00Z') + 0 * NOW, { admits: () => true })).toBeDefined()
    expect(MACHINE_RULES_FROM_SOURCE).toBeUndefined()
  })
  it('an awake member with no work stays when its machine goes offline: it moves once it holds work', () => {
    const f = fixture([machine('mac', { receivedAt: NOW - MOVE_OFFLINE_MS - 1 }), machine('mini')], SLEEP_IDLE_MS - 60_000)
    expect(kinds(f.state)).toEqual([])
  })
})

// Task #3662 (anh Dee, 11-06): Wren stopped three times in an hour on an account with about 10% weekly quota left, and Aspen on one
// at about 12%; the keeper reopened Wren each time and its task waited 5 minutes per stop. A member on an account below the routing
// quota floor moves to the best fallback instead of reopening in place, and one that keeps stopping is moved too.
describe('crewKeep: low quota and a member that keeps stopping (#3662)', () => {
  const moveOf = (state: SuperTabState, deps = everywhere) => crewKeep(state, NOW, deps).find(a => a.kind === 'move')
  const stuckOf = (state: SuperTabState) => crewKeep(state, NOW, everywhere).find(a => a.kind === 'stuck')

  it('moves a member off an account that runs the PM when its weekly quota is below 20%, not above it', () => {
    // The PM runs on 'mac', Ada with it: 10% left is under the 20% an account running a PM keeps.
    expect(moveOf(fixture([machine('mac', { usedPercent: 90 }), machine('mini')]).state)).toMatchObject({ toMachineId: 'mini', reason: 'quota-low' })
    expect(moveOf(fixture([machine('mac', { usedPercent: 70 }), machine('mini')]).state)).toBeUndefined()
  })

  it('moves a member off any other account below 5%, and keeps it above', () => {
    const onMini = (used: number) => {
      const f = fixture([machine('mac'), machine('mini', { usedPercent: used })])
      f.state.agents.find(a => a.id === f.ada.id)!.machineId = 'mini'
      return f.state
    }
    expect(moveOf(onMini(96))).toMatchObject({ toMachineId: 'mac', reason: 'quota-low' })
    expect(moveOf(onMini(90))).toBeUndefined()
  })

  it('never picks a fallback that is itself below its floor', () => {
    const f = fixture([machine('mac', { usedPercent: 90 }), machine('mini', { signedIn: false })])
    f.state.agents.find(a => a.id === f.ada.id)!.machineId = 'mini'
    // Ada's own machine is signed out; the only other one runs the PM with 10% left, under its 20%.
    expect(moveOf(f.state)).toBeUndefined()
    expect(stuckOf(f.state)).toMatchObject({ reason: 'signed-out' })
  })

  it('says nothing of a quota it cannot read', () => {
    expect(moveOf(fixture([machine('mac', { usedPercent: 99, known: false }), machine('mini')]).state)).toBeUndefined()
  })

  it('moves a member that stopped three times in the last hour, not two, and counts only the last hour', () => {
    const stopped = (times: number[]) => {
      const f = fixture()
      f.state.agents.find(a => a.id === f.ada.id)!.stops = times
      return f.state
    }
    expect(moveOf(stopped([NOW - 50 * 60_000, NOW - 30 * 60_000, NOW - 5 * 60_000]))).toMatchObject({ toMachineId: 'mini', reason: 'unstable' })
    expect(moveOf(stopped([NOW - 30 * 60_000, NOW - 5 * 60_000]))).toBeUndefined()
    expect(moveOf(stopped([NOW - 61 * 60_000, NOW - 30 * 60_000, NOW - 5 * 60_000]))).toBeUndefined()
  })

  it('records a stop when a reading first shows the session gone, once however long it stays down', () => {
    const f = fixture()
    recordAgentPresence(f.state, 'mac', [], NOW)
    recordAgentPresence(f.state, 'mac', [], NOW + 30_000)
    const ada = () => f.state.agents.find(a => a.id === f.ada.id)!
    expect(ada().stops).toEqual([NOW])
    recordAgentPresence(f.state, 'mac', [ada().harnessId], NOW + 60_000)
    recordAgentPresence(f.state, 'mac', [], NOW + 90_000)
    expect(ada().stops).toEqual([NOW, NOW + 90_000])
    // Only the last hour is kept, and at most a few.
    recordAgentPresence(f.state, 'mac', [ada().harnessId], NOW + 2 * 3_600_000)
    recordAgentPresence(f.state, 'mac', [], NOW + 2 * 3_600_000 + 1_000)
    expect(ada().stops).toEqual([NOW + 2 * 3_600_000 + 1_000])
  })
})