Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

39.9 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
  DEFAULT_SERVICE_OPTIONS, ENGINE_LIVE_ENV, ENGINE_RUNTIME_ENV, ENGINE_SCREEN_ENV, ENGINE_MODEL_CONTROL_ENV, masterRunsEngineModelControl, ENGINE_QUESTION_CONTROL_ENV, masterRunsEngineQuestionControl, ENGINE_SUBMISSION_ENV, masterRunsEngineSubmission, ENGINE_NATIVE_CONTROL_ENV, masterRunsEngineNativeControl, KNOWN_SERVICES, SERVICE_HOSTS, SERVICE_PROCESSES_ENV, ServiceSupervisor, UPDATER_HOST, masterRunsLiveEngines, masterRunsEngineRuntime, masterRunsEngineScreen, serviceOptions, serviceProcessesEnv, serviceSpecs, servicesTheMasterRuns, SWARM_ENV, servicesOn, swarmOn,
  type ServiceSpec, type ServiceSupervisorDeps, type ServiceSupervisorOptions,
} from './services.js'
import type { CoreHandle } from './supervisor.js'

const MIB = 1024 * 1024

class FakeService implements CoreHandle {
  readonly kills: NodeJS.Signals[] = []
  private messageListeners: Array<(message: unknown) => void> = []
  private exitListeners: Array<(code: number | null, signal: NodeJS.Signals | null) => void> = []
  constructor(readonly pid: number | undefined, readonly spec: ServiceSpec, readonly env: Record<string, string>) {}
  readonly sent: unknown[] = []
  send(message: unknown): void { this.sent.push(message) }
  kill(signal: NodeJS.Signals): void { this.kills.push(signal) }
  onMessage(listener: (message: unknown) => void): void { this.messageListeners.push(listener) }
  onExit(listener: (code: number | null, signal: NodeJS.Signals | null) => void): void { this.exitListeners.push(listener) }
  say(message: unknown): void { for (const listener of this.messageListeners) listener(message) }
  beat(rssBytes = 100 * MIB, heapUsedBytes = rssBytes / 2): void { this.say({ type: 'harnessd:heartbeat', rssBytes, heapUsedBytes, loopDelayMs: 2 }) }
  exit(code: number | null, signal: NodeJS.Signals | null = null): void { for (const listener of this.exitListeners) listener(code, signal) }
}

const options: ServiceSupervisorOptions = {
  ...DEFAULT_SERVICE_OPTIONS,
  heartbeatTimeoutMs: 6_000,
  stopGraceMs: 1_000,
  initialBackoffMs: 1_000,
  maxBackoffMs: 4_000,
  backoffResetMs: 20_000,
  heapRestartPercent: 75,
  parkCrashes: 3,
  parkWindowMs: 60_000,
  parkRetryMs: 120_000,
}
const search: ServiceSpec = { name: 'search', services: ['search'], heapLimitMiB: 512, rssLimitMiB: 1_024 }
const devices: ServiceSpec = { name: 'devices', services: ['devices'], heapLimitMiB: 256, rssLimitMiB: 0 }

describe('ServiceSupervisor', () => {
  let children: FakeService[]
  let lines: string[]
  const latest = (name = 'search') => children.filter((child) => child.spec.name === name).at(-1)!
  const staged: Array<{ version?: string; ptyVersion?: string }> = []
  const make = (specs: ServiceSpec[] = [search], overrides: Partial<ServiceSupervisorOptions> = {}, env: Record<string, string> = {}, tell = true, more: Partial<ServiceSupervisorDeps> = {}) =>
    new ServiceSupervisor(specs, {
      ...(tell ? { staged: (what: { version?: string; ptyVersion?: string }) => { staged.push(what) } } : {}),
      spawnService: (spec, extra) => {
        const child = new FakeService(2000 + children.length, spec, extra)
        children.push(child)
        return child
      },
      now: () => performance.now(),
      wallClock: () => Date.now(),
      setTimer: (run, ms) => setTimeout(run, ms),
      clearTimer: (timer) => clearTimeout(timer as ReturnType<typeof setTimeout>),
      log: (line) => lines.push(line),
      ...more,
    }, { ...options, ...overrides }, env)

  beforeEach(() => {
    vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date', 'performance'] })
    children = []
    lines = []
  })
  afterEach(() => vi.useRealTimers())

  it('on Windows, asks each service to leave rather than signalling it, and ends it after the grace', () => {
    const supervisor = make([search], {}, {}, true, { stopByMessage: true })
    supervisor.start()
    const done = vi.fn()
    supervisor.stop(done)
    expect(latest().kills).toEqual([])
    expect(latest().sent).toEqual([{ type: 'harnessd:shutdown' }])
    vi.advanceTimersByTime(options.stopGraceMs)
    expect(latest().kills).toEqual(['SIGKILL'])
    latest().exit(null, 'SIGKILL')
    expect(done).toHaveBeenCalledOnce()
  })

  it('starts every service with its name, its token and its watchdog, and none waits on another', () => {
    const supervisor = make([search, devices], {}, { HARNESSD_SERVICE_TOKEN: 'secret' })
    supervisor.start()
    expect(children.map((child) => child.spec.name)).toEqual(['search', 'devices'])
    expect(latest().env).toEqual({ HARNESSD_SERVICE_TOKEN: 'secret', HARNESSD_SERVICE: 'search', HARNESSD_RESTARTS: '0', HARNESSD_WATCHDOG_MS: '6000' })
    expect(supervisor.status().map((status) => status.state)).toEqual(['starting', 'starting'])
    latest().beat()
    expect(supervisor.status()[0]).toMatchObject({ name: 'search', state: 'running', pid: 2000, restarts: 0, lastExit: null })
    // Anything that is not a heartbeat is ignored.
    latest().say({ type: 'harnessd:bound', protocol: 2, port: 1 })
    latest().say('noise')
    expect(supervisor.status()[0].state).toBe('running')
  })

  it('passes on what the updater staged, and starts it again at once when it asks: not a crash', () => {
    const updater: ServiceSpec = { name: 'updater', ...UPDATER_HOST }
    staged.length = 0
    const supervisor = make([updater], { parkCrashes: 2 })
    supervisor.start()
    latest('updater').beat()
    latest('updater').say({ type: 'harnessd:staged', version: '9.9.9' })
    expect(staged).toEqual([{ version: '9.9.9' }])
    expect(lines).toContain('[harnessd] service updater staged 9.9.9')
    // A terminal host (Windows), alone or with a build.
    latest('updater').say({ type: 'harnessd:staged', ptyVersion: '0.11.0' })
    latest('updater').say({ type: 'harnessd:staged', version: '9.9.9', ptyVersion: '0.11.0' })
    expect(staged.slice(1)).toEqual([{ ptyVersion: '0.11.0' }, { version: '9.9.9', ptyVersion: '0.11.0' }])
    expect(lines).toContain('[harnessd] service updater staged terminal host 0.11.0')
    expect(lines).toContain('[harnessd] service updater staged 9.9.9 and terminal host 0.11.0')
    staged.length = 1
    for (let i = 0; i < 3; i++) {
      latest('updater').exit(75)
      vi.advanceTimersByTime(0)
    }
    expect(children).toHaveLength(4)
    expect(supervisor.status()[0]).toMatchObject({ state: 'restarting', restarts: 3, lastExit: 'code 75', lastExitReason: 'restart' })
    expect(lines).toContain('[harnessd] service updater asked to start again (code 75) — restarting')
    // Killed for hanging, its 75 is no request.
    latest('updater').beat()
    vi.advanceTimersByTime(options.heartbeatTimeoutMs * 2)
    latest('updater').exit(75)
    expect(supervisor.status()[0].lastExitReason).toBe('hung')
    // A master that was not told how to pass a staging on only says it.
    const quiet = make([updater], {}, {}, false)
    quiet.start()
    latest('updater').say({ type: 'harnessd:staged', version: '9.9.10' })
    expect(staged).toEqual([{ version: '9.9.9' }])
  })

  it('takes a staged build, or a request to start again at once, from the updater alone', () => {
    // A staged build restarts the core. From any other service it is a bug or worse: said once, nothing done.
    staged.length = 0
    const supervisor = make([search], { parkCrashes: 2 })
    supervisor.start()
    latest().beat()
    latest().say({ type: 'harnessd:staged', version: '6.6.6' })
    latest().say({ type: 'harnessd:staged', version: '6.6.7' })
    expect(staged).toEqual([])
    expect(lines.filter((line) => line.includes('staged'))).toEqual(['[harnessd] service search said it staged 6.6.6, which only the updater may — ignored'])
    // Its exit 75 is a crash like any other: restarted after the backoff, and parked when it keeps on.
    latest().exit(75)
    expect(supervisor.status()[0]).toMatchObject({ state: 'restarting', lastExit: 'code 75', lastExitReason: 'crashed' })
    vi.advanceTimersByTime(0)
    expect(children).toHaveLength(1)
    vi.advanceTimersByTime(options.initialBackoffMs)
    expect(children).toHaveLength(2)
    latest().exit(75)
    expect(supervisor.status()[0].state).toBe('parked')
  })

  it('says a Windows exit code signed: a service TerminateProcess ended exited -1', () => {
    const supervisor = make([search], { parkCrashes: 99 })
    supervisor.start()
    latest().beat()
    latest().exit(0xffff_ffff)
    expect(supervisor.status()[0]).toMatchObject({ lastExit: 'code -1', lastExitReason: 'crashed' })
    expect(lines.some((line) => line.includes('exited (code -1, crashed) — restarting in'))).toBe(true)
  })

  it('restarts a crashed service with a backoff that doubles, caps, and is forgiven after a long run', () => {
    const supervisor = make([search], { parkCrashes: 99 })
    supervisor.start()
    latest().beat()
    latest().exit(1)
    expect(supervisor.status()[0]).toMatchObject({ state: 'restarting', restarts: 1, lastExit: 'code 1', lastExitReason: 'crashed', pid: null })
    vi.advanceTimersByTime(999)
    expect(children).toHaveLength(1)
    vi.advanceTimersByTime(1)
    expect(children).toHaveLength(2)
    expect(latest().env.HARNESSD_RESTARTS).toBe('1')
    expect(supervisor.status()[0].state).toBe('restarting')
    latest().beat()
    expect(supervisor.status()[0].state).toBe('running')
    for (const delay of [2_000, 4_000, 4_000]) {
      latest().exit(null, 'SIGSEGV')
      vi.advanceTimersByTime(delay - 1)
      const before = children.length
      vi.advanceTimersByTime(1)
      expect(children.length).toBe(before + 1)
    }
    expect(lines.some((line) => line.includes('exited (signal SIGSEGV, crashed) — restarting in 4000 ms'))).toBe(true)
    // Up long enough: the next crash restarts after the initial delay again.
    for (let left = 21_000; left > 0; left -= 1_000) { vi.advanceTimersByTime(1_000); latest().beat() }
    latest().exit(1)
    const before = children.length
    vi.advanceTimersByTime(1_000)
    expect(children.length).toBe(before + 1)
  })

  it('kills a service that stops beating, from its start too, and restarts it as hung', () => {
    const supervisor = make()
    supervisor.start()
    // Never beat at all: hung from the start, once one beat's time more has passed.
    vi.advanceTimersByTime(6_000)
    expect(latest().kills).toEqual([])
    vi.advanceTimersByTime(2_000)
    expect(latest().kills).toEqual(['SIGKILL'])
    latest().exit(null, 'SIGKILL')
    expect(supervisor.status()[0]).toMatchObject({ lastExitReason: 'hung', state: 'restarting' })
    vi.advanceTimersByTime(1_000)
    const second = latest()
    for (let i = 0; i < 3; i++) { vi.advanceTimersByTime(5_000); second.beat() }
    expect(second.kills).toEqual([])
    vi.advanceTimersByTime(8_000)
    expect(second.kills).toEqual(['SIGKILL'])
    expect(lines.some((line) => line.includes('sent no heartbeat for 8000 ms — it is hung'))).toBe(true)
  })

  it('takes no silence it was paused through for a hang: the beat a resumed service sends keeps it', () => {
    const supervisor = make()
    supervisor.start()
    const child = latest()
    child.beat()
    // Paused with the master: on resume the silence runs out at once and the overdue beat follows.
    vi.advanceTimersByTime(6_000)
    vi.advanceTimersByTime(10)
    child.beat()
    for (let i = 0; i < 6; i++) { vi.advanceTimersByTime(5_000); child.beat() }
    expect(child.kills).toEqual([])
    expect(supervisor.status()[0]).toMatchObject({ restarts: 0 })
  })

  it('restarts a service that outgrows its heap share or its resident budget, and counts it like a crash', () => {
    const supervisor = make([search, devices], { parkCrashes: 99 })
    supervisor.start()
    latest('search').beat(200 * MIB, 380 * MIB)
    expect(latest('search').kills).toEqual([])
    latest('search').beat(200 * MIB, 390 * MIB)
    expect(latest('search').kills).toEqual(['SIGTERM'])
    // While it is going, more beats do not ask again.
    latest('search').beat(200 * MIB, 500 * MIB)
    expect(latest('search').kills).toEqual(['SIGTERM'])
    latest('search').exit(0)
    expect(supervisor.status()[0]).toMatchObject({ lastExitReason: 'memory', restarts: 1 })
    vi.advanceTimersByTime(1_000)
    latest('search').beat(1_100 * MIB, 10 * MIB)
    expect(latest('search').kills).toEqual(['SIGTERM'])
    expect(lines.some((line) => line.includes('over its 1024 MiB budget'))).toBe(true)
    // No resident budget: only the heap share counts.
    latest('devices').beat(9_000 * MIB, 10 * MIB)
    expect(latest('devices').kills).toEqual([])
    expect(lines.some((line) => line.includes('past 75% of its 512 MiB limit'))).toBe(true)
  })

  it('SIGKILLs a service that outlives its grace to stop for memory', () => {
    make().start()
    latest().beat(100 * MIB, 500 * MIB)
    expect(latest().kills).toEqual(['SIGTERM'])
    vi.advanceTimersByTime(1_000)
    expect(latest().kills).toEqual(['SIGTERM', 'SIGKILL'])
    expect(lines.some((line) => line.includes('outlived its 1000 ms to stop'))).toBe(true)
  })

  it('parks a service that keeps crashing, says so, and tries it again later', () => {
    const supervisor = make()
    supervisor.start()
    for (let crash = 1; crash <= 2; crash++) {
      latest().exit(1)
      vi.advanceTimersByTime(4_000)
    }
    latest().exit(1)
    expect(supervisor.status()[0]).toMatchObject({ state: 'parked', restarts: 3 })
    expect(lines.some((line) => line.includes('ended 3 times in 1 min (code 1, crashed) — parked; trying again in 2 min'))).toBe(true)
    const parked = children.length
    vi.advanceTimersByTime(119_999)
    expect(children.length).toBe(parked)
    vi.advanceTimersByTime(1)
    expect(children.length).toBe(parked + 1)
    // A fresh start: its next crash is one, not the fourth.
    latest().exit(1)
    expect(supervisor.status()[0].state).toBe('restarting')
  })

  it('crashes far apart never park a service', () => {
    const supervisor = make()
    supervisor.start()
    for (let crash = 0; crash < 5; crash++) {
      latest().exit(1)
      vi.advanceTimersByTime(61_000)
    }
    expect(supervisor.status()[0].state).not.toBe('parked')
  })

  it('stops every service, SIGKILLing one that outlives its grace, and says when all are gone', () => {
    const supervisor = make([search, devices])
    supervisor.start()
    const done = vi.fn()
    supervisor.stop(done)
    expect(supervisor.status().map((status) => status.state)).toEqual(['stopping', 'stopping'])
    expect(latest('search').kills).toEqual(['SIGTERM'])
    latest('search').exit(0)
    expect(done).not.toHaveBeenCalled()
    vi.advanceTimersByTime(1_000)
    expect(latest('devices').kills).toEqual(['SIGTERM', 'SIGKILL'])
    latest('devices').exit(null, 'SIGKILL')
    expect(done).toHaveBeenCalledOnce()
    expect(supervisor.status()).toEqual([
      expect.objectContaining({ name: 'search', state: 'stopped', lastExitReason: 'stopped', lastExit: 'code 0' }),
      expect.objectContaining({ name: 'devices', state: 'stopped', lastExitReason: 'stopped', lastExit: 'signal SIGKILL' }),
    ])
    // Stopping again, or a late exit from a process already let go, changes nothing.
    const again = vi.fn()
    supervisor.stop(again)
    expect(again).toHaveBeenCalledOnce()
  })

  it('a second stop while stopping waits for the same end, and sends no second signal', () => {
    const supervisor = make()
    supervisor.start()
    const first = vi.fn()
    const second = vi.fn()
    supervisor.stop(first)
    supervisor.stop(second)
    expect(latest().kills).toEqual(['SIGTERM'])
    latest().exit(0)
    expect(first).toHaveBeenCalledOnce()
    expect(second).toHaveBeenCalledOnce()
  })

  it('stops a service waiting to restart, or parked, without starting it', () => {
    const supervisor = make([search, devices])
    supervisor.start()
    latest('search').beat()
    for (let crash = 0; crash < 3; crash++) { latest('devices').exit(1); vi.advanceTimersByTime(4_000); latest('search').beat() }
    latest('search').exit(1)
    expect(supervisor.status().map((status) => status.state)).toEqual(['restarting', 'parked'])
    const started = children.length
    const done = vi.fn()
    supervisor.stop(done)
    expect(done).toHaveBeenCalledOnce()
    vi.advanceTimersByTime(300_000)
    expect(children.length).toBe(started)
    expect(supervisor.status().map((status) => status.state)).toEqual(['stopped', 'stopped'])
  })

  it('leaves the new reader workers off for old cores that already read inline, and starts only the requested engine', () => {
    const specs = serviceSpecs({}, SERVICE_HOSTS).filter(spec => spec.name.startsWith('engine-'))
    expect(specs.map(spec => spec.name)).toEqual(['engine-claude', 'engine-codex'])
    const supervisor = make(specs)
    supervisor.start()
    for (const protocol of [0, 1, 2, 3, 4]) supervisor.unasked(protocol)
    expect(children).toHaveLength(0)
    supervisor.want('engine-codex')
    expect(children.map(child => child.spec.name)).toEqual(['engine-codex'])
  })

  it('starts an experiment\'s process only once the core asks for one of its services, and keeps it like any other', () => {
    const experiment: ServiceSpec = { name: 'experiments', services: ['orchestrator', 'messages'], heapLimitMiB: 256, rssLimitMiB: 512, onDemand: true }
    const supervisor = make([search, experiment])
    supervisor.start()
    expect(children.map((child) => child.spec.name)).toEqual(['search'])
    expect(supervisor.status().map((status) => status.state)).toEqual(['starting', 'off'])
    supervisor.want('search')
    supervisor.want('nobody')
    expect(children).toHaveLength(1)
    supervisor.want('messages')
    expect(latest('experiments').env.HARNESSD_SERVICE).toBe('experiments')
    // Asked again, while it runs or after it crashed: the master's to restart, once.
    supervisor.want('orchestrator')
    latest('experiments').exit(1)
    supervisor.want('messages')
    expect(children.filter((child) => child.spec.name === 'experiments')).toHaveLength(1)
    vi.advanceTimersByTime(1_000)
    expect(children.filter((child) => child.spec.name === 'experiments')).toHaveLength(2)
    expect(lines.filter((line) => line.includes('service experiments started'))).toHaveLength(2)
  })

  it('starts every process on demand a core too old to ask for it would never ask for, and stops one never started at once', () => {
    const experiment: ServiceSpec = { name: 'orchestrator', services: ['orchestrator'], heapLimitMiB: 256, rssLimitMiB: 512, onDemand: true }
    const devices: ServiceSpec = { name: 'devices', services: ['devices', 'wifi'], heapLimitMiB: 256, rssLimitMiB: 512, onDemand: true, askedSince: 4 }
    const idle: ServiceSpec = { ...experiment, name: 'messages', services: ['messages'] }
    // A core of protocol 3 asks for the experiments, not for the devices: theirs starts as it binds.
    const three = make([search, experiment, devices])
    three.start()
    three.unasked(3)
    expect(children.map((child) => child.spec.name)).toEqual(['search', 'devices'])
    three.unasked(4)
    expect(children).toHaveLength(2)
    // A core from before `want` asks for none.
    children.length = 0
    const supervisor = make([experiment, devices])
    supervisor.start()
    supervisor.unasked(2)
    expect(children.map((child) => child.spec.name)).toEqual(['orchestrator', 'devices'])
    const never = make([idle])
    never.start()
    const done = vi.fn()
    never.stop(done)
    expect(done).toHaveBeenCalledOnce()
    expect(never.status()[0].state).toBe('stopped')
    // Started again with the others (a re-execution the new bundle refused): off, and started when asked.
    never.start()
    expect(never.status()[0].state).toBe('off')
    never.unasked(2)
    expect(children.map((child) => child.spec.name)).toEqual(['orchestrator', 'devices', 'messages'])
    // One running then is stopped and started with the others, and waits to be asked for again.
    never.stop(vi.fn())
    latest('messages').exit(0)
    never.start()
    expect(never.status()[0].state).toBe('off')
  })

  it('with no services, starting does nothing and stopping is done at once', () => {
    const supervisor = make([])
    supervisor.start()
    const done = vi.fn()
    supervisor.stop(done)
    expect(done).toHaveBeenCalledOnce()
    expect(supervisor.status()).toEqual([])
  })

  it('names a process with no pid, and ignores what a process it let go of says', () => {
    const supervisor = new ServiceSupervisor([search], {
      spawnService: (spec, env) => { const child = new FakeService(undefined, spec, env); children.push(child); return child },
      now: () => performance.now(),
      wallClock: () => Date.now(),
      setTimer: (run, ms) => setTimeout(run, ms),
      clearTimer: (timer) => clearTimeout(timer as ReturnType<typeof setTimeout>),
      log: (line) => lines.push(line),
    }, options)
    supervisor.start()
    expect(lines[0]).toContain('started (pid ?)')
    const first = latest()
    first.exit(1)
    vi.advanceTimersByTime(1_000)
    first.beat(9_000 * MIB, 9_000 * MIB)
    first.exit(1)
    expect(latest().kills).toEqual([])
    expect(supervisor.status()[0].restarts).toBe(1)
  })

  it('reads its timings from the environment, keeping the defaults for what is unset or invalid', () => {
    expect(serviceOptions({})).toEqual(DEFAULT_SERVICE_OPTIONS)
    expect(serviceOptions({
      HARNESSD_SERVICE_HEARTBEAT_TIMEOUT_MS: '2000', HARNESSD_SERVICE_STOP_GRACE_MS: '50', HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '10',
      HARNESSD_SERVICE_MAX_BACKOFF_MS: '20', HARNESSD_SERVICE_BACKOFF_RESET_MS: '30', HARNESSD_SERVICE_HEAP_RESTART_PERCENT: '150',
      HARNESSD_SERVICE_PARK_CRASHES: '2', HARNESSD_SERVICE_PARK_WINDOW_MS: '40', HARNESSD_SERVICE_PARK_RETRY_MS: '60',
    })).toEqual({ heartbeatTimeoutMs: 2_000, stopGraceMs: 50, initialBackoffMs: 10, maxBackoffMs: 20, backoffResetMs: 30, heapRestartPercent: 100, parkCrashes: 2, parkWindowMs: 40, parkRetryMs: 60 })
    // A heartbeat patience under a second, or a value that is not a number, keeps the default.
    expect(serviceOptions({ HARNESSD_SERVICE_HEARTBEAT_TIMEOUT_MS: '10', HARNESSD_SERVICE_PARK_CRASHES: 'many' }))
      .toEqual(DEFAULT_SERVICE_OPTIONS)
  })

  it('reads the services to run from the environment, keeping only known ones, once each', () => {
    const known = { search: { services: ['search'], heapLimitMiB: 512, rssLimitMiB: 1_024 }, devices: { services: ['devices'], heapLimitMiB: 256, rssLimitMiB: 0 } }
    expect(serviceSpecs({ HARNESSD_SERVICES: ' devices, nope ,search,devices,' }, known)).toEqual([search, devices])
    // Every service in its own process unless told otherwise: isolation is the default.
    expect(serviceSpecs({}, known)).toEqual([search, devices])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'none' }, known)).toEqual([])
    expect(serviceSpecs({ HARNESSD_SERVICES: '' }, known)).toEqual([])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'none,search' }, known)).toEqual([search])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'toString,constructor' }, known)).toEqual([])
    // One heap limit for every service, when given as a whole number of MiB.
    expect(serviceSpecs({ HARNESSD_SERVICES: 'search', HARNESSD_SERVICE_HEAP_LIMIT_MIB: '96' }, known)).toEqual([{ ...search, heapLimitMiB: 96 }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'search', HARNESSD_SERVICE_HEAP_LIMIT_MIB: 'lots' }, known)).toEqual([search])
  })

  it('runs the services a host shares in one process: those named of them, or all of them by its name', () => {
    const edge = { services: ['workspaces', 'usage', 'monitor'], heapLimitMiB: 256, rssLimitMiB: 512 }
    const hosts = { search: { services: ['search'], heapLimitMiB: 512, rssLimitMiB: 1_024 }, edge }
    expect(serviceSpecs({}, hosts)).toEqual([search, { name: 'edge', ...edge }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'monitor,workspaces' }, hosts)).toEqual([{ name: 'edge', ...edge, services: ['workspaces', 'monitor'] }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'edge,search' }, hosts)).toEqual([search, { name: 'edge', ...edge }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'usage' }, hosts)).toEqual([{ name: 'edge', ...edge, services: ['usage'] }])
  })

  it('leaves an experiment\'s process to the core\'s asking, unless it is named: then it starts with the others', () => {
    const orchestrator = { services: ['orchestrator'], heapLimitMiB: 256, rssLimitMiB: 512, onDemand: true }
    const hosts = { search: { services: ['search'], heapLimitMiB: 512, rssLimitMiB: 1_024 }, orchestrator }
    expect(serviceSpecs({}, hosts)).toEqual([search, { name: 'orchestrator', ...orchestrator }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'search,orchestrator' }, hosts))
      .toEqual([search, { name: 'orchestrator', services: ['orchestrator'], heapLimitMiB: 256, rssLimitMiB: 512 }])
    expect(serviceSpecs({ HARNESSD_SERVICES: 'search' }, hosts)).toEqual([search])
  })

  it('runs the devices only once there is one, asked for by a core of protocol 4, and from the start when named (tests)', () => {
    const [devices] = serviceSpecs({}, SERVICE_HOSTS).filter((spec) => spec.name === 'devices')
    expect(devices).toMatchObject({ services: ['devices', 'wifi', 'machineAddons'], onDemand: true, askedSince: 4 })
    const [named] = serviceSpecs({ HARNESSD_SERVICES: 'devices' }, SERVICE_HOSTS)
    expect(named.services).toEqual(['devices', 'wifi', 'machineAddons'])
    expect(named.onDemand).toBeUndefined()
  })

  it('runs models only once grid is in use or asked for, by a core of protocol 4, and from the start when named', () => {
    const [models] = serviceSpecs({}, SERVICE_HOSTS).filter((spec) => spec.name === 'models')
    expect(models).toMatchObject({ services: ['models'], onDemand: true, askedSince: 4 })
    const [named] = serviceSpecs({ HARNESSD_SERVICES: 'models' }, SERVICE_HOSTS)
    expect(named.services).toEqual(['models'])
    expect(named.onDemand).toBeUndefined()
  })

  it('runs the viewers and the Store only once something needs them, asked for by a core of protocol 5, and from the start when named', () => {
    const [viewers] = serviceSpecs({}, SERVICE_HOSTS).filter((spec) => spec.name === 'viewers')
    expect(viewers).toMatchObject({ services: ['viewers', 'store'], onDemand: true, askedSince: 5 })
    // Named, as tests and support name it: from the start, as every master before it ran them.
    const [named] = serviceSpecs({ HARNESSD_SERVICES: 'viewers' }, SERVICE_HOSTS)
    expect(named.services).toEqual(['viewers', 'store'])
    expect(named.onDemand).toBeUndefined()
    // Asked for by either of its services.
    const supervisor = make([search, viewers])
    supervisor.start()
    expect(children.map((child) => child.spec.name)).toEqual(['search'])
    supervisor.want('store')
    supervisor.want('viewers')
    expect(children.map((child) => child.spec.name)).toEqual(['search', 'viewers'])
  })

  it('starts the viewers as a core of protocol 4 binds, which never asks for them, and leaves the rest to it', () => {
    const onDemand = serviceSpecs({}, SERVICE_HOSTS).filter((spec) => spec.onDemand)
    const four = make(onDemand)
    four.start()
    expect(children).toHaveLength(0)
    // Models, the devices and the gateway it asks for itself (protocol 4); the engines' workers are new to it.
    four.unasked(4)
    expect(children.map((child) => child.spec.name)).toEqual(['viewers'])
    // A core of this protocol asks for each itself.
    children.length = 0
    const five = make(onDemand)
    five.start()
    five.unasked(5)
    expect(children).toHaveLength(0)
  })

  it('knows every service each process this build runs hosts, and each in one process only', () => {
    const hosted = Object.values(SERVICE_HOSTS).flatMap((host) => host.services)
    expect(KNOWN_SERVICES).toEqual(hosted)
    expect(new Set(hosted).size).toBe(hosted.length)
    expect(SERVICE_HOSTS.edge.services).toEqual(['workspaces', 'usage', 'monitor', 'projects', 'handoff', 'recaps', 'windowNames', 'shell', 'connectors', 'sharedMachines', 'agentCleanup', 'messages', 'osDevice'])
    // Agent Messages has no process of its own: one cost every computer it was on 62 MiB at idle (2026-10-12).
    expect(Object.keys(SERVICE_HOSTS)).not.toContain('messages')
  })

  // Agent Messages left a process of its own, started on demand, for the edge host (2026-10-12). Where it runs is the
  // master's to say, by service (`serviceProcessesEnv`), and a core routes it there and runs none of it itself, whichever
  // build each is: two at once would both type the same messages into the same panes.
  it('Agent Messages under a master of this build: in the edge host from the start; a core of either build routes it there, and its ask starts nothing', () => {
    const specs = serviceSpecs({}, SERVICE_HOSTS)
    const hosts = specs.filter((spec) => spec.services.includes('messages'))
    expect(hosts.map((spec) => [spec.name, spec.onDemand])).toEqual([['edge', undefined]])
    const env = { HARNESSD_SUPERVISED: '1', HARNESSD_SERVICE_TOKEN: 'token', ...serviceProcessesEnv(specs, 42) }
    // Both builds know the name: a core from before, which ran it as an experiment, routes it to the link the edge
    // host opens as `messages` and starts none of it in its own process; so does a core of this build.
    expect(servicesTheMasterRuns(env, KNOWN_SERVICES).has('messages')).toBe(true)
    const supervisor = make(specs)
    supervisor.start()
    const started = children.length
    expect(children.filter((child) => child.spec.services.includes('messages')).map((child) => child.spec.name)).toEqual(['edge'])
    // Either core asks for it as it starts (core/messagesLink.ts, core/experiments.ts): no second process.
    supervisor.want('messages')
    supervisor.want('messages')
    expect(children).toHaveLength(started)
  })

  it('Agent Messages under a master from before: its own process, started once when a core of this build asks, and routed there', () => {
    // The master as 3cff043360 ran it: a process of its own on demand, and the edge host without it.
    const before = { ...SERVICE_HOSTS, edge: { ...SERVICE_HOSTS.edge, services: SERVICE_HOSTS.edge.services.filter((name) => name !== 'messages') },
      messages: { services: ['messages'], heapLimitMiB: 256, rssLimitMiB: 512, onDemand: true } }
    const specs = serviceSpecs({}, before)
    const env = { HARNESSD_SUPERVISED: '1', HARNESSD_SERVICE_TOKEN: 'token', ...serviceProcessesEnv(specs, 42) }
    expect(servicesTheMasterRuns(env, KNOWN_SERVICES).has('messages')).toBe(true)
    const supervisor = make(specs)
    supervisor.start()
    expect(children.filter((child) => child.spec.services.includes('messages'))).toHaveLength(0)
    // This build's core asks for it as it starts, whatever its setting says: once, one process.
    supervisor.want('messages')
    supervisor.want('messages')
    expect(children.filter((child) => child.spec.services.includes('messages')).map((child) => [child.spec.name, child.spec.services])).toEqual([['messages', ['messages']]])
  })
})

describe('the Swarm switch', () => {
  it('is on unless HARNESS_SWARM says off, so nothing changes for anyone who does not set it', () => {
    expect(SWARM_ENV).toBe('HARNESS_SWARM')
    expect(swarmOn({})).toBe(true)
    for (const on of ['', 'on', '1', 'true', 'yes', 'anything']) expect(swarmOn({ HARNESS_SWARM: on }), on).toBe(true)
    for (const off of ['off', 'OFF', ' off ', '0', 'false', 'no']) expect(swarmOn({ HARNESS_SWARM: off }), off).toBe(false)
  })

  it('off, leaves the Swarm\'s service out of the services this computer runs, and out of every process the master starts', () => {
    expect(servicesOn({}, ['search', 'superTabs', 'cloud'])).toEqual(['search', 'superTabs', 'cloud'])
    expect(servicesOn({ HARNESS_SWARM: 'off' }, ['search', 'superTabs', 'cloud'])).toEqual(['search', 'cloud'])
    const processes = (env: NodeJS.ProcessEnv): string[] => serviceSpecs(env, SERVICE_HOSTS).flatMap(spec => spec.services)
    expect(processes({})).toContain('superTabs')
    expect(processes({ HARNESS_SWARM: 'off' })).not.toContain('superTabs')
    // Named outright, it still stays off: the switch is the computer's, HARNESSD_SERVICES only says where services run.
    expect(serviceSpecs({ HARNESSD_SERVICES: 'superTabs,search', HARNESS_SWARM: 'off' }, SERVICE_HOSTS).map(spec => spec.name)).toEqual(['search'])
    expect(processes({ HARNESS_SWARM: 'off' })).toEqual(processes({}).filter(service => service !== 'superTabs'))
  })
})

describe('which services the core leaves to its master', () => {
  const known = ['search', 'viewers', 'workspaces', 'messages']
  const supervised = { HARNESSD_SUPERVISED: '1', HARNESSD_SERVICE_TOKEN: 'token' }

  it('is what the master says it runs, told in a form a core from before the list reads too', () => {
    const specs = serviceSpecs({ HARNESSD_SERVICES: 'search,workspaces' }, SERVICE_HOSTS)
    expect(serviceProcessesEnv(specs, 42)).toEqual({ [SERVICE_PROCESSES_ENV]: 'search,workspaces', HARNESSD_SERVICES: 'search,workspaces', [ENGINE_LIVE_ENV]: '42:1', [ENGINE_RUNTIME_ENV]: '42:1', [ENGINE_SCREEN_ENV]: '42:1', [ENGINE_MODEL_CONTROL_ENV]: '42:1', [ENGINE_QUESTION_CONTROL_ENV]: '42:1', [ENGINE_SUBMISSION_ENV]: '42:1', [ENGINE_NATIVE_CONTROL_ENV]: '42:1' })
    // A process named as one of its services is named whole: the viewers' process runs the Store beside them.
    expect(serviceProcessesEnv(serviceSpecs({ HARNESSD_SERVICES: 'viewers' }, SERVICE_HOSTS), 42)).toMatchObject({ [SERVICE_PROCESSES_ENV]: 'viewers,store' })
    expect(serviceProcessesEnv(serviceSpecs({ HARNESSD_SERVICES: 'store' }, SERVICE_HOSTS), 42)).toMatchObject({ [SERVICE_PROCESSES_ENV]: 'store' })
    // By service, not by process: a core from before the edge host routes the services it knows of it.
    const hosted = serviceSpecs({ HARNESSD_SERVICES: 'edge' }, SERVICE_HOSTS)
    expect(serviceProcessesEnv(hosted, 42)).toEqual({ [SERVICE_PROCESSES_ENV]: 'workspaces,usage,monitor,projects,handoff,recaps,windowNames,shell,connectors,sharedMachines,agentCleanup,messages,osDevice', HARNESSD_SERVICES: 'workspaces,usage,monitor,projects,handoff,recaps,windowNames,shell,connectors,sharedMachines,agentCleanup,messages,osDevice', [ENGINE_LIVE_ENV]: '42:1', [ENGINE_RUNTIME_ENV]: '42:1', [ENGINE_SCREEN_ENV]: '42:1', [ENGINE_MODEL_CONTROL_ENV]: '42:1', [ENGINE_QUESTION_CONTROL_ENV]: '42:1', [ENGINE_SUBMISSION_ENV]: '42:1', [ENGINE_NATIVE_CONTROL_ENV]: '42:1' })
    expect([...servicesTheMasterRuns({ ...supervised, ...serviceProcessesEnv(hosted, 42) }, known)]).toEqual(['workspaces', 'messages'])
    expect(serviceProcessesEnv([], 42)).toEqual({ [SERVICE_PROCESSES_ENV]: '', HARNESSD_SERVICES: 'none', [ENGINE_LIVE_ENV]: '42:1', [ENGINE_RUNTIME_ENV]: '42:1', [ENGINE_SCREEN_ENV]: '42:1', [ENGINE_MODEL_CONTROL_ENV]: '42:1', [ENGINE_QUESTION_CONTROL_ENV]: '42:1', [ENGINE_SUBMISSION_ENV]: '42:1', [ENGINE_NATIVE_CONTROL_ENV]: '42:1' })
    expect([...servicesTheMasterRuns({ ...supervised, ...serviceProcessesEnv(specs, 42) }, known)]).toEqual(['search', 'workspaces'])
    expect([...servicesTheMasterRuns({ ...supervised, ...serviceProcessesEnv([], 42) }, known)]).toEqual([])
    // What the master says wins over whatever HARNESSD_SERVICES the core inherited; names it does not know are its own.
    expect([...servicesTheMasterRuns({ ...supervised, [SERVICE_PROCESSES_ENV]: ' messages, nothing ', HARNESSD_SERVICES: 'search' }, known)]).toEqual(['messages'])
  })

  it('under a master too old to say, is what HARNESSD_SERVICES names, and nothing when it names none: never every service', () => {
    // A released master (0.3.58) runs no service process unless HARNESSD_SERVICES names one, and does not
    // re-execute on an update: the core it starts must not take every service for out of process.
    expect([...servicesTheMasterRuns(supervised, known)]).toEqual([])
    expect([...servicesTheMasterRuns({ ...supervised, HARNESSD_SERVICES: 'search' }, known)]).toEqual(['search'])
  })

  it('accepts live engine support only from the actual supervising parent', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsLiveEngines(env, 42)).toBe(true)
    expect(masterRunsLiveEngines(env, 43)).toBe(false)
    expect(masterRunsLiveEngines(supervised, 42)).toBe(false)
    expect(masterRunsLiveEngines({ ...env, [ENGINE_LIVE_ENV]: '1' }, 42)).toBe(false)
    expect(masterRunsLiveEngines({ ...env, [ENGINE_LIVE_ENV]: '42:2' }, 42)).toBe(false)
    expect(masterRunsLiveEngines({ ...env, HARNESSD_SUPERVISED: undefined }, 42)).toBe(false)
    expect(masterRunsLiveEngines({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
  })

  it('accepts runtime profile support only from the actual supervising parent, independently of live support', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsEngineRuntime(env, 42)).toBe(true)
    expect(masterRunsEngineRuntime(env, 43)).toBe(false)
    expect(masterRunsEngineRuntime(supervised, 42)).toBe(false)
    expect(masterRunsEngineRuntime({ ...env, [ENGINE_RUNTIME_ENV]: '1' }, 42)).toBe(false)
    expect(masterRunsEngineRuntime({ ...env, [ENGINE_RUNTIME_ENV]: '42:2' }, 42)).toBe(false)
    expect(masterRunsEngineRuntime({ ...env, HARNESSD_SUPERVISED: undefined }, 42)).toBe(false)
    expect(masterRunsEngineRuntime({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
    expect(masterRunsLiveEngines({ ...env, [ENGINE_RUNTIME_ENV]: undefined }, 42)).toBe(true)
  })

  it('negotiates model controls only from a capable supervising parent', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsEngineQuestionControl(env, 42)).toBe(true)
    expect(masterRunsEngineQuestionControl(env, 43)).toBe(false)
    expect(masterRunsEngineQuestionControl({ ...env, [ENGINE_QUESTION_CONTROL_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineQuestionControl({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
    expect(masterRunsEngineModelControl(env, 42)).toBe(true)
    expect(masterRunsEngineModelControl(env, 43)).toBe(false)
    expect(masterRunsEngineModelControl({ ...env, [ENGINE_MODEL_CONTROL_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineModelControl({ ...env, [ENGINE_LIVE_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineModelControl({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
  })

  it('negotiates screen readers only from a capable supervising parent', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsEngineScreen(env, 42)).toBe(true)
    expect(masterRunsEngineScreen(env, 43)).toBe(false)
    expect(masterRunsEngineScreen({ ...env, [ENGINE_SCREEN_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineScreen({ ...env, [ENGINE_LIVE_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineScreen({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
  })

  it('negotiates native control connections only from a capable supervising parent', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsEngineNativeControl(env, 42)).toBe(true)
    expect(masterRunsEngineNativeControl(env, 43)).toBe(false)
    // An older master never set it: the core composes the inline control, as it does for the others.
    expect(masterRunsEngineNativeControl({ ...env, [ENGINE_NATIVE_CONTROL_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineNativeControl({ ...env, [ENGINE_LIVE_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineNativeControl({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
  })

  it('negotiates submission readers only from a capable supervising parent', () => {
    const env = { ...supervised, ...serviceProcessesEnv([], 42) }
    expect(masterRunsEngineSubmission(env, 42)).toBe(true)
    expect(masterRunsEngineSubmission(env, 43)).toBe(false)
    // An older master never set it: the core composes the inline readers, the way it does for the others.
    expect(masterRunsEngineSubmission({ ...env, [ENGINE_SUBMISSION_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineSubmission({ ...env, [ENGINE_LIVE_ENV]: undefined }, 42)).toBe(false)
    expect(masterRunsEngineSubmission({ ...env, HARNESSD_SERVICE_TOKEN: undefined }, 42)).toBe(false)
  })

  it('is none without a master, or without the token its services would connect with', () => {
    expect([...servicesTheMasterRuns({ [SERVICE_PROCESSES_ENV]: 'search' }, known)]).toEqual([])
    expect([...servicesTheMasterRuns({ HARNESSD_SUPERVISED: '1', [SERVICE_PROCESSES_ENV]: 'search' }, known)]).toEqual([])
  })
})