Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

harnessd: a paused daemon, a master lost at start, non-POSIX shells, updated engines; e2e in parallel (#773)

Deef55e3f4b9800
diff --git a/cli/e2e/harness/bundle.ts b/cli/e2e/harness/bundle.ts
new file mode 100644
index 000000000..557988462
--- /dev/null
+++ b/cli/e2e/harness/bundle.ts
@@ -0,0 +1,27 @@
+/**
+ * `E2E_BUNDLE=1`: every daemon in the run starts from one bundle built here, as the release ships,
+ * instead of from the sources through tsx (vitest's global setup; `daemon.ts` reads the path).
+ *
+ * Why: tsx transpiles the whole tree at every start, and a parallel run starts hundreds of daemons —
+ * on a 12-core Mac four files at a time drove the load average past 100, and tests timed out that
+ * pass alone. A daemon from the bundle starts on a fraction of the CPU. And the run tests the code as
+ * it was when it began: from the sources, an edit made mid-run reached every daemon started after it,
+ * and 21 tests failed on a half-written change.
+ */
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..')
+
+export default function setup(): (() => void) | undefined {
+ const asked = process.env.E2E_BUNDLE
+ if (!asked || asked === '0') return undefined
+ const out = mkdtempSync(join(tmpdir(), 'harness-e2e-bundle-'))
+ execFileSync(process.execPath, ['build-bundle.mjs'], { cwd: CLI_ROOT, env: { ...process.env, BUNDLE_OUT_DIR: out }, stdio: 'pipe' })
+ // Read by every daemon the run starts (`IsolatedDaemon.start`); the workers inherit it.
+ process.env.E2E_BUNDLE_PATH = join(out, 'cli.js')
+ return () => rmSync(out, { recursive: true, force: true })
+}
diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.ts
index 426f3d06c..49cafe880 100644
--- a/cli/e2e/harness/daemon.ts
+++ b/cli/e2e/harness/daemon.ts
@@ -152,7 +152,9 @@ export class IsolatedDaemon {
const entry = this.options.noMaster ? '__run' : '__harnessd'
// Readiness is judged from what this start prints, never from an earlier boot's lines.
const from = this.output.length
- const script = this.options.scriptPath ? [this.options.scriptPath] : ['--import', 'tsx', 'src/cli.ts']
+ // A test's own bundle (an old release), the run's bundle (`E2E_BUNDLE`, see bundle.ts), or the sources.
+ const bundle = this.options.scriptPath ?? process.env.E2E_BUNDLE_PATH
+ const script = bundle ? [bundle] : ['--import', 'tsx', 'src/cli.ts']
const child = spawn(process.execPath, [...heap, ...script, entry], {
cwd: CLI_ROOT, env: this.env, stdio: ['ignore', 'pipe', 'pipe'],
})
@@ -229,7 +231,18 @@ export class IsolatedDaemon {
}
async close(): Promise<void> {
+ const core = this.options.noMaster ? null : this.corePid()
await this.stop().catch(() => {})
+ // A core whose master died during the test must not outlive it: 20 such cores, left by a run
+ // whose masters crashed, spun on their closed output pipes for the rest of a parallel suite. Only
+ // this test's core is killed — its pid, still running this daemon's entry, orphaned or ours.
+ if (core && IsolatedDaemon.alive(core)) {
+ const { stdout } = await exec('ps', ['-o', 'ppid=,command=', '-p', String(core)]).catch(() => ({ stdout: '' }))
+ const [ppid, ...command] = stdout.trim().split(/\s+/)
+ if (command.join(' ').includes('__run') && (ppid === '1' || Number(ppid) === this.pid)) {
+ try { process.kill(core, 'SIGKILL') } catch { /* gone */ }
+ }
+ }
await this.tmux.close()
}
}
diff --git a/cli/e2e/orphan.e2e.ts b/cli/e2e/orphan.e2e.ts
new file mode 100644
index 000000000..4b729f15e
--- /dev/null
+++ b/cli/e2e/orphan.e2e.ts
@@ -0,0 +1,88 @@
+/**
+ * A core whose master dies, for Claude Code and Codex. The master is the one process `harness start`
+ * leaves behind and the one the pid file names; a core without it is supervised by nobody, and must
+ * leave so nothing holds the port the next start needs. The hard moment is a master killed while its
+ * core is still starting: the core has bound and begun to beat, and is a second from finishing.
+ */
+import { mkdirSync } from 'node:fs'
+import { createConnection } from 'node:net'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
+ const cwd = join(d.projectsDir, folder)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
+ return until(`${folder} to bind its conversation`, async () => {
+ const agent = await row(client, created.agent.id)
+ return agent?.sessionId && agent.status === 'active' ? agent : null
+ }, 60_000, 500)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+/** Whether anything accepts connections on the port. */
+const answers = (port: number) => new Promise<boolean>((resolve) => {
+ const socket = createConnection({ host: '127.0.0.1', port })
+ socket.once('connect', () => { socket.destroy(); resolve(true) })
+ socket.once('error', () => resolve(false))
+})
+
+describe('a core whose master dies', () => {
+ let daemon: IsolatedDaemon | undefined
+ const cores: number[] = []
+ afterEach(async () => {
+ // A core this test failed to see leave must not outlive it.
+ for (const pid of cores.splice(0)) { try { process.kill(pid, 'SIGKILL') } catch { /* gone */ } }
+ await daemon?.close(); daemon = undefined
+ })
+
+ it.each([
+ ['while its core is still starting', 'bound'],
+ ['once its core is up', 'ready'],
+ ] as const)('killed %s: the core leaves, the port is free, and the next start has every agent', async (_when, moment) => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })
+ await d.start()
+ let client = await LocalClient.connect(d)
+ const agents = [await create(d, client, 'claude', 'orphan-claude'), await create(d, client, 'codex', 'orphan-codex')]
+ client.close()
+ await d.stop()
+
+ const from = d.log().length
+ await d.start({ ready: 'none' })
+ const line = moment === 'bound' ? /\[harnessd\] core bound/ : /\[cli\] ready/
+ await until(`the core to be ${moment}`, () => line.test(d.log().slice(from)) || null, 60_000, 10)
+ const core = d.corePid()!
+ cores.push(core)
+ await d.kill()
+ await until('the core to leave', () => !IsolatedDaemon.alive(core) || null, 60_000, 200)
+ await until('the port to be free', async () => !(await answers(d.port)) || null, 10_000, 200)
+
+ await d.start()
+ client = await LocalClient.connect(d)
+ for (const agent of agents) {
+ await until(`${agent.engine} back`, async () => {
+ const now = await row(client, agent.id)
+ return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null
+ }, 60_000, 500)
+ await turn(client, agent.id, `after the master died (${agent.engine})`)
+ }
+ client.close()
+ }, 300_000)
+})
diff --git a/cli/e2e/paused.e2e.ts b/cli/e2e/paused.e2e.ts
new file mode 100644
index 000000000..d82fcbbba
--- /dev/null
+++ b/cli/e2e/paused.e2e.ts
@@ -0,0 +1,96 @@
+/**
+ * The daemon paused while its engines go on, for Claude Code and Codex: the master and the core are
+ * stopped (SIGSTOP), as Ctrl-Z on a daemon run in a terminal, a paused virtual machine or a swap storm
+ * can leave them, for longer than the master's 30 s heartbeat watchdog. The engines live in tmux and
+ * keep working. When the daemon resumes, nothing the person did meanwhile may be lost: a turn typed
+ * into a pane is in the conversation once, a message an app sent into the paused socket is taken once,
+ * and every agent works as before.
+ */
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {
+ const cwd = join(d.projectsDir, folder)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
+ return until(`${folder} to bind its conversation`, async () => {
+ const agent = await row(client, created.agent.id)
+ return agent?.sessionId && agent.status === 'active' ? agent : null
+ }, 60_000, 500)
+}
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+/** What the person said in a conversation, oldest first, as the apps read it back. */
+async function said(client: LocalClient, sessionId: string): Promise<string[]> {
+ const page = await client.request('session_get', { sessionId, limit: 50 }, 60_000)
+ expect(page.error, JSON.stringify(page)).toBeUndefined()
+ return (page.events as Row[]).filter((event) => event.type === 'user_message').map((event) => String(event.payload.content))
+}
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+
+describe('the daemon paused while its engines go on', () => {
+ let daemon: IsolatedDaemon | undefined
+ const paused: number[] = []
+ afterEach(async () => {
+ // A stopped daemon left behind would hold the test's close until its SIGKILL.
+ for (const pid of paused.splice(0)) { try { process.kill(pid, 'SIGCONT') } catch { /* gone */ } }
+ await daemon?.close(); daemon = undefined
+ })
+
+ it('paused past the watchdog: the typed turn and the app\'s message are each in the conversation once, and the core is not taken for hung', async () => {
+ const d = await IsolatedDaemon.create()
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })
+ await d.start()
+ let client = await LocalClient.connect(d)
+ const claude = await create(d, client, 'claude', 'paused-claude')
+ const codex = await create(d, client, 'codex', 'paused-codex')
+ await turn(client, claude.id, 'before the pause (claude)')
+ await turn(client, codex.id, 'before the pause (codex)')
+
+ const master = d.pid!, core = d.corePid()!
+ for (const pid of [master, core]) { process.kill(pid, 'SIGSTOP'); paused.push(pid) }
+ // An app's message into the paused socket: the kernel holds it until the core reads again.
+ client.send('message', { agentId: codex.id, content: 'sent while the daemon was paused' })
+ // The person types into a pane meanwhile, and the engine takes the turn without the daemon.
+ await d.tmux.run('send-keys', '-t', claude.tmuxPane, 'typed while the daemon was paused', 'Enter')
+ await sleep(40_000)
+ // Everything resumes at once, as it does after Ctrl-Z and fg or a virtual machine's pause.
+ for (const pid of paused.splice(0)) process.kill(pid, 'SIGCONT')
+
+ if (client.closed) client = await LocalClient.connect(d)
+ await until('the app\'s message to be taken', async () =>
+ (await said(client, codex.sessionId).catch(() => [])).includes('sent while the daemon was paused') || null, 60_000, 500)
+ await until('the typed turn to be read', async () =>
+ (await said(client, claude.sessionId).catch(() => [])).includes('typed while the daemon was paused') || null, 60_000, 500)
+ await sleep(5_000)
+ expect(await said(client, codex.sessionId)).toEqual(['before the pause (codex)', 'sent while the daemon was paused'])
+ expect(await said(client, claude.sessionId)).toEqual(['before the pause (claude)', 'typed while the daemon was paused'])
+ for (const agent of [claude, codex]) {
+ const now = await row(client, agent.id)
+ expect(now?.status, agent.engine).toBe('active')
+ expect(now?.sessionId, agent.engine).toBe(agent.sessionId)
+ await turn(client, agent.id, `after the pause (${agent.engine})`)
+ }
+ // The master was paused too, so the silence was not the core's: it is the same core.
+ expect(d.log()).not.toMatch(/it is hung/)
+ expect(d.coresStarted()).toBe(1)
+ client.close()
+ }, 240_000)
+})
diff --git a/cli/e2e/serviceProcesses.e2e.ts b/cli/e2e/serviceProcesses.e2e.ts
index d6d65d745..b37e5dbb3 100644
--- a/cli/e2e/serviceProcesses.e2e.ts
+++ b/cli/e2e/serviceProcesses.e2e.ts
@@ -33,11 +33,16 @@ async function turn(client: LocalClient, agentId: string, content: string): Prom
client.send('message', { agentId, content })
await ended
}
-/** The search service's own process, from the process table: it is `harnessd search`. */
-function searchPids(): number[] {
+/**
+ * This daemon's search service processes: titled `harnessd search` AND started by its own master, by
+ * the pid the master logged. The title alone matched every daemon's search on the machine — another
+ * file's in a parallel run, another checkout's — and the tests below stop and kill what it returns.
+ */
+function searchPids(d: IsolatedDaemon): number[] {
+ const ours = new Set([...d.log().matchAll(/\[harnessd\] service search started \(pid (\d+)\)/g)].map((match) => Number(match[1])))
const table = execFileSync('ps', ['-A', '-o', 'pid=,command=']).toString().trim().split('\n')
return table.map((line) => line.trim().match(/^(\d+)\s+(.*)$/)).filter((match): match is RegExpMatchArray => !!match)
- .filter(([, , command]) => command.trim() === 'harnessd search').map(([, pid]) => Number(pid))
+ .filter(([, pid, command]) => command.trim() === 'harnessd search' && ours.has(Number(pid))).map(([, pid]) => Number(pid))
}
const finds = async (client: LocalClient, word: string, sessionId: string): Promise<boolean> =>
JSON.stringify(await client.request('session_search', { query: word }, 30_000)).includes(sessionId)
@@ -63,7 +68,7 @@ describe('services in their own processes', () => {
const d = await fresh()
const client = await LocalClient.connect(d)
await until('search to connect to the core', () => d.log().includes('[services] search connected') || null, 30_000, 200)
- expect(searchPids().length).toBeGreaterThanOrEqual(1)
+ expect(searchPids(d).length).toBeGreaterThanOrEqual(1)
for (const engine of ['claude', 'codex'] as const) {
const agent = await create(d, client, engine, `search-out-${engine}`)
const word = `axolotl${engine}`
@@ -80,7 +85,7 @@ describe('services in their own processes', () => {
const agent = await create(d, client, 'claude', 'search-killed')
await turn(client, agent.id, 'remember the pangolin')
await until('search to find it', () => finds(client, 'pangolin', agent.sessionId) || null, 60_000, 1_000)
- const before = searchPids()
+ const before = searchPids(d)
expect(before.length).toBeGreaterThanOrEqual(1)
for (const pid of before) process.kill(pid, 'SIGKILL')
// Asked while it is down: an answer, not a hang.
@@ -90,7 +95,7 @@ describe('services in their own processes', () => {
await turn(client, agent.id, 'while search was gone')
await until('the master to restart search', () => restarts(d) >= 1 || null, 30_000, 200)
await until('search to answer again', () => finds(client, 'pangolin', agent.sessionId) || null, 60_000, 1_000)
- expect(searchPids().some((pid) => !before.includes(pid))).toBe(true)
+ expect(searchPids(d).some((pid) => !before.includes(pid))).toBe(true)
expect(d.coresStarted()).toBe(1)
client.close()
})
@@ -101,7 +106,7 @@ describe('services in their own processes', () => {
const agent = await create(d, client, 'codex', 'search-hung')
await until('search to connect', () => d.log().includes('[services] search connected') || null, 30_000, 200)
// Hung: a stopped process neither beats nor answers, as one stuck in a loop would not.
- for (const pid of searchPids()) process.kill(pid, 'SIGSTOP')
+ for (const pid of searchPids(d)) process.kill(pid, 'SIGSTOP')
await until('the master to find search hung', () => d.log().includes('[harnessd] service search sent no heartbeat') || null, 30_000, 200)
await until('search to be started again', () => restarts(d) >= 1 || null, 30_000, 200)
await turn(client, agent.id, 'the core never waited on search')
@@ -140,10 +145,10 @@ describe('services in their own processes', () => {
await turn(client, agent.id, 'remember the quokka')
await until('search to find it', () => finds(client, 'quokka', agent.sessionId) || null, 60_000, 1_000)
// Down: killed, and the purge happens before the master has it back.
- for (const pid of searchPids()) process.kill(pid, 'SIGSTOP')
+ for (const pid of searchPids(d)) process.kill(pid, 'SIGSTOP')
const createdAt = Date.parse(agent.createdAt)
const review = await client.request('agent_purge', { agentId: agent.id, sessionId: agent.sessionId, createdAt, mode: 'inspect' }, 60_000)
- for (const pid of searchPids()) process.kill(pid, 'SIGKILL')
+ for (const pid of searchPids(d)) process.kill(pid, 'SIGKILL')
const deleted = await client.request('agent_purge', { agentId: agent.id, sessionId: agent.sessionId, createdAt, mode: 'delete', reviewId: review.reviewId }, 90_000)
expect(deleted, JSON.stringify(deleted)).toMatchObject({ deleted: true })
await until('search to be back', () => restarts(d) >= 1 && d.log().split('[services] search connected').length >= 3 || null, 60_000, 250)
diff --git a/cli/e2e/shells.e2e.ts b/cli/e2e/shells.e2e.ts
new file mode 100644
index 000000000..6ab753fd2
--- /dev/null
+++ b/cli/e2e/shells.e2e.ts
@@ -0,0 +1,75 @@
+/**
+ * The person's login shell, for Claude Code and Codex. The daemon launches each engine through the
+ * shell the person uses (`$SHELL`), so their PATH, version managers and startup files apply. zsh and
+ * bash speak the POSIX shell language the launch is written in; fish, tcsh and nushell do not. tcsh
+ * ships with macOS, so it stands in for every shell that is not POSIX: an agent must start, in its
+ * own folder, and work, whichever shell it is.
+ */
+import { mkdirSync, realpathSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
+import { LocalClient, type Frame } from './harness/client.js'
+import { IsolatedDaemon, until } from './harness/daemon.js'
+
+type Engine = 'claude' | 'codex'
+type Row = Record<string, any>
+
+const rows = async (client: LocalClient): Promise<Row[]> =>
+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents
+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)
+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
+ client.send('message', { agentId, content })
+ expect((await started).payload?.userMessage).toBe(content)
+ await ended
+}
+
+const SHELLS = ['/bin/bash', '/bin/tcsh'] as const
+
+describe('the person\'s login shell', () => {
+ let daemon: IsolatedDaemon | undefined
+ afterEach(async () => { await daemon?.close(); daemon = undefined })
+
+ it.each(SHELLS)('a terminal tile under %s is that shell, in the folder asked for', async (shell) => {
+ const d = await IsolatedDaemon.create({ env: { SHELL: shell } })
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })
+ await d.start()
+ const client = await LocalClient.connect(d)
+ const cwd = join(d.projectsDir, 'tile')
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine: 'terminal', cwd }, 60_000)
+ expect(created.error, JSON.stringify(created)).toBeUndefined()
+ const info = await until(`the tile's ${shell} to be running`, async () => {
+ const answer = await client.request('terminal_info', { agentId: created.agent.id }, 10_000)
+ return answer.command && !answer.error && answer.command !== 'sh' ? answer : null
+ }, 30_000, 500)
+ expect(info.command).toBe(shell.split('/').at(-1))
+ expect(realpathSync(info.path)).toBe(realpathSync(cwd))
+ client.close()
+ }, 120_000)
+
+ for (const shell of SHELLS) {
+ it.each(['claude', 'codex'] as const)(`%s under ${shell}: an agent starts in its own folder and works`, async (engine: Engine) => {
+ const d = await IsolatedDaemon.create({ env: { SHELL: shell } })
+ daemon = d
+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })
+ await d.start()
+ const client = await LocalClient.connect(d)
+ const cwd = join(d.projectsDir, `shell-${engine}`)
+ mkdirSync(cwd, { recursive: true })
+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 120_000)
+ expect(created.error, JSON.stringify(created)).toBeUndefined()
+ const agent = await until(`the ${engine} agent to bind under ${shell}`, async () => {
+ const now = await row(client, created.agent.id)
+ return now?.sessionId && now.status === 'active' ? now : null
+ }, 60_000, 500)
+ const info = await client.request('terminal_info', { agentId: agent.id }, 10_000)
+ expect(realpathSync(info.path)).toBe(realpathSync(cwd))
+ await turn(client, agent.id, `under ${shell}`)
+ client.close()
+ }, 240_000)
+ }
+})
diff --git a/cli/src/harnessd/coreLink.spec.ts b/cli/src/harnessd/coreLink.spec.ts
index 184dd8e34..d7fd0e7af 100644
--- a/cli/src/harnessd/coreLink.spec.ts
+++ b/cli/src/harnessd/coreLink.spec.ts
@@ -8,6 +8,7 @@ class FakeChannel implements MasterChannel {
private disconnect: Array<() => void> = []
rss = 100
parentPid = 4242
+ connected = true
send?: (message: CoreMessage) => unknown = (message) => { this.sent.push(message) }
once(_event: 'disconnect', listener: () => void): void { this.disconnect.push(listener) }
on(_event: 'message', listener: (message: unknown) => void): void { this.messageListeners.push(listener) }
@@ -113,6 +114,34 @@ describe('connectToMaster', () => {
expect(gone).toHaveBeenCalledOnce()
})
+ // runForeground subscribes long after the core has bound: a master killed in between went unnoticed,
+ // and its core ran on for good, holding the port.
+ it('hears a master that went before anyone asked, and only once', async () => {
+ const channel = new FakeChannel()
+ const link = connectToMaster(channel, supervised)
+ link.bound(1)
+ channel.leave()
+ const gone = vi.fn()
+ link.onMasterGone(gone)
+ expect(gone).not.toHaveBeenCalled()
+ await Promise.resolve()
+ expect(gone).toHaveBeenCalledOnce()
+ const later = vi.fn()
+ link.onMasterGone(later)
+ await Promise.resolve()
+ expect(later).toHaveBeenCalledOnce()
+ expect(gone).toHaveBeenCalledOnce()
+ })
+
+ it('hears a master whose channel had already closed when the core started', async () => {
+ const channel = new FakeChannel()
+ channel.connected = false
+ const gone = vi.fn()
+ connectToMaster(channel, supervised).onMasterGone(gone)
+ await Promise.resolve()
+ expect(gone).toHaveBeenCalledOnce()
+ })
+
it('survives a send on a channel the master already closed', () => {
const channel = new FakeChannel()
channel.send = () => { throw new Error('ERR_IPC_CHANNEL_CLOSED') }
@@ -123,6 +152,7 @@ describe('connectToMaster', () => {
it('uses this process by default, which no master started', () => {
expect(connectToMaster().supervised).toBe(false)
expect(processChannel.memoryUsage().rss).toBeGreaterThan(0)
+ expect(processChannel.connected).toBe(process.connected)
expect(processChannel.parentPid).toBe(process.ppid)
})
diff --git a/cli/src/harnessd/coreLink.ts b/cli/src/harnessd/coreLink.ts
index bcafec68a..210f8f2ac 100644
--- a/cli/src/harnessd/coreLink.ts
+++ b/cli/src/harnessd/coreLink.ts
@@ -52,6 +52,8 @@ export interface MasterChannel {
memoryUsage(): { rss: number; heapUsed: number }
/** The process that started this one: the master, when it holds the channel. */
readonly parentPid: number
+ /** False once the channel to the master has closed (`process.connected`). */
+ readonly connected?: boolean
}
export interface CoreLink {
@@ -81,6 +83,7 @@ export const processChannel: MasterChannel = {
on: process.on.bind(process),
memoryUsage: () => process.memoryUsage(),
parentPid: process.ppid,
+ get connected() { return process.connected },
}
export function connectToMaster(
@@ -101,6 +104,20 @@ export function connectToMaster(
if (supervised) {
channel.on('message', (message) => { if (isMasterMessage(message)) status = message.status })
}
+ // ⚠️ Listened for from the start and remembered, not from whenever the caller subscribes.
+ // runForeground subscribes some 1,100 lines into start-up, long after this core has bound and begun
+ // to beat, and 'disconnect' fires once, for whoever listens then: a master that died in between —
+ // killed while its core was still starting — went unnoticed, and the core ran on for good, holding
+ // the port the next start needs. Measured: 20 cores whose master died as they bound were all still
+ // running ten minutes later.
+ let masterGone = supervised && channel.connected === false
+ const goneListeners: Array<() => void> = []
+ if (supervised) {
+ channel.once('disconnect', () => {
+ masterGone = true
+ for (const listener of goneListeners.splice(0)) listener()
+ })
+ }
return {
supervised,
masterPid: supervised ? channel.parentPid : null,
@@ -118,7 +135,11 @@ export function connectToMaster(
heartbeat = setInterval(beat, intervalMs)
heartbeat.unref()
},
- onMasterGone: (listener) => { if (supervised) channel.once('disconnect', listener) },
+ onMasterGone: (listener) => {
+ if (!supervised) return
+ if (masterGone) queueMicrotask(listener)
+ else goneListeners.push(listener)
+ },
status: () => status,
close: () => {
if (heartbeat) clearInterval(heartbeat)
diff --git a/cli/src/harnessd/protocol.ts b/cli/src/harnessd/protocol.ts
index 46de53fdb..d0756f1c4 100644
--- a/cli/src/harnessd/protocol.ts
+++ b/cli/src/harnessd/protocol.ts
@@ -26,6 +26,16 @@ export const CORE_EXIT_UPDATE = 75
*/
export const CORE_EXIT_STOP = 78
+/**
+ * How long a master waits for one more beat once a process's silence has run out, counted from then:
+ * a third of the silence, the longest a core or service ever goes between beats
+ * (`coreLink.heartbeatInterval`), so one that was only paused along with its master always lands one.
+ * Why there is a wait at all: `Supervisor.watchHeartbeat`.
+ */
+export function heartbeatGraceMs(timeoutMs: number): number {
+ return Math.ceil(timeoutMs / 3)
+}
+
export type CoreMessage =
/** The control port is bound: from here on the daemon answers, though start-up is not done. */
| { type: 'harnessd:bound'; protocol: number; port: number }
diff --git a/cli/src/harnessd/services.spec.ts b/cli/src/harnessd/services.spec.ts
index 282aade83..7e2c6c3d4 100644
--- a/cli/src/harnessd/services.spec.ts
+++ b/cli/src/harnessd/services.spec.ts
@@ -105,8 +105,10 @@ describe('ServiceSupervisor', () => {
it('kills a service that stops beating, from its start too, and restarts it as hung', () => {
const supervisor = make()
supervisor.start()
- // Never beat at all: hung from the start.
+ // Never beat at all: hung from the start, once one beat's time more has passed.
vi.advanceTimersByTime(6_000)
+ expect(latest().kills).toEqual([])
+ vi.advanceTimersByTime(2_000)
expect(latest().kills).toEqual(['SIGKILL'])
latest().exit(null, 'SIGKILL')
expect(supervisor.status()[0]).toMatchObject({ lastExitReason: 'hung', state: 'restarting' })
@@ -114,9 +116,23 @@ describe('ServiceSupervisor', () => {
const second = latest()
for (let i = 0; i < 3; i++) { vi.advanceTimersByTime(5_000); second.beat() }
expect(second.kills).toEqual([])
- vi.advanceTimersByTime(6_000)
+ vi.advanceTimersByTime(8_000)
expect(second.kills).toEqual(['SIGKILL'])
- expect(lines.some((line) => line.includes('sent no heartbeat for 6000 ms — it is hung'))).toBe(true)
+ expect(lines.some((line) => line.includes('sent no heartbeat for 8000 ms — it is hung'))).toBe(true)
+ })
+
+ it('takes no silence it was paused through for a hang: the beat a resumed service sends keeps it', () => {
+ const supervisor = make()
+ supervisor.start()
+ const child = latest()
+ child.beat()
+ // Paused with the master: on resume the silence runs out at once and the overdue beat follows.
+ vi.advanceTimersByTime(6_000)
+ vi.advanceTimersByTime(10)
+ child.beat()
+ for (let i = 0; i < 6; i++) { vi.advanceTimersByTime(5_000); child.beat() }
+ expect(child.kills).toEqual([])
+ expect(supervisor.status()[0]).toMatchObject({ restarts: 0 })
})
it('restarts a service that outgrows its heap share or its resident budget, and counts it like a crash', () => {
diff --git a/cli/src/harnessd/services.ts b/cli/src/harnessd/services.ts
index d11686d52..e57710319 100644
--- a/cli/src/harnessd/services.ts
+++ b/cli/src/harnessd/services.ts
@@ -11,7 +11,7 @@
* bind and no update to prove; it beats, or it is restarted. Everything that touches the operating
* system is injected, so every decision here is tested without one.
*/
-import { isCoreMessage } from './protocol.js'
+import { heartbeatGraceMs, isCoreMessage } from './protocol.js'
import type { CoreHandle } from './supervisor.js'
export interface ServiceSpec {
@@ -225,12 +225,18 @@ class Service {
}, this.options.stopGraceMs)
}
+ /** A silence that runs out gets one beat's time more, in case the master was paused too: see
+ * `Supervisor.watchHeartbeat`, where a 40 s pause cost the core and an app's message. */
private watchHeartbeat(child: CoreHandle): void {
+ const timeout = this.options.heartbeatTimeoutMs
+ const grace = heartbeatGraceMs(timeout)
this.armTimer('heartbeatTimer', () => {
- this.deps.log(`[harnessd] service ${this.spec.name} sent no heartbeat for ${this.options.heartbeatTimeoutMs} ms — it is hung; killing it`)
- this.killReason = 'hung'
- child.kill('SIGKILL')
- }, this.options.heartbeatTimeoutMs)
+ this.armTimer('heartbeatTimer', () => {
+ this.deps.log(`[harnessd] service ${this.spec.name} sent no heartbeat for ${timeout + grace} ms — it is hung; killing it`)
+ this.killReason = 'hung'
+ child.kill('SIGKILL')
+ }, grace)
+ }, timeout)
}
private setState(state: ServiceState): void {
diff --git a/cli/src/harnessd/supervisor.spec.ts b/cli/src/harnessd/supervisor.spec.ts
index b0b095580..a34e36ca5 100644
--- a/cli/src/harnessd/supervisor.spec.ts
+++ b/cli/src/harnessd/supervisor.spec.ts
@@ -158,20 +158,38 @@ describe('Supervisor', () => {
expect(supervisor.status()).toMatchObject({ restarts: 1, lastExitReason: 'not-ready' })
})
- it('kills a bound core whose heartbeats stop: it is hung', () => {
+ it('kills a bound core whose heartbeats stop, one beat\'s time after its silence runs out: it is hung', () => {
const supervisor = make()
supervisor.start()
core().up()
vi.advanceTimersByTime(5_000)
core().beat()
- vi.advanceTimersByTime(5_999)
+ vi.advanceTimersByTime(6_000)
+ expect(lines.at(-1)).toBe('[harnessd] core sent no heartbeat for 6000 ms — giving it 2000 ms more, in case this master was paused too')
+ vi.advanceTimersByTime(1_999)
expect(core().kills).toEqual([])
vi.advanceTimersByTime(1)
expect(core().kills).toEqual(['SIGKILL'])
+ expect(lines.at(-1)).toBe('[harnessd] core sent no heartbeat for 8000 ms — it is hung; — killing it')
core().exit(null, 'SIGKILL')
expect(supervisor.status().lastExitReason).toBe('hung')
})
+ it('takes no silence it was paused through for a hang: the beat a resumed core sends keeps it', () => {
+ // Master and core paused together read, on resume, as the silence running out at once and the
+ // core's overdue beat arriving a moment later (e2e/paused.e2e.ts).
+ const supervisor = make()
+ supervisor.start()
+ core().up()
+ vi.advanceTimersByTime(6_000)
+ expect(lines.at(-1)).toContain('giving it 2000 ms more')
+ vi.advanceTimersByTime(10)
+ core().beat()
+ live(30_000)
+ expect(core().kills).toEqual([])
+ expect(supervisor.status()).toMatchObject({ state: 'running', restarts: 0 })
+ })
+
it('backs off crash after crash, up to its cap, and starts over after a good run', () => {
const supervisor = make({ crashLoopCrashes: 100 })
supervisor.start()
diff --git a/cli/src/harnessd/supervisor.ts b/cli/src/harnessd/supervisor.ts
index 3dac90b27..4817d63c3 100644
--- a/cli/src/harnessd/supervisor.ts
+++ b/cli/src/harnessd/supervisor.ts
@@ -14,6 +14,7 @@ import {
CORE_EXIT_STOP,
CORE_EXIT_UPDATE,
HARNESSD_PROTOCOL,
+ heartbeatGraceMs,
isCoreMessage,
type CoreMessage,
type MasterMessage,
@@ -380,9 +381,22 @@ export class Supervisor {
}, this.options.stopGraceMs)
}
+ /**
+ * ⚠️ A silence the master slept through is not a hang. Ctrl-Z on a daemon run in a terminal, a
+ * paused virtual machine or a swap storm stops the master with its core, and on resume every timer
+ * that fell due fires at once, before the master reads the beat the resumed core sends a moment
+ * later. Measured end to end (`e2e/paused.e2e.ts`): paused for 40 s, the core was killed as hung the
+ * instant it resumed, and the message an app had sent into its socket was lost with it. So a silence
+ * that runs out gets one beat's time more, counted from now, while this master is running.
+ */
private watchHeartbeat(): void {
const core = this.core!
- this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${this.options.heartbeatTimeoutMs} ms — it is hung;`), this.options.heartbeatTimeoutMs)
+ const timeout = this.options.heartbeatTimeoutMs
+ const grace = heartbeatGraceMs(timeout)
+ this.armTimer('heartbeatTimer', () => {
+ this.deps.log(`[harnessd] core sent no heartbeat for ${timeout} ms — giving it ${grace} ms more, in case this master was paused too`)
+ this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${timeout + grace} ms — it is hung;`), grace)
+ }, timeout)
}
private finish(code: number): void {
diff --git a/cli/src/lib/engineLaunch.spec.ts b/cli/src/lib/engineLaunch.spec.ts
index 8cc80ae68..63c27bed5 100644
--- a/cli/src/lib/engineLaunch.spec.ts
+++ b/cli/src/lib/engineLaunch.spec.ts
@@ -1,8 +1,9 @@
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
import { execFileSync } from 'node:child_process'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
+import { env } from '../config/env.js'
import {
AGENT_NAME_RE,
BYPASS_PERMISSION_FLAGS,
@@ -16,6 +17,9 @@ import {
terminalHintLines,
buildEngineCommandArgv,
buildEngineLaunchArgv,
+ buildTerminalLaunchArgv,
+ interactiveEngineShell,
+ isPosixShell,
commandAvailableInInteractiveShell,
commandSupportsFlagInInteractiveShell,
dropPermissionFlagIfUnsupported,
@@ -493,6 +497,57 @@ describe('buildEngineLaunchArgv', () => {
})
})
+describe('a login shell that is not POSIX', () => {
+ let dataDir = ''
+ let savedDataDir = ''
+ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'launch-shell-')); savedDataDir = env.ADAPTER_DATA_DIR; env.ADAPTER_DATA_DIR = dataDir })
+ afterEach(() => { env.ADAPTER_DATA_DIR = savedDataDir; rmSync(dataDir, { recursive: true, force: true }) })
+
+ it('tells the POSIX shells from the rest by name', () => {
+ for (const shell of ['/bin/sh', '/bin/bash', '/bin/zsh', '/usr/bin/dash', '/bin/ksh', '/usr/local/bin/mksh', '/bin/ash', '/usr/bin/yash']) expect(isPosixShell(shell), shell).toBe(true)
+ for (const shell of ['/usr/local/bin/fish', '/bin/tcsh', '/bin/csh', '/opt/homebrew/bin/nu', '/usr/bin/xonsh', '/usr/bin/elvish']) expect(isPosixShell(shell), shell).toBe(false)
+ })
+
+ it('asks each its own way for the person\'s interactive startup files', () => {
+ expect(interactiveEngineShell('/bin/zsh')?.args).toEqual(['-lic'])
+ expect(interactiveEngineShell('/bin/bash')?.args).toEqual(['-ic'])
+ expect(interactiveEngineShell('/usr/bin/dash')?.args).toEqual(['-ic'])
+ expect(interactiveEngineShell('/usr/local/bin/fish')).toMatchObject({ args: ['-i', '-c'], label: 'fish shell' })
+ expect(interactiveEngineShell('/bin/tcsh')?.args).toEqual(['-i', '-c'])
+ expect(interactiveEngineShell('/opt/homebrew/bin/nu')?.args).toEqual(['-c'])
+ })
+
+ it('launches an engine through the shell, which hands a one-time POSIX script to /bin/sh', () => {
+ const argv = buildEngineLaunchArgv('claude', { cwd: "/work/it's here" }, '/usr/local/bin/fish', '/opt/node/bin/node', '/opt/grid/bin/grid', '/usr/bin/tmux')
+ expect(argv.slice(0, 3)).toEqual(['/usr/local/bin/fish', '-i', '-c'])
+ expect(argv).toHaveLength(4)
+ const file = /^exec \/bin\/sh '(.+)'$/.exec(argv[3])?.[1]
+ expect(file && file.startsWith(join(dataDir, 'launch'))).toBe(true)
+ const script = readFileSync(file!, 'utf8')
+ // It removes itself first, then takes the arguments a POSIX shell would have been given.
+ expect(script.startsWith(`rm -f -- "$0"\nset -- '/work/it'"'"'s here' '${engineBin('claude')}'`)).toBe(true)
+ expect(script).toContain('harness_engine "$@"')
+ // The same launch for zsh is unchanged: no file, the script straight to zsh.
+ const zsh = buildEngineLaunchArgv('claude', { cwd: '/work' }, '/bin/zsh', '/opt/node/bin/node', '/opt/grid/bin/grid', '/usr/bin/tmux')
+ expect(zsh.slice(0, 4)).toEqual(['/usr/bin/env', 'DISABLE_AUTO_UPDATE=true', '/bin/zsh', '-lic'])
+ })
+
+ it.skipIf(!existsSync('/bin/tcsh'))('runs that script for real under tcsh, with its arguments intact and the file gone', () => {
+ const argv = buildEngineLaunchArgv('claude', { cwd: dataDir, installFirst: undefined }, '/bin/tcsh', '/opt/node/bin/node', '/opt/grid/bin/grid', null)
+ const file = /^exec \/bin\/sh '(.+)'$/.exec(argv[argv.length - 1])![1]
+ // Run the file the way tcsh would hand it over, with the engine replaced by printf.
+ writeFileSync(file, readFileSync(file, 'utf8').replace(/harness_engine "\$@"\s*$/, 'printf "%s|" "$PWD" "$@"'))
+ const out = execFileSync('/bin/tcsh', ['-i', '-c', argv[argv.length - 1]], { encoding: 'utf8', env: { ...process.env, HOME: dataDir }, stdio: ['ignore', 'pipe', 'ignore'] })
+ expect(out).toContain(`${dataDir}|${engineBin('claude')}|`)
+ expect(existsSync(file)).toBe(false)
+ })
+
+ it('opens a terminal tile with /bin/sh entering the folder, then the person\'s own shell', () => {
+ expect(buildTerminalLaunchArgv({ cwd: '/work' }, '/usr/local/bin/fish')).toEqual(['/bin/sh', '-c', expect.stringContaining('exec "$@"'), 'harness-terminal', '/work', '/usr/local/bin/fish'])
+ expect(buildTerminalLaunchArgv({ cwd: '/work' }, '/bin/zsh').slice(0, 2)).toEqual(['/bin/zsh', '-c'])
+ })
+})
+
describe('a first prompt on launch', () => {
const PROMPT = 'Start a local model on this machine'
@@ -828,7 +883,8 @@ describe('commandSupportsFlagInInteractiveShell', () => {
fakeEngine(binDir, 'opencode', '--auto')
await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
- // Downgraded underneath us; the cached yes stands until the cache is reset.
+ // Downgraded underneath us where the daemon cannot see the file (only the probe shell's PATH has
+ // it): remembered by name alone, the yes stands until the cache is reset.
fakeEngine(binDir, 'opencode', '--auto-update')
await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
resetCommandFlagSupportCache()
@@ -838,6 +894,52 @@ describe('commandSupportsFlagInInteractiveShell', () => {
await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
})
+ // e2e/updates.e2e.ts: remembered by name alone, a yes outlived the update that dropped the flag,
+ // and the engine was launched with a flag it refused at once. A yes is kept for the file it was read from.
+ it('answers an unchanged engine from memory, and asks again once an update has changed its file', async () => {
+ const binDir = mkdtempSync(join(tmpdir(), 'harness-engine-update-'))
+ dirs.push(binDir)
+ process.env.HARNESS_ENGINE_TEST_PATH = binDir
+ const savedPath = process.env.PATH
+ process.env.PATH = `${binDir}:${savedPath ?? ''}`
+ try {
+ const shell = bashProbeShell()
+ const runs = join(binDir, 'runs')
+ const build = (path: string, help: string) => {
+ writeFileSync(path, `#!/bin/sh\necho run >> '${runs}'\nif [ "$1" = "--help" ]; then printf "%s\\n" '${help}'; exit 0; fi\nexit 2\n`)
+ chmodSync(path, 0o700)
+ }
+ const asked = () => existsSync(runs) ? readFileSync(runs, 'utf8').split('\n').filter(Boolean).length : 0
+ const opencode = join(binDir, 'opencode')
+
+ build(opencode, '--auto')
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
+ expect(asked()).toBe(1)
+ // Rewritten in place by an update that dropped the flag: asked again, and refused.
+ build(opencode, '--auto-update')
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('unsupported')
+ expect(asked()).toBe(2)
+
+ // Installed as versions and linked, the way Homebrew and native installs update: the same size,
+ // and only the link moves.
+ mkdirSync(join(binDir, 'v1')); mkdirSync(join(binDir, 'v2'))
+ build(join(binDir, 'v1', 'opencode'), '--auto')
+ build(join(binDir, 'v2', 'opencode'), '--nope')
+ rmSync(opencode)
+ symlinkSync(join(binDir, 'v1', 'opencode'), opencode)
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')
+ expect(asked()).toBe(3)
+ rmSync(opencode)
+ symlinkSync(join(binDir, 'v2', 'opencode'), opencode)
+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('unsupported')
+ expect(asked()).toBe(4)
+ } finally {
+ process.env.PATH = savedPath
+ }
+ })
+
// A shell's own failure exits 1 — as zsh's read-only `status` did — and must not read as a missing
// flag: only the probe's own "not in the help" exit refuses an engine.
it('does not read a shell failure as a missing flag', async () => {
diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.ts
index 00e3bb3fe..e5f09011d 100644
--- a/cli/src/lib/engineLaunch.ts
+++ b/cli/src/lib/engineLaunch.ts
@@ -1,12 +1,16 @@
import { execFile } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+import { mkdirSync, realpathSync, statSync, writeFileSync } from 'node:fs'
import { homedir, userInfo } from 'node:os'
import { isAbsolute, basename, dirname, join } from 'node:path'
+import { env } from '../config/env.js'
import { isTerminalEngine, type AgentEngine } from '../engines/types.js'
import { isOpencodeV2 } from '../engines/opencode/version.js'
import { binaryOnPath, resolveBinaryOnPath } from './binaryOnPath.js'
import { engineBin } from './engineBin.js'
import { engineInstallPaths, npmEnginePrefix, type EngineInstallRecipe } from './engineInstall.js'
import { GRID_NO_UPDATE_CHECK_VAR, gridBinaryPath } from './gridExec.js'
+import { loginShellEnvironment } from './loginShellEnv.js'
import { managedNodePath } from './nodeRuntime.js'
import { RAISE_OPEN_FILES_SH } from './openFiles.js'
import { ENGINE_EXIT_PANE_OPTION } from './tmux.js'
@@ -400,13 +404,32 @@ function currentUserShell(): string | undefined {
}
}
+/**
+ * The shells that speak the POSIX shell language every launch script here is written in. Any other
+ * login shell (fish, tcsh, csh, nushell, xonsh) cannot run one: handed the script with `-c`, it fails
+ * on the first `if … then` or `"$@"`, and the engine never starts. tcsh ships with macOS and stands
+ * in for all of them: before this, no agent started at all for a person whose shell was not POSIX,
+ * and no terminal tile either (e2e/shells.e2e.ts).
+ */
+const POSIX_SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'mksh', 'oksh', 'pdksh', 'ash', 'yash', 'posh', 'busybox'])
+/** Shells that are not POSIX but take `-i` to load the person's interactive startup files. */
+const INTERACTIVE_FLAG_SHELLS = new Set(['fish', 'tcsh', 'csh', 'xonsh'])
+
+export function isPosixShell(path: string): boolean {
+ return POSIX_SHELLS.has(basename(path).toLowerCase())
+}
+
export function interactiveEngineShell(shell: string | undefined = undefined): InteractiveEngineShell | null {
const candidate = shell === undefined ? currentUserShell() : shell
if (!candidate || !isAbsolute(candidate)) return null
- switch (basename(candidate).toLowerCase()) {
+ const name = basename(candidate).toLowerCase()
+ switch (name) {
case 'zsh': return { path: candidate, args: ['-lic'], label: 'zsh login shell' }
case 'bash': return { path: candidate, args: ['-ic'], label: 'bash interactive shell' }
- default: return { path: candidate, args: ['-ic'], label: `${basename(candidate)} interactive shell` }
+ default: return isPosixShell(candidate)
+ ? { path: candidate, args: ['-ic'], label: `${name} interactive shell` }
+ // Separate flags: not every one of these reads them combined.
+ : { path: candidate, args: INTERACTIVE_FLAG_SHELLS.has(name) ? ['-i', '-c'] : ['-c'], label: `${name} shell` }
}
}
@@ -415,12 +438,30 @@ export function interactiveEngineShell(shell: string | undefined = undefined): I
* DISABLE_UPDATE_PROMPT would auto-update instead; DISABLE_AUTO_UPDATE skips that work entirely.
* Keep ordinary terminal launches unchanged, and keep loading rc files for PATH/version managers. */
function engineShellArgv(shell: InteractiveEngineShell, args: readonly string[]): string[] {
+ if (!isPosixShell(shell.path)) return throughPosixShell(shell, args)
const prefix = basename(shell.path).toLowerCase() === 'zsh'
? ['/usr/bin/env', 'DISABLE_AUTO_UPDATE=true']
: []
return [...prefix, shell.path, ...shell.args, ...args]
}
+/**
+ * For a shell that is not POSIX: it loads the person's environment (their PATH, their version
+ * managers), then hands the script to /bin/sh. The script and its arguments go in a one-time file, so
+ * all the person's shell parses is `exec /bin/sh '<file>'`, which fish, tcsh, nushell and xonsh read
+ * alike; the file removes itself as it starts. It sits in the daemon's own data folder, private to
+ * this user, and holds no secret: those reach the pane through the session's environment, never its
+ * command. `args` is what a POSIX shell would take after `-c`: the script, `$0`, then the arguments.
+ */
+function throughPosixShell(shell: InteractiveEngineShell, args: readonly string[]): string[] {
+ const [script = '', , ...positional] = args
+ const directory = join(env.ADAPTER_DATA_DIR, 'launch')
+ mkdirSync(directory, { recursive: true, mode: 0o700 })
+ const file = join(directory, `${randomUUID()}.sh`)
+ writeFileSync(file, `rm -f -- "$0"\nset -- ${positional.map(shellSingleQuote).join(' ')}\n${script}`, { mode: 0o600 })
+ return [shell.path, ...shell.args, `exec /bin/sh ${shellSingleQuote(file)}`]
+}
+
/**
* Full argv for a fresh tmux pane. `exec` replaces the shell with the engine,
* preserving process discovery while loading the same startup files a user
@@ -644,7 +685,10 @@ export function buildTerminalLaunchArgv(
const hintPrelude = opts.terminalHint && process.env.HARNESS_OS !== '1'
? `printf '%s\\n' ${terminalHintLines(opts.terminalHint.machineName).map(shellSingleQuote).join(' ')}\n`
: ''
- return [path, '-c', RAISE_OPEN_FILES_SH + clearEnvPrelude(opts.clearEnv) + cwdPrelude + hintPrelude + 'shift\nexec "$@"', 'harness-terminal', opts.cwd ?? '', path, ...loginArgs]
+ // The prelude is a POSIX script. For a shell that is not POSIX, /bin/sh runs it and then execs the
+ // person's shell, which loads its own startup files as it always does.
+ const interpreter = isPosixShell(path) ? path : '/bin/sh'
+ return [interpreter, '-c', RAISE_OPEN_FILES_SH + clearEnvPrelude(opts.clearEnv) + cwdPrelude + hintPrelude + 'shift\nexec "$@"', 'harness-terminal', opts.cwd ?? '', path, ...loginArgs]
}
/**
@@ -909,7 +953,7 @@ export type CommandFlagSupport = 'supported' | 'unsupported' | 'unknown'
const FLAG_UNSUPPORTED_EXIT = 64
/**
- * The pairs this probe has seen the engine SUPPORT.
+ * The pairs this probe has seen the engine SUPPORT, each with the engine file it saw (`engineFileStamp`).
*
* Every relaunch asks, and a post-reboot restore asks once per agent, so the working case — which
* is nearly every case — is worth answering from memory instead of spawning `--help` again.
@@ -919,8 +963,32 @@ const FLAG_UNSUPPORTED_EXIT = 64
* `--auto`. A cached `unsupported` would go on refusing that upgraded engine until the daemon
* happened to restart, which is the one outcome worth more than the spawn it saves. `unknown` is
* not kept for the same reason at shorter range: one slow `--help` would turn Auto off machine-wide.
+ *
+ * ⚠️ And a kept `supported` is only good for the file it was read from. An update can drop a flag as
+ * well as add one, and remembered by name alone the answer outlived it: measured end to end
+ * (`e2e/updates.e2e.ts`), an engine updated to a build without its permission flag was still
+ * launched with it, and refused it at once — the row went ready, then stopped, with no reason given —
+ * where a daemon restarted after the update refused the create and said why.
+ */
+const flagSupportCache = new Map<string, string>()
+
+/**
+ * Which file a command name runs, as a stamp that changes when an update replaces or rewrites it:
+ * its real path (a Homebrew or native install points at a new version's folder), inode, size and
+ * modification time. Resolved on the login shell's PATH, the one a launch resolves it on, then the
+ * daemon's own. '' when neither finds it — remembered by name alone then, as before.
*/
-const flagSupportCache = new Set<string>()
+function engineFileStamp(command: string): string {
+ const path = resolveBinaryOnPath(command, { PATH: loginShellEnvironment().PATH })
+ ?? resolveBinaryOnPath(command)
+ if (!path) return ''
+ try {
+ const stat = statSync(path, { bigint: true })
+ return [realpathSync(path), stat.ino, stat.size, stat.mtimeNs].join('\u0000')
+ } catch {
+ return ''
+ }
+}
/** Test seam, and for a machine where the engine was just upgraded. */
export function resetCommandFlagSupportCache(): void { flagSupportCache.clear() }
@@ -936,7 +1004,8 @@ export async function commandSupportsFlagInInteractiveShell(
shell: string | undefined = undefined,
): Promise<CommandFlagSupport> {
const key = `${command}\u0000${flag}\u0000${shell ?? ''}`
- if (flagSupportCache.has(key)) return 'supported'
+ const stamp = engineFileStamp(command)
+ if (flagSupportCache.get(key) === stamp) return 'supported'
const interactive = interactiveEngineShell(shell)
if (!interactive) return 'unknown'
// `harness_help_status`, not `status`: in zsh `status` is a read-only special parameter (an alias
@@ -964,7 +1033,7 @@ export async function commandSupportsFlagInInteractiveShell(
const answer: CommandFlagSupport = !error
? 'supported'
: Number((error as { code?: number | string }).code) === FLAG_UNSUPPORTED_EXIT ? 'unsupported' : 'unknown'
- if (answer === 'supported') flagSupportCache.add(key)
+ if (answer === 'supported') flagSupportCache.set(key, stamp)
resolve(answer)
},
)
diff --git a/cli/src/lib/localModels.integration.spec.ts b/cli/src/lib/localModels.integration.spec.ts
index 7bf9f5682..6fb4a1508 100644
--- a/cli/src/lib/localModels.integration.spec.ts
+++ b/cli/src/lib/localModels.integration.spec.ts
@@ -14,7 +14,11 @@ it('discovers, downloads, starts, verifies, stops and restarts across real subpr
const server = createServer(async (req, res) => {
const chunks: Buffer[] = []
for await (const chunk of req) chunks.push(Buffer.from(chunk))
- const body = JSON.parse(Buffer.concat(chunks).toString())
+ // The engine probe's GET /health is the one request with no body, and this fixture serves no
+ // engine: parsing it threw outside the test, failing a unit run whose every test had passed.
+ const text = Buffer.concat(chunks).toString()
+ if (!text) { res.statusCode = 404; res.end(); return }
+ const body = JSON.parse(text)
res.setHeader('content-type', 'application/json')
if (req.url === '/v1/grid/catalog') {
expect(req.headers.authorization).toBe('Bearer catalog-fixture')
diff --git a/cli/vitest.e2e.config.ts b/cli/vitest.e2e.config.ts
index f0ae27724..919c1de4b 100644
--- a/cli/vitest.e2e.config.ts
+++ b/cli/vitest.e2e.config.ts
@@ -2,17 +2,29 @@ import { defineConfig } from 'vitest/config'
/**
* End-to-end: each file boots real daemons (see e2e/harness/daemon.ts) under throwaway homes with
- * private tmux servers and fake engines. Files run one at a time so their timing is the daemon's,
- * not the host's contention.
+ * private tmux servers and fake engines. Files run one at a time by default, so their timing is the
+ * daemon's, not the host's contention.
+ *
+ * `E2E_WORKERS=<n>` runs n files at a time. Each file has its own daemons, homes, ports and tmux
+ * servers, so they share nothing but the machine; one at a time, the suite took about 25 minutes on a
+ * 12-core Mac. The checks whose numbers depend on a quiet machine (the soak, scale) run on their own,
+ * when asked, either way.
+ *
+ * `E2E_BUNDLE=1` starts every daemon from one bundle built for the run (e2e/harness/bundle.ts):
+ * less CPU per start, the code frozen as the run began, and the build that ships under test.
*/
+const workers = Math.max(1, Number(process.env.E2E_WORKERS ?? 1) || 1)
+
export default defineConfig({
test: {
environment: 'node',
setupFiles: ['./vitest.setup.ts'],
+ globalSetup: ['./e2e/harness/bundle.ts'],
include: ['e2e/**/*.e2e.ts'],
testTimeout: 180_000,
hookTimeout: 180_000,
- fileParallelism: false,
+ fileParallelism: workers > 1,
+ maxWorkers: workers,
pool: 'forks',
},
})