The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
harnessd: a paused daemon, a master lost at start, non-POSIX shells, updated engines; e2e in parallel (#773)
Deef55e3f4b9800
diff --git a/cli/e2e/harness/bundle.ts b/cli/e2e/harness/bundle.tsnew file mode 100644index 000000000..557988462--- /dev/null+++ b/cli/e2e/harness/bundle.ts@@ -0,0 +1,27 @@+/**+ * `E2E_BUNDLE=1`: every daemon in the run starts from one bundle built here, as the release ships,+ * instead of from the sources through tsx (vitest's global setup; `daemon.ts` reads the path).+ *+ * Why: tsx transpiles the whole tree at every start, and a parallel run starts hundreds of daemons —+ * on a 12-core Mac four files at a time drove the load average past 100, and tests timed out that+ * pass alone. A daemon from the bundle starts on a fraction of the CPU. And the run tests the code as+ * it was when it began: from the sources, an edit made mid-run reached every daemon started after it,+ * and 21 tests failed on a half-written change.+ */+import { execFileSync } from 'node:child_process'+import { mkdtempSync, rmSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { dirname, join, resolve } from 'node:path'+import { fileURLToPath } from 'node:url'++const CLI_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..')++export default function setup(): (() => void) | undefined {+ const asked = process.env.E2E_BUNDLE+ if (!asked || asked === '0') return undefined+ const out = mkdtempSync(join(tmpdir(), 'harness-e2e-bundle-'))+ execFileSync(process.execPath, ['build-bundle.mjs'], { cwd: CLI_ROOT, env: { ...process.env, BUNDLE_OUT_DIR: out }, stdio: 'pipe' })+ // Read by every daemon the run starts (`IsolatedDaemon.start`); the workers inherit it.+ process.env.E2E_BUNDLE_PATH = join(out, 'cli.js')+ return () => rmSync(out, { recursive: true, force: true })+}diff --git a/cli/e2e/harness/daemon.ts b/cli/e2e/harness/daemon.tsindex 426f3d06c..49cafe880 100644--- a/cli/e2e/harness/daemon.ts+++ b/cli/e2e/harness/daemon.ts@@ -152,7 +152,9 @@ export class IsolatedDaemon {const entry = this.options.noMaster ? '__run' : '__harnessd'// Readiness is judged from what this start prints, never from an earlier boot's lines.const from = this.output.length- const script = this.options.scriptPath ? [this.options.scriptPath] : ['--import', 'tsx', 'src/cli.ts']+ // A test's own bundle (an old release), the run's bundle (`E2E_BUNDLE`, see bundle.ts), or the sources.+ const bundle = this.options.scriptPath ?? process.env.E2E_BUNDLE_PATH+ const script = bundle ? [bundle] : ['--import', 'tsx', 'src/cli.ts']const child = spawn(process.execPath, [...heap, ...script, entry], {cwd: CLI_ROOT, env: this.env, stdio: ['ignore', 'pipe', 'pipe'],})@@ -229,7 +231,18 @@ export class IsolatedDaemon {}async close(): Promise<void> {+ const core = this.options.noMaster ? null : this.corePid()await this.stop().catch(() => {})+ // A core whose master died during the test must not outlive it: 20 such cores, left by a run+ // whose masters crashed, spun on their closed output pipes for the rest of a parallel suite. Only+ // this test's core is killed — its pid, still running this daemon's entry, orphaned or ours.+ if (core && IsolatedDaemon.alive(core)) {+ const { stdout } = await exec('ps', ['-o', 'ppid=,command=', '-p', String(core)]).catch(() => ({ stdout: '' }))+ const [ppid, ...command] = stdout.trim().split(/\s+/)+ if (command.join(' ').includes('__run') && (ppid === '1' || Number(ppid) === this.pid)) {+ try { process.kill(core, 'SIGKILL') } catch { /* gone */ }+ }+ }await this.tmux.close()}}diff --git a/cli/e2e/orphan.e2e.ts b/cli/e2e/orphan.e2e.tsnew file mode 100644index 000000000..4b729f15e--- /dev/null+++ b/cli/e2e/orphan.e2e.ts@@ -0,0 +1,88 @@+/**+ * A core whose master dies, for Claude Code and Codex. The master is the one process `harness start`+ * leaves behind and the one the pid file names; a core without it is supervised by nobody, and must+ * leave so nothing holds the port the next start needs. The hard moment is a master killed while its+ * core is still starting: the core has bound and begun to beat, and is a second from finishing.+ */+import { mkdirSync } from 'node:fs'+import { createConnection } from 'node:net'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++type Engine = 'claude' | 'codex'+type Row = Record<string, any>++const rows = async (client: LocalClient): Promise<Row[]> =>+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {+ const cwd = join(d.projectsDir, folder)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()+ return until(`${folder} to bind its conversation`, async () => {+ const agent = await row(client, created.agent.id)+ return agent?.sessionId && agent.status === 'active' ? agent : null+ }, 60_000, 500)+}+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ expect((await started).payload?.userMessage).toBe(content)+ await ended+}+/** Whether anything accepts connections on the port. */+const answers = (port: number) => new Promise<boolean>((resolve) => {+ const socket = createConnection({ host: '127.0.0.1', port })+ socket.once('connect', () => { socket.destroy(); resolve(true) })+ socket.once('error', () => resolve(false))+})++describe('a core whose master dies', () => {+ let daemon: IsolatedDaemon | undefined+ const cores: number[] = []+ afterEach(async () => {+ // A core this test failed to see leave must not outlive it.+ for (const pid of cores.splice(0)) { try { process.kill(pid, 'SIGKILL') } catch { /* gone */ } }+ await daemon?.close(); daemon = undefined+ })++ it.each([+ ['while its core is still starting', 'bound'],+ ['once its core is up', 'ready'],+ ] as const)('killed %s: the core leaves, the port is free, and the next start has every agent', async (_when, moment) => {+ const d = await IsolatedDaemon.create()+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ let client = await LocalClient.connect(d)+ const agents = [await create(d, client, 'claude', 'orphan-claude'), await create(d, client, 'codex', 'orphan-codex')]+ client.close()+ await d.stop()++ const from = d.log().length+ await d.start({ ready: 'none' })+ const line = moment === 'bound' ? /\[harnessd\] core bound/ : /\[cli\] ready/+ await until(`the core to be ${moment}`, () => line.test(d.log().slice(from)) || null, 60_000, 10)+ const core = d.corePid()!+ cores.push(core)+ await d.kill()+ await until('the core to leave', () => !IsolatedDaemon.alive(core) || null, 60_000, 200)+ await until('the port to be free', async () => !(await answers(d.port)) || null, 10_000, 200)++ await d.start()+ client = await LocalClient.connect(d)+ for (const agent of agents) {+ await until(`${agent.engine} back`, async () => {+ const now = await row(client, agent.id)+ return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null+ }, 60_000, 500)+ await turn(client, agent.id, `after the master died (${agent.engine})`)+ }+ client.close()+ }, 300_000)+})diff --git a/cli/e2e/paused.e2e.ts b/cli/e2e/paused.e2e.tsnew file mode 100644index 000000000..d82fcbbba--- /dev/null+++ b/cli/e2e/paused.e2e.ts@@ -0,0 +1,96 @@+/**+ * The daemon paused while its engines go on, for Claude Code and Codex: the master and the core are+ * stopped (SIGSTOP), as Ctrl-Z on a daemon run in a terminal, a paused virtual machine or a swap storm+ * can leave them, for longer than the master's 30 s heartbeat watchdog. The engines live in tmux and+ * keep working. When the daemon resumes, nothing the person did meanwhile may be lost: a turn typed+ * into a pane is in the conversation once, a message an app sent into the paused socket is taken once,+ * and every agent works as before.+ */+import { mkdirSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++type Engine = 'claude' | 'codex'+type Row = Record<string, any>++const rows = async (client: LocalClient): Promise<Row[]> =>+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)+async function create(d: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Row> {+ const cwd = join(d.projectsDir, folder)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)+ expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()+ return until(`${folder} to bind its conversation`, async () => {+ const agent = await row(client, created.agent.id)+ return agent?.sessionId && agent.status === 'active' ? agent : null+ }, 60_000, 500)+}+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ expect((await started).payload?.userMessage).toBe(content)+ await ended+}+/** What the person said in a conversation, oldest first, as the apps read it back. */+async function said(client: LocalClient, sessionId: string): Promise<string[]> {+ const page = await client.request('session_get', { sessionId, limit: 50 }, 60_000)+ expect(page.error, JSON.stringify(page)).toBeUndefined()+ return (page.events as Row[]).filter((event) => event.type === 'user_message').map((event) => String(event.payload.content))+}+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))++describe('the daemon paused while its engines go on', () => {+ let daemon: IsolatedDaemon | undefined+ const paused: number[] = []+ afterEach(async () => {+ // A stopped daemon left behind would hold the test's close until its SIGKILL.+ for (const pid of paused.splice(0)) { try { process.kill(pid, 'SIGCONT') } catch { /* gone */ } }+ await daemon?.close(); daemon = undefined+ })++ it('paused past the watchdog: the typed turn and the app\'s message are each in the conversation once, and the core is not taken for hung', async () => {+ const d = await IsolatedDaemon.create()+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ let client = await LocalClient.connect(d)+ const claude = await create(d, client, 'claude', 'paused-claude')+ const codex = await create(d, client, 'codex', 'paused-codex')+ await turn(client, claude.id, 'before the pause (claude)')+ await turn(client, codex.id, 'before the pause (codex)')++ const master = d.pid!, core = d.corePid()!+ for (const pid of [master, core]) { process.kill(pid, 'SIGSTOP'); paused.push(pid) }+ // An app's message into the paused socket: the kernel holds it until the core reads again.+ client.send('message', { agentId: codex.id, content: 'sent while the daemon was paused' })+ // The person types into a pane meanwhile, and the engine takes the turn without the daemon.+ await d.tmux.run('send-keys', '-t', claude.tmuxPane, 'typed while the daemon was paused', 'Enter')+ await sleep(40_000)+ // Everything resumes at once, as it does after Ctrl-Z and fg or a virtual machine's pause.+ for (const pid of paused.splice(0)) process.kill(pid, 'SIGCONT')++ if (client.closed) client = await LocalClient.connect(d)+ await until('the app\'s message to be taken', async () =>+ (await said(client, codex.sessionId).catch(() => [])).includes('sent while the daemon was paused') || null, 60_000, 500)+ await until('the typed turn to be read', async () =>+ (await said(client, claude.sessionId).catch(() => [])).includes('typed while the daemon was paused') || null, 60_000, 500)+ await sleep(5_000)+ expect(await said(client, codex.sessionId)).toEqual(['before the pause (codex)', 'sent while the daemon was paused'])+ expect(await said(client, claude.sessionId)).toEqual(['before the pause (claude)', 'typed while the daemon was paused'])+ for (const agent of [claude, codex]) {+ const now = await row(client, agent.id)+ expect(now?.status, agent.engine).toBe('active')+ expect(now?.sessionId, agent.engine).toBe(agent.sessionId)+ await turn(client, agent.id, `after the pause (${agent.engine})`)+ }+ // The master was paused too, so the silence was not the core's: it is the same core.+ expect(d.log()).not.toMatch(/it is hung/)+ expect(d.coresStarted()).toBe(1)+ client.close()+ }, 240_000)+})diff --git a/cli/e2e/serviceProcesses.e2e.ts b/cli/e2e/serviceProcesses.e2e.tsindex d6d65d745..b37e5dbb3 100644--- a/cli/e2e/serviceProcesses.e2e.ts+++ b/cli/e2e/serviceProcesses.e2e.ts@@ -33,11 +33,16 @@ async function turn(client: LocalClient, agentId: string, content: string): Promclient.send('message', { agentId, content })await ended}-/** The search service's own process, from the process table: it is `harnessd search`. */-function searchPids(): number[] {+/**+ * This daemon's search service processes: titled `harnessd search` AND started by its own master, by+ * the pid the master logged. The title alone matched every daemon's search on the machine — another+ * file's in a parallel run, another checkout's — and the tests below stop and kill what it returns.+ */+function searchPids(d: IsolatedDaemon): number[] {+ const ours = new Set([...d.log().matchAll(/\[harnessd\] service search started \(pid (\d+)\)/g)].map((match) => Number(match[1])))const table = execFileSync('ps', ['-A', '-o', 'pid=,command=']).toString().trim().split('\n')return table.map((line) => line.trim().match(/^(\d+)\s+(.*)$/)).filter((match): match is RegExpMatchArray => !!match)- .filter(([, , command]) => command.trim() === 'harnessd search').map(([, pid]) => Number(pid))+ .filter(([, pid, command]) => command.trim() === 'harnessd search' && ours.has(Number(pid))).map(([, pid]) => Number(pid))}const finds = async (client: LocalClient, word: string, sessionId: string): Promise<boolean> =>JSON.stringify(await client.request('session_search', { query: word }, 30_000)).includes(sessionId)@@ -63,7 +68,7 @@ describe('services in their own processes', () => {const d = await fresh()const client = await LocalClient.connect(d)await until('search to connect to the core', () => d.log().includes('[services] search connected') || null, 30_000, 200)- expect(searchPids().length).toBeGreaterThanOrEqual(1)+ expect(searchPids(d).length).toBeGreaterThanOrEqual(1)for (const engine of ['claude', 'codex'] as const) {const agent = await create(d, client, engine, `search-out-${engine}`)const word = `axolotl${engine}`@@ -80,7 +85,7 @@ describe('services in their own processes', () => {const agent = await create(d, client, 'claude', 'search-killed')await turn(client, agent.id, 'remember the pangolin')await until('search to find it', () => finds(client, 'pangolin', agent.sessionId) || null, 60_000, 1_000)- const before = searchPids()+ const before = searchPids(d)expect(before.length).toBeGreaterThanOrEqual(1)for (const pid of before) process.kill(pid, 'SIGKILL')// Asked while it is down: an answer, not a hang.@@ -90,7 +95,7 @@ describe('services in their own processes', () => {await turn(client, agent.id, 'while search was gone')await until('the master to restart search', () => restarts(d) >= 1 || null, 30_000, 200)await until('search to answer again', () => finds(client, 'pangolin', agent.sessionId) || null, 60_000, 1_000)- expect(searchPids().some((pid) => !before.includes(pid))).toBe(true)+ expect(searchPids(d).some((pid) => !before.includes(pid))).toBe(true)expect(d.coresStarted()).toBe(1)client.close()})@@ -101,7 +106,7 @@ describe('services in their own processes', () => {const agent = await create(d, client, 'codex', 'search-hung')await until('search to connect', () => d.log().includes('[services] search connected') || null, 30_000, 200)// Hung: a stopped process neither beats nor answers, as one stuck in a loop would not.- for (const pid of searchPids()) process.kill(pid, 'SIGSTOP')+ for (const pid of searchPids(d)) process.kill(pid, 'SIGSTOP')await until('the master to find search hung', () => d.log().includes('[harnessd] service search sent no heartbeat') || null, 30_000, 200)await until('search to be started again', () => restarts(d) >= 1 || null, 30_000, 200)await turn(client, agent.id, 'the core never waited on search')@@ -140,10 +145,10 @@ describe('services in their own processes', () => {await turn(client, agent.id, 'remember the quokka')await until('search to find it', () => finds(client, 'quokka', agent.sessionId) || null, 60_000, 1_000)// Down: killed, and the purge happens before the master has it back.- for (const pid of searchPids()) process.kill(pid, 'SIGSTOP')+ for (const pid of searchPids(d)) process.kill(pid, 'SIGSTOP')const createdAt = Date.parse(agent.createdAt)const review = await client.request('agent_purge', { agentId: agent.id, sessionId: agent.sessionId, createdAt, mode: 'inspect' }, 60_000)- for (const pid of searchPids()) process.kill(pid, 'SIGKILL')+ for (const pid of searchPids(d)) process.kill(pid, 'SIGKILL')const deleted = await client.request('agent_purge', { agentId: agent.id, sessionId: agent.sessionId, createdAt, mode: 'delete', reviewId: review.reviewId }, 90_000)expect(deleted, JSON.stringify(deleted)).toMatchObject({ deleted: true })await until('search to be back', () => restarts(d) >= 1 && d.log().split('[services] search connected').length >= 3 || null, 60_000, 250)diff --git a/cli/e2e/shells.e2e.ts b/cli/e2e/shells.e2e.tsnew file mode 100644index 000000000..6ab753fd2--- /dev/null+++ b/cli/e2e/shells.e2e.ts@@ -0,0 +1,75 @@+/**+ * The person's login shell, for Claude Code and Codex. The daemon launches each engine through the+ * shell the person uses (`$SHELL`), so their PATH, version managers and startup files apply. zsh and+ * bash speak the POSIX shell language the launch is written in; fish, tcsh and nushell do not. tcsh+ * ships with macOS, so it stands in for every shell that is not POSIX: an agent must start, in its+ * own folder, and work, whichever shell it is.+ */+import { mkdirSync, realpathSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++type Engine = 'claude' | 'codex'+type Row = Record<string, any>++const rows = async (client: LocalClient): Promise<Row[]> =>+ (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 30_000)).agents+const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)+const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {+ const started = client.next(isTurn('turn_started', agentId), 45_000, `turn_started (${content})`)+ const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ expect((await started).payload?.userMessage).toBe(content)+ await ended+}++const SHELLS = ['/bin/bash', '/bin/tcsh'] as const++describe('the person\'s login shell', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ it.each(SHELLS)('a terminal tile under %s is that shell, in the folder asked for', async (shell) => {+ const d = await IsolatedDaemon.create({ env: { SHELL: shell } })+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ const client = await LocalClient.connect(d)+ const cwd = join(d.projectsDir, 'tile')+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine: 'terminal', cwd }, 60_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const info = await until(`the tile's ${shell} to be running`, async () => {+ const answer = await client.request('terminal_info', { agentId: created.agent.id }, 10_000)+ return answer.command && !answer.error && answer.command !== 'sh' ? answer : null+ }, 30_000, 500)+ expect(info.command).toBe(shell.split('/').at(-1))+ expect(realpathSync(info.path)).toBe(realpathSync(cwd))+ client.close()+ }, 120_000)++ for (const shell of SHELLS) {+ it.each(['claude', 'codex'] as const)(`%s under ${shell}: an agent starts in its own folder and works`, async (engine: Engine) => {+ const d = await IsolatedDaemon.create({ env: { SHELL: shell } })+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ await d.start()+ const client = await LocalClient.connect(d)+ const cwd = join(d.projectsDir, `shell-${engine}`)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 120_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const agent = await until(`the ${engine} agent to bind under ${shell}`, async () => {+ const now = await row(client, created.agent.id)+ return now?.sessionId && now.status === 'active' ? now : null+ }, 60_000, 500)+ const info = await client.request('terminal_info', { agentId: agent.id }, 10_000)+ expect(realpathSync(info.path)).toBe(realpathSync(cwd))+ await turn(client, agent.id, `under ${shell}`)+ client.close()+ }, 240_000)+ }+})diff --git a/cli/src/harnessd/coreLink.spec.ts b/cli/src/harnessd/coreLink.spec.tsindex 184dd8e34..d7fd0e7af 100644--- a/cli/src/harnessd/coreLink.spec.ts+++ b/cli/src/harnessd/coreLink.spec.ts@@ -8,6 +8,7 @@ class FakeChannel implements MasterChannel {private disconnect: Array<() => void> = []rss = 100parentPid = 4242+ connected = truesend?: (message: CoreMessage) => unknown = (message) => { this.sent.push(message) }once(_event: 'disconnect', listener: () => void): void { this.disconnect.push(listener) }on(_event: 'message', listener: (message: unknown) => void): void { this.messageListeners.push(listener) }@@ -113,6 +114,34 @@ describe('connectToMaster', () => {expect(gone).toHaveBeenCalledOnce()})+ // runForeground subscribes long after the core has bound: a master killed in between went unnoticed,+ // and its core ran on for good, holding the port.+ it('hears a master that went before anyone asked, and only once', async () => {+ const channel = new FakeChannel()+ const link = connectToMaster(channel, supervised)+ link.bound(1)+ channel.leave()+ const gone = vi.fn()+ link.onMasterGone(gone)+ expect(gone).not.toHaveBeenCalled()+ await Promise.resolve()+ expect(gone).toHaveBeenCalledOnce()+ const later = vi.fn()+ link.onMasterGone(later)+ await Promise.resolve()+ expect(later).toHaveBeenCalledOnce()+ expect(gone).toHaveBeenCalledOnce()+ })++ it('hears a master whose channel had already closed when the core started', async () => {+ const channel = new FakeChannel()+ channel.connected = false+ const gone = vi.fn()+ connectToMaster(channel, supervised).onMasterGone(gone)+ await Promise.resolve()+ expect(gone).toHaveBeenCalledOnce()+ })+it('survives a send on a channel the master already closed', () => {const channel = new FakeChannel()channel.send = () => { throw new Error('ERR_IPC_CHANNEL_CLOSED') }@@ -123,6 +152,7 @@ describe('connectToMaster', () => {it('uses this process by default, which no master started', () => {expect(connectToMaster().supervised).toBe(false)expect(processChannel.memoryUsage().rss).toBeGreaterThan(0)+ expect(processChannel.connected).toBe(process.connected)expect(processChannel.parentPid).toBe(process.ppid)})diff --git a/cli/src/harnessd/coreLink.ts b/cli/src/harnessd/coreLink.tsindex bcafec68a..210f8f2ac 100644--- a/cli/src/harnessd/coreLink.ts+++ b/cli/src/harnessd/coreLink.ts@@ -52,6 +52,8 @@ export interface MasterChannel {memoryUsage(): { rss: number; heapUsed: number }/** The process that started this one: the master, when it holds the channel. */readonly parentPid: number+ /** False once the channel to the master has closed (`process.connected`). */+ readonly connected?: boolean}export interface CoreLink {@@ -81,6 +83,7 @@ export const processChannel: MasterChannel = {on: process.on.bind(process),memoryUsage: () => process.memoryUsage(),parentPid: process.ppid,+ get connected() { return process.connected },}export function connectToMaster(@@ -101,6 +104,20 @@ export function connectToMaster(if (supervised) {channel.on('message', (message) => { if (isMasterMessage(message)) status = message.status })}+ // ⚠️ Listened for from the start and remembered, not from whenever the caller subscribes.+ // runForeground subscribes some 1,100 lines into start-up, long after this core has bound and begun+ // to beat, and 'disconnect' fires once, for whoever listens then: a master that died in between —+ // killed while its core was still starting — went unnoticed, and the core ran on for good, holding+ // the port the next start needs. Measured: 20 cores whose master died as they bound were all still+ // running ten minutes later.+ let masterGone = supervised && channel.connected === false+ const goneListeners: Array<() => void> = []+ if (supervised) {+ channel.once('disconnect', () => {+ masterGone = true+ for (const listener of goneListeners.splice(0)) listener()+ })+ }return {supervised,masterPid: supervised ? channel.parentPid : null,@@ -118,7 +135,11 @@ export function connectToMaster(heartbeat = setInterval(beat, intervalMs)heartbeat.unref()},- onMasterGone: (listener) => { if (supervised) channel.once('disconnect', listener) },+ onMasterGone: (listener) => {+ if (!supervised) return+ if (masterGone) queueMicrotask(listener)+ else goneListeners.push(listener)+ },status: () => status,close: () => {if (heartbeat) clearInterval(heartbeat)diff --git a/cli/src/harnessd/protocol.ts b/cli/src/harnessd/protocol.tsindex 46de53fdb..d0756f1c4 100644--- a/cli/src/harnessd/protocol.ts+++ b/cli/src/harnessd/protocol.ts@@ -26,6 +26,16 @@ export const CORE_EXIT_UPDATE = 75*/export const CORE_EXIT_STOP = 78+/**+ * How long a master waits for one more beat once a process's silence has run out, counted from then:+ * a third of the silence, the longest a core or service ever goes between beats+ * (`coreLink.heartbeatInterval`), so one that was only paused along with its master always lands one.+ * Why there is a wait at all: `Supervisor.watchHeartbeat`.+ */+export function heartbeatGraceMs(timeoutMs: number): number {+ return Math.ceil(timeoutMs / 3)+}+export type CoreMessage =/** The control port is bound: from here on the daemon answers, though start-up is not done. */| { type: 'harnessd:bound'; protocol: number; port: number }diff --git a/cli/src/harnessd/services.spec.ts b/cli/src/harnessd/services.spec.tsindex 282aade83..7e2c6c3d4 100644--- a/cli/src/harnessd/services.spec.ts+++ b/cli/src/harnessd/services.spec.ts@@ -105,8 +105,10 @@ describe('ServiceSupervisor', () => {it('kills a service that stops beating, from its start too, and restarts it as hung', () => {const supervisor = make()supervisor.start()- // Never beat at all: hung from the start.+ // Never beat at all: hung from the start, once one beat's time more has passed.vi.advanceTimersByTime(6_000)+ expect(latest().kills).toEqual([])+ vi.advanceTimersByTime(2_000)expect(latest().kills).toEqual(['SIGKILL'])latest().exit(null, 'SIGKILL')expect(supervisor.status()[0]).toMatchObject({ lastExitReason: 'hung', state: 'restarting' })@@ -114,9 +116,23 @@ describe('ServiceSupervisor', () => {const second = latest()for (let i = 0; i < 3; i++) { vi.advanceTimersByTime(5_000); second.beat() }expect(second.kills).toEqual([])- vi.advanceTimersByTime(6_000)+ vi.advanceTimersByTime(8_000)expect(second.kills).toEqual(['SIGKILL'])- expect(lines.some((line) => line.includes('sent no heartbeat for 6000 ms — it is hung'))).toBe(true)+ expect(lines.some((line) => line.includes('sent no heartbeat for 8000 ms — it is hung'))).toBe(true)+ })++ it('takes no silence it was paused through for a hang: the beat a resumed service sends keeps it', () => {+ const supervisor = make()+ supervisor.start()+ const child = latest()+ child.beat()+ // Paused with the master: on resume the silence runs out at once and the overdue beat follows.+ vi.advanceTimersByTime(6_000)+ vi.advanceTimersByTime(10)+ child.beat()+ for (let i = 0; i < 6; i++) { vi.advanceTimersByTime(5_000); child.beat() }+ expect(child.kills).toEqual([])+ expect(supervisor.status()[0]).toMatchObject({ restarts: 0 })})it('restarts a service that outgrows its heap share or its resident budget, and counts it like a crash', () => {diff --git a/cli/src/harnessd/services.ts b/cli/src/harnessd/services.tsindex d11686d52..e57710319 100644--- a/cli/src/harnessd/services.ts+++ b/cli/src/harnessd/services.ts@@ -11,7 +11,7 @@* bind and no update to prove; it beats, or it is restarted. Everything that touches the operating* system is injected, so every decision here is tested without one.*/-import { isCoreMessage } from './protocol.js'+import { heartbeatGraceMs, isCoreMessage } from './protocol.js'import type { CoreHandle } from './supervisor.js'export interface ServiceSpec {@@ -225,12 +225,18 @@ class Service {}, this.options.stopGraceMs)}+ /** A silence that runs out gets one beat's time more, in case the master was paused too: see+ * `Supervisor.watchHeartbeat`, where a 40 s pause cost the core and an app's message. */private watchHeartbeat(child: CoreHandle): void {+ const timeout = this.options.heartbeatTimeoutMs+ const grace = heartbeatGraceMs(timeout)this.armTimer('heartbeatTimer', () => {- this.deps.log(`[harnessd] service ${this.spec.name} sent no heartbeat for ${this.options.heartbeatTimeoutMs} ms — it is hung; killing it`)- this.killReason = 'hung'- child.kill('SIGKILL')- }, this.options.heartbeatTimeoutMs)+ this.armTimer('heartbeatTimer', () => {+ this.deps.log(`[harnessd] service ${this.spec.name} sent no heartbeat for ${timeout + grace} ms — it is hung; killing it`)+ this.killReason = 'hung'+ child.kill('SIGKILL')+ }, grace)+ }, timeout)}private setState(state: ServiceState): void {diff --git a/cli/src/harnessd/supervisor.spec.ts b/cli/src/harnessd/supervisor.spec.tsindex b0b095580..a34e36ca5 100644--- a/cli/src/harnessd/supervisor.spec.ts+++ b/cli/src/harnessd/supervisor.spec.ts@@ -158,20 +158,38 @@ describe('Supervisor', () => {expect(supervisor.status()).toMatchObject({ restarts: 1, lastExitReason: 'not-ready' })})- it('kills a bound core whose heartbeats stop: it is hung', () => {+ it('kills a bound core whose heartbeats stop, one beat\'s time after its silence runs out: it is hung', () => {const supervisor = make()supervisor.start()core().up()vi.advanceTimersByTime(5_000)core().beat()- vi.advanceTimersByTime(5_999)+ vi.advanceTimersByTime(6_000)+ expect(lines.at(-1)).toBe('[harnessd] core sent no heartbeat for 6000 ms — giving it 2000 ms more, in case this master was paused too')+ vi.advanceTimersByTime(1_999)expect(core().kills).toEqual([])vi.advanceTimersByTime(1)expect(core().kills).toEqual(['SIGKILL'])+ expect(lines.at(-1)).toBe('[harnessd] core sent no heartbeat for 8000 ms — it is hung; — killing it')core().exit(null, 'SIGKILL')expect(supervisor.status().lastExitReason).toBe('hung')})+ it('takes no silence it was paused through for a hang: the beat a resumed core sends keeps it', () => {+ // Master and core paused together read, on resume, as the silence running out at once and the+ // core's overdue beat arriving a moment later (e2e/paused.e2e.ts).+ const supervisor = make()+ supervisor.start()+ core().up()+ vi.advanceTimersByTime(6_000)+ expect(lines.at(-1)).toContain('giving it 2000 ms more')+ vi.advanceTimersByTime(10)+ core().beat()+ live(30_000)+ expect(core().kills).toEqual([])+ expect(supervisor.status()).toMatchObject({ state: 'running', restarts: 0 })+ })+it('backs off crash after crash, up to its cap, and starts over after a good run', () => {const supervisor = make({ crashLoopCrashes: 100 })supervisor.start()diff --git a/cli/src/harnessd/supervisor.ts b/cli/src/harnessd/supervisor.tsindex 3dac90b27..4817d63c3 100644--- a/cli/src/harnessd/supervisor.ts+++ b/cli/src/harnessd/supervisor.ts@@ -14,6 +14,7 @@ import {CORE_EXIT_STOP,CORE_EXIT_UPDATE,HARNESSD_PROTOCOL,+ heartbeatGraceMs,isCoreMessage,type CoreMessage,type MasterMessage,@@ -380,9 +381,22 @@ export class Supervisor {}, this.options.stopGraceMs)}+ /**+ * ⚠️ A silence the master slept through is not a hang. Ctrl-Z on a daemon run in a terminal, a+ * paused virtual machine or a swap storm stops the master with its core, and on resume every timer+ * that fell due fires at once, before the master reads the beat the resumed core sends a moment+ * later. Measured end to end (`e2e/paused.e2e.ts`): paused for 40 s, the core was killed as hung the+ * instant it resumed, and the message an app had sent into its socket was lost with it. So a silence+ * that runs out gets one beat's time more, counted from now, while this master is running.+ */private watchHeartbeat(): void {const core = this.core!- this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${this.options.heartbeatTimeoutMs} ms — it is hung;`), this.options.heartbeatTimeoutMs)+ const timeout = this.options.heartbeatTimeoutMs+ const grace = heartbeatGraceMs(timeout)+ this.armTimer('heartbeatTimer', () => {+ this.deps.log(`[harnessd] core sent no heartbeat for ${timeout} ms — giving it ${grace} ms more, in case this master was paused too`)+ this.armTimer('heartbeatTimer', () => this.kill(core, 'hung', `sent no heartbeat for ${timeout + grace} ms — it is hung;`), grace)+ }, timeout)}private finish(code: number): void {diff --git a/cli/src/lib/engineLaunch.spec.ts b/cli/src/lib/engineLaunch.spec.tsindex 8cc80ae68..63c27bed5 100644--- a/cli/src/lib/engineLaunch.spec.ts+++ b/cli/src/lib/engineLaunch.spec.ts@@ -1,8 +1,9 @@import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'import { execFileSync } from 'node:child_process'import { tmpdir } from 'node:os'import { join } from 'node:path'+import { env } from '../config/env.js'import {AGENT_NAME_RE,BYPASS_PERMISSION_FLAGS,@@ -16,6 +17,9 @@ import {terminalHintLines,buildEngineCommandArgv,buildEngineLaunchArgv,+ buildTerminalLaunchArgv,+ interactiveEngineShell,+ isPosixShell,commandAvailableInInteractiveShell,commandSupportsFlagInInteractiveShell,dropPermissionFlagIfUnsupported,@@ -493,6 +497,57 @@ describe('buildEngineLaunchArgv', () => {})})+describe('a login shell that is not POSIX', () => {+ let dataDir = ''+ let savedDataDir = ''+ beforeEach(() => { dataDir = mkdtempSync(join(tmpdir(), 'launch-shell-')); savedDataDir = env.ADAPTER_DATA_DIR; env.ADAPTER_DATA_DIR = dataDir })+ afterEach(() => { env.ADAPTER_DATA_DIR = savedDataDir; rmSync(dataDir, { recursive: true, force: true }) })++ it('tells the POSIX shells from the rest by name', () => {+ for (const shell of ['/bin/sh', '/bin/bash', '/bin/zsh', '/usr/bin/dash', '/bin/ksh', '/usr/local/bin/mksh', '/bin/ash', '/usr/bin/yash']) expect(isPosixShell(shell), shell).toBe(true)+ for (const shell of ['/usr/local/bin/fish', '/bin/tcsh', '/bin/csh', '/opt/homebrew/bin/nu', '/usr/bin/xonsh', '/usr/bin/elvish']) expect(isPosixShell(shell), shell).toBe(false)+ })++ it('asks each its own way for the person\'s interactive startup files', () => {+ expect(interactiveEngineShell('/bin/zsh')?.args).toEqual(['-lic'])+ expect(interactiveEngineShell('/bin/bash')?.args).toEqual(['-ic'])+ expect(interactiveEngineShell('/usr/bin/dash')?.args).toEqual(['-ic'])+ expect(interactiveEngineShell('/usr/local/bin/fish')).toMatchObject({ args: ['-i', '-c'], label: 'fish shell' })+ expect(interactiveEngineShell('/bin/tcsh')?.args).toEqual(['-i', '-c'])+ expect(interactiveEngineShell('/opt/homebrew/bin/nu')?.args).toEqual(['-c'])+ })++ it('launches an engine through the shell, which hands a one-time POSIX script to /bin/sh', () => {+ const argv = buildEngineLaunchArgv('claude', { cwd: "/work/it's here" }, '/usr/local/bin/fish', '/opt/node/bin/node', '/opt/grid/bin/grid', '/usr/bin/tmux')+ expect(argv.slice(0, 3)).toEqual(['/usr/local/bin/fish', '-i', '-c'])+ expect(argv).toHaveLength(4)+ const file = /^exec \/bin\/sh '(.+)'$/.exec(argv[3])?.[1]+ expect(file && file.startsWith(join(dataDir, 'launch'))).toBe(true)+ const script = readFileSync(file!, 'utf8')+ // It removes itself first, then takes the arguments a POSIX shell would have been given.+ expect(script.startsWith(`rm -f -- "$0"\nset -- '/work/it'"'"'s here' '${engineBin('claude')}'`)).toBe(true)+ expect(script).toContain('harness_engine "$@"')+ // The same launch for zsh is unchanged: no file, the script straight to zsh.+ const zsh = buildEngineLaunchArgv('claude', { cwd: '/work' }, '/bin/zsh', '/opt/node/bin/node', '/opt/grid/bin/grid', '/usr/bin/tmux')+ expect(zsh.slice(0, 4)).toEqual(['/usr/bin/env', 'DISABLE_AUTO_UPDATE=true', '/bin/zsh', '-lic'])+ })++ it.skipIf(!existsSync('/bin/tcsh'))('runs that script for real under tcsh, with its arguments intact and the file gone', () => {+ const argv = buildEngineLaunchArgv('claude', { cwd: dataDir, installFirst: undefined }, '/bin/tcsh', '/opt/node/bin/node', '/opt/grid/bin/grid', null)+ const file = /^exec \/bin\/sh '(.+)'$/.exec(argv[argv.length - 1])![1]+ // Run the file the way tcsh would hand it over, with the engine replaced by printf.+ writeFileSync(file, readFileSync(file, 'utf8').replace(/harness_engine "\$@"\s*$/, 'printf "%s|" "$PWD" "$@"'))+ const out = execFileSync('/bin/tcsh', ['-i', '-c', argv[argv.length - 1]], { encoding: 'utf8', env: { ...process.env, HOME: dataDir }, stdio: ['ignore', 'pipe', 'ignore'] })+ expect(out).toContain(`${dataDir}|${engineBin('claude')}|`)+ expect(existsSync(file)).toBe(false)+ })++ it('opens a terminal tile with /bin/sh entering the folder, then the person\'s own shell', () => {+ expect(buildTerminalLaunchArgv({ cwd: '/work' }, '/usr/local/bin/fish')).toEqual(['/bin/sh', '-c', expect.stringContaining('exec "$@"'), 'harness-terminal', '/work', '/usr/local/bin/fish'])+ expect(buildTerminalLaunchArgv({ cwd: '/work' }, '/bin/zsh').slice(0, 2)).toEqual(['/bin/zsh', '-c'])+ })+})+describe('a first prompt on launch', () => {const PROMPT = 'Start a local model on this machine'@@ -828,7 +883,8 @@ describe('commandSupportsFlagInInteractiveShell', () => {fakeEngine(binDir, 'opencode', '--auto')await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')- // Downgraded underneath us; the cached yes stands until the cache is reset.+ // Downgraded underneath us where the daemon cannot see the file (only the probe shell's PATH has+ // it): remembered by name alone, the yes stands until the cache is reset.fakeEngine(binDir, 'opencode', '--auto-update')await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')resetCommandFlagSupportCache()@@ -838,6 +894,52 @@ describe('commandSupportsFlagInInteractiveShell', () => {await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')})+ // e2e/updates.e2e.ts: remembered by name alone, a yes outlived the update that dropped the flag,+ // and the engine was launched with a flag it refused at once. A yes is kept for the file it was read from.+ it('answers an unchanged engine from memory, and asks again once an update has changed its file', async () => {+ const binDir = mkdtempSync(join(tmpdir(), 'harness-engine-update-'))+ dirs.push(binDir)+ process.env.HARNESS_ENGINE_TEST_PATH = binDir+ const savedPath = process.env.PATH+ process.env.PATH = `${binDir}:${savedPath ?? ''}`+ try {+ const shell = bashProbeShell()+ const runs = join(binDir, 'runs')+ const build = (path: string, help: string) => {+ writeFileSync(path, `#!/bin/sh\necho run >> '${runs}'\nif [ "$1" = "--help" ]; then printf "%s\\n" '${help}'; exit 0; fi\nexit 2\n`)+ chmodSync(path, 0o700)+ }+ const asked = () => existsSync(runs) ? readFileSync(runs, 'utf8').split('\n').filter(Boolean).length : 0+ const opencode = join(binDir, 'opencode')++ build(opencode, '--auto')+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')+ expect(asked()).toBe(1)+ // Rewritten in place by an update that dropped the flag: asked again, and refused.+ build(opencode, '--auto-update')+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('unsupported')+ expect(asked()).toBe(2)++ // Installed as versions and linked, the way Homebrew and native installs update: the same size,+ // and only the link moves.+ mkdirSync(join(binDir, 'v1')); mkdirSync(join(binDir, 'v2'))+ build(join(binDir, 'v1', 'opencode'), '--auto')+ build(join(binDir, 'v2', 'opencode'), '--nope')+ rmSync(opencode)+ symlinkSync(join(binDir, 'v1', 'opencode'), opencode)+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('supported')+ expect(asked()).toBe(3)+ rmSync(opencode)+ symlinkSync(join(binDir, 'v2', 'opencode'), opencode)+ await expect(commandSupportsFlagInInteractiveShell('opencode', '--auto', shell)).resolves.toBe('unsupported')+ expect(asked()).toBe(4)+ } finally {+ process.env.PATH = savedPath+ }+ })+// A shell's own failure exits 1 — as zsh's read-only `status` did — and must not read as a missing// flag: only the probe's own "not in the help" exit refuses an engine.it('does not read a shell failure as a missing flag', async () => {diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.tsindex 00e3bb3fe..e5f09011d 100644--- a/cli/src/lib/engineLaunch.ts+++ b/cli/src/lib/engineLaunch.ts@@ -1,12 +1,16 @@import { execFile } from 'node:child_process'+import { randomUUID } from 'node:crypto'+import { mkdirSync, realpathSync, statSync, writeFileSync } from 'node:fs'import { homedir, userInfo } from 'node:os'import { isAbsolute, basename, dirname, join } from 'node:path'+import { env } from '../config/env.js'import { isTerminalEngine, type AgentEngine } from '../engines/types.js'import { isOpencodeV2 } from '../engines/opencode/version.js'import { binaryOnPath, resolveBinaryOnPath } from './binaryOnPath.js'import { engineBin } from './engineBin.js'import { engineInstallPaths, npmEnginePrefix, type EngineInstallRecipe } from './engineInstall.js'import { GRID_NO_UPDATE_CHECK_VAR, gridBinaryPath } from './gridExec.js'+import { loginShellEnvironment } from './loginShellEnv.js'import { managedNodePath } from './nodeRuntime.js'import { RAISE_OPEN_FILES_SH } from './openFiles.js'import { ENGINE_EXIT_PANE_OPTION } from './tmux.js'@@ -400,13 +404,32 @@ function currentUserShell(): string | undefined {}}+/**+ * The shells that speak the POSIX shell language every launch script here is written in. Any other+ * login shell (fish, tcsh, csh, nushell, xonsh) cannot run one: handed the script with `-c`, it fails+ * on the first `if … then` or `"$@"`, and the engine never starts. tcsh ships with macOS and stands+ * in for all of them: before this, no agent started at all for a person whose shell was not POSIX,+ * and no terminal tile either (e2e/shells.e2e.ts).+ */+const POSIX_SHELLS = new Set(['sh', 'bash', 'zsh', 'dash', 'ksh', 'mksh', 'oksh', 'pdksh', 'ash', 'yash', 'posh', 'busybox'])+/** Shells that are not POSIX but take `-i` to load the person's interactive startup files. */+const INTERACTIVE_FLAG_SHELLS = new Set(['fish', 'tcsh', 'csh', 'xonsh'])++export function isPosixShell(path: string): boolean {+ return POSIX_SHELLS.has(basename(path).toLowerCase())+}+export function interactiveEngineShell(shell: string | undefined = undefined): InteractiveEngineShell | null {const candidate = shell === undefined ? currentUserShell() : shellif (!candidate || !isAbsolute(candidate)) return null- switch (basename(candidate).toLowerCase()) {+ const name = basename(candidate).toLowerCase()+ switch (name) {case 'zsh': return { path: candidate, args: ['-lic'], label: 'zsh login shell' }case 'bash': return { path: candidate, args: ['-ic'], label: 'bash interactive shell' }- default: return { path: candidate, args: ['-ic'], label: `${basename(candidate)} interactive shell` }+ default: return isPosixShell(candidate)+ ? { path: candidate, args: ['-ic'], label: `${name} interactive shell` }+ // Separate flags: not every one of these reads them combined.+ : { path: candidate, args: INTERACTIVE_FLAG_SHELLS.has(name) ? ['-i', '-c'] : ['-c'], label: `${name} shell` }}}@@ -415,12 +438,30 @@ export function interactiveEngineShell(shell: string | undefined = undefined): I* DISABLE_UPDATE_PROMPT would auto-update instead; DISABLE_AUTO_UPDATE skips that work entirely.* Keep ordinary terminal launches unchanged, and keep loading rc files for PATH/version managers. */function engineShellArgv(shell: InteractiveEngineShell, args: readonly string[]): string[] {+ if (!isPosixShell(shell.path)) return throughPosixShell(shell, args)const prefix = basename(shell.path).toLowerCase() === 'zsh'? ['/usr/bin/env', 'DISABLE_AUTO_UPDATE=true']: []return [...prefix, shell.path, ...shell.args, ...args]}+/**+ * For a shell that is not POSIX: it loads the person's environment (their PATH, their version+ * managers), then hands the script to /bin/sh. The script and its arguments go in a one-time file, so+ * all the person's shell parses is `exec /bin/sh '<file>'`, which fish, tcsh, nushell and xonsh read+ * alike; the file removes itself as it starts. It sits in the daemon's own data folder, private to+ * this user, and holds no secret: those reach the pane through the session's environment, never its+ * command. `args` is what a POSIX shell would take after `-c`: the script, `$0`, then the arguments.+ */+function throughPosixShell(shell: InteractiveEngineShell, args: readonly string[]): string[] {+ const [script = '', , ...positional] = args+ const directory = join(env.ADAPTER_DATA_DIR, 'launch')+ mkdirSync(directory, { recursive: true, mode: 0o700 })+ const file = join(directory, `${randomUUID()}.sh`)+ writeFileSync(file, `rm -f -- "$0"\nset -- ${positional.map(shellSingleQuote).join(' ')}\n${script}`, { mode: 0o600 })+ return [shell.path, ...shell.args, `exec /bin/sh ${shellSingleQuote(file)}`]+}+/*** Full argv for a fresh tmux pane. `exec` replaces the shell with the engine,* preserving process discovery while loading the same startup files a user@@ -644,7 +685,10 @@ export function buildTerminalLaunchArgv(const hintPrelude = opts.terminalHint && process.env.HARNESS_OS !== '1'? `printf '%s\\n' ${terminalHintLines(opts.terminalHint.machineName).map(shellSingleQuote).join(' ')}\n`: ''- return [path, '-c', RAISE_OPEN_FILES_SH + clearEnvPrelude(opts.clearEnv) + cwdPrelude + hintPrelude + 'shift\nexec "$@"', 'harness-terminal', opts.cwd ?? '', path, ...loginArgs]+ // The prelude is a POSIX script. For a shell that is not POSIX, /bin/sh runs it and then execs the+ // person's shell, which loads its own startup files as it always does.+ const interpreter = isPosixShell(path) ? path : '/bin/sh'+ return [interpreter, '-c', RAISE_OPEN_FILES_SH + clearEnvPrelude(opts.clearEnv) + cwdPrelude + hintPrelude + 'shift\nexec "$@"', 'harness-terminal', opts.cwd ?? '', path, ...loginArgs]}/**@@ -909,7 +953,7 @@ export type CommandFlagSupport = 'supported' | 'unsupported' | 'unknown'const FLAG_UNSUPPORTED_EXIT = 64/**- * The pairs this probe has seen the engine SUPPORT.+ * The pairs this probe has seen the engine SUPPORT, each with the engine file it saw (`engineFileStamp`).** Every relaunch asks, and a post-reboot restore asks once per agent, so the working case — which* is nearly every case — is worth answering from memory instead of spawning `--help` again.@@ -919,8 +963,32 @@ const FLAG_UNSUPPORTED_EXIT = 64* `--auto`. A cached `unsupported` would go on refusing that upgraded engine until the daemon* happened to restart, which is the one outcome worth more than the spawn it saves. `unknown` is* not kept for the same reason at shorter range: one slow `--help` would turn Auto off machine-wide.+ *+ * ⚠️ And a kept `supported` is only good for the file it was read from. An update can drop a flag as+ * well as add one, and remembered by name alone the answer outlived it: measured end to end+ * (`e2e/updates.e2e.ts`), an engine updated to a build without its permission flag was still+ * launched with it, and refused it at once — the row went ready, then stopped, with no reason given —+ * where a daemon restarted after the update refused the create and said why.+ */+const flagSupportCache = new Map<string, string>()++/**+ * Which file a command name runs, as a stamp that changes when an update replaces or rewrites it:+ * its real path (a Homebrew or native install points at a new version's folder), inode, size and+ * modification time. Resolved on the login shell's PATH, the one a launch resolves it on, then the+ * daemon's own. '' when neither finds it — remembered by name alone then, as before.*/-const flagSupportCache = new Set<string>()+function engineFileStamp(command: string): string {+ const path = resolveBinaryOnPath(command, { PATH: loginShellEnvironment().PATH })+ ?? resolveBinaryOnPath(command)+ if (!path) return ''+ try {+ const stat = statSync(path, { bigint: true })+ return [realpathSync(path), stat.ino, stat.size, stat.mtimeNs].join('\u0000')+ } catch {+ return ''+ }+}/** Test seam, and for a machine where the engine was just upgraded. */export function resetCommandFlagSupportCache(): void { flagSupportCache.clear() }@@ -936,7 +1004,8 @@ export async function commandSupportsFlagInInteractiveShell(shell: string | undefined = undefined,): Promise<CommandFlagSupport> {const key = `${command}\u0000${flag}\u0000${shell ?? ''}`- if (flagSupportCache.has(key)) return 'supported'+ const stamp = engineFileStamp(command)+ if (flagSupportCache.get(key) === stamp) return 'supported'const interactive = interactiveEngineShell(shell)if (!interactive) return 'unknown'// `harness_help_status`, not `status`: in zsh `status` is a read-only special parameter (an alias@@ -964,7 +1033,7 @@ export async function commandSupportsFlagInInteractiveShell(const answer: CommandFlagSupport = !error? 'supported': Number((error as { code?: number | string }).code) === FLAG_UNSUPPORTED_EXIT ? 'unsupported' : 'unknown'- if (answer === 'supported') flagSupportCache.add(key)+ if (answer === 'supported') flagSupportCache.set(key, stamp)resolve(answer)},)diff --git a/cli/src/lib/localModels.integration.spec.ts b/cli/src/lib/localModels.integration.spec.tsindex 7bf9f5682..6fb4a1508 100644--- a/cli/src/lib/localModels.integration.spec.ts+++ b/cli/src/lib/localModels.integration.spec.ts@@ -14,7 +14,11 @@ it('discovers, downloads, starts, verifies, stops and restarts across real subprconst server = createServer(async (req, res) => {const chunks: Buffer[] = []for await (const chunk of req) chunks.push(Buffer.from(chunk))- const body = JSON.parse(Buffer.concat(chunks).toString())+ // The engine probe's GET /health is the one request with no body, and this fixture serves no+ // engine: parsing it threw outside the test, failing a unit run whose every test had passed.+ const text = Buffer.concat(chunks).toString()+ if (!text) { res.statusCode = 404; res.end(); return }+ const body = JSON.parse(text)res.setHeader('content-type', 'application/json')if (req.url === '/v1/grid/catalog') {expect(req.headers.authorization).toBe('Bearer catalog-fixture')diff --git a/cli/vitest.e2e.config.ts b/cli/vitest.e2e.config.tsindex f0ae27724..919c1de4b 100644--- a/cli/vitest.e2e.config.ts+++ b/cli/vitest.e2e.config.ts@@ -2,17 +2,29 @@ import { defineConfig } from 'vitest/config'/*** End-to-end: each file boots real daemons (see e2e/harness/daemon.ts) under throwaway homes with- * private tmux servers and fake engines. Files run one at a time so their timing is the daemon's,- * not the host's contention.+ * private tmux servers and fake engines. Files run one at a time by default, so their timing is the+ * daemon's, not the host's contention.+ *+ * `E2E_WORKERS=<n>` runs n files at a time. Each file has its own daemons, homes, ports and tmux+ * servers, so they share nothing but the machine; one at a time, the suite took about 25 minutes on a+ * 12-core Mac. The checks whose numbers depend on a quiet machine (the soak, scale) run on their own,+ * when asked, either way.+ *+ * `E2E_BUNDLE=1` starts every daemon from one bundle built for the run (e2e/harness/bundle.ts):+ * less CPU per start, the code frozen as the run began, and the build that ships under test.*/+const workers = Math.max(1, Number(process.env.E2E_WORKERS ?? 1) || 1)+export default defineConfig({test: {environment: 'node',setupFiles: ['./vitest.setup.ts'],+ globalSetup: ['./e2e/harness/bundle.ts'],include: ['e2e/**/*.e2e.ts'],testTimeout: 180_000,hookTimeout: 180_000,- fileParallelism: false,+ fileParallelism: workers > 1,+ maxWorkers: workers,pool: 'forks',},})