The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Core boundary, part 10: a failing service cannot take the core down, and four failures found end to end (#736)
Dee5f56e9f738ed
diff --git a/cli/e2e/robustness.e2e.ts b/cli/e2e/robustness.e2e.tsnew file mode 100644index 000000000..e776fa09c--- /dev/null+++ b/cli/e2e/robustness.e2e.ts@@ -0,0 +1,210 @@+/**+ * What the daemon must survive, on the real daemon under a throwaway home and a private tmux server:+ * hostile local clients, strangers and malformed bodies at the hook door, an agent's pane killed from+ * outside, a restart in the middle of a turn, and state files that are garbage when it starts. After+ * each, the same core is serving and a well-behaved client still works.+ */+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'+import { join } from 'node:path'+import WebSocket from 'ws'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++const row = async (client: LocalClient, agentId: string) =>+ ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true })).agents)+ .find((agent) => agent.id === agentId)++async function boundAgent(daemon: IsolatedDaemon, client: LocalClient, name: string): Promise<Record<string, any>> {+ const cwd = join(daemon.projectsDir, name)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const agentId: string = created.agent.id+ return until('the agent to bind its conversation', async () => {+ const agent = await row(client, agentId)+ return agent?.sessionId ? agent : null+ }, 45_000, 500)+}++const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId++async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {+ const started = client.next(isTurn('turn_started', agentId), 30_000, `turn_started (${content})`)+ const ended = client.next(isTurn('turn_ended', agentId), 30_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ await started+ await ended+}++/** A raw local WebSocket, past the handshake or not, for sending what no app would. */+async function rawSocket(daemon: IsolatedDaemon): Promise<WebSocket> {+ const ws = new WebSocket(`ws+unix://${daemon.socketPath}:/api/local-ws`)+ await new Promise<void>((resolve, reject) => { ws.once('open', () => resolve()); ws.once('error', reject) })+ ws.on('error', () => {})+ return ws+}++const hook = (daemon: IsolatedDaemon, path: string, body: string | Buffer, headers: Record<string, string> = {}) =>+ fetch(`http://127.0.0.1:${daemon.port}${path}`, { method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body })++describe('what the daemon survives', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ const fresh = async (options: Parameters<typeof IsolatedDaemon.create>[0] = {}) => {+ const d = await IsolatedDaemon.create(options)+ daemon = d+ onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-150).join('\n')}`) })+ return d+ }++ it('hostile local clients: garbage, wrong shapes, unknown requests and a flood hurt neither the daemon nor other clients', async () => {+ const d = await fresh()+ await d.start()+ const good = await LocalClient.connect(d)+ expect((await good.request('agents_list', {})).agents).toEqual([])++ // Before the handshake, and after it: nothing a client sends can take the daemon down.+ const early = await rawSocket(d)+ for (const junk of ['not json', '{', '[]', 'null', '42', '{"type":5}', JSON.stringify({ type: 'agents_list', payload: 'nope' })]) early.send(junk)+ early.send(Buffer.from([0xde, 0xad, 0xbe, 0xef]), { binary: true })++ const bad = await LocalClient.connect(d)+ const unsupported = await bad.request('no_such_request', { anything: true }, 10_000).catch((error: Error) => ({ error: error.message }))+ expect(unsupported).toBeTruthy()+ const shapes: Array<Record<string, unknown>> = [+ { type: 'agents_list', payload: null }, { type: 'agents_list' }, { type: 'message', payload: { agentId: 42, content: { not: 'text' } } },+ { type: 'agent_create', payload: { engine: 'not-an-engine', cwd: 7 } }, { type: 'machine_select', payload: { machineId: 'someone-else' } },+ { type: 'session_search', payload: { requestId: 'x', query: 'a'.repeat(100_000) } },+ ]+ const rawBad = await rawSocket(d)+ rawBad.send(JSON.stringify({ type: 'machine_select', payload: { machineId: d.computerId, localProtocolVersion: 1 } }))+ for (const frame of shapes) rawBad.send(JSON.stringify(frame))+ // A flood of well-formed requests from one client.+ for (let i = 0; i < 500; i++) rawBad.send(JSON.stringify({ type: 'agents_list', payload: { requestId: `flood-${i}` } }))+ // One frame larger than any app sends: that connection is cut, nobody else's.+ const huge = await rawSocket(d)+ const cut = new Promise<number>((resolve) => huge.once('close', (code) => resolve(code)))+ huge.send('x'.repeat(64 * 1024 * 1024))+ expect(await cut).toBeGreaterThan(0)++ // The good client never noticed, and it is the same core.+ expect((await good.request('agents_list', {}, 15_000)).agents).toEqual([])+ const agent = await boundAgent(d, good, 'after-hostile-clients')+ await turn(good, agent.id, 'still works')+ expect(d.coresStarted()).toBe(1)+ for (const ws of [early, rawBad]) ws.close()+ bad.close()+ good.close()+ })++ it('strangers and malformed bodies at the hook door are refused or answered, and real hooks still bind', async () => {+ const d = await fresh()+ await d.start()+ const token = { 'x-harness-hook-token': d.hookCredential() }+ const valid = JSON.stringify({ engine: 'claude', sessionId: 'not-a-real-session', tmuxPane: '%999', cwd: '/nowhere' })+ expect((await hook(d, '/api/hook/session-start', valid)).status).toBe(401)+ expect((await hook(d, '/api/hook/session-start', valid, { 'x-harness-hook-token': 'wrong' })).status).toBe(401)+ for (const path of ['/api/hook/session-start', '/api/hook/session-end', '/api/hook/tool-start', '/api/hook/turn-stop']) {+ expect((await hook(d, path, '{not json', token)).status, path).toBe(400)+ expect((await hook(d, path, JSON.stringify({ engine: 42 }), token)).status, path).toBeGreaterThanOrEqual(400)+ // Larger than any hook sends: read and dropped, never held in memory.+ expect((await hook(d, path, Buffer.alloc(2 * 1024 * 1024, 'a'), token)).status, path).toBe(400)+ }+ // A well-formed hook for a pane and a session nobody has: ignored or refused, said out loud.+ const stray = await hook(d, '/api/hook/session-start', valid, token)+ expect([200, 403]).toContain(stray.status)+ const unknown = await fetch(`http://127.0.0.1:${d.port}/api/hook/does-not-exist`, { method: 'POST', headers: token, body: '{}' })+ expect(unknown.status).toBe(404)++ const client = await LocalClient.connect(d)+ const agent = await boundAgent(d, client, 'after-hook-abuse')+ await turn(client, agent.id, 'hooks still bind')+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('an agent whose pane is killed from outside goes offline, and the daemon carries on', async () => {+ const d = await fresh()+ await d.start()+ const client = await LocalClient.connect(d)+ const survivor = await boundAgent(d, client, 'keeps-its-pane')+ const agent = await boundAgent(d, client, 'killed-from-outside')+ expect(agent.status).toBe('active')+ const pane = String(agent.terminal?.runtimes?.[0]?.paneId ?? agent.tmuxPane)+ await d.tmux.run('kill-pane', '-t', pane)+ const gone = await until('the agent to stop counting as active', async () => {+ const now = await row(client, agent.id)+ return !now || now.status !== 'active' ? (now ?? { status: 'removed' }) : null+ }, 45_000, 500)+ expect(['offline', 'stopped', 'removed']).toContain(gone.status)+ expect((await row(client, survivor.id))?.status).toBe('active')+ const other = await boundAgent(d, client, 'after-the-kill')+ await turn(client, other.id, 'the next agent is fine')+ expect(d.coresStarted()).toBe(1)+ client.close()+ })++ it('a turn that outlives a restart ends live, once, and the agent takes the next message', async () => {+ const d = await fresh()+ await d.start()+ const client = await LocalClient.connect(d)+ const agent = await boundAgent(d, client, 'restart-mid-turn')+ const started = client.next(isTurn('turn_started', agent.id), 30_000, 'turn_started')+ client.send('message', { agentId: agent.id, content: '!slow 8000' })+ await started+ await d.restart()+ const again = await LocalClient.connect(d)+ const ended = await again.waitFor(isTurn('turn_ended', agent.id), 45_000, 'the slow turn to end after the restart')+ expect(ended.replay, 'the end of a turn that finished after the restart is live news').toBeFalsy()+ // Never announced twice as a turn starting now.+ const liveStarts = again.frames.filter((frame) => isTurn('turn_started', agent.id)(frame) && !frame.replay && !(frame.payload as any)?.replay)+ expect(liveStarts).toHaveLength(0)+ await turn(again, agent.id, 'after the restart')+ again.close()+ client.close()+ })++ it('a turn that ends while the daemon is down is not left working, and the agent takes the next message', async () => {+ const d = await fresh()+ await d.start()+ const client = await LocalClient.connect(d)+ const agent = await boundAgent(d, client, 'ends-while-down')+ const started = client.next(isTurn('turn_started', agent.id), 30_000, 'turn_started')+ client.send('message', { agentId: agent.id, content: '!slow 1000' })+ await started+ await d.restart()+ const again = await LocalClient.connect(d)+ const settled = await until('the agent to read as not working', async () => {+ const now = await row(again, agent.id)+ return now && now.activity?.state !== 'working' ? now : null+ }, 30_000, 500)+ expect(settled.status).toBe('active')+ await turn(again, agent.id, 'after a turn that ended while the daemon was down')+ again.close()+ client.close()+ })++ it('state files that are garbage at start: the daemon starts, keeps what it can, and runs a new agent', async () => {+ const d = await fresh()+ await d.start()+ const first = await LocalClient.connect(d)+ await boundAgent(d, first, 'before-the-corruption')+ first.close()+ await d.stop()+ // Every file the core reads at start, written over with garbage of different kinds.+ writeFileSync(join(d.dataDir, 'registry.json'), '{"agents": [ {"agentId": "half', { mode: 0o600 })+ writeFileSync(join(d.dataDir, 'agent-names.json'), '\u0000\u0001 not json', { mode: 0o600 })+ mkdirSync(join(d.dataDir, 'stopped-agents'), { recursive: true, mode: 0o700 })+ writeFileSync(join(d.dataDir, 'stopped-agents', 'garbage.json'), '[[[', { mode: 0o600 })+ writeFileSync(join(d.dataDir, 'registry-boot'), 'not a boot id', { mode: 0o600 })+ await d.start()+ const client = await LocalClient.connect(d)+ expect(Array.isArray((await client.request('agents_list', { includeStopped: true })).agents)).toBe(true)+ const agent = await boundAgent(d, client, 'after-the-corruption')+ await turn(client, agent.id, 'runs on garbage state')+ client.close()+ void mkdtempSync+ })+})diff --git a/cli/e2e/services.e2e.ts b/cli/e2e/services.e2e.tsnew file mode 100644index 000000000..7f13a5673--- /dev/null+++ b/cli/e2e/services.e2e.ts@@ -0,0 +1,108 @@+/**+ * A failing service never takes the core down (core/serviceHost.ts), proven on the real daemon: every+ * service made to fail as it starts, and services made to fail on every call, while a client starts an+ * agent, the agent binds, messages become turns that start and end, and the daemon restarts.+ */+import { mkdirSync } from 'node:fs'+import { join } from 'node:path'+import { afterEach, describe, expect, it, onTestFailed } from 'vitest'+import { LocalClient, type Frame } from './harness/client.js'+import { IsolatedDaemon, until } from './harness/daemon.js'++const row = async (client: LocalClient, agentId: string) =>+ ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true })).agents)+ .find((agent) => agent.id === agentId)++/** Start a Claude Code agent and wait for it to bind its conversation. */+async function boundAgent(daemon: IsolatedDaemon, client: LocalClient, name: string): Promise<string> {+ const cwd = join(daemon.projectsDir, name)+ mkdirSync(cwd, { recursive: true })+ const created = await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)+ expect(created.error, JSON.stringify(created)).toBeUndefined()+ const agentId: string = created.agent.id+ await until('the agent to bind its conversation', async () => (await row(client, agentId))?.sessionId || null, 45_000, 500)+ return agentId+}++/** One message, one turn: it starts and it ends. */+async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {+ const isTurn = (type: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId+ const started = client.next(isTurn('turn_started'), 30_000, `turn_started (${content})`)+ const ended = client.next(isTurn('turn_ended'), 30_000, `turn_ended (${content})`)+ client.send('message', { agentId, content })+ expect((await started).payload?.userMessage).toBe(content)+ await ended+}++describe('a failing service never takes the core down', () => {+ let daemon: IsolatedDaemon | undefined+ afterEach(async () => { await daemon?.close(); daemon = undefined })++ it('with every service failing to start, the core starts, runs an agent through turns and a restart, and says search is off', async () => {+ daemon = await IsolatedDaemon.create({ env: { HARNESSD_TEST_FAULTS: 'search,viewers,models,workspaces' } })+ onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-80).join('\n')}`) })+ await daemon.start()+ for (const service of ['search', 'viewers', 'models', 'workspaces']) {+ expect(daemon.log()).toContain(`[services] ${service} did not start · injected fault: ${service} · the core runs without it`)+ }+ const client = await LocalClient.connect(daemon)+ const agentId = await boundAgent(daemon, client, 'no-services')+ await turn(client, agentId, 'first, with no services')+ expect((await client.request('session_search', { query: 'first' })).error).toBe('SEARCH_UNAVAILABLE')++ await daemon.restart()+ const again = await LocalClient.connect(daemon)+ await until('the agent to be back after a restart', async () => {+ const agent = await row(again, agentId)+ return agent?.sessionId && agent.status !== 'stopped' ? agent : null+ }, 45_000, 250)+ await turn(again, agentId, 'second, after a restart')+ again.close()+ client.close()+ })++ it('with services failing on every call, they are switched off and every turn still reaches the client', async () => {+ daemon = await IsolatedDaemon.create({+ env: { HARNESSD_TEST_FAULTS: 'search.touch,search.search,viewers.frameContext,viewers.attach,workspaces.nameBranches' },+ })+ onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-120).join('\n')}`) })+ await daemon.start()+ const client = await LocalClient.connect(daemon)+ const agentId = await boundAgent(daemon, client, 'failing-services')++ // Each turn calls search at its start and its end: the fifth failure switches search off, and the+ // turns after it are delivered exactly as the ones before.+ for (let i = 1; i <= 4; i++) await turn(client, agentId, `turn ${i}`)+ await until('search to be switched off', () => daemon!.log().includes('[services] search switched off after 5 failures'), 15_000)+ await until('the viewers to be switched off', () => daemon!.log().includes('[services] viewers switched off after 5 failures'), 15_000)+ expect(daemon.log()).toContain('[services] search.touch failed · injected fault: search.touch')+ expect(daemon.log()).toContain('[services] viewers.frameContext failed · injected fault: viewers.frameContext')++ // Off, search says so; the agent's row still carries everything the core owns.+ expect((await client.request('session_search', { query: 'turn' })).error).toBe('SEARCH_UNAVAILABLE')+ const agent = await row(client, agentId)+ expect(agent?.sessionId).toBeTruthy()+ expect(agent?.status).not.toBe('stopped')+ await turn(client, agentId, 'after search went off')++ // One core all along: nothing failing in a service restarted it.+ expect(daemon.coresStarted()).toBe(1)+ client.close()+ })++ it('a search request that fails before search is switched off answers that request, and the next one too', async () => {+ daemon = await IsolatedDaemon.create({ env: { HARNESSD_TEST_FAULTS: 'search.search' } })+ onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-80).join('\n')}`) })+ await daemon.start()+ const client = await LocalClient.connect(daemon)+ const first = await client.request('session_search', { query: 'anything' })+ expect(first).toMatchObject({ error: 'SERVICE_UNAVAILABLE', service: 'search', retryable: true })+ const answers = []+ for (let i = 0; i < 5; i++) answers.push((await client.request('session_search', { query: `again ${i}` })).error)+ // The fifth failure switched search off: from then on the request says search is off.+ expect(answers.slice(0, 3)).toEqual(['SERVICE_UNAVAILABLE', 'SERVICE_UNAVAILABLE', 'SERVICE_UNAVAILABLE'])+ expect(answers.at(-1)).toBe('SEARCH_UNAVAILABLE')+ expect((await client.request('agents_list', {})).agents).toEqual([])+ client.close()+ })+})diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.tsindex 4ac43f28d..dcdbc573b 100644--- a/cli/src/backendSocket.spec.ts+++ b/cli/src/backendSocket.spec.ts@@ -6,6 +6,7 @@ import { homedir, tmpdir } from 'os'import { join } from 'path'import { fileURLToPath } from 'url'import { AGENT_OPENED_THROTTLE_MS, BackendSocket, compactRuntimePickerModels, deviceAgentListItem, deviceAgentRow, grokHistoryPage } from './backendSocket.js'+import { ServiceUnavailableError } from './core/serviceHost.js'import { AuthSessionError, type AuthSessionManager } from './lib/authSession.js'import { WS_IDLE_DEADLINE_MS as IDLE_DEADLINE_MS } from './lib/wsLiveness.js'import { TerminalStreamManager } from './lib/terminalStreamManager.js'@@ -836,6 +837,17 @@ describe('BackendSocket outbound queue', () => {await vi.waitFor(() => {expect(wrapReply).toHaveBeenCalledWith('web-1', 'session_search_result', 's-2', { error: 'SEARCH_UNAVAILABLE' })})++ // A search service that failed (core/serviceHost.ts) answers this one request, and says the+ // client may ask again — never INTERNAL, and never the whole socket.+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ socket.sessionSearchProvider = () => { throw new ServiceUnavailableError('search', new Error('disk I/O error')) }+ unwrap.mockReturnValueOnce({ type: 'session_search', payload: { requestId: 's-3', query: 'dial' } })+ ws.message({ t: 'down', connId: 'web-1', frame: { type: 'session_search', payload: envelope } })+ await vi.waitFor(() => {+ expect(wrapReply).toHaveBeenCalledWith('web-1', 'session_search_result', 's-3', { error: 'SERVICE_UNAVAILABLE', service: 'search', retryable: true })+ })+ expect(warn).toHaveBeenCalledWith('[backend] session_search: the search service is unavailable')await socket.stop()})diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.tsindex 1c2d4ca29..09491f346 100644--- a/cli/src/backendSocket.ts+++ b/cli/src/backendSocket.ts@@ -1,4 +1,5 @@import type { PurgeAgentService } from './lib/purgeAgentService.js'+import { ServiceUnavailableError } from './core/serviceHost.js'import type { ActivityFrame } from './lib/turnActivity.js'import { readSessionGitPullRequest } from './lib/sessionGitPullRequest.js'import { MonitorCompletions, type MonitorActivity } from './lib/harnessMonitor.js'@@ -3489,6 +3490,13 @@ export class BackendSocket {return}} catch (err) {+ // A service on the core boundary that failed or is off (core/serviceHost.ts): the host has logged+ // it, and the client may ask again — the service can be back after the daemon restarts.+ if (err instanceof ServiceUnavailableError) {+ console.warn(`[backend] ${type}: ${err.message}`)+ if (requestId !== undefined) reply(type, requestId, { error: 'SERVICE_UNAVAILABLE', service: err.service, retryable: true })+ return+ }console.error(`[backend] dispatch ${type} failed:`, err)if (requestId !== undefined) reply(type, requestId, { error: 'INTERNAL' })}diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex b7139066a..924c62d73 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -160,7 +160,8 @@ import { createAgentClosing } from './core/agents/close.js'import { createEngineHooks, installEngineHooks } from './core/engines/hooks.js'import { createCursorTaskHooks } from './core/engines/cursorTasks.js'import { databaseHistory } from './core/transcripts/databaseHistory.js'-import { createCoreApi, emptyPorts } from './core/api.js'+import { createCoreApi, emptyPorts, MODELS_FALLBACKS, SEARCH_FALLBACKS, VIEWERS_FALLBACKS, WORKSPACES_FALLBACKS } from './core/api.js'+import { createServiceHost, testFaults } from './core/serviceHost.js'import { startSearch } from './services/search.js'import { startViewers } from './services/viewers.js'import { startModels } from './services/models.js'@@ -1778,14 +1779,17 @@ async function runForeground(session: AuthSession | null): Promise<void> {accessToken: () => new AuthSessionManager(backendHttpBase()).accessToken(),})const ports = emptyPorts()+ // Each service starts and is called through the host, so one that fails is logged and left off and+ // the core carries on without it (core/serviceHost.ts).+ const serviceHost = createServiceHost(ports, { faults: testFaults(process.env.HARNESSD_TEST_FAULTS) })// The DSH companions: each harness agent's viewer and verdict watch (services/viewers.ts).- startViewers(coreApi, ports)+ serviceHost.start('viewers', startViewers, coreApi, VIEWERS_FALLBACKS)dshFrameContextRef = (s) => ports.viewers?.frameContext(s) ?? nullconst attachDsh = (s: RegisteredSession): void => ports.viewers?.attach(s)const detachDsh = (agentId: string): void => ports.viewers?.detach(agentId)// The folders agents work in: branch names and unused worktrees (services/workspaces.ts).- startWorkspaces(coreApi, ports)+ serviceHost.start('workspaces', startWorkspaces, coreApi, WORKSPACES_FALLBACKS)const syncTerminalTitles = async (): Promise<void> => {ports.workspaces?.nameBranches()const titles = await terminals.titles()@@ -1851,12 +1855,14 @@ async function runForeground(session: AuthSession | null): Promise<void> {ensureBundledCoreHarnesses()// Models: grid access, the model pictures on agents' frames, the keystroke prewarm (services/models.ts).- startModels(coreApi, ports)+ serviceHost.start('models', startModels, coreApi, MODELS_FALLBACKS)const models = ports.modelsbackend.ensureGrid = models ? (request) => models.ensure(request) : null// Offline, for every list read: is there a `grid` here holding a sign-in? What decides whether the// picker offers local and shared models or a Set up row.backend.gridSetUp = models ? () => models.setUp() : null+ // Models switched off: the socket reads grid as it stands, as it does with no models at all.+ serviceHost.onOff('models', () => { backend.ensureGrid = null; backend.gridSetUp = null })/*** Is ANY device surface watching this machine?@@ -2126,10 +2132,12 @@ async function runForeground(session: AuthSession | null): Promise<void> {backend.handoffProvider = (req) => prepareAgentHandoff(handoffDeps, req)// Session search (services/search.ts).- startSearch(coreApi, ports)+ serviceHost.start('search', startSearch, coreApi, SEARCH_FALLBACKS)const sessionSearch = ports.searchbackend.sessionSearchProvider = sessionSearch ? (query, options) => sessionSearch.search(query, options) : nullbackend.sessionTailProvider = sessionSearch ? (sessionId, options) => sessionSearch.tail(sessionId, options) : null+ // Search switched off: both requests say search is unavailable, as on a Node without an index.+ serviceHost.onOff('search', () => { backend.sessionSearchProvider = null; backend.sessionTailProvider = null })const runtimeController = new RuntimeProfileController({manager: runtimeProfiles,diff --git a/cli/src/core/api.ts b/cli/src/core/api.tsindex 7396b13fe..6e0bc1435 100644--- a/cli/src/core/api.ts+++ b/cli/src/core/api.ts@@ -15,6 +15,7 @@ import { projectDisplayName, type registry, type RegisteredSession } from '../liimport type { ExternalSessions, OpenSessions } from '../lib/sessionSearch/external.js'import type { SessionSearchIndex } from '../lib/sessionSearch/indexer.js'import type { StoppedAgentStore } from '../lib/stoppedAgents.js'+import { FAIL, later, type PortFallbacks } from './serviceHost.js'export interface CoreApi {/** The daemon's data folder; a service keeps its own files in it. */@@ -62,9 +63,15 @@ export interface CoreApi {}/** The core's calls into session search: index a session at its turn boundaries, forget a purged- * conversation, the title it indexed for one being adopted, and the two requests it answers- * (`session_search`, `session_tail`). */-export type SearchPort = Pick<SessionSearchIndex, 'touch' | 'deleteHistory' | 'session' | 'search' | 'tail'>+ * conversation, the title it indexed for one being adopted, the two requests it answers+ * (`session_search`, `session_tail`), and stopping its sweeps. */+export type SearchPort = Pick<SessionSearchIndex, 'touch' | 'deleteHistory' | 'session' | 'search' | 'tail' | 'stop'>++/** What the core gets when search fails: nothing indexed, no title, and the requests answered with+ * an error. */+export const SEARCH_FALLBACKS: PortFallbacks<SearchPort> = {+ touch: undefined, deleteHistory: undefined, session: undefined, search: FAIL, tail: later(FAIL), stop: undefined,+}/** The core's calls into the DSH viewers: each harness agent's viewer server and verdict watch. */export interface ViewersPort {@@ -81,6 +88,12 @@ export interface ViewersPort {stop(): Promise<void>}+/** What the core gets when the viewers fail: agents' frames carry no DSH context and the windows+ * no viewer, and a restart or shutdown goes on. */+export const VIEWERS_FALLBACKS: PortFallbacks<ViewersPort> = {+ attach: undefined, detach: undefined, frameContext: null, forwardingUrl: null, stop: later(undefined),+}+/** The core's calls into models (grid): have grid ready, whether it is set up, and the two things* the core tells it — someone is typing to an agent on a grid, and the sign-in ended. */export interface ModelsPort {@@ -94,6 +107,12 @@ export interface ModelsPort {signedOut(): void}+/** What the core gets when models fails: a grid request answered with an error, grid read as not+ * set up, and no prewarm. */+export const MODELS_FALLBACKS: PortFallbacks<ModelsPort> = {+ ensure: later(FAIL), setUp: false, prewarm: undefined, signedOut: undefined,+}+/** The core's calls into workspaces: name made-up worktree branches after their sessions, on each* terminal-title pass, and sweep the worktrees nothing uses, when the core says it is time. */export interface WorkspacesPort {@@ -101,6 +120,9 @@ export interface WorkspacesPort {sweepUnused(): void}+/** What the core gets when workspaces fails: branches keep their names and nothing is swept. */+export const WORKSPACES_FALLBACKS: PortFallbacks<WorkspacesPort> = { nameBranches: undefined, sweepUnused: undefined }+/** Each port is filled by the service that owns it when that service starts, and is null while the* service is off: the core never waits on one. */export interface CorePorts {diff --git a/cli/src/core/serviceHost.spec.ts b/cli/src/core/serviceHost.spec.tsnew file mode 100644index 000000000..67bddcb78--- /dev/null+++ b/cli/src/core/serviceHost.spec.ts@@ -0,0 +1,238 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'+import { fakeCore } from '../testing/fakeCore.js'+import { emptyPorts, type CorePorts, type SearchPort, type ViewersPort, type WorkspacesPort } from './api.js'+import { createServiceHost, FAIL, later, ServiceUnavailableError, testFaults, type PortFallbacks } from './serviceHost.js'++// A port whose members cover every shape: sync and async, value and FAIL fallbacks, a stop.+class FakeSearch {+ readonly calls: string[] = []+ touch(sessionId: string): void { this.calls.push(`touch ${sessionId}`) }+ deleteHistory(sessionId: string): void { this.calls.push(`delete ${sessionId}`) }+ session(sessionId: string) { return { title: `title of ${sessionId}` } as never }+ search(query: string) { return { hits: [query] } as never }+ async tail(sessionId: string) { return { sessionId } as never }+ stop(): void { this.calls.push('stop') }+}+const SEARCH: PortFallbacks<SearchPort> = {+ touch: undefined, deleteHistory: undefined, session: undefined, search: FAIL, tail: later(FAIL), stop: undefined,+}+const VIEWERS: PortFallbacks<ViewersPort> = {+ attach: undefined, detach: undefined, frameContext: null, forwardingUrl: null, stop: later(undefined),+}++function host(options: Parameters<typeof createServiceHost>[1] = {}) {+ const ports = emptyPorts()+ const lines: string[] = []+ let at = 0+ const services = createServiceHost(ports, { log: (line) => lines.push(line), now: () => at, ...options })+ return { ports, lines, services, advance: (ms: number) => { at += ms } }+}+function startSearch(h: ReturnType<typeof host>, port: Partial<SearchPort> = new FakeSearch()) {+ h.services.start('search', (_core, ports) => { ports.search = port as SearchPort }, fakeCore(), SEARCH)+ return port as SearchPort+}++describe('hosting services so one that fails cannot take the core down', () => {+ afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers() })++ describe('starting', () => {+ it('installs the port a service fills, guarded, with every call reaching it as made', async () => {+ const h = host()+ const real = new FakeSearch()+ startSearch(h, real)+ const port = h.ports.search!+ expect(port).not.toBe(real)+ port.touch('s1')+ port.deleteHistory('s2')+ expect(real.calls).toEqual(['touch s1', 'delete s2'])+ expect(port.session('s1')).toEqual({ title: 'title of s1' })+ expect(port.search('fix', {})).toEqual({ hits: ['fix'] })+ expect(await port.tail('s1', {})).toEqual({ sessionId: 's1' })+ expect(h.services.isOff('search')).toBe(false)+ expect(h.lines).toEqual([])+ })++ it('leaves off, and says so, a service whose start throws, and the rest of the ports alone', () => {+ const h = host()+ const viewers = { stop: vi.fn() } as unknown as ViewersPort+ h.ports.viewers = viewers+ h.services.start('search', () => { throw new Error('index locked') }, fakeCore(), SEARCH)+ h.services.start('workspaces', () => { throw 'no home folder' }, fakeCore(), { nameBranches: undefined, sweepUnused: undefined })+ expect(h.ports.search).toBeNull()+ expect(h.ports.workspaces).toBeNull()+ expect(h.ports.viewers).toBe(viewers)+ expect(h.services.isOff('search')).toBe(true)+ expect(h.lines).toEqual([+ '[services] search did not start · index locked · the core runs without it',+ '[services] workspaces did not start · no home folder · the core runs without it',+ ])+ })++ it('counts a service that leaves its own port empty as off, without a word of its own', () => {+ const h = host()+ h.services.start('search', () => {}, fakeCore(), SEARCH)+ expect(h.ports.search).toBeNull()+ expect(h.services.isOff('search')).toBe(true)+ expect(h.lines).toEqual([])+ })++ it('takes only the port a service owns: what it writes into another is dropped', () => {+ const h = host()+ const stray = { nameBranches: vi.fn(), sweepUnused: vi.fn() } as WorkspacesPort+ h.services.start('search', (_core, ports) => { ports.search = new FakeSearch(); ports.workspaces = stray }, fakeCore(), SEARCH)+ expect(h.ports.search).not.toBeNull()+ expect(h.ports.workspaces).toBeNull()+ })+ })++ describe('a call that fails', () => {+ it('is logged and answered with its fallback, so the caller carries on', async () => {+ const h = host()+ const port = startSearch(h, {+ ...new FakeSearch(),+ touch: () => { throw new Error('store closed') },+ session: () => { throw 'bad row' },+ })+ expect(h.ports.search!.touch('s1')).toBeUndefined()+ expect(h.ports.search!.session('s1')).toBeUndefined()+ expect(port).toBeDefined()+ expect(h.lines).toEqual(['[services] search.touch failed · store closed', '[services] search.session failed · bad row'])+ })++ it('goes back to the one request when its fallback is FAIL, sync or async', async () => {+ const h = host()+ const cause = new Error('disk I/O error')+ startSearch(h, { ...new FakeSearch(), search: () => { throw cause }, tail: async () => { throw cause } })+ let thrown: unknown+ try { h.ports.search!.search('x', {}) } catch (error) { thrown = error }+ expect(thrown).toBeInstanceOf(ServiceUnavailableError)+ expect(thrown).toMatchObject({ service: 'search', cause, name: 'ServiceUnavailableError', message: 'the search service is unavailable' })+ await expect(h.ports.search!.tail('s1', {})).rejects.toMatchObject({ service: 'search', cause })+ })++ it('resolves an async member to its fallback whether it rejects or throws before its promise', async () => {+ const h = host()+ let viewersStop: () => Promise<void> = async () => { throw new Error('viewer hung') }+ h.services.start('viewers', (_core, ports) => {+ ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: () => viewersStop() }+ }, fakeCore(), VIEWERS)+ await expect(h.ports.viewers!.stop()).resolves.toBeUndefined()+ viewersStop = () => { throw new Error('sync throw in async member') }+ await expect(h.ports.viewers!.stop()).resolves.toBeUndefined()+ expect(h.lines).toEqual(['[services] viewers.stop failed · viewer hung', '[services] viewers.stop failed · sync throw in async member'])+ })++ it('passes a promise that resolves straight through', async () => {+ const h = host()+ h.services.start('viewers', (_core, ports) => {+ ports.viewers = { attach: vi.fn(), detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => 'http://v'), stop: async () => {} }+ }, fakeCore(), VIEWERS)+ expect(h.ports.viewers!.forwardingUrl('a1')).toBe('http://v')+ await expect(h.ports.viewers!.stop()).resolves.toBeUndefined()+ })+ })++ describe('switching a failing service off', () => {+ it('after five failures within the window: stopped, unbound, and its calls answered without reaching it', async () => {+ const h = host()+ const real = new FakeSearch()+ real.touch = () => { throw new Error('store closed') }+ startSearch(h, real)+ const captured = h.ports.search!+ const unbind = vi.fn()+ h.services.onOff('search', unbind)+ for (let i = 0; i < 4; i++) { captured.touch('s1'); h.advance(1_000) }+ expect(h.services.isOff('search')).toBe(false)+ captured.touch('s1')+ expect(h.services.isOff('search')).toBe(true)+ expect(h.ports.search).toBeNull()+ expect(real.calls).toEqual(['stop'])+ expect(unbind).toHaveBeenCalledTimes(1)+ expect(h.lines.at(-1)).toBe('[services] search switched off after 5 failures in 60s · it stays off until the daemon restarts')+ // A reference the core took before it went off now answers without running the service.+ const spy = vi.spyOn(real, 'deleteHistory')+ expect(captured.deleteHistory('s1')).toBeUndefined()+ expect(spy).not.toHaveBeenCalled()+ expect(() => captured.search('x', {})).toThrow(ServiceUnavailableError)+ await expect(captured.tail('s1', {})).rejects.toBeInstanceOf(ServiceUnavailableError)+ expect(h.lines).toHaveLength(6)+ })++ it('forgets failures older than the window, so a slow trickle never switches a service off', () => {+ const h = host({ maxFailures: 3, windowMs: 10_000 })+ startSearch(h, { ...new FakeSearch(), touch: () => { throw new Error('flaky') } })+ for (let i = 0; i < 10; i++) { h.ports.search!.touch('s1'); h.advance(6_000) }+ expect(h.services.isOff('search')).toBe(false)+ })++ it('switches off once, even when a call begun before it fails after', async () => {+ const h = host({ maxFailures: 2 })+ const rejects: Array<(error: Error) => void> = []+ startSearch(h, { ...new FakeSearch(), tail: () => new Promise((_resolve, reject) => { rejects.push(reject) }) })+ const calls = [h.ports.search!.tail('a', {}), h.ports.search!.tail('b', {}), h.ports.search!.tail('c', {})]+ for (const reject of rejects) reject(new Error('timeout'))+ await Promise.allSettled(calls)+ expect(h.lines.filter((line) => line.includes('switched off'))).toHaveLength(1)+ })++ it('carries on when an unbinding or a stop fails, and stops nothing it cannot', async () => {+ const h = host({ maxFailures: 1 })+ const late = vi.fn()+ startSearch(h, { ...new FakeSearch(), touch: () => { throw new Error('x') }, stop: () => { throw new Error('stop failed') } })+ h.services.onOff('search', () => { throw new Error('unbind failed') })+ h.services.onOff('search', late)+ h.ports.search!.touch('s1')+ expect(late).toHaveBeenCalled()+ expect(h.lines).toContain('[services] switching search off: stop failed')+ expect(h.lines).toContain('[services] switching search off: unbind failed')++ const async = host({ maxFailures: 1 })+ async.services.start('viewers', (_core, ports) => {+ ports.viewers = { attach: () => { throw new Error('x') }, detach: vi.fn(), frameContext: vi.fn(() => null), forwardingUrl: vi.fn(() => null), stop: async () => { throw new Error('viewer stuck') } }+ }, fakeCore(), VIEWERS)+ async.ports.viewers!.attach({} as never)+ await new Promise((resolve) => setImmediate(resolve))+ expect(async.services.isOff('viewers')).toBe(true)++ const stopless = host({ maxFailures: 1 })+ stopless.services.start('workspaces', (_core, ports) => {+ ports.workspaces = { nameBranches: () => { throw new Error('x') }, sweepUnused: vi.fn() }+ }, fakeCore(), { nameBranches: undefined, sweepUnused: undefined })+ stopless.ports.workspaces!.nameBranches()+ expect(stopless.services.isOff('workspaces')).toBe(true)+ })++ it('by default after five failures in sixty seconds, logged as a warning', () => {+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const ports: CorePorts = emptyPorts()+ const services = createServiceHost(ports)+ services.start('workspaces', (_core, p) => { p.workspaces = { nameBranches: () => { throw new Error('x') }, sweepUnused: vi.fn() } }, fakeCore(), { nameBranches: undefined, sweepUnused: undefined })+ for (let i = 0; i < 5; i++) ports.workspaces?.nameBranches()+ expect(services.isOff('workspaces')).toBe(true)+ expect(warn).toHaveBeenLastCalledWith('[services] workspaces switched off after 5 failures in 60s · it stays off until the daemon restarts')+ })+ })++ describe('faults injected for the end-to-end suite', () => {+ beforeEach(() => { vi.spyOn(console, 'warn').mockImplementation(() => {}) })++ it('reads a comma-separated list, and none when unset', () => {+ expect([...testFaults(' search , viewers.attach,,')]).toEqual(['search', 'viewers.attach'])+ expect(testFaults(undefined).size).toBe(0)+ })++ it('fails a service\'s start, or one member on every call, as a real fault would', () => {+ const started = host({ faults: testFaults('search') })+ startSearch(started)+ expect(started.ports.search).toBeNull()+ expect(started.lines).toEqual(['[services] search did not start · injected fault: search · the core runs without it'])++ const member = host({ faults: testFaults('search.touch') })+ const real = startSearch(member, new FakeSearch()) as unknown as FakeSearch+ member.ports.search!.touch('s1')+ member.ports.search!.deleteHistory('s1')+ expect(real.calls).toEqual(['delete s1'])+ expect(member.lines).toEqual(['[services] search.touch failed · injected fault: search.touch'])+ })+ })+})diff --git a/cli/src/core/serviceHost.ts b/cli/src/core/serviceHost.tsnew file mode 100644index 000000000..c307b6eba--- /dev/null+++ b/cli/src/core/serviceHost.ts@@ -0,0 +1,174 @@+/**+ * Hosting services in the core's process so that a failing one cannot take the core down with it+ * (docs/design/2026-10-03-harnessd.md, the core boundary). In process, a service shares the core's+ * heap and event loop; this is the isolation a function boundary can give, and a service moved out+ * of process gets the rest.+ *+ * - **Start.** A service whose start throws is left off: its port stays null, it is logged, and the+ * core starts without it.+ * - **Calls.** Every call into a port is guarded. One that throws, or whose promise rejects, is logged+ * and answered with that member's fallback, so the core's caller (the event funnel, restore, a+ * frame being built) carries on. A member whose fallback is `FAIL` answers a request: its failure+ * goes back to that one request, which the socket answers with an error.+ * - **Switching off.** A service that fails `maxFailures` times within `windowMs` is switched off+ * for the rest of the daemon's life: it is stopped when its port can stop, its port goes null, the+ * core's calls answer their fallbacks without reaching it, and the `onOff` handlers unbind what+ * clients reach it through. A restart of the daemon starts it again.+ */+import type { CoreApi, CorePorts } from './api.js'++/** The fallback of a member whose failure belongs to its caller: it throws (or rejects). */+export const FAIL = Symbol('fail')++const LATER = Symbol('later')+/** The fallback of a member that returns a promise: resolve to `value` (or reject, for `FAIL`). */+export interface Later<T> { readonly [LATER]: T }+export function later<T>(value: T): Later<T> {+ return { [LATER]: value }+}++/** For every member of a port, what the core gets when that member fails or its service is off. */+export type PortFallbacks<P> = {+ [K in keyof P]-?: P[K] extends (...args: never[]) => infer R+ ? R extends PromiseLike<infer T> ? Later<T | typeof FAIL> : R | typeof FAIL+ : never+}++/** Why a call into a service was not answered: the service is off, or the call failed. */+export class ServiceUnavailableError extends Error {+ constructor(readonly service: string, readonly cause?: unknown) {+ super(`the ${service} service is unavailable`)+ this.name = 'ServiceUnavailableError'+ }+}++export interface ServiceHostOptions {+ /** Failures within `windowMs` that switch a service off. */+ maxFailures?: number+ windowMs?: number+ now?: () => number+ log?: (line: string) => void+ /**+ * Faults to inject, for the end-to-end suite only (`HARNESSD_TEST_FAULTS`): `name` makes that+ * service's start throw, `name.member` makes that member throw on every call.+ */+ faults?: ReadonlySet<string>+}++type PortName = keyof CorePorts+type Port<K extends PortName> = NonNullable<CorePorts[K]>+type Member = (...args: unknown[]) => unknown++const describe = (error: unknown): string => (error instanceof Error ? error.message : String(error))++/** `HARNESSD_TEST_FAULTS`: a comma-separated list of `service` and `service.member` names. */+export function testFaults(value: string | undefined): ReadonlySet<string> {+ return new Set((value ?? '').split(',').map((entry) => entry.trim()).filter(Boolean))+}++export function createServiceHost(ports: CorePorts, options: ServiceHostOptions = {}) {+ const maxFailures = options.maxFailures ?? 5+ const windowMs = options.windowMs ?? 60_000+ const now = options.now ?? Date.now+ const log = options.log ?? ((line: string) => console.warn(line))+ const faults = options.faults ?? new Set<string>()+ const off = new Set<PortName>()+ const failures = new Map<PortName, number[]>()+ const offHandlers = new Map<PortName, Array<() => void>>()+ const stoppers = new Map<PortName, () => void>()++ const inject = (target: string): void => {+ if (faults.has(target)) throw new Error(`injected fault: ${target}`)+ }++ const switchOff = (name: PortName, failed: number): void => {+ if (off.has(name)) return+ off.add(name)+ ports[name] = null+ log(`[services] ${name} switched off after ${failed} failures in ${Math.round(windowMs / 1000)}s · it stays off until the daemon restarts`)+ for (const handler of [stoppers.get(name), ...offHandlers.get(name) ?? []]) {+ try { handler?.() } catch (error) { log(`[services] switching ${name} off: ${describe(error)}`) }+ }+ }++ const failed = (name: PortName, member: string, error: unknown): void => {+ log(`[services] ${name}.${member} failed · ${describe(error)}`)+ const at = now()+ const recent = (failures.get(name) ?? []).filter((t) => at - t < windowMs)+ recent.push(at)+ failures.set(name, recent)+ if (recent.length >= maxFailures) switchOff(name, recent.length)+ }++ const guard = <K extends PortName>(name: K, port: Port<K>, fallbacks: PortFallbacks<Port<K>>): Port<K> => {+ const guarded: Record<string, Member> = {}+ for (const [member, fallback] of Object.entries(fallbacks) as Array<[string, unknown]>) {+ const deferred = typeof fallback === 'object' && fallback !== null && LATER in fallback+ const value = deferred ? (fallback as Later<unknown>)[LATER] : fallback+ const answer = (cause?: unknown): unknown => {+ if (value === FAIL) {+ const error = new ServiceUnavailableError(name, cause)+ if (deferred) return Promise.reject(error)+ throw error+ }+ return deferred ? Promise.resolve(value) : value+ }+ guarded[member] = (...args: unknown[]): unknown => {+ if (off.has(name)) return answer()+ let result: unknown+ try {+ inject(`${name}.${member}`)+ result = ((port as unknown as Record<string, Member>)[member]).apply(port, args)+ } catch (error) {+ failed(name, member, error)+ return answer(error)+ }+ if (!deferred) return result+ return Promise.resolve(result).catch((error: unknown) => {+ failed(name, member, error)+ return answer(error)+ })+ }+ }+ return guarded as unknown as Port<K>+ }++ return {+ /**+ * Start one service. It fills its own port (`ports[name]`) in a staging copy; what it filled is+ * installed guarded, and anything else it wrote is ignored. A service that leaves its port null+ * has said why itself and is simply off.+ */+ start<K extends PortName>(name: K, start: (core: CoreApi, ports: CorePorts) => void, core: CoreApi, fallbacks: PortFallbacks<Port<K>>): void {+ const staging: CorePorts = { ...ports, [name]: null }+ try {+ inject(name)+ start(core, staging)+ } catch (error) {+ off.add(name)+ ports[name] = null+ log(`[services] ${name} did not start · ${describe(error)} · the core runs without it`)+ return+ }+ const port = staging[name] as Port<K> | null+ if (!port) {+ off.add(name)+ ports[name] = null+ return+ }+ const stop = (port as unknown as { stop?: () => unknown }).stop+ if (typeof stop === 'function') stoppers.set(name, () => { void Promise.resolve(stop.call(port)).catch(() => {}) })+ ports[name] = guard(name, port, fallbacks) as CorePorts[K]+ },+ /** Run `handler` once, when `name` is switched off. */+ onOff(name: PortName, handler: () => void): void {+ offHandlers.set(name, [...offHandlers.get(name) ?? [], handler])+ },+ /** Whether `name` is off: it did not start, or it was switched off. */+ isOff(name: PortName): boolean {+ return off.has(name)+ },+ }+}++export type ServiceHost = ReturnType<typeof createServiceHost>diff --git a/cli/src/core/turns/funnel.spec.ts b/cli/src/core/turns/funnel.spec.tsindex 22e7f1a8a..3ccfd7b23 100644--- a/cli/src/core/turns/funnel.spec.ts+++ b/cli/src/core/turns/funnel.spec.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, it, vi } from 'vitest'import { deviceErrorText } from '../../lib/deviceErrors.js'import type { RegisteredSession } from '../../lib/registry.js'-import { createEventFunnel, funnelFor, type FunnelDeps } from './funnel.js'+import { createEventFunnel, funnelFor, outsideConsumers, type FunnelDeps } from './funnel.js'type Event = Parameters<ReturnType<typeof funnelFor>>[1][number]const started = (userMessage = 'go'): Event => ({ type: 'turn_started', payload: { userMessage } }) as Event@@ -40,6 +40,28 @@ function setup(engine = 'claude', over: Partial<FunnelDeps> = {}) {describe('the event funnel', () => {afterEach(() => vi.restoreAllMocks())+ it('carries on past a teams or device service that throws: every consumer after it still gets the event', () => {+ vi.spyOn(console, 'log').mockImplementation(() => {})+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})+ const boom = (what: string) => vi.fn(() => { throw new Error(`${what} unplugged`) })+ const { funnel, calls } = setup('claude', {+ teams: { started: boom('teams') },+ device: () => ({ turnStarted: boom('device'), turnEnded: boom('device'), stream: boom('device') }),+ })+ funnel('s1', [started(), text(), ended()])+ expect(calls).toEqual([+ 'send turn_started', 'search',+ 'input started', 'deviceInput started', 'heartbeat', 'questions start', 'questions pre-turn',+ 'send text_delta',+ 'send turn_ended', 'search',+ 'deviceInput ended', 'input ended', 'questions stop',+ 'mirror',+ ])+ // Once a minute per consumer: the devices failed three times and said so once.+ expect(error.mock.calls.map(([line]) => line)).toEqual(['[funnel] teams failed · teams unplugged', '[funnel] devices failed · device unplugged'])+ })++it('fans a live turn out in its exact order: the app first, the recaps after the batch, the device stream last', () => {vi.spyOn(console, 'log').mockImplementation(() => {})const { funnel, calls } = setup()@@ -186,3 +208,33 @@ describe('createEventFunnel', () => {expect(String(log.mock.calls[0][0])).toContain('aborted by devin error')})})++describe('a consumer outside the core', () => {+ it('is logged at most once a minute, with how many failures went unsaid, and never stops the caller', () => {+ const lines: string[] = []+ let at = 0+ const outside = outsideConsumers((line) => lines.push(line), () => at)+ const ran: string[] = []+ outside('devices', () => { throw new Error('serial port gone') })+ at += 10_000+ outside('devices', () => { throw 'still gone' })+ outside('teams', () => { ran.push('teams') })+ at += 60_000+ outside('devices', () => { throw 'back and gone again' })+ at += 60_001+ outside('devices', () => { throw new Error('once more') })+ expect(ran).toEqual(['teams'])+ expect(lines).toEqual([+ '[funnel] devices failed · serial port gone',+ '[funnel] devices failed · back and gone again · 1 more since',+ '[funnel] devices failed · once more',+ ])+ })++ it('reports to the error log by default', () => {+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})+ outsideConsumers()('teams', () => { throw new Error('x') })+ expect(error).toHaveBeenCalledWith('[funnel] teams failed · x')+ error.mockRestore()+ })+})diff --git a/cli/src/core/turns/funnel.ts b/cli/src/core/turns/funnel.tsindex 731b5ea2d..b5e5761d1 100644--- a/cli/src/core/turns/funnel.ts+++ b/cli/src/core/turns/funnel.ts@@ -52,12 +52,36 @@ export interface FunnelDeps {startHeartbeat: (sessionId: string) => voidquestionWatcher: Pick<QuestionWatcher, 'start' | 'noteTurnStart' | 'stop'>mirror: Pick<CommanderMirror, 'ingest'>+ /** How a consumer outside the core is called ([outsideConsumers]). */+ outside?: OutsideConsumers+}++type OutsideConsumers = (consumer: string, call: () => void) => void++/**+ * A consumer outside the core — the teams and the device service — answers for itself: one that+ * throws is logged, at most once a minute per consumer and with how many were not, and every consumer+ * after it still gets the event. The heartbeat, the question watcher and the recaps must not miss a+ * turn because a device was unplugged mid-call. The services on the boundary are guarded the same way+ * (core/serviceHost.ts).+ */+export function outsideConsumers(log: (line: string) => void = (line) => console.error(line), now: () => number = Date.now): OutsideConsumers {+ const said = new Map<string, { at: number; quiet: number }>()+ return (consumer, call) => {+ try { call() } catch (error) {+ const at = now()+ const last = said.get(consumer)+ if (last && at - last.at < 60_000) { last.quiet++; return }+ log(`[funnel] ${consumer} failed · ${error instanceof Error ? error.message : String(error)}${last?.quiet ? ` · ${last.quiet} more since` : ''}`)+ said.set(consumer, { at, quiet: 0 })+ }+ }}/** The funnel itself, over what it feeds. */export function funnelFor({bySession, tokenUsage, agentIdFor, turnActivity, isSubagentSession, clients, search, turnStartedAt, input, teams,- deviceInput, device, startHeartbeat, questionWatcher, mirror,+ deviceInput, device, startHeartbeat, questionWatcher, mirror, outside = outsideConsumers(),}: FunnelDeps) {return (sessionId: string, events: Events, opts?: EmitOptions): void => {if (!events.length || !bySession(sessionId)?.active) return@@ -94,9 +118,9 @@ export function funnelFor({turnStartedAt.set(sessionId, Date.now())console.log(`[turn] ${sid(sessionId)} started · engine=${bySession(sessionId)?.engine ?? 'claude'} · bytes=${Buffer.byteLength(event.payload.userMessage, 'utf8')}`)input.onTurnStarted(agentIdFor(sessionId), event.payload.userMessage)- if (!opts?.resumed && !opts?.replay) teams.started(agentId, event.payload.userMessage, 'transcript', bySession(sessionId)?.engine)+ if (!opts?.resumed && !opts?.replay) outside('teams', () => teams.started(agentId, event.payload.userMessage, 'transcript', bySession(sessionId)?.engine))deviceInput.onTurnStarted(agentId, event.payload.userMessage)- device()?.turnStarted(agentId)+ outside('devices', () => device()?.turnStarted(agentId))startHeartbeat(sessionId)questionWatcher.start(sessionId) // Claude opens its dialog INSIDE a turn// ...and anything already drawn belongs to the turn BEFORE this one — unless this is a turn the@@ -114,7 +138,7 @@ export function funnelFor({)// Filter only the Device receipt view; shared normalizers, mirror, and local input stay unchanged.if (!isDeviceInputBoundary(usageSession?.engine ?? '', events, eventIndex)) {- device()?.turnEnded(agentId, event.payload.aborted === true)+ outside('devices', () => device()?.turnEnded(agentId, event.payload.aborted === true))deviceInput.onTurnEnded(agentId)}input.onTurnEnded(agentIdFor(sessionId))@@ -130,7 +154,7 @@ export function funnelFor({}mirror.ingest(events, sessionId, { replay: !!(opts?.resumed || opts?.replay) })// Subscribed devices only (lib/autonomous-device/stream.ts). A transcript re-read is history, not live.- if (!opts?.replay) device()?.stream(agentIdFor(sessionId), events)+ if (!opts?.replay) outside('devices', () => device()?.stream(agentIdFor(sessionId), events))}}diff --git a/cli/src/hookServer.spec.ts b/cli/src/hookServer.spec.tsindex db262ee58..f36507631 100644--- a/cli/src/hookServer.spec.ts+++ b/cli/src/hookServer.spec.ts@@ -426,6 +426,32 @@ describe('requests must name this server', () => {expect(onLogs).not.toHaveBeenCalled()})+ it('answers every request even when its handler throws, instead of leaving the caller waiting', async () => {+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})+ const onStop = vi.fn(() => { throw new Error('stop failed after the answer') })+ const { base } = await start({+ onStatus: async () => { throw new Error('status read failed') },+ onLogs: () => { throw 'log file gone' },+ onStop,+ })+ // Before the answer: a 500 the caller can act on, at once.+ const status = await fetch(`${base}/api/status`)+ expect(status.status).toBe(500)+ expect(await status.json()).toEqual({ error: 'INTERNAL' })+ // After the headers went out: the response is ended, not left open.+ const logs = await fetch(`${base}/api/logs`)+ expect(logs.status).toBe(200)+ expect(await logs.text()).toBe('')+ // After the response ended: nothing more to send, and nothing breaks.+ const stop = await fetch(`${base}/api/stop`, { method: 'POST', headers: { 'x-adapter-local': '1' } })+ expect(await stop.json()).toEqual({ ok: true })+ await vi.waitFor(() => expect(onStop).toHaveBeenCalled())+ expect(error).toHaveBeenCalledWith('[hooks] GET /api/status failed:', 'status read failed')+ expect(error).toHaveBeenCalledWith('[hooks] GET /api/logs failed:', 'log file gone')+ await vi.waitFor(() => expect(error).toHaveBeenCalledWith('[hooks] POST /api/stop failed:', 'stop failed after the answer'))+ error.mockRestore()+ })+it('serves a status that has to read before it answers', async () => {// A harness's `updatedAt` is when its conversation last moved, which is read from its transcript.const { base } = await start({ onStatus: async () => ({ sessions: [{ id: 'a', updatedAt: 42 }] }) })diff --git a/cli/src/hookServer.ts b/cli/src/hookServer.tsindex dc38ab115..97d24461f 100644--- a/cli/src/hookServer.ts+++ b/cli/src/hookServer.ts@@ -877,7 +877,15 @@ export function startHookServer(}json(404, { error: 'not found' })- })()+ })().catch((error: unknown) => {+ // Whatever a handler throws, the request is answered: an engine's hook that waits on this server+ // holds up that engine's turn until its own timeout, and the desktop waits out thirty seconds.+ console.error(`[hooks] ${req.method} ${(req.url ?? '').split('?')[0].slice(0, 80)} failed:`, error instanceof Error ? error.message : error)+ if (!res.headersSent) {+ res.writeHead(500, { 'Content-Type': 'application/json' })+ res.end(JSON.stringify({ error: 'INTERNAL' }))+ } else if (!res.writableEnded) res.end()+ })}const server = http.createServer(handle)diff --git a/cli/src/lib/registry.quarantine.spec.ts b/cli/src/lib/registry.quarantine.spec.tsnew file mode 100644index 000000000..18fadb1df--- /dev/null+++ b/cli/src/lib/registry.quarantine.spec.ts@@ -0,0 +1,82 @@+import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'fs'+import { tmpdir } from 'os'+import { join } from 'path'+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'++// The one way a corrupt registry is still refused: it cannot even be moved aside.+const rename = vi.hoisted(() => ({ fails: false as false | 'error' | 'string' }))+vi.mock('fs', async (real) => {+ const actual = await real<typeof import('fs')>()+ return {+ ...actual,+ renameSync: (from: string, to: string) => {+ if (rename.fails === 'error' && String(to).includes('.corrupt-')) throw Object.assign(new Error('EXDEV: cross-device link not permitted'), { code: 'EXDEV' })+ if (rename.fails === 'string' && String(to).includes('.corrupt-')) throw 'read-only file system'+ return actual.renameSync(from, to)+ },+ }+})++let dataDir = ''+const processIdentity = (pid: number) => ({ pid, executable: 'claude', startMarker: `Mon Aug 10 10:00:${String(pid % 60).padStart(2, '0')} 2026` })++async function loadRegistry() {+ vi.resetModules()+ process.env.ADAPTER_DATA_DIR = dataDir+ process.env.CLAUDE_PROJECTS_DIR = dataDir+ process.env.CODEX_HOME = dataDir+ process.env.CURSOR_HOME = dataDir+ return (await import('./registry.js')).registry+}++describe('a registry no version can load', () => {+ let error: ReturnType<typeof vi.spyOn>+ beforeEach(() => {+ dataDir = mkdtempSync(join(tmpdir(), 'adapter-registry-quarantine-'))+ rename.fails = false+ error = vi.spyOn(console, 'error').mockImplementation(() => {})+ })+ afterEach(() => { vi.restoreAllMocks() })++ it('with two rows claiming one agent: moved aside, and the registry starts empty and writable', async () => {+ const registry = await loadRegistry()+ registry.load()+ registry.openProcessAgent({ agentId: 'a1', engine: 'claude', tmuxPane: '%1', processIdentity: processIdentity(101) })+ registry.openProcessAgent({ agentId: 'a2', engine: 'claude', tmuxPane: '%2', processIdentity: processIdentity(102) })+ const file = join(dataDir, 'registry.json')+ const rows = JSON.parse(readFileSync(file, 'utf8')) as Array<Record<string, unknown>>+ rows[1] = { ...rows[1], agentId: rows[0].agentId }+ const duplicated = JSON.stringify(rows)+ writeFileSync(file, duplicated, { mode: 0o600 })++ const again = await loadRegistry()+ again.load()+ expect(again.list()).toEqual([])+ const aside = readdirSync(dataDir).filter((name) => name.startsWith('registry.json.corrupt-'))+ expect(aside).toHaveLength(1)+ expect(readFileSync(join(dataDir, aside[0]), 'utf8')).toBe(duplicated)+ again.openProcessAgent({ agentId: 'a3', engine: 'claude', tmuxPane: '%3', processIdentity: processIdentity(103) })+ expect(again.list().map((s) => s.agentId)).toEqual(['a3'])+ })++ it('that cannot be moved aside is left as it is, and nothing is written over it', async () => {+ const file = join(dataDir, 'registry.json')+ writeFileSync(file, '[{"agentId":', { mode: 0o600 })+ rename.fails = 'error'+ const registry = await loadRegistry()+ registry.load()+ expect(error).toHaveBeenCalledWith(expect.stringMatching(/^\[registry\] registry is not JSON \(.*\), and it could not be moved aside \(EXDEV: cross-device link not permitted\); refusing to overwrite it$/))+ registry.openProcessAgent({ agentId: 'must-not-write', engine: 'claude', tmuxPane: '%9', processIdentity: processIdentity(109) })+ expect(readFileSync(file, 'utf8')).toBe('[{"agentId":')+ expect(readdirSync(dataDir).filter((name) => name.startsWith('registry.json.corrupt-'))).toEqual([])+ })++ it('says why it could not be moved, whatever the failure was', async () => {+ const file = join(dataDir, 'registry.json')+ writeFileSync(file, '{"schemaVersion":2}', { mode: 0o600 })+ rename.fails = 'string'+ const registry = await loadRegistry()+ registry.load()+ expect(error).toHaveBeenCalledWith('[registry] registry root is not an array, and it could not be moved aside (read-only file system); refusing to overwrite it')+ })+})diff --git a/cli/src/lib/registry.spec.ts b/cli/src/lib/registry.spec.tsindex 39768e01e..c3c096daf 100644--- a/cli/src/lib/registry.spec.ts+++ b/cli/src/lib/registry.spec.ts@@ -1,5 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'-import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs'+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'fs'import { tmpdir } from 'os'import { join } from 'path'import { processStartMarker } from './processLiveness.js'@@ -214,7 +214,7 @@ describe('registry remote display names', () => {})})- it('persists Codex engine metadata and preserves a malformed v2 row for operator recovery', async () => {+ it('persists Codex engine metadata and moves a malformed v2 row aside for operator recovery', async () => {const sessionsDir = join(dataDir, 'sessions')const transcriptPath = join(sessionsDir, 'codex-session.jsonl')mkdirSync(sessionsDir)@@ -243,10 +243,16 @@ describe('registry remote display names', () => {const malformed = JSON.stringify(persisted)writeFileSync(join(dataDir, 'registry.json'), malformed)+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})registry.load()expect(registry.list()).toEqual([])expect(registry.get('invalid-session')).toBeUndefined()- expect(readFileSync(join(dataDir, 'registry.json'), 'utf-8')).toBe(malformed)+ // Kept byte for byte beside the registry, which starts empty and can be written again.+ const aside = readdirSync(dataDir).filter((name) => name.startsWith('registry.json.corrupt-'))+ expect(aside).toHaveLength(1)+ expect(readFileSync(join(dataDir, aside[0]), 'utf-8')).toBe(malformed)+ expect(existsSync(join(dataDir, 'registry.json'))).toBe(false)+ error.mockRestore()})it('migrates legacy tmux rows additively without changing agent identity or binding', async () => {@@ -292,12 +298,35 @@ describe('registry remote display names', () => {it.each([['truncated JSON', '[{"agentId":'],['non-array root', '{"schemaVersion":2}'],- ['unknown row schema', JSON.stringify([{ schemaVersion: 99, agentId: 'future' }])],- ['nonnumeric row schema', JSON.stringify([{ schemaVersion: '3', agentId: 'future' }])],['empty legacy row', JSON.stringify([{}])],['primitive legacy row', JSON.stringify([7])],['legacy row without runtime', JSON.stringify([{ agentId: 'legacy-agent' }])],- ])('preserves %s byte-for-byte and blocks later writes', async (_label, bytes) => {+ ])('moves %s aside byte-for-byte, starts empty, and goes on starting agents', async (_label, bytes) => {+ const file = join(dataDir, 'registry.json')+ writeLegacyStateFile(file, bytes)+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})+ const { registry } = await loadRegistryModule()+ registry.load()+ const aside = readdirSync(dataDir).filter((name) => name.startsWith('registry.json.corrupt-'))+ expect(aside).toHaveLength(1)+ expect(readFileSync(join(dataDir, aside[0]), 'utf8')).toBe(bytes)+ expect(statSync(join(dataDir, aside[0])).mode & 0o777).toBe(0o600)+ expect(error).toHaveBeenCalledWith(expect.stringMatching(/^\[registry\] registry .*; moved it aside to .*registry\.json\.corrupt-.* and started empty$/))+ registry.openProcessAgent({+ agentId: 'after-the-move',+ engine: 'claude',+ tmuxPane: '%88',+ processIdentity: processIdentity(888),+ })+ expect(registry.list().map((s) => s.agentId)).toEqual(['after-the-move'])+ expect(JSON.parse(readFileSync(file, 'utf8')).map((row: { agentId: string }) => row.agentId)).toEqual(['after-the-move'])+ error.mockRestore()+ })++ it.each([+ ['unknown row schema', JSON.stringify([{ schemaVersion: 99, agentId: 'future' }])],+ ['nonnumeric row schema', JSON.stringify([{ schemaVersion: '3', agentId: 'future' }])],+ ])('preserves a newer version\'s %s byte-for-byte and blocks later writes', async (_label, bytes) => {const file = join(dataDir, 'registry.json')writeLegacyStateFile(file, bytes)const { registry } = await loadRegistryModule()diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.tsindex b720fb42a..85bdacc78 100644--- a/cli/src/lib/registry.ts+++ b/cli/src/lib/registry.ts@@ -918,8 +918,7 @@ class Registry {try {const stored = JSON.parse(readPrivateStateFile(FILE)) as unknownif (!Array.isArray(stored)) {- this.writeBlocked = true- console.error('[registry] registry root is not an array; refusing to overwrite it')+ this.quarantine('root is not an array')return}const { rows: parsed, dropped, changed: strippedRetired } = withoutRetiredRows(stored)@@ -931,15 +930,13 @@ class Registry {}const legacyRows = parsed.filter((row) => !row || typeof row !== 'object' || !Object.hasOwn(row, 'schemaVersion'))if (legacyRows.some((row) => !validLegacyRegistryRow(row))) {- this.writeBlocked = true- console.error('[registry] registry contains a malformed legacy row; refusing to overwrite it')+ this.quarantine('contains a malformed legacy row')return}const v2Rows = parsed.filter((row) => !!row && typeof row === 'object'&& (row as { schemaVersion?: unknown }).schemaVersion === 2)if (v2Rows.length && !validatedRows(v2Rows)) {- this.writeBlocked = true- console.error('[registry] registry contains a malformed v2 row; refusing to overwrite it')+ this.quarantine('contains a malformed v2 row')return}const arr = parsed as Array<Partial<RegisteredSession>>@@ -1085,32 +1082,49 @@ class Registry {|| legacyLauncherId !== '') changed = trueif (this.agents.has(s.agentId)) {- this.agents.clear()- this.sessionIndex.clear()- this.runtimeIndex.clear()- this.processIndex.clear()- this.writeBlocked = true- console.error('[registry] registry contains duplicate agent identities; refusing to load or overwrite it')+ this.quarantine('contains duplicate agent identities')return}this.index(s)}if (!validatedRows(this.list().map(persistedRow))) {- this.agents.clear()- this.sessionIndex.clear()- this.runtimeIndex.clear()- this.processIndex.clear()- this.writeBlocked = true- console.error('[registry] registry violates global identity invariants; refusing to load or overwrite it')+ this.quarantine('violates global identity invariants')return}if (changed) this.save()} catch (error) {- if (existsSync(FILE)) {+ if (error instanceof SyntaxError) this.quarantine(`is not JSON (${error.message})`)+ else if (existsSync(FILE)) {this.writeBlocked = trueconsole.error('[registry] registry is unreadable; refusing to overwrite it:', error instanceof Error ? error.message : error)}- // A genuinely absent file starts empty. Any existing unreadable file is preserved byte-for-byte.+ // A genuinely absent file starts empty. A file that cannot be opened safely (a symlink, another+ // owner, open permissions) is left exactly as it is, and nothing is written over it.+ }+ }++ /**+ * Bytes no version of the registry can load — not JSON, the wrong shape, malformed or contradictory+ * rows — are moved aside, kept byte-for-byte for an operator, and the daemon starts empty: discovery+ * finds the agents still running in tmux again. Refusing every write instead left a daemon that could+ * not start a single agent until someone repaired the file by hand. A newer version's rows (an+ * unknown schema) are not corruption, and are still never touched.+ */+ private quarantine(reason: string): void {+ this.agents.clear()+ this.sessionIndex.clear()+ this.runtimeIndex.clear()+ this.processIndex.clear()+ this.persistedBaseline.clear()+ this.persistedContents = null+ const aside = `${FILE}.corrupt-${new Date().toISOString().replace(/[:.]/g, '-')}`+ try {+ renameSync(FILE, aside)+ this.writeBlocked = false+ console.error(`[registry] registry ${reason}; moved it aside to ${aside} and started empty`)+ } catch (error) {+ this.writeBlocked = true+ console.error(`[registry] registry ${reason}, and it could not be moved aside (${error instanceof Error ? error.message : error}); refusing to overwrite it`)}}diff --git a/cli/src/lib/tmux.paneLookup.spec.ts b/cli/src/lib/tmux.paneLookup.spec.tsnew file mode 100644index 000000000..decaa47c0--- /dev/null+++ b/cli/src/lib/tmux.paneLookup.spec.ts@@ -0,0 +1,46 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++// `tmux display-message` as each version and failure answers it, for a pane that dropped out of+// `tmux list-panes` — the question the reconciler asks before it counts a pane as missing.+const answer = vi.hoisted(() => ({ err: null as (Error & { killed?: boolean }) | null, stdout: '', stderr: '' }))+vi.mock('child_process', async (real) => {+ const actual = await real<typeof import('child_process')>()+ return {+ ...actual,+ execFile: (file: string, args: string[], _options: unknown, done: (err: Error | null, stdout: string, stderr: string) => void) => {+ if (file === 'tmux' && args[0] === 'display-message') { queueMicrotask(() => done(answer.err, answer.stdout, answer.stderr)); return {} }+ return (actual.execFile as unknown as (...a: unknown[]) => unknown)(file, args, _options, done)+ },+ }+})++const { lookupPaneEngineProcess } = await import('./tmux.js')++describe('asking tmux for a pane that is not in its list', () => {+ afterEach(() => { answer.err = null; answer.stdout = ''; answer.stderr = '' })++ it('is gone when tmux answers with nothing, as tmux 3.7 does for an unknown pane', async () => {+ answer.stdout = '\n'+ expect(await lookupPaneEngineProcess('%9', 'claude')).toEqual({ ok: false, unknown: false, reason: 'tmux has no pane %9' })+ })++ it('is gone when tmux says it cannot find the pane, as older versions do', async () => {+ answer.err = Object.assign(new Error('Command failed'), {})+ answer.stderr = "can't find pane: %9\n"+ expect(await lookupPaneEngineProcess('%9', 'claude')).toEqual({ ok: false, unknown: false, reason: 'tmux has no pane %9' })+ })++ it('is unknown when tmux could not be asked: no server on this socket, or a timeout', async () => {+ answer.err = new Error('Command failed')+ answer.stderr = 'no server running on /tmp/tmux-501/default\n'+ expect(await lookupPaneEngineProcess('%9', 'claude')).toEqual({ ok: false, unknown: true, reason: 'tmux could not resolve pane %9' })+ answer.err = Object.assign(new Error('timed out'), { killed: true })+ answer.stderr = ''+ expect(await lookupPaneEngineProcess('%9', 'claude')).toMatchObject({ unknown: true })+ })++ it('is unknown when tmux answers with something that is not a pid', async () => {+ answer.stdout = 'not-a-pid\n'+ expect(await lookupPaneEngineProcess('%9', 'claude')).toMatchObject({ ok: false, unknown: true })+ })+})diff --git a/cli/src/lib/tmux.ts b/cli/src/lib/tmux.tsindex b06a361a0..ca319d503 100644--- a/cli/src/lib/tmux.ts+++ b/cli/src/lib/tmux.ts@@ -453,15 +453,30 @@ export function processTreePids(rows: readonly ProcessRow[], rootPids: readonly}-function panePid(pane: string): Promise<number | null> {+/**+ * The pane's root pid; `'missing'` when tmux answered that it has no such pane; null when tmux could+ * not be asked (a timeout, no server, a socket it cannot reach) — which says nothing about the pane.+ * tmux 3.7 answers `display-message -t` for an unknown pane with an empty line and exit 0; older+ * versions say "can't find pane".+ */+function panePidLookup(pane: string): Promise<number | 'missing' | null> {return new Promise((resolve) => {- execFile('tmux', ['display-message', '-p', '-t', pane, '#{pane_pid}'], { timeout: 2000 }, (err, stdout) => {- const pid = Number(stdout.trim())- resolve(!err && Number.isSafeInteger(pid) && pid > 0 ? pid : null)+ execFile('tmux', ['display-message', '-p', '-t', pane, '#{pane_pid}'], { timeout: 2000 }, (err, stdout, stderr) => {+ const text = String(stdout ?? '').trim()+ const pid = Number(text)+ if (!err && Number.isSafeInteger(pid) && pid > 0) resolve(pid)+ else if (!err && text === '') resolve('missing')+ else if (err && /can't find pane/.test(String(stderr ?? ''))) resolve('missing')+ else resolve(null)})})}+async function panePid(pane: string): Promise<number | null> {+ const found = await panePidLookup(pane)+ return typeof found === 'number' ? found : null+}+/*** Linux caps `comm` at 15 bytes (TASK_COMM_LEN) and never prints a path there, while macOS prints the* full executable path. Measured on Ubuntu 24.04: a binary named `a-very-long-engine-name-binary` reads@@ -834,9 +849,11 @@ export async function lookupPaneEngineProcess(pane: string,engine: RegisteredSession['engine'],): Promise<PaneProcessLookup> {- const rootPid = await panePid(pane)- // The caller already confirmed the pane is in `tmux list-panes`, so a failure here is the tmux call,- // not a missing pane.+ const rootPid = await panePidLookup(pane)+ // tmux saying it has no such pane is an answer: the pane is gone. The reconciler asks exactly when a+ // pane has dropped out of `tmux list-panes`, and reading that as "could not ask" kept an agent whose+ // pane was killed counted as active for good. A failed call is still no evidence either way.+ if (rootPid === 'missing') return { ok: false, unknown: false, reason: `tmux has no pane ${pane}` }if (!rootPid) return { ok: false, unknown: true, reason: `tmux could not resolve pane ${pane}` }const rows = await processRows()if (!rows) return { ok: false, unknown: true, reason: 'the process table could not be read' }diff --git a/docs/design/2026-10-03-harnessd.md b/docs/design/2026-10-03-harnessd.mdindex 2c1979b36..fd2c0bab3 100644--- a/docs/design/2026-10-03-harnessd.md+++ b/docs/design/2026-10-03-harnessd.md@@ -407,6 +407,18 @@ Done so far:it under `cli/src/services`, held to 100% beside the core. A service holds no credential: mintingthe grid name and the access token go through `CoreApi.account`. Relay, sharing, teams anddevices, and step 14, remain.+- **A failing service cannot take the core down** (part 10, `cli/src/core/serviceHost.ts`). A+ service whose start throws is left off and the core starts without it. Every call into a port is+ guarded and answered with a fallback declared beside the port's contract, so the funnel, restore+ and frame building carry on; a request's failure goes back to that one request as+ `SERVICE_UNAVAILABLE` with `retryable: true`. Five failures in sixty seconds switch the service off+ until the daemon restarts: it is stopped, its port goes null and its socket bindings are unbound.+ The funnel calls the teams and the device service the same way. `HARNESSD_TEST_FAULTS` injects+ faults, and `e2e/services.e2e.ts` proves on the real daemon that agents bind, run their turns and+ come back after a restart with every service failing to start or failing on every call.+- **What the daemon survives** (`e2e/robustness.e2e.ts`): hostile local clients, strangers and+ malformed bodies at the hook door, a pane killed from outside, a turn that outlives a restart, a+ turn that ends while the daemon is down, and garbage state files at start.Next, in order:@@ -501,6 +513,15 @@ Found while mapping, and fixed on their own rather than inside a move:- Kilo's reader was skipped on cancel and on shutdown, so a cancelled Kilo agent could keepshowing as working. Fixed once the readers became one table (step 4).- `BackendSocket.onLocalClient` is never wired.+- Found by the end-to-end suite, and fixed (part 10):+ - A registry no version could load left the daemon refusing every write, so it could not start+ an agent until someone repaired the file. Such a file is now moved aside, kept byte for byte,+ and the daemon starts empty; a newer version's rows are still never touched.+ - A pane closed with `kill-pane` was never counted missing: tmux 3.7 answers an unknown pane with+ an empty line, which was read as "could not ask". Its agent now stops counting as active.+ - The hook server answered nothing when a handler threw outside its proxy routes, which held an+ engine's turn until its hook timed out. Every request is answered now.+ - One throwing teams or device consumer cost an event its heartbeat, question watcher and recaps.## Testing