Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

The core's event loop: no blocking canary or unpack, a deadline per reconcile pass, guarded transcript readers (#826)

Dee7b6af45cf3d2
diff --git a/cli/e2e/robustness.e2e.ts b/cli/e2e/robustness.e2e.ts
index 4d77b19a5..dd2533711 100644
--- a/cli/e2e/robustness.e2e.ts
+++ b/cli/e2e/robustness.e2e.ts
@@ -226,7 +226,12 @@ describe('what the daemon survives', () => {
mkdirSync(join(d.dataDir, 'stopped-agents'), { recursive: true, mode: 0o700 })
writeFileSync(join(d.dataDir, 'stopped-agents', 'garbage.json'), '[[[', { mode: 0o600 })
writeFileSync(join(d.dataDir, 'registry-boot'), 'not a boot id', { mode: 0o600 })
+ // The Wi-Fi device's (experimental): either one threw out of the core's start, into safe mode.
+ writeFileSync(join(d.dataDir, 'autonomous-device-connections.json'), '{"not": "a list"}', { mode: 0o600 })
+ writeFileSync(join(d.dataDir, 'device-results.json'), '{"version": 1, "entries": [', { mode: 0o600 })
await d.start()
+ await until('the device pieces to be left out', () => /the Wi-Fi device service could not be started/.test(d.log())
+ && /the Wi-Fi device link could not be started/.test(d.log()), 30_000)
const client = await LocalClient.connect(d)
expect(Array.isArray((await client.request('agents_list', { includeStopped: true })).agents)).toBe(true)
const agent = await boundAgent(d, client, 'after-the-corruption')
diff --git a/cli/src/backendSocket.spec.ts b/cli/src/backendSocket.spec.ts
index 48cca89b1..a88107343 100644
--- a/cli/src/backendSocket.spec.ts
+++ b/cli/src/backendSocket.spec.ts
@@ -1,7 +1,7 @@
import * as gitPullRequest from './lib/gitPullRequest.js'
import * as sessionGitPullRequest from './lib/sessionGitPullRequest.js'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
-import { readFileSync, mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'fs'
+import { existsSync, readFileSync, mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'fs'
import { homedir, tmpdir } from 'os'
import { join } from 'path'
import { fileURLToPath } from 'url'
@@ -643,6 +643,81 @@ describe('BackendSocket outbound queue', () => {
await socket.stop()
})
+ it('answers a turn end "no role" without building the orchestrator on a machine that has no project', async () => {
+ // Asked at every turn's end. Building the service makes its folder and parses every saved run on
+ // the core's event loop, and from then on every frame sent is handed to it.
+ const stateDir = join(process.env.ADAPTER_DATA_DIR!, 'orchestrator')
+ rmSync(stateDir, { recursive: true, force: true })
+ const socket = new BackendSocket('token')
+ try {
+ expect(socket.orchestratorRoleOf('agent-1')).toBeNull()
+ expect(socket.orchestratorRoleOf('agent-2')).toBeNull()
+ expect(existsSync(stateDir)).toBe(false)
+ expect((socket as unknown as { orchestratorService: unknown }).orchestratorService).toBeNull()
+ } finally { await socket.stop() }
+ })
+
+ it('answers a turn end from a project an earlier run of the daemon saved', async () => {
+ const stateDir = join(process.env.ADAPTER_DATA_DIR!, 'orchestrator')
+ rmSync(stateDir, { recursive: true, force: true })
+ mkdirSync(stateDir, { recursive: true, mode: 0o700 })
+ const id = 'a'.repeat(32)
+ const task = { id: 'part', title: 'Part', harness: 'cad', prompt: 'Draw the part', dependsOn: [], state: 'running', attempt: 1, agentId: 'agent-worker', cwd: stateDir, summary: '', error: null, uncertain: false, artifacts: [], inputs: {} }
+ writeFileSync(join(stateDir, `${id}.json`), JSON.stringify({
+ version: 1, id, fingerprint: 'f', prompt: 'Build the robot', engine: 'claude', bypassPermission: false, parallelism: 3,
+ root: stateDir, directorId: 'agent-director', directorWorking: false, state: 'active', error: null,
+ tasks: [task], messages: [], revision: 1, createdAt: 1, updatedAt: 1,
+ }), { mode: 0o600 })
+ const socket = new BackendSocket('token')
+ try {
+ expect(socket.orchestratorRoleOf('agent-worker')).toEqual({ role: 'worker' })
+ expect(socket.orchestratorRoleOf('agent-director')).toEqual({ role: 'director', busy: true })
+ expect(socket.orchestratorRoleOf('agent-other')).toBeNull()
+ } finally {
+ await socket.stop()
+ rmSync(stateDir, { recursive: true, force: true })
+ }
+ })
+
+ it('leaves an orchestrator folder it cannot list to the service to report, as before', async () => {
+ const stateDir = join(process.env.ADAPTER_DATA_DIR!, 'orchestrator')
+ rmSync(stateDir, { recursive: true, force: true })
+ writeFileSync(stateDir, 'not a folder')
+ const socket = new BackendSocket('token')
+ try {
+ expect(() => socket.orchestratorRoleOf('agent-1')).toThrow()
+ } finally {
+ await socket.stop()
+ rmSync(stateDir, { force: true })
+ }
+ })
+
+ it('signed out, seals and queues nothing for a cloud link that never opens, and still serves its windows', async () => {
+ const socket = new BackendSocket('token')
+ const internal = socket as unknown as { e2ee: { wrapUp(frame: unknown): unknown; wrapCommander(frame: unknown): unknown }; queue: unknown[] }
+ const frames: Array<{ type?: unknown }> = []
+ socket.registerLocalClient('local:window', { sendFrame: (frame) => { frames.push(frame); return true }, sendBinary: () => true })
+ try {
+ // Sent at start-up, before the daemon has looked for a session: queued, as for any daemon.
+ socket.send({ type: 'turn_started', dbSessionId: 's1', payload: { sessionId: 's1' } })
+ expect(internal.queue).toHaveLength(1)
+ const wrapUp = vi.spyOn(internal.e2ee, 'wrapUp')
+ const wrapCommander = vi.spyOn(internal.e2ee, 'wrapCommander')
+ socket.serveThisComputerOnly()
+ expect(internal.queue).toHaveLength(0)
+ for (let i = 0; i < 100; i++) socket.send({ type: 'text_delta', dbSessionId: 's1', payload: { content: `word ${i}` } })
+ socket.sendCommander({ type: 'commander_event', agentId: 's1', dbSessionId: 's1', payload: { kind: 'done', text: 'done' } })
+ socket.sendUser({ type: 'notification', payload: {} })
+ expect(wrapUp).not.toHaveBeenCalled()
+ expect(wrapCommander).not.toHaveBeenCalled()
+ expect(internal.queue).toHaveLength(0)
+ expect(frames.filter((frame) => frame.type === 'text_delta')).toHaveLength(100)
+ } finally {
+ await socket.unregisterLocalClient('local:window')
+ await socket.stop()
+ }
+ })
+
it('bounds the opening handshake and reconnects when it times out', async () => {
// A connect attempt whose TCP side came up but whose upgrade was never answered used to sit in
// CONNECTING forever: no 'open', so no heartbeat to terminate it, and `this.ws` set, so every
diff --git a/cli/src/backendSocket.ts b/cli/src/backendSocket.ts
index 47f052eb2..4c6103635 100644
--- a/cli/src/backendSocket.ts
+++ b/cli/src/backendSocket.ts
@@ -61,7 +61,7 @@ import { readGitProject } from './lib/gitProject.js'
import { agentFrame, type AgentDshContext, type AgentFrame } from './lib/agentFrame.js'
import { agentTokenUsage } from './lib/agentTokenUsage.js'
import { listInstalledDsh } from './dsh/installed.js'
-import { OrchestratorService } from './orchestrator/service.js'
+import { OrchestratorService, hasSavedProjects } from './orchestrator/service.js'
import { OrchestratorError } from './orchestrator/model.js'
import { orchestratorRequest } from './orchestrator/wire.js'
import { TeamService } from './teams/service.js'
@@ -428,10 +428,13 @@ export class BackendSocket {
try { this.teamMailbox().start(); this.teams().start() }
catch { console.warn('[teams] preserved unreadable team state; inspect Team for recovery') }
}
- /** The commander asks this for every turn that ends — see OrchestratorService.roleOf. */
+ /** The commander asks this for every turn that ends — see OrchestratorService.roleOf. "No role",
+ * without building the service, on a machine with no saved project (see hasSavedProjects). */
orchestratorRoleOf(agentId: string): ReturnType<OrchestratorService['roleOf']> {
+ if (!this.orchestratorService && !(this.orchestratorSaved ??= hasSavedProjects(join(env.ADAPTER_DATA_DIR, 'orchestrator')))) return null
return this.orchestration().roleOf(agentId)
}
+ private orchestratorSaved: boolean | null = null
private orchestration(): OrchestratorService {
return this.orchestratorService ??= new OrchestratorService({
stateDir: join(env.ADAPTER_DATA_DIR, 'orchestrator'),
@@ -999,9 +1002,15 @@ export class BackendSocket {
for (const [connId, sink] of this.localClients) {
if (!sink.sendFrame(frame)) void this.unregisterLocalClient(connId)
}
- this.enqueue({ t: 'up', frame: this.e2ee.wrapUp(frame) })
+ if (!this.thisComputerOnly) this.enqueue({ t: 'up', frame: this.e2ee.wrapUp(frame) })
}
+ /** Signed out: the cloud link is never dialed in this process's life (a sign-in restarts it), so what
+ * was queued for it is dropped and nothing more is sealed or queued: every frame, a text_delta's
+ * among them, was sealed and queued for a link that never opens, two thousand deep. */
+ serveThisComputerOnly(): void { this.thisComputerOnly = true; this.queue.length = 0 }
+ private thisComputerOnly = false
+
/** Send an up-frame to the LOOPBACK clients only — never to the cloud.
*
* For things that describe what is happening at THIS desk rather than what the machine is doing: the
@@ -1136,7 +1145,7 @@ export class BackendSocket {
this.onOutboundCommander?.(frame)
if (env.LOG_FRAMES) logFrame('→', 'device', frame)
deviceDump.record('out', 'commander', undefined, frame)
- this.enqueue({ t: 'up', webEligible: false, commanderEligible: true, frame: this.e2ee.wrapCommander(frame) })
+ if (!this.thisComputerOnly) this.enqueue({ t: 'up', webEligible: false, commanderEligible: true, frame: this.e2ee.wrapCommander(frame) })
}
/**
@@ -1222,6 +1231,7 @@ export class BackendSocket {
}
private enqueue(msg: OutboundEnvelope): void {
+ if (this.thisComputerOnly) return
const item: QueueItem = { id: this.nextQueueId++, data: JSON.stringify(msg), msg, attempts: 0 }
if (this.queue.length >= QUEUE_MAX) this.dropOneQueued()
if (this.queue.length >= QUEUE_MAX) {
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 7f0000712..b052b58e8 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -10,6 +10,7 @@ import { HarnessShareRelay, type SharedMachineReference } from './sharing/relay.
import { SharedViewerPool } from './sharing/viewer.js'
import { fingerprint as e2eeCoreFingerprint, b64d as e2eeCoreDecode } from './lib/e2ee/core.js'
import { AutonomousDeviceDirect } from './lib/autonomous-device/direct.js'
+import { runningDevicePart, startDevicePart } from './lib/autonomous-device/parts.js'
/**
* machine-adapter CLI (the `harness` command) — connect this computer to a "remote" agent.
*
@@ -123,7 +124,7 @@ import { TerminalBackendCoordinator } from './lib/terminalBackendCoordinator.js'
import { TerminalStreamManager } from './lib/terminalStreamManager.js'
import { terminalRouteKey, terminalRuntimeLabel } from './lib/terminalRuntime.js'
import { probeTerminalAgents } from './lib/terminalAgentDiscovery.js'
-import { TerminalAgentReconciler } from './lib/terminalAgentReconciler.js'
+import { RECONCILE_PASS_DEADLINE_MS, TerminalAgentReconciler } from './lib/terminalAgentReconciler.js'
import { remoteCommand } from './remoteCommand.js'
import { tuiCommand } from './tui/index.js'
import { newCommand } from './lib/newCommand.js'
@@ -1445,7 +1446,7 @@ const RESOLVE_ON_START_TIMEOUT_MS = 10_000
async function downloadCanaryStage(entry: UpdateEntry, dir: string, log: (m: string) => void): Promise<boolean> {
const cliBuf = await downloadVerified(entry.cli)
const notifyBuf = await downloadVerified(entry.notify)
- if (!canary(cliBuf, dir, entry.version)) { log(` ✗ the new build failed its self-check — keeping v${VERSION}`); return false }
+ if (!await canary(cliBuf, dir, entry.version)) { log(` ✗ the new build failed its self-check — keeping v${VERSION}`); return false }
// Asked for by name, so a version this machine once rolled back is installed and no longer rejected.
stage(dir, cliBuf, notifyBuf, entry.version)
confirmUpdate(dir) // canary passed + bytes already verified ⇒ drop the .prev backups
@@ -2525,7 +2526,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
current: () => registry.list(),
backends: terminalBackends,
backendOrder: terminalConfig.backends,
- transaction: (apply) => registry.transaction(apply),
+ transaction: (apply) => registry.transaction(apply, { holdSavesMs: RECONCILE_PASS_DEADLINE_MS }),
...(slowProbeMs > 0 ? {
probe: async (hints: Parameters<typeof probeTerminalAgents>[3]) => {
const probe = await probeTerminalAgents(terminalBackends, terminalConfig.backends, process.pid, hints)
@@ -2720,8 +2721,8 @@ async function runForeground(session: AuthSession | null): Promise<void> {
onAutonomousDeviceRequest: async (method, target, body) => {
if (!autonomousDeviceService) return { status: 503, body: { error: { code: 'UNAVAILABLE', message: 'Autonomous device service is starting' } } }
return autonomousDeviceLocalRequest({
- discover: async () => ({ devices: await autonomousDeviceDirect!.discover() }),
- pairStart: ({ code, device }) => autonomousDeviceDirect!.pair(device, code),
+ discover: async () => ({ devices: await runningDevicePart(autonomousDeviceDirect, 'Wi-Fi device link').discover() }),
+ pairStart: ({ code, device }) => runningDevicePart(autonomousDeviceDirect, 'Wi-Fi device link').pair(device, code),
pairStatus: () => {
const pending = backend.pendingPair()
return pending?.role === 'device' ? { state: pending.active ? 'running' : 'waiting', pairId: pending.pairId, deviceLabel: pending.label, expiresAt: pending.expiresAt } : { state: 'idle' }
@@ -3772,6 +3773,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
backend.connect()
console.log(`[cli] dialing ${env.BACKEND_WS_URL}/api/adapter-ws · watching registered sessions for ${ENGINES.length} engines`)
} else {
+ backend.serveThisComputerOnly()
console.log(`[cli] not signed in — serving this computer only · watching registered sessions for ${ENGINES.length} engines`)
}
@@ -3997,7 +3999,7 @@ async function runForeground(session: AuthSession | null): Promise<void> {
})
deviceStoreRef = deviceStore
deviceStore.startUiDelivery()
- autonomousDeviceService = new AutonomousDeviceService({
+ autonomousDeviceService = startDevicePart('Wi-Fi device service', () => new AutonomousDeviceService({
store: deviceStore,
resultJournal: new DeviceResultJournal(join(env.ADAPTER_DATA_DIR, 'device-results.json')),
inputConsumed: (id, text) => deviceInput.onTurnStarted(id, text),
@@ -4027,10 +4029,10 @@ async function runForeground(session: AuthSession | null): Promise<void> {
recent: (id, n) => mirror.recent(registry.byAgent(id)?.sessionId ?? id, n),
fullText: id => mirror.lastFullText(registry.byAgent(id)?.sessionId ?? id),
emit: (frame, deviceId) => backend.emitAutonomousDeviceEvent(frame, deviceId),
- })
- if (appVoiceFocus) autonomousDeviceService.appFocus(appVoiceFocus.machineId, appVoiceFocus.agentId, appVoiceFocus.connId)
- backend.setAutonomousDeviceService(autonomousDeviceService)
- autonomousDeviceDirect = new AutonomousDeviceDirect({
+ }))
+ if (appVoiceFocus) autonomousDeviceService?.appFocus(appVoiceFocus.machineId, appVoiceFocus.agentId, appVoiceFocus.connId)
+ if (autonomousDeviceService) backend.setAutonomousDeviceService(autonomousDeviceService)
+ autonomousDeviceDirect = startDevicePart('Wi-Fi device link', () => new AutonomousDeviceDirect({
machineId: backend.machineId, label: hostname(),
receive: (connId, frame, pairing) => backend.receiveDirectDevice(connId, frame, pairing),
attach: (connId, send) => backend.attachDirectDevice(connId, send),
@@ -4039,9 +4041,9 @@ async function runForeground(session: AuthSession | null): Promise<void> {
authenticatedFingerprint: connId => { const pub = backend.e2ee.sessionIdentity(connId); return pub ? e2eeCoreFingerprint(e2eeCoreDecode(pub)) : null },
pairedFingerprint: connId => backend.pairedDirectFingerprint(connId),
pair: code => backend.pair(code), paired: () => backend.listPairs(),
- }, env.ADAPTER_DATA_DIR)
+ }, env.ADAPTER_DATA_DIR))
backend.onDirectDeviceRevoked = fp => autonomousDeviceDirect?.revoked(fp)
- autonomousDeviceDirect.start()
+ autonomousDeviceDirect?.start()
// Worktrees Harness made that no live or stopped harness uses and nothing would miss
// (services/workspaces.ts): a few minutes after start, once restored agents are back in the
diff --git a/cli/src/core/transcripts/attach.spec.ts b/cli/src/core/transcripts/attach.spec.ts
index 28a77cdac..4521759d9 100644
--- a/cli/src/core/transcripts/attach.spec.ts
+++ b/cli/src/core/transcripts/attach.spec.ts
@@ -196,6 +196,33 @@ describe('attaching a session', () => {
expect(await none.attach.attachSession(session('pi', transcript([{}])))).toBe(true)
})
+ it('attaches a session whose device or runtime profile cannot take in its lines, on either path', async () => {
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ // Asking the device whether it watches the session throws: the attach goes on without it.
+ const asking = setup()
+ asking.service.needsTranscript.mockImplementation(() => { throw new Error('device state unreadable') })
+ expect(await asking.attach.attachSession(session('pi', transcript([{ events: [] }])))).toBe(true)
+ expect(asking.normalizers.piNormalizers.has('pi-s')).toBe(true)
+ expect(asking.service.observeTranscript).not.toHaveBeenCalled()
+ // The device watches, but cannot take a line in; the profile cannot be hydrated either.
+ const run = setup()
+ run.service.needsTranscript.mockReturnValue(true)
+ run.service.observeTranscript.mockImplementation(() => { throw new Error('evidence unreadable') })
+ vi.mocked(run.deps.runtimeProfiles.hydrate).mockImplementation(() => { throw 'profile unreadable' })
+ expect(await run.attach.attachSession(session('pi', transcript([{ events: [] }, { events: [] }])))).toBe(true)
+ expect(await run.attach.attachSession(session('claude', transcript([CLAUDE_PROMPT])))).toBe(true)
+ expect(run.normalizers.piNormalizers.has('pi-s')).toBe(true)
+ expect(run.service.observeTranscript).toHaveBeenCalledWith('claude-agent', 'claude-s', 'claude', expect.any(String))
+ expect(run.profile.commit).toHaveBeenCalled()
+ expect(error.mock.calls).toEqual([
+ ['[transcripts] the device could not take in a line of pi-s; the line goes on to its engine: device state unreadable'],
+ ['[transcripts] the device could not take in a line of pi-s; the line goes on to its engine: evidence unreadable'],
+ ['[transcripts] the runtime profile could not take in a line of pi-s; the line goes on to its engine: profile unreadable'],
+ ['[transcripts] the device could not take in a line of claude-s; the line goes on to its engine: evidence unreadable'],
+ ])
+ })
+
it('folds a resumed conversation only up to where its relaunched engine began, and tails the rest live', async () => {
vi.spyOn(console, 'log').mockImplementation(() => {})
const take = vi.fn((_sessionId: string): RelaunchMark | undefined => undefined)
diff --git a/cli/src/core/transcripts/attach.ts b/cli/src/core/transcripts/attach.ts
index 4f7d151a0..6fb891295 100644
--- a/cli/src/core/transcripts/attach.ts
+++ b/cli/src/core/transcripts/attach.ts
@@ -36,6 +36,7 @@ import { tailFileCapped, WHOLE_READ_CAP_BYTES } from '../../lib/transcriptTail.j
import type { TailHold, Watcher } from '../../watcher/watcher.js'
import type { SessionNormalizers } from './normalizers.js'
import type { RelaunchMarks } from './relaunch.js'
+import { createSideReads } from './sideReads.js'
export interface AttachDeps {
/** Whether the session's terminal is known to be gone (core/terminals/control.ts `terminalGone`). */
@@ -74,6 +75,7 @@ export function createAttach({
turnStates, codexNormalizers, cursorNormalizers, opencodeReaders, kiloReaders, museNormalizers, ampNormalizers,
grokNormalizers, agyNormalizers, copilotNormalizers, piNormalizers, hermesReaders, devinReaders, commandcodeNormalizers,
} = normalizers
+ const sideRead = createSideReads()
/**
* Sessions that attached before their transcript existed, so nothing was folded and nothing has ever
* been streamed for them.
@@ -159,8 +161,10 @@ export function createAttach({
|| (session.engine === 'cursor' && replayCursorFromStart))
const historyEvents: LiveEvent[] = []
let historyTurnOpen = false
- const observe = device()?.needsTranscript(session.agentId, session.sessionId, session.engine)
- ? (line: string): void => device()?.observeTranscript(session.agentId, session.sessionId, session.engine, line)
+ let observed = false
+ sideRead('device', session.sessionId, () => { observed = !!device()?.needsTranscript(session.agentId, session.sessionId, session.engine) })
+ const observe = observed
+ ? (line: string): void => sideRead('device', session.sessionId, () => device()?.observeTranscript(session.agentId, session.sessionId, session.engine, line))
: undefined
// Returns `turnOpen` rather than assigning it: every engine folds exactly once, and a second call
// quietly overwriting the first is the kind of mistake a returned value makes impossible to write.
@@ -237,7 +241,7 @@ export function createAttach({
}
if (!fromEnd) {
if (observe) for (const line of lines) observe(line)
- runtimeProfiles.hydrate(session, lines)
+ sideRead('runtime profile', session.sessionId, () => runtimeProfiles.hydrate(session, lines))
}
await runtimeProfiles.ingestConfig(session, true)
// From here to the release in `attachSession` nothing is awaited for a held tail, so the hold cannot
diff --git a/cli/src/core/transcripts/ingest.spec.ts b/cli/src/core/transcripts/ingest.spec.ts
index c487aa512..25e18ac6c 100644
--- a/cli/src/core/transcripts/ingest.spec.ts
+++ b/cli/src/core/transcripts/ingest.spec.ts
@@ -65,6 +65,41 @@ describe('ingesting a transcript line', () => {
expect(none.ingest.ingestLine(line('s1', 'claude', CLAUDE_PROMPT))).toEqual(expect.any(Array))
})
+ it('gives a line to its engine though the device or the runtime profile cannot take it in, and says so once a session', () => {
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const clean = setup({ s1: 'claude' })
+ const expected = clean.ingest.ingestLine(line('s1', 'claude', CLAUDE_PROMPT))
+ expect(expected).not.toEqual([])
+ const { ingest, deps, service } = setup({ s1: 'claude', s2: 'claude' })
+ service.needsTranscript.mockReturnValue(true)
+ service.observeTranscript.mockImplementation(() => { throw new Error('evidence unreadable') })
+ vi.mocked(deps.runtimeProfiles.ingest).mockImplementation(() => { throw 'profile unreadable' })
+ // The line's events are the ones it has without the two readers: a turn's start, here.
+ expect(ingest.ingestLine(line('s1', 'claude', CLAUDE_PROMPT))).toEqual(expected)
+ ingest.ingestLine(line('s1', 'claude', '{}'))
+ ingest.ingestLine(line('s2', 'claude', CLAUDE_PROMPT))
+ expect(deps.runtimeProfiles.ingest).toHaveBeenCalledTimes(3)
+ expect(error.mock.calls).toEqual([
+ ['[transcripts] the device could not take in a line of s1; the line goes on to its engine: evidence unreadable'],
+ ['[transcripts] the runtime profile could not take in a line of s1; the line goes on to its engine: profile unreadable'],
+ ['[transcripts] the device could not take in a line of s2; the line goes on to its engine: evidence unreadable'],
+ ['[transcripts] the runtime profile could not take in a line of s2; the line goes on to its engine: profile unreadable'],
+ ])
+ })
+
+ it('keeps every line of a catch-up batch though a reader throws on the first', () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ const run = setup({ s1: 'claude' })
+ const watcher = new EventEmitter()
+ run.ingest.wireWatcher(watcher as unknown as Pick<Watcher, 'on'>)
+ vi.mocked(run.deps.runtimeProfiles.ingest).mockImplementationOnce(() => { throw new Error('bad record') })
+ const reply = JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: 'hi' }] } })
+ watcher.emit('history', { sessionId: 's1', lines: [line('s1', 'claude', CLAUDE_PROMPT), line('s1', 'claude', reply)] } as HistoryEvent)
+ const clean = setup({ s1: 'claude' })
+ const expected = [...clean.ingest.ingestLine(line('s1', 'claude', CLAUDE_PROMPT))!, ...clean.ingest.ingestLine(line('s1', 'claude', reply))!]
+ expect(vi.mocked(run.deps.emit).mock.calls).toEqual([['s1', expected, { replay: true }]])
+ })
+
it('creates each file engine\'s normalizer on the session\'s first line, and keeps it', () => {
const engines = {
codex: CodexNormalizer, cursor: CursorNormalizer, muse: MuseNormalizer, amp: AmpNormalizer, grok: GrokNormalizer,
@@ -126,9 +161,9 @@ describe('the watcher wiring', () => {
run.watcher.emit('line', line('nobody', 'claude', CLAUDE_PROMPT))
expect(vi.mocked(run.deps.emit).mock.calls).toHaveLength(1)
expect(vi.mocked(run.deps.emit).mock.calls[0][0]).toBe('s1')
- vi.mocked(run.deps.runtimeProfiles.ingest).mockImplementationOnce(() => { throw new Error('bad line') })
+ vi.mocked(run.deps.tokenUsage.changed).mockImplementationOnce(() => { throw new Error('bad line') })
run.watcher.emit('line', line('s1', 'claude', CLAUDE_PROMPT))
- vi.mocked(run.deps.runtimeProfiles.ingest).mockImplementationOnce(() => { throw 'worse' })
+ vi.mocked(run.deps.tokenUsage.changed).mockImplementationOnce(() => { throw 'worse' })
run.watcher.emit('line', line('s1', 'claude', CLAUDE_PROMPT))
expect(error.mock.calls).toEqual([
['[cli] line handler error (session s1):', 'bad line'],
@@ -144,9 +179,9 @@ describe('the watcher wiring', () => {
run.watcher.emit('history', batch([line('nobody', 'claude')]))
expect(vi.mocked(run.deps.emit).mock.calls).toHaveLength(1)
expect(vi.mocked(run.deps.emit).mock.calls[0][2]).toEqual({ replay: true })
- vi.mocked(run.deps.runtimeProfiles.ingest).mockImplementationOnce(() => { throw new Error('bad batch') })
+ vi.mocked(run.deps.tokenUsage.changed).mockImplementationOnce(() => { throw new Error('bad batch') })
run.watcher.emit('history', batch([line('s1', 'claude', CLAUDE_PROMPT)]))
- vi.mocked(run.deps.runtimeProfiles.ingest).mockImplementationOnce(() => { throw 'worse' })
+ vi.mocked(run.deps.tokenUsage.changed).mockImplementationOnce(() => { throw 'worse' })
run.watcher.emit('history', batch([line('s1', 'claude', CLAUDE_PROMPT)]))
expect(error.mock.calls).toEqual([
['[cli] history handler error (session s1):', 'bad batch'],
diff --git a/cli/src/core/transcripts/ingest.ts b/cli/src/core/transcripts/ingest.ts
index 9e2732bb3..d7c931b9d 100644
--- a/cli/src/core/transcripts/ingest.ts
+++ b/cli/src/core/transcripts/ingest.ts
@@ -25,6 +25,7 @@ import type { RegisteredSession } from '../../lib/registry.js'
import type { RuntimeProfileManager } from '../../lib/runtimeProfile.js'
import type { HistoryEvent, LineEvent, RewrittenEvent, Watcher } from '../../watcher/watcher.js'
import type { SessionNormalizers } from './normalizers.js'
+import { createSideReads } from './sideReads.js'
export interface IngestDeps {
has: (sessionId: string) => boolean
@@ -46,6 +47,7 @@ export function createIngest({
turnStates, codexNormalizers, cursorNormalizers, museNormalizers, ampNormalizers, grokNormalizers, agyNormalizers,
copilotNormalizers, piNormalizers, commandcodeNormalizers,
} = normalizers
+ const sideRead = createSideReads()
// JSONL watcher → normalize each appended line → stream up. ONE lineToEvents pass feeds BOTH
// audiences: web (send, ServerEvents) and device (mirror.ingest → curated commander_event cards).
/** One transcript line through its engine's normalizer. The events, not yet emitted — the two
@@ -55,11 +57,13 @@ export function createIngest({
const session = bySession(evt.sessionId)
if (!session || session.engine !== evt.engine) return null
tokenUsage.changed(session)
- const service = device()
- if (service?.needsTranscript(session.agentId, evt.sessionId, session.engine)) {
- service.observeTranscript(session.agentId, evt.sessionId, session.engine, evt.text)
- }
- runtimeProfiles.ingest(session, evt.text)
+ sideRead('device', evt.sessionId, () => {
+ const service = device()
+ if (service?.needsTranscript(session.agentId, evt.sessionId, session.engine)) {
+ service.observeTranscript(session.agentId, evt.sessionId, session.engine, evt.text)
+ }
+ })
+ sideRead('runtime profile', evt.sessionId, () => runtimeProfiles.ingest(session, evt.text))
let events
if (session.engine === 'codex') {
let normalizer = codexNormalizers.get(evt.sessionId)
diff --git a/cli/src/core/transcripts/sideReads.ts b/cli/src/core/transcripts/sideReads.ts
new file mode 100644
index 000000000..9cd8bfb38
--- /dev/null
+++ b/cli/src/core/transcripts/sideReads.ts
@@ -0,0 +1,26 @@
+/**
+ * The readers that see a transcript's lines beside its engine's normalizer: the device's delivery
+ * evidence (experimental) and the runtime profile (model, effort, mode).
+ *
+ * Neither may cost a line its events. Live, a throw from either dropped the line before its
+ * normalizer saw it, and a turn's end with it, so the agent read as working until its next turn; a
+ * catch-up batch lost every line after it; an attach failed. A reader that throws is reported once
+ * per session and asked again on the next line.
+ */
+import { sid } from '../../lib/log.js'
+
+export type SideRead = (reader: string, sessionId: string, read: () => void) => void
+
+export function createSideReads(): SideRead {
+ const reported = new Set<string>()
+ return (reader, sessionId, read) => {
+ try {
+ read()
+ } catch (error) {
+ const key = `${reader}\n${sessionId}`
+ if (reported.has(key)) return
+ reported.add(key)
+ console.error(`[transcripts] the ${reader} could not take in a line of ${sid(sessionId)}; the line goes on to its engine: ${error instanceof Error ? error.message : String(error)}`)
+ }
+ }
+}
diff --git a/cli/src/lib/autonomous-device/localApi.ts b/cli/src/lib/autonomous-device/localApi.ts
index 69456e629..902d3ad6f 100644
--- a/cli/src/lib/autonomous-device/localApi.ts
+++ b/cli/src/lib/autonomous-device/localApi.ts
@@ -24,7 +24,7 @@ const routes: Record<string, string> = {
const errorStatus: Record<string, number> = {
BAD_REQUEST: 400, UNKNOWN_DEVICE: 404, ALREADY_PAIRED: 409, BUSY: 409,
DEVICE_NOT_FOUND: 404, BAD_CODE: 400, NO_INTENT: 409, STALE_PAIR: 409, CODE_MISMATCH: 403, BACKEND_DOWN: 503, TIMEOUT: 504, RATE_LIMITED: 429, EXPIRED: 410, CANCELLED: 409,
- LOCAL_NETWORK_BLOCKED: 503,
+ LOCAL_NETWORK_BLOCKED: 503, UNAVAILABLE: 503,
}
function failure(status: number, code: string, message: string): AutonomousDeviceLocalResponse {
diff --git a/cli/src/lib/autonomous-device/parts.spec.ts b/cli/src/lib/autonomous-device/parts.spec.ts
new file mode 100644
index 000000000..43ed64170
--- /dev/null
+++ b/cli/src/lib/autonomous-device/parts.spec.ts
@@ -0,0 +1,62 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { AutonomousDeviceDirect, type DirectDeviceHost } from './direct.js'
+import { autonomousDeviceLocalRequest } from './localApi.js'
+import { runningDevicePart, startDevicePart } from './parts.js'
+import { DeviceResultJournal } from './resultJournal.js'
+import { AutonomousDeviceService } from './service.js'
+
+const dirs: string[] = []
+afterEach(() => {
+ vi.restoreAllMocks()
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true })
+})
+function dataDir(): string {
+ const dir = mkdtempSync(join(tmpdir(), 'device-parts-'))
+ dirs.push(dir)
+ return dir
+}
+const host = {} as DirectDeviceHost
+
+describe('the Wi-Fi device pieces the core starts', () => {
+ it('leaves out a piece whose state file does not parse, and says so, instead of failing the core\'s start', () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const dir = dataDir()
+ writeFileSync(join(dir, 'autonomous-device-connections.json'), '{"not": "a list"}', { mode: 0o600 })
+ writeFileSync(join(dir, 'device-results.json'), '{"version": 1, "machineId": "machine", "entries": "garbage"', { mode: 0o600 })
+ // Built bare, each throws: before, out of the core's start, which put the daemon in safe mode.
+ expect(() => new AutonomousDeviceDirect(host, dir)).toThrow('Invalid device connection metadata')
+ const service = () => new AutonomousDeviceService({ machineId: 'machine', resultJournal: new DeviceResultJournal(join(dir, 'device-results.json')) } as unknown as ConstructorParameters<typeof AutonomousDeviceService>[0])
+ expect(service).toThrow()
+ expect(startDevicePart('Wi-Fi device link', () => new AutonomousDeviceDirect(host, dir))).toBeUndefined()
+ expect(startDevicePart('Wi-Fi device service', service)).toBeUndefined()
+ expect(startDevicePart('test piece', () => { throw 'not an Error' })).toBeUndefined()
+ expect(warn.mock.calls.map((call) => call[0])).toEqual([
+ '[device] the Wi-Fi device link could not be started, and this daemon runs without it: Invalid device connection metadata',
+ expect.stringMatching(/^\[device\] the Wi-Fi device service could not be started, and this daemon runs without it: /),
+ '[device] the test piece could not be started, and this daemon runs without it: not an Error',
+ ])
+ })
+
+ it('starts a piece whose state reads', () => {
+ const direct = startDevicePart('Wi-Fi device link', () => new AutonomousDeviceDirect(host, dataDir()))
+ expect(direct).toBeInstanceOf(AutonomousDeviceDirect)
+ expect(runningDevicePart(direct, 'Wi-Fi device link')).toBe(direct)
+ })
+
+ it('answers a request that needs a piece that is not running UNAVAILABLE', async () => {
+ const missing = undefined as AutonomousDeviceDirect | undefined
+ const management = {
+ discover: async () => ({ devices: await runningDevicePart(missing, 'Wi-Fi device link').discover() }),
+ pairStart: () => runningDevicePart(missing, 'Wi-Fi device link').pair('device', 'ABC234'),
+ pairStatus: () => ({}), list: () => ({}), status: () => ({}), revoke: () => ({}), receipt: () => ({}),
+ }
+ expect(await autonomousDeviceLocalRequest(management, 'GET', '/api/autonomous-device/discover', undefined)).toEqual({
+ status: 503,
+ body: { error: { code: 'UNAVAILABLE', message: 'The Wi-Fi device link is not running on this computer: its state could not be read. See the daemon\'s log.' } },
+ })
+ expect((await autonomousDeviceLocalRequest(management, 'POST', '/api/autonomous-device/pair/start', { code: 'ABC234', device: 'device' })).status).toBe(503)
+ })
+})
diff --git a/cli/src/lib/autonomous-device/parts.ts b/cli/src/lib/autonomous-device/parts.ts
new file mode 100644
index 000000000..aa1a7edcb
--- /dev/null
+++ b/cli/src/lib/autonomous-device/parts.ts
@@ -0,0 +1,23 @@
+/**
+ * The Wi-Fi device's pieces, as the core starts them: each one, or none.
+ *
+ * The Wi-Fi device is experimental, and its service and its direct link read their state files when
+ * they are built. One that did not parse threw out of the core's start, and a core whose start throws
+ * sits in safe mode: every terminal on the computer down, for a feature most never use. A piece that
+ * cannot be built is now left out, and said so; the core starts without it, and the requests that
+ * need it answer UNAVAILABLE. The files are left as they are, for whoever looks into them.
+ */
+export function startDevicePart<T>(what: string, build: () => T): T | undefined {
+ try {
+ return build()
+ } catch (error) {
+ console.warn(`[device] the ${what} could not be started, and this daemon runs without it: ${error instanceof Error ? error.message : String(error)}`)
+ return undefined
+ }
+}
+
+/** [part], or an UNAVAILABLE refusal for a request that needs the piece that could not be started. */
+export function runningDevicePart<T>(part: T | undefined, what: string): T {
+ if (part === undefined) throw Object.assign(new Error(`The ${what} is not running on this computer: its state could not be read. See the daemon's log.`), { code: 'UNAVAILABLE' })
+ return part
+}
diff --git a/cli/src/lib/registry.spec.ts b/cli/src/lib/registry.spec.ts
index 495e44973..652055cfd 100644
--- a/cli/src/lib/registry.spec.ts
+++ b/cli/src/lib/registry.spec.ts
@@ -1851,6 +1851,37 @@ describe('registry across a reboot and pane loss', () => {
expect(saved.map((row) => row.agentId).sort()).toEqual([a, b].sort())
})
+ it('stops holding saves back for a transaction that outlives its hold, and saves what it has changed', async () => {
+ // A reconcile pass stuck inside its transaction held every save back for as long as it ran.
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const { registry } = await loadRegistryModule()
+ registry.load()
+ const saved = () => (JSON.parse(readFileSync(join(dataDir, 'registry.json'), 'utf-8')) as Array<Record<string, unknown>>).map((row) => row.agentId)
+ let finish!: () => void
+ let inside!: string
+ const stuck = registry.transaction(async () => {
+ inside = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%7', processIdentity: processIdentity(71) })!.entry.agentId
+ await new Promise<void>((resolve) => { finish = resolve })
+ }, { holdSavesMs: 50 })
+ expect(existsSync(join(dataDir, 'registry.json')) ? saved() : []).not.toContain(inside)
+ await vi.waitFor(() => expect(saved()).toContain(inside))
+ expect(warn).toHaveBeenCalledWith('[registry] a transaction has held saves back for 50 ms; saving without waiting for it')
+ // Saved as they happen from here on, and the stuck apply finishing changes nothing about that.
+ const after = registry.openProcessAgent({ engine: 'claude', tmuxPane: '%8', processIdentity: processIdentity(72) })!.entry.agentId
+ expect(saved()).toContain(after)
+ finish()
+ await stuck
+ // One that finishes in time holds its saves to the end, as before.
+ let during: string[] = []
+ await registry.transaction(() => {
+ registry.openProcessAgent({ engine: 'claude', tmuxPane: '%9', processIdentity: processIdentity(73) })
+ during = saved() as string[]
+ }, { holdSavesMs: 60_000 })
+ expect(during).toHaveLength(2)
+ expect(saved()).toHaveLength(3)
+ warn.mockRestore()
+ })
+
it('adopts a new process into an agent whose identity was cleared, keeping its id and session', async () => {
const transcriptPath = join(dataDir, 'session-c.jsonl')
writeFileSync(transcriptPath, '{}\n')
diff --git a/cli/src/lib/registry.ts b/cli/src/lib/registry.ts
index f7f04411f..070481aaa 100644
--- a/cli/src/lib/registry.ts
+++ b/cli/src/lib/registry.ts
@@ -2110,17 +2110,36 @@ class Registry {
return this.list().filter((entry) => this.terminalAvailableAgents.has(entry.agentId))
}
- async transaction<T>(apply: () => T | Promise<T>): Promise<T> {
+ /**
+ * Saves are held back while [apply] runs, and made once when it is done. [holdSavesMs] bounds the
+ * hold: a transaction whose apply has not finished by then stops holding saves back — what it has
+ * changed so far is saved, and what it changes after is saved as it changes — instead of the
+ * registry going unsaved for as long as it runs. A reconcile pass stuck on an engine's files did
+ * that (RECONCILE_PASS_DEADLINE_MS).
+ */
+ async transaction<T>(apply: () => T | Promise<T>, options: { holdSavesMs?: number } = {}): Promise<T> {
this.transactionDepth++
- try {
- return await apply()
- } finally {
+ let holding = true
+ const release = (): void => {
+ if (!holding) return
+ holding = false
this.transactionDepth--
if (this.transactionDepth === 0 && this.savePending) {
this.savePending = false
this.save()
}
}
+ const timer = options.holdSavesMs === undefined ? undefined : setTimeout(() => {
+ console.warn(`[registry] a transaction has held saves back for ${options.holdSavesMs} ms; saving without waiting for it`)
+ release()
+ }, options.holdSavesMs)
+ timer?.unref?.()
+ try {
+ return await apply()
+ } finally {
+ clearTimeout(timer)
+ release()
+ }
}
flush(): void {
diff --git a/cli/src/lib/runtimeInstall.spec.ts b/cli/src/lib/runtimeInstall.spec.ts
index ca276a008..e0968b61e 100644
--- a/cli/src/lib/runtimeInstall.spec.ts
+++ b/cli/src/lib/runtimeInstall.spec.ts
@@ -23,12 +23,12 @@ async function load() {
/** A real gzipped tarball laid out the way every managed runtime ships — `<name>-<version>-<key>/bin/<name>`
* (nodejs.org's own shape, which tmux and grid copy) — with a runnable binary inside. */
-function buildArchive(version: string, key: string, name: string = 'node'): { bytes: Buffer; root: string } {
+function buildArchive(version: string, key: string, name: string = 'node', script: string = '#!/bin/sh\nexit 0\n'): { bytes: Buffer; root: string } {
const archiveRoot = `${name}-${version}-${key}`
const source = join(root, 'src', archiveRoot)
rmSync(source, { recursive: true, force: true })
mkdirSync(join(source, archiveRoot, 'bin'), { recursive: true })
- writeFileSync(join(source, archiveRoot, 'bin', name), '#!/bin/sh\nexit 0\n', { mode: 0o755 })
+ writeFileSync(join(source, archiveRoot, 'bin', name), script, { mode: 0o755 })
const archive = join(root, `${archiveRoot}.tar.gz`)
execFileSync('/usr/bin/tar', ['-czf', archive, '-C', source, archiveRoot])
return { bytes: readFileSync(archive), root: archiveRoot }
@@ -137,6 +137,35 @@ describe('ensureManagedRuntime', () => {
expect(readdirSync(runtimeDir).filter((n) => n.startsWith('.node-staging-'))).toEqual([])
})
+ it('keeps the event loop it shares with every session turning while a new runtime answers --version', async () => {
+ // The core asks this on every start and, for grid, every ten minutes while it runs. Asked
+ // synchronously, a slow first run — a onefile grid unpacking itself takes most of a minute —
+ // held up every client, heartbeat and transcript for as long as it took.
+ const key = currentPlatformKey()
+ const { bytes, root: archiveRoot } = buildArchive('v22.23.4', key, 'node', '#!/bin/sh\nsleep 1\nexit 0\n')
+ stubFetch(manifestFor('node', key, 'v22.23.4', bytes, archiveRoot), bytes)
+ const { ensureManagedRuntime } = await load()
+
+ let ticks = 0
+ const ticker = setInterval(() => { ticks += 1 }, 20)
+ try {
+ expect(await ensureManagedRuntime()).toBe(join(runtimeDir, `node-v22.23.4-${key}`, 'bin', 'node'))
+ } finally { clearInterval(ticker) }
+ // A second of --version is fifty ticks of 20 ms; held synchronously it is none.
+ expect(ticks).toBeGreaterThan(10)
+ })
+
+ it('keeps what it has when the archive does not unpack, and leaves no staging behind', async () => {
+ const key = currentPlatformKey()
+ const bytes = Buffer.from('not a gzipped tarball')
+ stubFetch(manifestFor('node', key, 'v22.23.5', bytes, `node-v22.23.5-${key}`), bytes)
+ const { ensureManagedRuntime } = await load()
+
+ expect(await ensureManagedRuntime()).toBeNull()
+ expect(existsSync(join(runtimeDir, 'current-node'))).toBe(false)
+ expect(readdirSync(runtimeDir).filter((n) => n.startsWith('.node-staging-'))).toEqual([])
+ })
+
it('downloads a runtime with no deadline, only a silence of five minutes: a slow link must be able to finish it', async () => {
// The bundle's own limits (a quarter of an hour in all) would cut a runtime of tens of MB off on a
// slow link, and every daemon runs on it (lib/selfUpdate.ts RUNTIME_DOWNLOAD_LIMITS).
@@ -215,6 +244,36 @@ describe('ensureManagedRuntime', () => {
* that already has an older one. Everything else — the archive shape, the staging, the pointer —
* is the runtime convention Node and tmux already use.
*/
+describe('finishesCleanly', () => {
+ it('says whether a child exits 0, without holding up the event loop while it runs', async () => {
+ const { finishesCleanly } = await import('./runtimeInstall.js')
+ let ticks = 0
+ const ticker = setInterval(() => { ticks += 1 }, 20)
+ try {
+ expect(await finishesCleanly('/bin/sh', ['-c', 'sleep 0.6; exit 0'], 10_000)).toBe(true)
+ } finally { clearInterval(ticker) }
+ expect(ticks).toBeGreaterThan(5)
+ expect(await finishesCleanly('/bin/sh', ['-c', 'exit 3'], 10_000)).toBe(false)
+ // Neither a missing executable nor one spawn refuses outright is an exception for its caller.
+ expect(await finishesCleanly(join(tmpdir(), 'no-such-runtime-binary'), [], 10_000)).toBe(false)
+ expect(await finishesCleanly('/bin/sh\0', [], 10_000)).toBe(false)
+ })
+
+ it('gives up on a child at its deadline, and SIGKILLs one that ignores the SIGTERM', async () => {
+ const { finishesCleanly } = await import('./runtimeInstall.js')
+ let started = performance.now()
+ // A child that exits 0 after its deadline is still a no: the deadline is the answer.
+ expect(await finishesCleanly('/bin/sh', ['-c', 'sleep 30'], 200)).toBe(false)
+ expect(performance.now() - started).toBeLessThan(10_000)
+ started = performance.now()
+ expect(await finishesCleanly('/bin/sh', ['-c', 'trap "" TERM; exec sleep 30'], 200)).toBe(false)
+ const took = performance.now() - started
+ // The SIGTERM is ignored; the SIGKILL two seconds on is not.
+ expect(took).toBeGreaterThan(2_000)
+ expect(took).toBeLessThan(15_000)
+ }, 30_000)
+})
+
describe('ensureManagedGrid', () => {
const key = currentPlatformKey()
diff --git a/cli/src/lib/runtimeInstall.ts b/cli/src/lib/runtimeInstall.ts
index 049980560..21d96f593 100644
--- a/cli/src/lib/runtimeInstall.ts
+++ b/cli/src/lib/runtimeInstall.ts
@@ -1,4 +1,4 @@
-import { execFileSync } from 'node:child_process'
+import { spawn } from 'node:child_process'
import { accessSync, chmodSync, constants, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { dirname, join, sep } from 'node:path'
@@ -47,15 +47,54 @@ function artifactFor(document: unknown, name: string, key: string): RuntimeArtif
return { version: raw.version, url: raw.url, sha256: raw.sha256, size: raw.size, archiveRoot: raw.archiveRoot }
}
+/** How long a child that outlived its deadline has between SIGTERM and SIGKILL. */
+const KILL_GRACE_MS = 2_000
+
+/**
+ * Runs [file] to its end and says whether it exited 0 within [timeoutMs] — WITHOUT holding up the
+ * event loop while it runs.
+ *
+ * This module runs inside the core: on every start, and every ten minutes while it runs
+ * ([startGridPinRecheck]). A synchronous exec here froze the core for as long as the child took — up
+ * to two minutes for an unpack and one for a onefile grid's first `--version` — and a core that does
+ * not turn its event loop answers no client, beats no heartbeat, and is restarted by its master. A
+ * child that outlives its deadline is sent SIGTERM, as the synchronous exec did, and SIGKILL if it
+ * is still there [KILL_GRACE_MS] later; the answer waits for it to be gone, so a staging directory is
+ * never removed under a tar that is still writing into it.
+ */
+export function finishesCleanly(file: string, args: string[], timeoutMs: number, childEnv?: NodeJS.ProcessEnv): Promise<boolean> {
+ return new Promise((resolve) => {
+ let child: ReturnType<typeof spawn>
+ try {
+ child = spawn(file, args, { stdio: 'ignore', ...(childEnv ? { env: childEnv } : {}) })
+ } catch {
+ resolve(false)
+ return
+ }
+ let timedOut = false
+ let grace: NodeJS.Timeout | undefined
+ const deadline = setTimeout(() => {
+ timedOut = true
+ child.kill('SIGTERM')
+ grace = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS)
+ }, timeoutMs)
+ let settled = false
+ const settle = (ok: boolean) => {
+ if (settled) return
+ settled = true
+ clearTimeout(deadline)
+ clearTimeout(grace)
+ resolve(ok)
+ }
+ child.once('error', () => settle(false))
+ child.once('close', (code) => settle(!timedOut && code === 0))
+ })
+}
+
/** Does `binary --version` answer? `env` and `timeout` for a runtime whose first run is slow — a
* onefile grid unpacks itself the first time, and must not be told to update itself while it does. */
-function runs(binary: string, timeout: number = 15_000, childEnv?: NodeJS.ProcessEnv): boolean {
- try {
- execFileSync(binary, ['--version'], { timeout, stdio: 'ignore', ...(childEnv ? { env: childEnv } : {}) })
- return true
- } catch {
- return false
- }
+function runs(binary: string, timeout: number = 15_000, childEnv?: NodeJS.ProcessEnv): Promise<boolean> {
+ return finishesCleanly(binary, ['--version'], timeout, childEnv)
}
/**
@@ -83,7 +122,7 @@ interface ManagedArchive {
/** A version the manifest may not go below — this daemon's floor, which a manifest cannot lower. */
acceptsVersion?: (version: string) => boolean
/** Does the laid-down binary run on this computer? Asked before the pointer is written. */
- runs: (binary: string) => boolean
+ runs: (binary: string) => Promise<boolean>
/**
* Lay the binary and its directory down read-only. For a runtime whose own updater would replace
* the file in place — `grid update` is an os.replace INTO the directory — a directory it cannot
@@ -132,7 +171,7 @@ async function ensureManagedArchive(spec: ManagedArchive): Promise<string | null
mkdirSync(staging, { recursive: true, mode: 0o700 })
const archive = join(staging, `${spec.name}.tar.gz`)
writeFileSync(archive, bytes)
- execFileSync('/usr/bin/tar', ['-xzf', archive, '-C', staging], { timeout: 120_000, stdio: 'ignore' })
+ if (!await finishesCleanly('/usr/bin/tar', ['-xzf', archive, '-C', staging], 120_000)) return installed
const unpacked = join(staging, artifact.archiveRoot)
if (!existsSync(join(unpacked, spec.binary))) return installed
// Another start may have won the race; theirs is as good as ours.
@@ -153,7 +192,7 @@ async function ensureManagedArchive(spec: ManagedArchive): Promise<string | null
// hung exec each time until the manifest moved. A new pin is a new directory, and starts clean.
const unrunnable = join(target, '.unrunnable')
if (spec.followsPin && existsSync(unrunnable)) return installed
- if (!spec.runs(binary)) {
+ if (!await spec.runs(binary)) {
if (spec.followsPin) {
try { writeFileSync(unrunnable, `${new Date().toISOString()}\n`, { mode: 0o600 }) } catch { /* the retry is the cost */ }
}
diff --git a/cli/src/lib/selfUpdate.spec.ts b/cli/src/lib/selfUpdate.spec.ts
index cafe20b7c..bbb0c8603 100644
--- a/cli/src/lib/selfUpdate.spec.ts
+++ b/cli/src/lib/selfUpdate.spec.ts
@@ -23,11 +23,11 @@ describe('selfUpdate packaging', () => {
dirs = []
})
- it('canary runs installed cli.js as ESM', () => {
+ it('canary runs installed cli.js as ESM', async () => {
const dir = tempDir()
const cli = Buffer.from('#!/usr/bin/env node\nimport { createRequire } from "module";\nconsole.log(createRequire(import.meta.url) ? "1.2.3" : "nope")\n')
- expect(canary(cli, dir)).toBe(true)
+ expect(await canary(cli, dir)).toBe(true)
})
it('stages the module package metadata next to cli.js', () => {
@@ -383,14 +383,14 @@ describe('the updater when things go wrong', () => {
.rejects.toThrow('sha256 mismatch')
})
- it('fails the canary of a build that will not run, or that it cannot even write down', () => {
+ it('fails the canary of a build that will not run, or that it cannot even write down', async () => {
const broken = Buffer.from('#!/usr/bin/env node\nprocess.exit(3)\n')
const dir = tempDir()
- expect(canary(broken, dir)).toBe(false)
+ expect(await canary(broken, dir)).toBe(false)
expect(readdirSync(dir), 'the canary cleans up after itself').toEqual([])
const file = join(dir, 'not-a-folder')
writeFileSync(file, '')
- expect(canary(Buffer.from('console.log(1)\n'), file)).toBe(false)
+ expect(await canary(Buffer.from('console.log(1)\n'), file)).toBe(false)
})
it('skips a build that fails its canary, says why, and stages nothing', async () => {
@@ -556,28 +556,46 @@ describe('the canary\'s verdicts', () => {
dirs = []
})
- it('tells a build that failed or names another version from a canary that could not be written', () => {
+ it('tells a build that failed or names another version from a canary that could not be written', async () => {
const dir = tempDir()
- expect(runCanary(Buffer.from('console.log("9.9.9")\n'), dir, '9.9.9')).toEqual({ ok: true })
- expect(runCanary(Buffer.from('console.log("loading…"); console.log("9.9.9")\n'), dir, '9.9.9')).toEqual({ ok: true })
- expect(runCanary(Buffer.from('console.log("9.9.8")\n'), dir, '9.9.9')).toEqual({ ok: false, problem: 'wrong-version', detail: 'it says it is 9.9.8' })
- expect(runCanary(Buffer.from(''), dir, '9.9.9')).toEqual({ ok: false, problem: 'wrong-version', detail: 'it says it is nothing' })
- expect(runCanary(Buffer.from('process.exit(3)\n'), dir, '9.9.9')).toEqual({ ok: false, problem: 'failed', detail: 'exit 3' })
- expect(runCanary(Buffer.from('process.kill(process.pid, "SIGKILL")\n'), dir)).toEqual({ ok: false, problem: 'failed', detail: 'signal SIGKILL' })
- expect(canary(Buffer.from('console.log("anything")\n'), dir)).toBe(true)
- expect(canary(Buffer.from('console.log("9.9.8")\n'), dir, '9.9.9')).toBe(false)
+ expect(await runCanary(Buffer.from('console.log("9.9.9")\n'), dir, '9.9.9')).toEqual({ ok: true })
+ expect(await runCanary(Buffer.from('console.log("loading…"); console.log("9.9.9")\n'), dir, '9.9.9')).toEqual({ ok: true })
+ expect(await runCanary(Buffer.from('console.log("9.9.8")\n'), dir, '9.9.9')).toEqual({ ok: false, problem: 'wrong-version', detail: 'it says it is 9.9.8' })
+ expect(await runCanary(Buffer.from(''), dir, '9.9.9')).toEqual({ ok: false, problem: 'wrong-version', detail: 'it says it is nothing' })
+ expect(await runCanary(Buffer.from('process.exit(3)\n'), dir, '9.9.9')).toEqual({ ok: false, problem: 'failed', detail: 'exit 3' })
+ expect(await runCanary(Buffer.from('process.kill(process.pid, "SIGKILL")\n'), dir)).toEqual({ ok: false, problem: 'failed', detail: 'signal SIGKILL' })
+ expect(await canary(Buffer.from('console.log("anything")\n'), dir)).toBe(true)
+ expect(await canary(Buffer.from('console.log("9.9.8")\n'), dir, '9.9.9')).toBe(false)
const file = join(dir, 'not-a-folder')
writeFileSync(file, '')
- expect(runCanary(Buffer.from('console.log(1)\n'), file, '1')).toMatchObject({ ok: false, problem: 'unwritable' })
+ expect(await runCanary(Buffer.from('console.log(1)\n'), file, '1')).toMatchObject({ ok: false, problem: 'unwritable' })
expect(readdirSync(dir)).toEqual(['not-a-folder'])
})
+ it('keeps the event loop it shares with every session turning while the new build loads', async () => {
+ // It ran with spawnSync inside the core: no terminal byte, hook or heartbeat for as long as the new
+ // build took to answer, up to 15 s.
+ let ticks = 0
+ const timer = setInterval(() => { ticks++ }, 20)
+ try {
+ expect(await runCanary(Buffer.from('setTimeout(() => console.log("9.9.9"), 600)\n'), tempDir(), '9.9.9')).toEqual({ ok: true })
+ } finally { clearInterval(timer) }
+ expect(ticks).toBeGreaterThan(10)
+ })
+
+ it('gives up on a build that does not answer in time, in seconds as in milliseconds', async () => {
+ expect(await runCanary(Buffer.from('setInterval(() => {}, 1000)\n'), tempDir(), '9.9.9', 300))
+ .toEqual({ ok: false, problem: 'failed', detail: 'no answer within 300 ms' })
+ expect(await runCanary(Buffer.from('setInterval(() => {}, 1000)\n'), tempDir(), '9.9.9', 1_000))
+ .toEqual({ ok: false, problem: 'failed', detail: 'no answer within 1 s' })
+ })
+
it('says why a canary that never ran failed', async () => {
vi.resetModules()
vi.doMock('./nodeRuntime.js', () => ({ managedNodePath: () => '/nonexistent/node' }))
try {
const { runCanary: runWithoutNode } = await import('./selfUpdate.js')
- expect(runWithoutNode(Buffer.from('console.log(1)\n'), tempDir(), '1')).toMatchObject({ ok: false, problem: 'failed', detail: expect.stringContaining('ENOENT') })
+ expect(await runWithoutNode(Buffer.from('console.log(1)\n'), tempDir(), '1')).toMatchObject({ ok: false, problem: 'failed', detail: expect.stringContaining('ENOENT') })
} finally {
vi.doUnmock('./nodeRuntime.js')
vi.resetModules()
diff --git a/cli/src/lib/selfUpdate.ts b/cli/src/lib/selfUpdate.ts
index 95ba5f614..62dec6fff 100644
--- a/cli/src/lib/selfUpdate.ts
+++ b/cli/src/lib/selfUpdate.ts
@@ -15,7 +15,7 @@
import { createHash } from 'crypto'
import { existsSync, linkSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'fs'
-import { spawnSync } from 'child_process'
+import { execFile } from 'child_process'
import { join } from 'path'
import { SpawnLockBusyError, describeSpawnLockFailure } from './daemonSpawnLock.js'
import { managedNodePath } from './nodeRuntime.js'
@@ -236,14 +236,21 @@ export async function downloadVerified(ref: FileRef, limits: TransferLimits = DO
*/
export type CanaryResult = { ok: true } | { ok: false; problem: 'unwritable' | 'failed' | 'wrong-version'; detail: string }
+/** How long the canary may take to answer. */
+export const CANARY_TIMEOUT_MS = 15_000
+
/** Cheap runnability check: write the new bundle into a temp install-shaped dir and run
* `node cli.js version`. This catches broken ESM/CJS packaging before the live install is touched.
*
* Given the version the manifest names, the build must also say it is that version. A manifest that
* named an older build's bytes under a newer version (published by mistake, e2e/updateHostile.e2e.ts)
* was otherwise installed, came up, still reported the older version, found the newer one in the
- * manifest again and installed it again: a restart a minute on every machine until it was fixed. */
-export function runCanary(cliBuf: Buffer, dir: string, version?: string): CanaryResult {
+ * manifest again and installed it again: a restart a minute on every machine until it was fixed.
+ *
+ * Asynchronous. It ran with `spawnSync` inside the daemon's own updater, on the core's event loop: for
+ * as long as the new build took to load and answer, up to its 15 s timeout, no terminal byte, hook,
+ * heartbeat or request was handled, and the master kills a core that stops beating as hung. */
+export async function runCanary(cliBuf: Buffer, dir: string, version?: string, timeoutMs = CANARY_TIMEOUT_MS): Promise<CanaryResult> {
const tmpDir = join(dir, `.canary-${process.pid}-${Date.now()}`)
const tmpCli = join(tmpDir, CLI)
try {
@@ -256,8 +263,20 @@ export function runCanary(cliBuf: Buffer, dir: string, version?: string): Canary
}
// The interpreter the NEXT daemon will run on — see managedNodePath(). Canarying on this
// process's interpreter would assert about a Node the new build may never be started with.
- const r = spawnSync(managedNodePath(), [tmpCli, 'version'], { timeout: 15_000, stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf8' })
- if (r.status !== 0) return { ok: false, problem: 'failed', detail: r.error ? r.error.message : r.signal ? `signal ${r.signal}` : `exit ${r.status}` }
+ const r = await new Promise<{ failure: string | null; stdout: string }>((resolve) => {
+ execFile(managedNodePath(), [tmpCli, 'version'], { timeout: timeoutMs, encoding: 'utf8' }, (error, stdout) => {
+ if (!error) { resolve({ failure: null, stdout }); return }
+ const failed = error as NodeJS.ErrnoException & { killed?: boolean; signal?: NodeJS.Signals | null; code?: number | string }
+ resolve({
+ failure: failed.killed && failed.signal === 'SIGTERM' ? `no answer within ${timeoutMs >= 1000 ? `${Math.round(timeoutMs / 1000)} s` : `${timeoutMs} ms`}`
+ : typeof failed.code === 'number' ? `exit ${failed.code}`
+ : failed.signal ? `signal ${failed.signal}`
+ : failed.message,
+ stdout: '',
+ })
+ })
+ })
+ if (r.failure !== null) return { ok: false, problem: 'failed', detail: r.failure }
const lines = r.stdout.trim().split('\n')
const said = lines[lines.length - 1]
if (version !== undefined && said !== version) return { ok: false, problem: 'wrong-version', detail: `it says it is ${said || 'nothing'}` }
@@ -267,8 +286,8 @@ export function runCanary(cliBuf: Buffer, dir: string, version?: string): Canary
}
}
-export function canary(cliBuf: Buffer, dir: string, version?: string): boolean {
- return runCanary(cliBuf, dir, version).ok
+export async function canary(cliBuf: Buffer, dir: string, version?: string): Promise<boolean> {
+ return (await runCanary(cliBuf, dir, version)).ok
}
/**
@@ -522,7 +541,7 @@ export function startSelfUpdater(opts: {
verified = { entry: key, version: entry.version, cliBuf, notifyBuf, canaried: false }
}
if (!verified.canaried) {
- const result = runCanary(verified.cliBuf, opts.dir, entry.version)
+ const result = await runCanary(verified.cliBuf, opts.dir, entry.version)
if (!result.ok && result.problem === 'unwritable') {
console.error(`[update] could not write the canary for ${entry.version} (${result.detail}) — trying again next check`)
return
diff --git a/cli/src/lib/terminalAgentReconciler.spec.ts b/cli/src/lib/terminalAgentReconciler.spec.ts
index fd8500692..a8dc9fc70 100644
--- a/cli/src/lib/terminalAgentReconciler.spec.ts
+++ b/cli/src/lib/terminalAgentReconciler.spec.ts
@@ -654,3 +654,52 @@ describe('a terminal pane (engine `terminal`)', () => {
expect(onRemoved).toHaveBeenCalledWith(current, 'terminal runtime absent after 2 confirmed scans')
})
})
+
+describe('a pass with a deadline', () => {
+ const available = (roots: Array<{ runtime: TerminalRuntimeRef; rootPid: number; cwd: string }> = []) =>
+ [{ instanceId: 'tmux:default', result: { state: 'available' as const, roots } }]
+
+ it('gives up a probe that has not answered by the deadline, applies nothing, and probes again on the next pass', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const onRemoved = vi.fn()
+ const onDormant = vi.fn()
+ let answer: TerminalAgentProbe | null = null
+ const probed = vi.fn(() => answer ? Promise.resolve(answer) : new Promise<TerminalAgentProbe>(() => {}))
+ const reconciler = new TerminalAgentReconciler({
+ current: () => [session()], backends: [], backendOrder: ['tmux'],
+ onDiscovered: vi.fn(), onObserved: vi.fn(), onDormant, onRemoved, probe: probed, passDeadlineMs: 50,
+ })
+ // Before, this never returned: not to a hook waiting to bind, nor to anything after it.
+ await reconciler.triggerHint(tmux, 'claude')
+ expect(warn).toHaveBeenCalledWith('[discovery] the terminal probe has not answered in 50 ms; this pass is given up, every agent kept as it is')
+ expect(onRemoved).not.toHaveBeenCalled()
+ expect(onDormant).not.toHaveBeenCalled()
+ answer = probe(available())
+ await reconciler.trigger()
+ expect(probed).toHaveBeenCalledTimes(2)
+ // The hint the given-up pass did not use is still there for the one that answered.
+ expect(probed.mock.calls[1]).toEqual([new Map([[terminalRouteKey(tmux), 'claude']])])
+ warn.mockRestore()
+ })
+
+ it('lets whoever waits for a pass go on when its apply does not finish, and keeps passes one at a time', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ let release!: () => void
+ const stuck = new Promise<void>((resolve) => { release = resolve })
+ const onDiscovered = vi.fn(() => stuck)
+ const probed = vi.fn(async () => probe(available([{ runtime: tmux, rootPid: 1, cwd: '/work' }]), [observed([tmux])]))
+ const reconciler = new TerminalAgentReconciler({
+ current: () => [], backends: [], backendOrder: ['tmux'],
+ onDiscovered, onObserved: vi.fn(), onDormant: vi.fn(), onRemoved: vi.fn(), probe: probed, passDeadlineMs: 50,
+ })
+ await reconciler.trigger()
+ expect(onDiscovered).toHaveBeenCalledTimes(1)
+ await reconciler.triggerHint(tmux, 'claude')
+ expect(warn.mock.calls).toEqual([['[discovery] a pass has run for 50 ms; whoever waits for it goes on without it']])
+ // No second pass beside the stuck one; the one asked for runs once it is done.
+ expect(probed).toHaveBeenCalledTimes(1)
+ release()
+ await vi.waitFor(() => expect(probed).toHaveBeenCalledTimes(2))
+ warn.mockRestore()
+ })
+})
diff --git a/cli/src/lib/terminalAgentReconciler.ts b/cli/src/lib/terminalAgentReconciler.ts
index 786406dd9..ca9ae2a20 100644
--- a/cli/src/lib/terminalAgentReconciler.ts
+++ b/cli/src/lib/terminalAgentReconciler.ts
@@ -12,6 +12,33 @@ import type { TerminalRuntimeRef } from './terminalTypes.js'
const MISS_LIMIT = 2
+/**
+ * How long one reconcile pass may take before the core stops waiting on it.
+ *
+ * A pass asks tmux and ps (the probe), then applies what they said inside a registry transaction.
+ * Nothing bounded the whole: a probe that never answered, or an apply stuck on an engine's files,
+ * held the pass forever, and with it every hook waiting for a pass to bind its agent, every later
+ * pass (they are serial), and every registry save (a transaction holds them back). Past this, a probe
+ * that has not answered is given up — nothing is applied, every agent kept as it is, and the next pass
+ * probes again — whoever waits for the pass goes on without it, and the registry stops holding saves
+ * back for it. A pass that is merely slow still finishes, and passes stay one at a time.
+ */
+export const RECONCILE_PASS_DEADLINE_MS = 30_000
+
+const GIVEN_UP = Symbol('given up')
+/** [work]'s value, or GIVEN_UP once [ms] have passed without it. A rejection after that is dropped. */
+function withinDeadline<T>(work: Promise<T>, ms: number): Promise<T | typeof GIVEN_UP> {
+ let timer: NodeJS.Timeout | undefined
+ const deadline = new Promise<typeof GIVEN_UP>((resolve) => {
+ timer = setTimeout(() => resolve(GIVEN_UP), ms)
+ timer.unref?.()
+ })
+ return Promise.race([work, deadline]).then((value) => {
+ if (value === GIVEN_UP) work.catch(() => {})
+ return value
+ }).finally(() => clearTimeout(timer))
+}
+
export interface TerminalAgentReconcilerDeps {
current: () => RegisteredSession[]
backends: readonly TerminalBackend[]
@@ -28,6 +55,8 @@ export interface TerminalAgentReconcilerDeps {
/** Hooks may arrive while reboot restoration is still allocating panes. Keep their hints,
* but do not scan or retire any saved owners until start() opens discovery. */
deferUntilStart?: boolean
+ /** [RECONCILE_PASS_DEADLINE_MS], shorter in tests. */
+ passDeadlineMs?: number
}
function currentProcessKey(session: RegisteredSession): string | null {
@@ -227,8 +256,18 @@ export class TerminalAgentReconciler {
// restore is trying to launch. The opening pass consumes every retained hint after restore.
if (this.waitingForStart) return Promise.resolve()
if (!this.inFlight) this.inFlight = this.drain().finally(() => { this.inFlight = null })
- return this.inFlight
+ return this.waitFor(this.inFlight)
+ }
+
+ /** A pass, waited for until the pass deadline and no longer: see [RECONCILE_PASS_DEADLINE_MS]. */
+ private async waitFor(pass: Promise<void>): Promise<void> {
+ if (await withinDeadline(pass, this.passDeadlineMs) !== GIVEN_UP) return
+ if (this.overdue === pass) return
+ this.overdue = pass
+ console.warn(`[discovery] a pass has run for ${this.passDeadlineMs} ms; whoever waits for it goes on without it`)
}
+ private overdue: Promise<void> | null = null
+ private get passDeadlineMs(): number { return this.deps.passDeadlineMs ?? RECONCILE_PASS_DEADLINE_MS }
private async drain(): Promise<void> {
while (this.pending) {
@@ -253,14 +292,18 @@ export class TerminalAgentReconciler {
// anything older matters to no probe still to come.
const probeSeq = this.routeSeq
for (const [key, seq] of this.routeTouched) if (seq <= probeSeq) this.routeTouched.delete(key)
- const probe = await (this.deps.probe
+ const probe = await withinDeadline(this.deps.probe
? this.deps.probe(hints)
: probeTerminalAgents(
this.deps.backends,
this.deps.backendOrder,
this.deps.daemonPid ?? process.pid,
hints,
- ))
+ ), this.passDeadlineMs)
+ if (probe === GIVEN_UP) {
+ console.warn(`[discovery] the terminal probe has not answered in ${this.passDeadlineMs} ms; this pass is given up, every agent kept as it is`)
+ return
+ }
const availableTargets = probe.targets.filter((target) => target.result.state === 'available')
const livePlacements = new Set(availableTargets.flatMap((target) =>
target.result.state === 'available'
diff --git a/cli/src/lib/tmuxBackend.spec.ts b/cli/src/lib/tmuxBackend.spec.ts
index 04960dd8a..11d766c09 100644
--- a/cli/src/lib/tmuxBackend.spec.ts
+++ b/cli/src/lib/tmuxBackend.spec.ts
@@ -16,7 +16,9 @@ afterEach(() => {
resetTmuxVersionCache()
process.env.PATH = originalPath
delete process.env.TMUX_BACKEND_CALLS
- for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true })
+ // Retried: listing panes also remembers the tmux server in the background (lib/tmux.ts), and that
+ // fake tmux can still be writing its call into the folder as the test ends (ENOTEMPTY on CI's Linux).
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 })
})
describe('TmuxBackend lifecycle', () => {
diff --git a/cli/src/orchestrator/service.ts b/cli/src/orchestrator/service.ts
index ce4522a8a..a51835f45 100644
--- a/cli/src/orchestrator/service.ts
+++ b/cli/src/orchestrator/service.ts
@@ -29,6 +29,28 @@ export interface OrchestratorDependencies {
changed?(id: string, revision: number): void
}
+/** A saved project's file: `<run id>.json`. */
+const RUN_FILE = /^[a-f0-9]{32}\.json$/
+
+/**
+ * Whether [stateDir] holds a saved project, asked WITHOUT building the service.
+ *
+ * The daemon asks a project's role of every turn that ends, and building the service to answer makes
+ * its folder and reads and parses every saved run, synchronously, on the core's event loop — after
+ * which every frame the daemon sends is handed to it as well. Asked that way, all of it happened at
+ * the first turn end of every daemon, nearly all of which never use the orchestrator. A project is
+ * only ever made through the service, so a folder found without one stays without one until
+ * something builds it. A folder that cannot be listed counts as one: the service reports it, as it
+ * always did.
+ */
+export function hasSavedProjects(stateDir: string): boolean {
+ try {
+ return readdirSync(stateDir).some(name => RUN_FILE.test(name))
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code !== 'ENOENT'
+ }
+}
+
/** Owns tasks, not terminals. A tab closing has no effect on this service. */
export class OrchestratorService {
private readonly runs = new Map<string, Run>()
@@ -44,7 +66,7 @@ export class OrchestratorService {
private load(): void {
if (this.loaded) return
secureStateDirectory(this.deps.stateDir)
- for (const name of readdirSync(this.deps.stateDir).filter(n => /^[a-f0-9]{32}\.json$/.test(n))) {
+ for (const name of readdirSync(this.deps.stateDir).filter(n => RUN_FILE.test(n))) {
try {
const run = Run.parse(JSON.parse(readPrivateStateFile(join(this.deps.stateDir, name), 8 * 1024 * 1024)))
requireThat(name === `${run.id}.json`, 'CORRUPT_STATE', 'Project identity does not match its file.')