Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

serialize worker recovery and daemon shutdown (#408)

kt16e97e6dea79
diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md
index 546faa9d40..18b0108fd4 100644
--- a/packages/coding-agent/CHANGELOG.md
+++ b/packages/coding-agent/CHANGELOG.md
@@ -9,6 +9,7 @@
- Changed tool expansion hints to appear only on the latest tool row instead of every tool call ([ENG-4583](https://linear.app/primeintellect/issue/ENG-4583/too-many-ctrlo-alerts)).
- Changed IPython kernels to set `NO_COLOR=1`, preventing ANSI color escapes from inflating `%%bash` output.
- Fixed update restarts starting concurrent daemon supervisors or unlinking a replacement supervisor's socket ([ENG-4600](https://linear.app/primeintellect/issue/ENG-4600/prevent-concurrent-daemon-supervisors-after-update-restart)).
+- Fixed worker recovery races and made daemon shutdown-all converge across hidden supervisors ([ENG-4603](https://linear.app/primeintellect/issue/ENG-4603/serialize-worker-recovery-and-make-shutdown-all-converge)).
- Changed provider, model, and MCP setup to use one tabbed configuration menu ([ENG-4539](https://linear.app/primeintellect/issue/ENG-4539/unify-providers-models-and-mcp-connections-menu)).
- Changed the shared configuration menu to show prominent, responsive tabs with configurable navigation shortcuts ([ENG-4534](https://linear.app/primeintellect/issue/ENG-4534/make-login-tabs-more-obvious)).
- Fixed IPython edit diffs replacing syntax highlighting with a single foreground color ([ENG-4616](https://linear.app/primeintellect/issue/ENG-4616/syntax-highlighting-is-overridden-in-diff-view)).
diff --git a/packages/coding-agent/src/cli/daemon-ps.ts b/packages/coding-agent/src/cli/daemon-ps.ts
index c376bfc1b4..1b25850aab 100644
--- a/packages/coding-agent/src/cli/daemon-ps.ts
+++ b/packages/coding-agent/src/cli/daemon-ps.ts
@@ -1,11 +1,13 @@
import { spawnSync } from "node:child_process";
import { existsSync, lstatSync, readdirSync, unlinkSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { basename, dirname, join, resolve } from "node:path";
import chalk from "chalk";
import { APP_NAME, VERSION } from "../config.js";
+import { getProcessStartId } from "../core/session-lease.js";
import { DaemonClient } from "../modes/daemon/daemon-client.js";
import { DAEMON_PROTOCOL_VERSION } from "../modes/daemon/daemon-protocol.js";
import { defaultDaemonSocketDir, defaultDaemonSocketPath } from "../modes/daemon/daemon-socket.js";
+import { acquireDaemonShutdownAdmission } from "../modes/daemon/daemon-supervisor-ownership.js";
import { formatDaemonListTable } from "./daemon-ps-format.js";
/**
@@ -182,6 +184,7 @@ interface ProbeResult {
version?: string;
protocolVersion?: number;
sessionCount?: number;
+ supervisorPid?: number;
reachable: boolean;
}
@@ -196,11 +199,13 @@ async function probeDaemon(socketPath: string): Promise<ProbeResult> {
try {
let version: string | undefined;
let protocolVersion: number | undefined;
+ let supervisorPid: number | undefined;
let greeted = false;
try {
const hello = await client.waitForHello(1500);
version = hello.appVersion;
protocolVersion = hello.protocol.version;
+ supervisorPid = hello.supervisorPid;
greeted = true;
} catch {
// Connected but no recognizable greeting: an old/foreign daemon.
@@ -217,7 +222,7 @@ async function probeDaemon(socketPath: string): Promise<ProbeResult> {
} catch {
// Leave sessionCount undefined when the daemon will not answer list.
}
- return { version, protocolVersion, sessionCount, reachable: true };
+ return { version, protocolVersion, sessionCount, supervisorPid, reachable: true };
} finally {
client.close();
}
@@ -363,10 +368,22 @@ const SHUTDOWN_ALL_ACTION_ORDER: Record<ReapAction["kind"], number> = {
};
export async function runShutdownAll(json: boolean): Promise<void> {
- const daemons = await discoverDaemons();
+ const admission = await acquireDaemonShutdownAdmission();
+ try {
+ await runShutdownAllConverging(json, () => admission.assertOrRenew());
+ } finally {
+ await admission.release();
+ }
+}
+
+async function runShutdownAllConverging(json: boolean, assertAdmission: () => Promise<void>): Promise<void> {
const stopped: Array<{ socketPath: string; action: string }> = [];
const failed: Array<{ socketPath: string; reason: string }> = [];
const handledPids = new Set<number>();
+ const reportedFailures = new Set<string>();
+
+ await stopHiddenSupervisors(stopped, failed, handledPids, reportedFailures, assertAdmission);
+ const daemons = (await discoverDaemons()).filter((daemon) => !isWorkerSocketPath(daemon.socketPath));
const actions = [...planShutdownAll(daemons)].sort(
(left, right) => SHUTDOWN_ALL_ACTION_ORDER[left.kind] - SHUTDOWN_ALL_ACTION_ORDER[right.kind],
@@ -375,6 +392,7 @@ export async function runShutdownAll(json: boolean): Promise<void> {
for (const action of actions) {
const { socketPath, pid } = action.daemon;
if (pid !== undefined && handledPids.has(pid)) {
+ await assertAdmission();
removeSocketFile(socketPath);
stopped.push({ socketPath, action: `already stopped (pid ${pid})` });
continue;
@@ -382,36 +400,59 @@ export async function runShutdownAll(json: boolean): Promise<void> {
switch (action.kind) {
case "remove-file": {
if ((await probeDaemon(socketPath)).reachable) {
- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);
- } else if (removeSocketFile(socketPath)) {
- stopped.push({ socketPath, action: "removed stale socket file" });
+ apply(
+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),
+ socketPath,
+ stopped,
+ failed,
+ );
} else {
- failed.push({ socketPath, reason: "could not remove socket file" });
+ await assertAdmission();
+ if (removeSocketFile(socketPath)) {
+ stopped.push({ socketPath, action: "removed stale socket file" });
+ } else {
+ failed.push({ socketPath, reason: "could not remove socket file" });
+ }
}
break;
}
case "kill": {
if ((await probeDaemon(socketPath)).reachable) {
- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);
+ apply(
+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),
+ socketPath,
+ stopped,
+ failed,
+ );
} else if (isDaemonProcessListening(pid!, socketPath)) {
+ await assertAdmission();
await forceKillDaemon(pid!);
handledPids.add(pid!);
+ await assertAdmission();
removeSocketFile(socketPath);
stopped.push({ socketPath, action: `killed unreachable daemon (pid ${pid})` });
} else {
+ await assertAdmission();
removeSocketFile(socketPath);
stopped.push({ socketPath, action: "daemon already stopped" });
}
break;
}
case "shutdown":
- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);
+ apply(
+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),
+ socketPath,
+ stopped,
+ failed,
+ );
break;
case "skip":
break;
}
}
+ await terminateVerifiedResiduals(stopped, failed, handledPids, reportedFailures, assertAdmission);
+
if (json) {
console.log(JSON.stringify({ stopped, failed }, null, 2));
return;
@@ -428,11 +469,187 @@ export async function runShutdownAll(json: boolean): Promise<void> {
}
}
+async function stopHiddenSupervisors(
+ stopped: Array<{ socketPath: string; action: string }>,
+ failed: Array<{ socketPath: string; reason: string }>,
+ handledPids: Set<number>,
+ reportedFailures: Set<string>,
+ assertAdmission: () => Promise<void>,
+): Promise<void> {
+ while (true) {
+ const listeners = scanListeningDaemons().filter((listener) => !isWorkerSocketPath(listener.socketPath));
+ const bySocket = new Map<string, DiscoveredDaemonProcess[]>();
+ for (const listener of listeners) {
+ const group = bySocket.get(listener.socketPath) ?? [];
+ group.push(listener);
+ bySocket.set(listener.socketPath, group);
+ }
+ const hidden: DiscoveredDaemonProcess[] = [];
+ for (const [socketPath, group] of bySocket) {
+ if (new Set(group.map((listener) => listener.pid)).size < 2) {
+ continue;
+ }
+ const currentPid = (await probeDaemon(socketPath)).supervisorPid;
+ if (currentPid === undefined || !group.some((listener) => listener.pid === currentPid)) {
+ recordShutdownFailure(
+ failed,
+ reportedFailures,
+ socketPath,
+ "could not identify the current same-path daemon",
+ );
+ continue;
+ }
+ hidden.push(...group.filter((listener) => listener.pid !== currentPid));
+ }
+ if (hidden.length === 0) {
+ return;
+ }
+ const before = daemonListenerSignature(hidden);
+ for (const listener of hidden) {
+ if (await terminateVerifiedListener(listener, failed, reportedFailures, assertAdmission)) {
+ handledPids.add(listener.pid);
+ stopped.push({ socketPath: listener.socketPath, action: `stopped hidden daemon (pid ${listener.pid})` });
+ }
+ }
+ const afterHidden = scanListeningDaemons().filter(
+ (listener) =>
+ !isWorkerSocketPath(listener.socketPath) &&
+ hidden.some((candidate) => candidate.pid === listener.pid && candidate.socketPath === listener.socketPath),
+ );
+ if (afterHidden.length === 0 || daemonListenerSignature(afterHidden) === before) {
+ return;
+ }
+ }
+}
+
+async function terminateVerifiedResiduals(
+ stopped: Array<{ socketPath: string; action: string }>,
+ failed: Array<{ socketPath: string; reason: string }>,
+ handledPids: Set<number>,
+ reportedFailures: Set<string>,
+ assertAdmission: () => Promise<void>,
+): Promise<void> {
+ let previousSignature: string | undefined;
+ while (true) {
+ const listeners = scanListeningDaemons();
+ if (listeners.length === 0) {
+ return;
+ }
+ const signature = daemonListenerSignature(listeners);
+ if (signature === previousSignature) {
+ for (const listener of listeners) {
+ const processStartId = getProcessStartId(listener.pid);
+ if (processStartId && getProcessStartId(listener.pid) === processStartId) {
+ recordShutdownFailure(
+ failed,
+ reportedFailures,
+ listener.socketPath,
+ `daemon process remained after shutdown (pid ${listener.pid}, start ${processStartId})`,
+ );
+ }
+ }
+ return;
+ }
+ previousSignature = signature;
+ const seenPids = new Set<number>();
+ for (const listener of listeners) {
+ if (seenPids.has(listener.pid)) {
+ continue;
+ }
+ seenPids.add(listener.pid);
+ const alreadyReported = handledPids.has(listener.pid);
+ if (await terminateVerifiedListener(listener, failed, reportedFailures, assertAdmission)) {
+ handledPids.add(listener.pid);
+ if (!alreadyReported) {
+ stopped.push({
+ socketPath: listener.socketPath,
+ action: `stopped residual daemon process (pid ${listener.pid})`,
+ });
+ }
+ }
+ }
+ }
+}
+
+async function terminateVerifiedListener(
+ listener: DiscoveredDaemonProcess,
+ failed: Array<{ socketPath: string; reason: string }>,
+ reportedFailures: Set<string>,
+ assertAdmission: () => Promise<void>,
+): Promise<boolean> {
+ const processStartId = getProcessStartId(listener.pid);
+ if (!processStartId) {
+ recordShutdownFailure(
+ failed,
+ reportedFailures,
+ listener.socketPath,
+ `could not verify daemon process identity (pid ${listener.pid})`,
+ );
+ return false;
+ }
+ if (getProcessStartId(listener.pid) !== processStartId) {
+ return false;
+ }
+ await assertAdmission();
+ if (getProcessStartId(listener.pid) !== processStartId) {
+ return false;
+ }
+ killDaemon(listener.pid);
+ const deadline = Date.now() + 1000;
+ while (getProcessStartId(listener.pid) === processStartId && Date.now() < deadline) {
+ await delay(50);
+ }
+ if (getProcessStartId(listener.pid) === processStartId) {
+ await assertAdmission();
+ if (getProcessStartId(listener.pid) !== processStartId) {
+ return false;
+ }
+ try {
+ process.kill(listener.pid, "SIGKILL");
+ } catch {
+ // The verified process exited between the identity check and signal.
+ }
+ }
+ return getProcessStartId(listener.pid) !== processStartId;
+}
+
+function daemonListenerSignature(listeners: readonly DiscoveredDaemonProcess[]): string {
+ return listeners
+ .map((listener) => `${listener.pid}:${getProcessStartId(listener.pid) ?? "unknown"}:${listener.socketPath}`)
+ .sort()
+ .join("\n");
+}
+
+function recordShutdownFailure(
+ failed: Array<{ socketPath: string; reason: string }>,
+ reportedFailures: Set<string>,
+ socketPath: string,
+ reason: string,
+): void {
+ const key = `${socketPath}\0${reason}`;
+ if (reportedFailures.has(key)) {
+ return;
+ }
+ reportedFailures.add(key);
+ failed.push({ socketPath, reason });
+}
+
+function isWorkerSocketPath(socketPath: string): boolean {
+ return (
+ process.platform !== "win32" &&
+ resolve(dirname(socketPath)) === resolve(defaultDaemonSocketDir()) &&
+ basename(socketPath).startsWith("worker-") &&
+ basename(socketPath).endsWith(".sock")
+ );
+}
+
async function stopDaemonForcefully(
socketPath: string,
pid: number | undefined,
handledPids: Set<number>,
+ assertAdmission: () => Promise<void>,
): Promise<ReapOutcome> {
+ await assertAdmission();
if (await shutdownDaemon(socketPath)) {
if (pid !== undefined) {
handledPids.add(pid);
@@ -440,14 +657,17 @@ async function stopDaemonForcefully(
return { reaped: `stopped daemon${pid ? ` (pid ${pid})` : ""}` };
}
if (!(await canConnectToSocket(socketPath, 250))) {
+ await assertAdmission();
removeSocketFile(socketPath);
return { reaped: "daemon already stopped" };
}
if (pid === undefined) {
return { skipped: "still listening but no pid to kill" };
}
+ await assertAdmission();
await forceKillDaemon(pid);
handledPids.add(pid);
+ await assertAdmission();
removeSocketFile(socketPath);
return { reaped: `force-killed unresponsive daemon (pid ${pid})` };
}
diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts
index 3b359fa36c..56bdfe9d91 100644
--- a/packages/coding-agent/src/main.ts
+++ b/packages/coding-agent/src/main.ts
@@ -71,6 +71,7 @@ import {
DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,
isDaemonWorkerProcess,
requireDaemonWorkerAuthenticationToken,
+ waitForDaemonWorkerStartupGate,
} from "./modes/daemon/daemon-worker-protocol.js";
import {
type AgentConnection,
@@ -1091,6 +1092,9 @@ export interface MainOptions {
export async function main(args: string[], options?: MainOptions) {
resetTimings();
+ if (isDaemonWorkerProcess()) {
+ waitForDaemonWorkerStartupGate();
+ }
installFileLogSink();
if (isDaemonCatalogProcess()) {
await runDaemonCatalogProcess();
diff --git a/packages/coding-agent/src/modes/daemon/daemon-client.ts b/packages/coding-agent/src/modes/daemon/daemon-client.ts
index b51a8de475..a23a065b78 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-client.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-client.ts
@@ -288,7 +288,8 @@ export class DaemonClient {
}
async authenticateWorker(token: string, timeoutMs = 3000): Promise<void> {
- const response = await this.requestWire({ type: "worker_auth", token }, timeoutMs, {}, false);
+ const legacyAuthentication = { type: "worker_auth", token } as DaemonWorkerCommandBody;
+ const response = await this.requestWire(legacyAuthentication, timeoutMs, {}, false);
if (!response.success) {
throw new Error(response.error);
}
diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
index d5ac89d1c9..47f341a843 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
@@ -144,6 +144,7 @@ import {
prepareDaemonSocketPath,
restrictDaemonSocketPath,
} from "./daemon-socket.js";
+import { assertDaemonSupervisorOwnerCurrent, isDaemonShutdownAdmissionActive } from "./daemon-supervisor-ownership.js";
import {
DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,
DAEMON_WORKER_RECOVERY_JOURNAL_ENV,
@@ -272,6 +273,7 @@ const DAEMON_SERVER_CAPABILITIES: readonly DaemonClientCapability[] = [
const DAEMON_CLIENT_CAPABILITY_SET: ReadonlySet<string> = new Set(DAEMON_SERVER_CAPABILITIES);
const UPDATE_RESTART_ABORT_BASH_TIMEOUT_MS = 5000;
+const SUPERVISOR_FENCE_POLL_MS = 250;
const UPDATE_RESTART_MARKER =
"<prime_agent_update_interrupted>\n" +
"Prime Agent was updated and intentionally interrupted this session. Continue from the saved transcript and restored tool/kernel state. Any running model, tool, bash, or child-agent work may have been partially completed.\n" +
@@ -300,6 +302,12 @@ function delay(ms: number): Promise<void> {
}
type RuntimeOpenGuard = () => boolean | Promise<boolean>;
+type SupervisorGenerationClaim = Omit<Extract<DaemonWorkerCommand, { type: "worker_auth" }>, "id" | "type" | "token">;
+
+interface BoundSupervisorGenerationClaim {
+ claim: SupervisorGenerationClaim;
+ ownerFingerprint: string;
+}
class RuntimeOpenCancelledError extends Error {}
@@ -342,7 +350,9 @@ export class AgentDaemon {
private readonly bindingSessions = new Set<string>();
private restoreActiveSessionId: string | undefined;
private supervisorMonitorTimer?: ReturnType<typeof setTimeout>;
+ private supervisorFenceTimer?: ReturnType<typeof setTimeout>;
private supervisorLaunchInProgress = false;
+ private readonly supervisorClaims = new Map<DaemonSocketClient, BoundSupervisorGenerationClaim>();
private agentMessagesPaused = false;
private readonly summarizer = new DaemonSessionSummarizer(
() => [...this.sessions.values()],
@@ -472,7 +482,11 @@ export class AgentDaemon {
}
this.supervisorMonitorTimer = setTimeout(() => {
this.supervisorMonitorTimer = undefined;
- void this.checkSupervisorAvailability(supervisorSocketPath);
+ void this.checkSupervisorAvailability(supervisorSocketPath).catch(() => {
+ if (!this.shuttingDown && !this.hasAuthenticatedSupervisorConnection()) {
+ this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);
+ }
+ });
}, delayMs);
}
@@ -480,6 +494,10 @@ export class AgentDaemon {
if (this.shuttingDown || this.hasAuthenticatedSupervisorConnection()) {
return;
}
+ if (await isDaemonShutdownAdmissionActive()) {
+ this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);
+ return;
+ }
if (await this.canConnectToSupervisor(supervisorSocketPath)) {
return;
}
@@ -490,7 +508,7 @@ export class AgentDaemon {
}
private hasAuthenticatedSupervisorConnection(): boolean {
- return [...this.clients].some((client) => client.authenticated === true);
+ return this.supervisorClaims.size > 0;
}
private clearSupervisorAvailabilityCheck(): void {
@@ -498,6 +516,49 @@ export class AgentDaemon {
clearTimeout(this.supervisorMonitorTimer);
this.supervisorMonitorTimer = undefined;
}
+ if (this.supervisorFenceTimer) {
+ clearTimeout(this.supervisorFenceTimer);
+ this.supervisorFenceTimer = undefined;
+ }
+ }
+
+ private scheduleSupervisorFenceCheck(): void {
+ if (this.shuttingDown || this.supervisorFenceTimer || this.supervisorClaims.size === 0) {
+ return;
+ }
+ this.supervisorFenceTimer = setTimeout(() => {
+ this.supervisorFenceTimer = undefined;
+ void this.checkSupervisorFences();
+ }, SUPERVISOR_FENCE_POLL_MS);
+ }
+
+ private async checkSupervisorFences(): Promise<void> {
+ for (const [client, boundClaim] of this.supervisorClaims) {
+ try {
+ boundClaim.ownerFingerprint = await this.assertSupervisorClaimCurrent(
+ boundClaim.claim,
+ boundClaim.ownerFingerprint,
+ );
+ } catch {
+ client.socket.end();
+ }
+ }
+ this.scheduleSupervisorFenceCheck();
+ }
+
+ private assertSupervisorClaimCurrent(
+ claim: SupervisorGenerationClaim,
+ validatedFingerprint?: string,
+ ): Promise<string> {
+ return assertDaemonSupervisorOwnerCurrent(
+ {
+ generation: claim.supervisorGeneration,
+ pid: claim.supervisorPid,
+ ...(claim.supervisorProcessStartId ? { processStartId: claim.supervisorProcessStartId } : {}),
+ socketPath: claim.supervisorSocketPath,
+ },
+ validatedFingerprint,
+ );
}
private canConnectToSupervisor(socketPath: string): Promise<boolean> {
@@ -570,6 +631,9 @@ export class AgentDaemon {
if (await this.canConnectToSupervisor(supervisorSocketPath)) {
return;
}
+ if (await isDaemonShutdownAdmissionActive()) {
+ return;
+ }
const launch = createCliSubprocessLaunchSpec(["--mode", "daemon", "--daemon-socket", supervisorSocketPath]);
const environment = { ...process.env };
delete environment[DAEMON_WORKER_ROLE_ENV];
@@ -1718,6 +1782,7 @@ export class AgentDaemon {
this.detachClient(client);
client.detachInput();
this.clients.delete(client);
+ this.supervisorClaims.delete(client);
const supervisorSocketPath = process.env[DAEMON_WORKER_SUPERVISOR_SOCKET_ENV];
if (this.options.worker && client.authenticated === true && supervisorSocketPath) {
this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 100);
@@ -1744,6 +1809,10 @@ export class AgentDaemon {
id?: unknown;
type?: unknown;
token?: unknown;
+ supervisorGeneration?: unknown;
+ supervisorPid?: unknown;
+ supervisorProcessStartId?: unknown;
+ supervisorSocketPath?: unknown;
activeSessionId?: unknown;
capabilities?: unknown;
supportsExtensionUi?: unknown;
@@ -1754,16 +1823,79 @@ export class AgentDaemon {
}
if (this.options.worker && client.authenticated !== true) {
const commandId = typeof parsed.id === "string" ? parsed.id : undefined;
- if (parsed.type !== "worker_auth" || parsed.token !== this.options.worker.authenticationToken) {
+ if (
+ parsed.type !== "worker_auth" ||
+ parsed.token !== this.options.worker.authenticationToken ||
+ typeof parsed.supervisorGeneration !== "string" ||
+ !Number.isInteger(parsed.supervisorPid) ||
+ (parsed.supervisorPid as number) <= 0 ||
+ (parsed.supervisorProcessStartId !== undefined && typeof parsed.supervisorProcessStartId !== "string") ||
+ typeof parsed.supervisorSocketPath !== "string"
+ ) {
this.write(client, failure(commandId, "worker_auth", "Worker authentication failed"));
client.socket.end();
return;
}
+ const claim: SupervisorGenerationClaim = {
+ supervisorGeneration: parsed.supervisorGeneration,
+ supervisorPid: parsed.supervisorPid as number,
+ ...(typeof parsed.supervisorProcessStartId === "string"
+ ? { supervisorProcessStartId: parsed.supervisorProcessStartId }
+ : {}),
+ supervisorSocketPath: parsed.supervisorSocketPath,
+ };
+ let ownerFingerprint: string;
+ try {
+ ownerFingerprint = await this.assertSupervisorClaimCurrent(claim);
+ } catch {
+ this.write(client, failure(commandId, "worker_auth", "supervisor_generation_stale"));
+ client.socket.end();
+ return;
+ }
+ for (const previous of this.supervisorClaims.keys()) {
+ if (previous !== client) {
+ previous.socket.end();
+ }
+ }
client.authenticated = true;
+ this.supervisorClaims.set(client, { claim, ownerFingerprint });
this.clearSupervisorAvailabilityCheck();
+ this.scheduleSupervisorFenceCheck();
this.write(client, { id: commandId, type: "response", command: "worker_auth", success: true });
return;
}
+ if (this.options.worker) {
+ const boundClaim = this.supervisorClaims.get(client);
+ if (!boundClaim) {
+ this.write(
+ client,
+ failure(
+ typeof parsed.id === "string" ? parsed.id : undefined,
+ "worker_auth",
+ "supervisor_generation_stale",
+ ),
+ );
+ client.socket.end();
+ return;
+ }
+ try {
+ boundClaim.ownerFingerprint = await this.assertSupervisorClaimCurrent(
+ boundClaim.claim,
+ boundClaim.ownerFingerprint,
+ );
+ } catch {
+ this.write(
+ client,
+ failure(
+ typeof parsed.id === "string" ? parsed.id : undefined,
+ typeof parsed.type === "string" ? parsed.type : "worker_auth",
+ "supervisor_generation_stale",
+ ),
+ );
+ client.socket.end();
+ return;
+ }
+ }
if (this.options.worker && typeof parsed.type === "string" && parsed.type.startsWith("worker_")) {
await this.handleWorkerCommand(client, parsed as DaemonWorkerCommand);
return;
@@ -4135,6 +4267,10 @@ export class AgentDaemon {
clearTimeout(this.supervisorMonitorTimer);
this.supervisorMonitorTimer = undefined;
}
+ if (this.supervisorFenceTimer) {
+ clearTimeout(this.supervisorFenceTimer);
+ this.supervisorFenceTimer = undefined;
+ }
this.log(`shutting down (exit ${exitCode}); closing ${this.sessions.size} active session(s)`);
const closingReason: DaemonClosingReason = this.updateRestartPreparing ? "update" : "shutdown";
for (const client of this.clients) {
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts
index 3e989e0fc9..84334a0222 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts
@@ -22,6 +22,10 @@ const REGISTRY_LOCK_UPDATE_MS = 1000;
const REGISTRY_LOCK_RETRIES = 500;
const REGISTRY_LOCK_RETRY_MS = 10;
const STARTUP_FENCE_POLL_MS = 250;
+const SHUTDOWN_ADMISSION_FILE_NAME = "shutdown-admission.json";
+const SHUTDOWN_ADMISSION_LEASE_MS = 5000;
+const SHUTDOWN_ADMISSION_REFRESH_MS = 1000;
+const SHUTDOWN_ADMISSION_WAIT_MS = 50;
type DaemonSupervisorOwnerPhase = "starting" | "owner" | "stopping";
@@ -44,6 +48,14 @@ interface DaemonSupervisorOwnerRecord extends ProcessIdentity {
updatedAt: string;
}
+interface DaemonShutdownAdmissionRecord extends ProcessIdentity {
+ version: 1;
+ token: string;
+ createdAt: string;
+ updatedAt: string;
+ expiresAt: string;
+}
+
interface DaemonSupervisorOwnerScope {
version: 1;
role: "supervisor";
@@ -88,6 +100,24 @@ class DaemonSupervisorAlreadyRunningError extends Error {
}
}
+class DaemonSupervisorOwnershipLostError extends Error {
+ readonly code = "supervisor_generation_stale" as const;
+
+ constructor(generation: string) {
+ super(`Daemon supervisor generation ${generation} no longer owns its registry entry`);
+ this.name = "DaemonSupervisorOwnershipLostError";
+ }
+}
+
+class DaemonShutdownAdmissionError extends Error {
+ readonly code = "daemon_shutdown_in_progress" as const;
+
+ constructor(message = "Daemon shutdown is in progress") {
+ super(message);
+ this.name = "DaemonShutdownAdmissionError";
+ }
+}
+
class DaemonSupervisorOwnership {
private released = false;
@@ -97,6 +127,16 @@ class DaemonSupervisorOwnership {
private readonly ownerDirectory: string,
) {}
+ async assertCurrent(): Promise<void> {
+ if (this.released) {
+ throw new DaemonSupervisorOwnershipLostError(this.record.generation);
+ }
+ const current = readOwnerRecord(this.ownerDirectory);
+ if (!current || !sameOwnerRecord(current, this.record)) {
+ throw new DaemonSupervisorOwnershipLostError(this.record.generation);
+ }
+ }
+
async updatePhase(phase: DaemonSupervisorOwnerPhase): Promise<void> {
if (this.released) {
return;
@@ -139,6 +179,73 @@ class DaemonSupervisorOwnership {
}
}
+class DaemonShutdownAdmission {
+ private released = false;
+ private lost = false;
+ private refreshPromise?: Promise<void>;
+ private readonly refreshTimer: ReturnType<typeof setInterval>;
+
+ constructor(
+ private readonly record: DaemonShutdownAdmissionRecord,
+ private readonly registryDir: string,
+ ) {
+ this.refreshTimer = setInterval(() => {
+ this.refreshPromise ??= this.assertOrRenew()
+ .catch(() => undefined)
+ .finally(() => {
+ this.refreshPromise = undefined;
+ });
+ }, SHUTDOWN_ADMISSION_REFRESH_MS);
+ this.refreshTimer.unref();
+ }
+
+ async assertOrRenew(): Promise<void> {
+ if (this.released || this.lost) {
+ throw new DaemonShutdownAdmissionError("Daemon shutdown admission was lost");
+ }
+ try {
+ await withDaemonSupervisorRegistryGuard(this.registryDir, () => {
+ const path = shutdownAdmissionPath(this.registryDir);
+ const current = readShutdownAdmission(path);
+ if (
+ !current ||
+ current.token !== this.record.token ||
+ current.pid !== this.record.pid ||
+ current.processStartId !== this.record.processStartId ||
+ Date.parse(current.expiresAt) <= Date.now() ||
+ !matchesExactProcessIdentity(this.record)
+ ) {
+ throw new DaemonShutdownAdmissionError("Daemon shutdown admission was lost");
+ }
+ const now = Date.now();
+ this.record.updatedAt = new Date(now).toISOString();
+ this.record.expiresAt = new Date(now + SHUTDOWN_ADMISSION_LEASE_MS).toISOString();
+ writeJsonAtomically(path, this.record);
+ });
+ } catch (error) {
+ this.lost = true;
+ clearInterval(this.refreshTimer);
+ throw error;
+ }
+ }
+
+ async release(): Promise<void> {
+ if (this.released) {
+ return;
+ }
+ this.released = true;
+ clearInterval(this.refreshTimer);
+ await this.refreshPromise;
+ await withDaemonSupervisorRegistryGuard(this.registryDir, () => {
+ const path = shutdownAdmissionPath(this.registryDir);
+ const current = readShutdownAdmission(path);
+ if (current?.token === this.record.token) {
+ rmSync(path, { force: true });
+ }
+ });
+ }
+}
+
function defaultDaemonSupervisorRegistryDir(environment: NodeJS.ProcessEnv = process.env): string {
return environment[DAEMON_SUPERVISOR_REGISTRY_DIR_ENV] ?? resolve(defaultDaemonSocketDir(), "supervisor-owners");
}
@@ -224,6 +331,9 @@ export async function acquireDaemonSupervisorOwnership(
writeOwnerScope(candidateDirectory, record);
writeOwnerRecord(candidateDirectory, record);
await withDaemonSupervisorRegistryGuard(registryDir, () => {
+ if (readActiveShutdownAdmission(registryDir)) {
+ throw new DaemonShutdownAdmissionError();
+ }
for (const directory of listOwnerDirectories(registryDir)) {
const owner = readOwnerRecordForScope(directory, (scope) => ownerConflicts(scope, record));
if (!owner) {
@@ -252,6 +362,66 @@ export async function acquireDaemonSupervisorOwnership(
return new DaemonSupervisorOwnership(record, registryDir, ownerDirectory);
}
+export async function assertDaemonSupervisorOwnerCurrent(
+ owner: {
+ generation: string;
+ pid: number;
+ processStartId?: string;
+ socketPath: string;
+ },
+ validatedFingerprint?: string,
+): Promise<string> {
+ const registryDir = defaultDaemonSupervisorRegistryDir();
+ const current = readOwnerRecord(ownerDirectoryPath(registryDir, owner.generation));
+ if (
+ !current ||
+ current.pid !== owner.pid ||
+ current.processStartId !== owner.processStartId ||
+ current.socketPath !== normalizeSocketPath(owner.socketPath) ||
+ !isProcessAlive(current.pid)
+ ) {
+ throw new DaemonSupervisorOwnershipLostError(owner.generation);
+ }
+ const fingerprint = ownerRecordFingerprint(current);
+ if (fingerprint !== validatedFingerprint && !isProcessIdentityAlive(current)) {
+ throw new DaemonSupervisorOwnershipLostError(owner.generation);
+ }
+ return fingerprint;
+}
+
+export async function acquireDaemonShutdownAdmission(): Promise<DaemonShutdownAdmission> {
+ const registryDir = defaultDaemonSupervisorRegistryDir();
+ const processStartId = getProcessStartId(process.pid);
+ while (true) {
+ let acquired: DaemonShutdownAdmissionRecord | undefined;
+ await withDaemonSupervisorRegistryGuard(registryDir, () => {
+ if (readActiveShutdownAdmission(registryDir)) {
+ return;
+ }
+ const now = Date.now();
+ acquired = {
+ version: OWNER_VERSION,
+ token: randomUUID(),
+ pid: process.pid,
+ ...(processStartId ? { processStartId } : {}),
+ createdAt: new Date(now).toISOString(),
+ updatedAt: new Date(now).toISOString(),
+ expiresAt: new Date(now + SHUTDOWN_ADMISSION_LEASE_MS).toISOString(),
+ };
+ writeJsonAtomically(shutdownAdmissionPath(registryDir), acquired);
+ });
+ if (acquired) {
+ return new DaemonShutdownAdmission(acquired, registryDir);
+ }
+ await delay(SHUTDOWN_ADMISSION_WAIT_MS);
+ }
+}
+
+export async function isDaemonShutdownAdmissionActive(): Promise<boolean> {
+ const registryDir = defaultDaemonSupervisorRegistryDir();
+ return withDaemonSupervisorRegistryGuard(registryDir, () => readActiveShutdownAdmission(registryDir) !== undefined);
+}
+
export async function persistDaemonStartupFenceFromOwner(
socketPath: string,
hello: DaemonSupervisorHelloIdentity,
@@ -349,10 +519,8 @@ export async function waitForDaemonStartupFence(
}
function isProcessIdentityAlive(identity: ProcessIdentity): boolean {
- try {
- process.kill(identity.pid, 0);
- } catch (error) {
- return (error as NodeJS.ErrnoException).code !== "ESRCH";
+ if (!isProcessAlive(identity.pid)) {
+ return false;
}
if (!identity.processStartId) {
return true;
@@ -361,6 +529,22 @@ function isProcessIdentityAlive(identity: ProcessIdentity): boolean {
return observed === undefined || observed === identity.processStartId;
}
+function matchesExactProcessIdentity(identity: ProcessIdentity): boolean {
+ if (!isProcessAlive(identity.pid)) {
+ return false;
+ }
+ return identity.processStartId === undefined || getProcessStartId(identity.pid) === identity.processStartId;
+}
+
+function isProcessAlive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code !== "ESRCH";
+ }
+ return true;
+}
+
function normalizeSocketPath(socketPath: string): string {
if (process.platform === "win32") {
return socketPath.toLowerCase();
@@ -395,6 +579,20 @@ function ownerConflicts(left: DaemonSupervisorOwnerScope, right: DaemonSuperviso
return left.socketPath === right.socketPath || left.descriptorDir === right.descriptorDir;
}
+function sameOwnerRecord(left: DaemonSupervisorOwnerRecord, right: DaemonSupervisorOwnerRecord): boolean {
+ return (
+ left.token === right.token &&
+ left.generation === right.generation &&
+ left.pid === right.pid &&
+ left.processStartId === right.processStartId &&
+ left.socketPath === right.socketPath
+ );
+}
+
+function ownerRecordFingerprint(record: DaemonSupervisorOwnerRecord): string {
+ return createHash("sha256").update(JSON.stringify(record)).digest("hex");
+}
+
function listOwnerDirectories(registryDir: string): string[] {
return readdirSync(registryDir)
.filter((name) => name.endsWith(".owner"))
@@ -535,6 +733,48 @@ function readStartupFence(path: string): DaemonStartupFenceRecord | undefined {
}
}
+function readActiveShutdownAdmission(registryDir: string): DaemonShutdownAdmissionRecord | undefined {
+ const path = shutdownAdmissionPath(registryDir);
+ const admission = readShutdownAdmission(path);
+ if (!admission) {
+ return undefined;
+ }
+ if (Date.parse(admission.expiresAt) > Date.now() && isProcessIdentityAlive(admission)) {
+ return admission;
+ }
+ rmSync(path, { force: true });
+ return undefined;
+}
+
+function readShutdownAdmission(path: string): DaemonShutdownAdmissionRecord | undefined {
+ try {
+ const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
+ if (!value || typeof value !== "object") {
+ throw new Error(`Invalid daemon shutdown admission: ${path}`);
+ }
+ const admission = value as Partial<DaemonShutdownAdmissionRecord>;
+ if (
+ admission.version !== OWNER_VERSION ||
+ typeof admission.token !== "string" ||
+ !Number.isInteger(admission.pid) ||
+ (admission.pid ?? 0) <= 0 ||
+ (admission.processStartId !== undefined && typeof admission.processStartId !== "string") ||
+ typeof admission.createdAt !== "string" ||
+ typeof admission.updatedAt !== "string" ||
+ typeof admission.expiresAt !== "string" ||
+ !Number.isFinite(Date.parse(admission.expiresAt))
+ ) {
+ throw new Error(`Invalid daemon shutdown admission: ${path}`);
+ }
+ return admission as DaemonShutdownAdmissionRecord;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+ return undefined;
+ }
+ throw error;
+ }
+}
+
function writeJsonAtomically(path: string, value: unknown): void {
const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
try {
@@ -551,6 +791,10 @@ function startupFencePath(directory: string, socketPath: string): string {
return resolve(directory, `${key}.json`);
}
+function shutdownAdmissionPath(registryDir: string): string {
+ return resolve(registryDir, SHUTDOWN_ADMISSION_FILE_NAME);
+}
+
function delay(ms: number): Promise<void> {
return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
}
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
index 0b2cc45f96..831253f705 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
@@ -3,6 +3,7 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";
import { chmodSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { createServer, type Server, type Socket } from "node:net";
import { dirname, isAbsolute, join, resolve } from "node:path";
+import { Writable } from "node:stream";
import { getLogger } from "@earendil-works/pi-ai";
import { createCliSubprocessLaunchSpec } from "../../cli/subprocess-launch.js";
import {
@@ -65,12 +66,18 @@ import {
prepareDaemonSocketPath,
restrictDaemonSocketPath,
} from "./daemon-socket.js";
-import { acquireDaemonSupervisorOwnership, waitForDaemonStartupFence } from "./daemon-supervisor-ownership.js";
+import {
+ acquireDaemonSupervisorOwnership,
+ isDaemonShutdownAdmissionActive,
+ waitForDaemonStartupFence,
+} from "./daemon-supervisor-ownership.js";
import { DaemonWorkerClient } from "./daemon-worker-client.js";
import {
DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,
DAEMON_WORKER_RECOVERY_JOURNAL_ENV,
DAEMON_WORKER_ROLE_ENV,
+ DAEMON_WORKER_STARTUP_GATE_COMMIT,
+ DAEMON_WORKER_STARTUP_GATE_FD_ENV,
DAEMON_WORKER_SUPERVISOR_SOCKET_ENV,
DAEMON_WORKER_TOKEN_ENV,
type DaemonCreateCommand,
@@ -89,7 +96,9 @@ const structuredLog = getLogger("coding-agent.daemon-supervisor");
const WORKER_CONNECT_TIMEOUT_MS = 30_000;
const WORKER_REQUEST_TIMEOUT_MS = 24 * 60 * 60 * 1000;
const WORKER_RETRY_DELAYS_MS = [250, 1000, 5000] as const;
+const DEFERRED_RECOVERY_RECHECK_MS = 5000;
const SUPERVISOR_CONFIG_FILE_NAME = "supervisor-config";
+const WORKER_STARTUP_GATE_FD = 3;
const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set([
"ack_result",
@@ -193,6 +202,7 @@ interface ResidentWorker {
>;
snapshotLoads: Map<string, Promise<DaemonAttachResult>>;
recovery?: Promise<void>;
+ deferredRecovery?: Promise<void>;
intentionalStop: boolean;
stopRevision: number;
}
@@ -225,10 +235,62 @@ interface WorkerAttachData {
worker: ResidentWorker;
}
+class SupervisorRecoveryCancelledError extends Error {
+ readonly code = "supervisor_recovery_cancelled" as const;
+}
+
+function isSupervisorGenerationStale(error: unknown): boolean {
+ return (
+ typeof error === "object" &&
+ error !== null &&
+ "code" in error &&
+ (error as { code?: unknown }).code === "supervisor_generation_stale"
+ );
+}
+
+function isSupervisorRecoveryCancelled(error: unknown): boolean {
+ return isSupervisorShutdownAdmissionCancelled(error) || isSupervisorGenerationStale(error);
+}
+
+function isSupervisorShutdownAdmissionCancelled(error: unknown): boolean {
+ return (
+ error instanceof SupervisorRecoveryCancelledError ||
+ (typeof error === "object" &&
+ error !== null &&
+ "code" in error &&
+ (error as { code?: unknown }).code === "supervisor_recovery_cancelled")
+ );
+}
+
function delay(ms: number): Promise<void> {
return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
}
+function unrefDelay(ms: number): Promise<void> {
+ return new Promise((resolveDelay) => setTimeout(resolveDelay, ms).unref());
+}
+
+function commitWorkerStartupGate(gate: Writable): Promise<void> {
+ return new Promise((resolveCommit, rejectCommit) => {
+ let settled = false;
+ const finish = (error?: Error | null) => {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ if (error) {
+ rejectCommit(error);
+ } else {
+ resolveCommit();
+ }
+ };
+ const onError = (error: Error) => finish(error);
+ gate.on("error", onError);
+ gate.once("close", () => gate.off("error", onError));
+ gate.end(DAEMON_WORKER_STARTUP_GATE_COMMIT, (error?: Error | null) => finish(error));
+ });
+}
+
function withoutCommandId(command: DaemonCommand): DaemonCommandBody {
const { id: _id, ...body } = command;
return body as DaemonCommandBody;
@@ -585,6 +647,41 @@ export class DaemonSupervisor {
appendRotatingLog(getDaemonLogPath(this.socketPath), `[${new Date().toISOString()}] supervisor: ${message}`);
}
+ private async assertCurrentOwnership(): Promise<void> {
+ const ownership = this.ownership;
+ if (!ownership) {
+ const error = new Error(`Daemon supervisor generation ${this.generation} no longer owns its registry entry`);
+ Object.assign(error, { code: "supervisor_generation_stale" as const });
+ throw error;
+ }
+ await ownership.assertCurrent();
+ }
+
+ private async assertRecoveryAllowed(): Promise<void> {
+ await this.assertCurrentOwnership();
+ if (await isDaemonShutdownAdmissionActive()) {
+ throw new SupervisorRecoveryCancelledError("Daemon shutdown admission cancelled worker recovery");
+ }
+ }
+
+ private supervisorAuthenticationClaim(): {
+ supervisorGeneration: string;
+ supervisorPid: number;
+ supervisorProcessStartId?: string;
+ supervisorSocketPath: string;
+ } {
+ const record = this.ownership?.record;
+ if (!record) {
+ throw new SupervisorRecoveryCancelledError("Daemon supervisor ownership is unavailable");
+ }
+ return {
+ supervisorGeneration: this.generation,
+ supervisorPid: record.pid,
+ ...(record.processStartId ? { supervisorProcessStartId: record.processStartId } : {}),
+ supervisorSocketPath: record.socketPath,
+ };
+ }
+
private loadWorkerDescriptors(): void {
for (const name of readdirSync(this.descriptorDir)) {
if (name === SUPERVISOR_CONFIG_FILE_NAME || !name.endsWith(".json")) {
@@ -778,6 +875,13 @@ export class DaemonSupervisor {
return;
}
+ try {
+ await this.assertCurrentOwnership();
+ } catch (error) {
+ this.write(client, failure(command.id, command.type, error));
+ return;
+ }
+
const journalIdentity =
envelopeClientId && command.id && isDaemonMutatingCommand(command)
? { clientId: envelopeClientId, commandId: command.id }
@@ -804,15 +908,21 @@ export class DaemonSupervisor {
const response = await this.handleCommand(client, command);
if (response) {
if (journalIdentity) {
+ await this.assertCurrentOwnership();
this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);
}
this.write(client, response);
}
} catch (error) {
this.log(`Supervisor command ${command.type} failed: ${error instanceof Error ? error.stack : String(error)}`);
- const response = failure(command.id, command.type, error, serializeDaemonError(error));
- if (journalIdentity) {
- this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);
+ let response = failure(command.id, command.type, error, serializeDaemonError(error));
+ if (journalIdentity && !isSupervisorGenerationStale(error)) {
+ try {
+ await this.assertCurrentOwnership();
+ this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);
+ } catch (ownershipError) {
+ response = failure(command.id, command.type, ownershipError, serializeDaemonError(ownershipError));
+ }
}
this.write(client, response);
}
@@ -1130,6 +1240,7 @@ export class DaemonSupervisor {
}
private async launchWorker(command: DaemonCreateCommand, existing?: ResidentWorker): Promise<ResidentWorker> {
+ await this.assertRecoveryAllowed();
if (existing && this.isWorkerRecoveryCancelled(existing)) {
throw new Error(`Session worker ${existing.descriptor.workerId} recovery was cancelled`);
}
@@ -1149,6 +1260,7 @@ export class DaemonSupervisor {
const orphanProcessJournalPath =
existing?.descriptor.orphanProcessJournalPath ?? join(this.descriptorDir, `${workerId}.orphans.jsonl`);
const launch = createCliSubprocessLaunchSpec(["--mode", "daemon", "--daemon-socket", socketPath]);
+ await this.assertRecoveryAllowed();
const child: ChildProcess = spawn(launch.command, launch.args, {
cwd: createCommand.config?.cwd ?? process.cwd(),
detached: true,
@@ -1159,59 +1271,104 @@ export class DaemonSupervisor {
[DAEMON_WORKER_ACTIVE_SESSION_ID_ENV]: rootActiveSessionId,
[DAEMON_WORKER_SUPERVISOR_SOCKET_ENV]: this.socketPath,
[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]: recoveryJournalPath,
+ [DAEMON_WORKER_STARTUP_GATE_FD_ENV]: String(WORKER_STARTUP_GATE_FD),
[ORPHAN_PROCESS_JOURNAL_ENV]: orphanProcessJournalPath,
[SESSION_LEASES_ENABLED_ENV]: "1",
[SESSION_LEASE_OWNER_ID_ENV]: rootActiveSessionId,
},
- stdio: "ignore",
+ stdio: ["ignore", "ignore", "ignore", "pipe"],
});
+ const childClosed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));
child.on("error", (error) => {
this.log(
`Session worker ${workerId} process error: ${error instanceof Error ? error.message : String(error)}`,
);
});
- if (!child.pid) {
- throw new Error("Failed to obtain daemon session worker pid");
+ const startupGate = child.stdio[WORKER_STARTUP_GATE_FD];
+ const previousDescriptor = existing?.descriptor;
+ const previousIntentionalStop = existing?.intentionalStop;
+ let descriptorAssigned = false;
+ let childPid: number;
+ let childProcessStartId: string | undefined;
+ let worker: ResidentWorker;
+ try {
+ if (!child.pid) {
+ throw new Error("Failed to obtain daemon session worker pid");
+ }
+ if (!(startupGate instanceof Writable)) {
+ throw new Error("Failed to create daemon session worker startup gate");
+ }
+ childPid = child.pid;
+ childProcessStartId = getProcessStartId(childPid);
+ await this.assertRecoveryAllowed();
+
+ const descriptor: DaemonWorkerDescriptor = {
+ version: 1,
+ workerId,
+ pid: childPid,
+ ...(childProcessStartId ? { processStartId: childProcessStartId } : {}),
+ socketPath,
+ recoveryJournalPath,
+ orphanProcessJournalPath,
+ supervisorSocketPath: this.socketPath,
+ authenticationToken: token,
+ rootActiveSessionId,
+ createdAt: existing?.descriptor.createdAt ?? now,
+ updatedAt: now,
+ lifecycle: "starting",
+ createCommand: { ...createCommand, id: undefined },
+ consecutiveFailures: existing?.descriptor.consecutiveFailures ?? 0,
+ };
+ worker = existing ?? {
+ descriptor,
+ descriptorPath,
+ summaries: new Map(),
+ snapshotCache: new Map(),
+ transcriptCaches: new Map(),
+ incomingTranscriptActiveSessionIds: new Set(),
+ duplicateIncomingTranscriptChunkIndexes: new Map(),
+ snapshotTransferFrames: new Map(),
+ snapshotLoads: new Map(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+ await this.assertRecoveryAllowed();
+ worker.descriptor = descriptor;
+ descriptorAssigned = true;
+ this.persistWorker(worker);
+ worker.intentionalStop = false;
+ this.workers.set(workerId, worker);
+ } catch (error) {
+ if (startupGate instanceof Writable) {
+ startupGate.destroy();
+ }
+ await childClosed;
+ child.unref();
+ try {
+ rmSync(`${descriptorPath}.${process.pid}.tmp`, { force: true });
+ } catch (cleanupError) {
+ this.reportCleanupFailure(`worker launch temp ${workerId}`, cleanupError);
+ }
+ if (existing && descriptorAssigned && previousDescriptor) {
+ try {
+ existing.descriptor = previousDescriptor;
+ } catch (cleanupError) {
+ this.reportCleanupFailure(`worker launch descriptor ${workerId}`, cleanupError);
+ }
+ }
+ throw error;
}
- const childProcessStartId = getProcessStartId(child.pid);
- child.unref();
-
- const descriptor: DaemonWorkerDescriptor = {
- version: 1,
- workerId,
- pid: child.pid,
- ...(childProcessStartId ? { processStartId: childProcessStartId } : {}),
- socketPath,
- recoveryJournalPath,
- orphanProcessJournalPath,
- supervisorSocketPath: this.socketPath,
- authenticationToken: token,
- rootActiveSessionId,
- createdAt: existing?.descriptor.createdAt ?? now,
- updatedAt: now,
- lifecycle: "starting",
- createCommand: { ...createCommand, id: undefined },
- consecutiveFailures: existing?.descriptor.consecutiveFailures ?? 0,
- };
- const worker = existing ?? {
- descriptor,
- descriptorPath,
- summaries: new Map(),
- snapshotCache: new Map(),
- transcriptCaches: new Map(),
- incomingTranscriptActiveSessionIds: new Set(),
- duplicateIncomingTranscriptChunkIndexes: new Map(),
- snapshotTransferFrames: new Map(),
- snapshotLoads: new Map(),
- intentionalStop: false,
- stopRevision: 0,
- };
- worker.descriptor = descriptor;
- worker.intentionalStop = false;
- this.workers.set(workerId, worker);
- this.persistWorker(worker);
try {
+ try {
+ await commitWorkerStartupGate(startupGate);
+ } catch (error) {
+ startupGate.destroy();
+ await childClosed;
+ throw error;
+ } finally {
+ child.unref();
+ }
const client = await this.connectWorker(worker, WORKER_CONNECT_TIMEOUT_MS);
const response = await client.request(withoutCommandId(createCommand), WORKER_REQUEST_TIMEOUT_MS);
if (!response.success) {
@@ -1228,10 +1385,11 @@ export class DaemonSupervisor {
worker.descriptor.rootSessionId = summary.sessionId;
worker.descriptor.sessionFile = summary.sessionFile;
await this.subscribeWorker(worker, rootActiveSessionId);
- await this.refreshWorkerSummaries(worker);
+ await this.refreshWorkerSummaries(worker, true);
if (existing && (this.isWorkerRecoveryCancelled(worker) || worker.stopRevision !== recoveryStopRevision)) {
throw new Error(`Session worker ${workerId} recovery was cancelled`);
}
+ await this.assertRecoveryAllowed();
worker.descriptor.lifecycle = "ready";
worker.descriptor.consecutiveFailures = 0;
worker.descriptor.lastError = undefined;
@@ -1239,6 +1397,43 @@ export class DaemonSupervisor {
await this.syncAgentPeers();
return worker;
} catch (error) {
+ if (isSupervisorGenerationStale(error)) {
+ throw error;
+ }
+ if (isSupervisorShutdownAdmissionCancelled(error)) {
+ let rolledBack = false;
+ try {
+ await this.stopWorker(worker, existing === undefined, true, false, existing !== undefined, {
+ child,
+ closed: childClosed,
+ });
+ rolledBack = true;
+ } catch (cleanupError) {
+ this.reportCleanupFailure(`cancelled worker launch ${workerId}`, cleanupError);
+ }
+ const mappedWorker = this.workers.get(workerId);
+ if (
+ rolledBack &&
+ existing &&
+ previousDescriptor &&
+ !this.shuttingDown &&
+ existing.stopRevision === recoveryStopRevision &&
+ existing.descriptor.stopRequestedAt === undefined &&
+ (mappedWorker === undefined || mappedWorker === existing)
+ ) {
+ existing.descriptor = previousDescriptor;
+ existing.intentionalStop = previousIntentionalStop ?? false;
+ this.workers.set(workerId, existing);
+ try {
+ this.persistWorker(existing);
+ } catch (cleanupError) {
+ this.reportCleanupFailure(`cancelled worker recovery ${workerId}`, cleanupError);
+ }
+ this.deferWorkerRecovery(existing, error instanceof Error ? error : new Error(String(error)));
+ }
+ throw error;
+ }
+ await this.assertRecoveryAllowed();
const shouldResumeRecovery =
existing !== undefined &&
!this.shuttingDown &&
@@ -1253,6 +1448,7 @@ export class DaemonSupervisor {
worker.descriptor.stopRequestedAt === undefined &&
worker.stopRevision === recoveryStopRevision
) {
+ await this.assertRecoveryAllowed();
worker.intentionalStop = false;
worker.descriptor.lifecycle = "recovering";
this.workers.set(workerId, worker);
@@ -1266,19 +1462,28 @@ export class DaemonSupervisor {
const deadline = Date.now() + timeoutMs;
let lastError: unknown;
while (Date.now() < deadline) {
+ await this.assertRecoveryAllowed();
const client = new DaemonWorkerClient(worker.descriptor.socketPath);
try {
await client.connect(Math.min(500, Math.max(50, deadline - Date.now())));
await client.waitForHello(1000);
- await client.authenticateWorker(worker.descriptor.authenticationToken, 1000);
+ await client.authenticateWorker(
+ worker.descriptor.authenticationToken,
+ this.supervisorAuthenticationClaim(),
+ 1000,
+ );
+ await this.assertRecoveryAllowed();
client.onFrame((frame) => this.handleWorkerFrame(worker, frame));
- client.onClose((error) => this.handleWorkerClose(worker, client, error));
+ client.onClose((error) => void this.handleWorkerClose(worker, client, error));
worker.client?.close();
worker.client = client;
return client;
} catch (error) {
lastError = error;
client.close();
+ if (isSupervisorRecoveryCancelled(error)) {
+ throw error;
+ }
await delay(25);
}
}
@@ -1306,6 +1511,7 @@ export class DaemonSupervisor {
}
private async adoptOrRecoverWorker(worker: ResidentWorker): Promise<void> {
+ await this.assertRecoveryAllowed();
if (worker.descriptor.stopRequestedAt) {
try {
// A tombstoned worker must not run long enough to elect another
@@ -1328,20 +1534,24 @@ export class DaemonSupervisor {
const observedProcessStartId = getProcessStartId(worker.descriptor.pid);
await this.connectWorker(worker, 2000);
await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId);
- await this.refreshWorkerSummaries(worker);
+ await this.refreshWorkerSummaries(worker, true);
if (worker.descriptor.processStartId === undefined && observedProcessStartId) {
worker.descriptor.processStartId = observedProcessStartId;
}
+ await this.assertRecoveryAllowed();
worker.descriptor.lifecycle = "ready";
worker.descriptor.consecutiveFailures = 0;
this.persistWorker(worker);
} catch (error) {
+ if (isSupervisorRecoveryCancelled(error)) {
+ return;
+ }
this.log(`Could not adopt worker ${worker.descriptor.workerId}: ${String(error)}`);
await this.recoverWorker(worker);
}
}
- private handleWorkerClose(worker: ResidentWorker, client: DaemonWorkerClient, error: Error): void {
+ private async handleWorkerClose(worker: ResidentWorker, client: DaemonWorkerClient, error: Error): Promise<void> {
if (worker.client !== client) {
return;
}
@@ -1363,6 +1573,17 @@ export class DaemonSupervisor {
if (this.shuttingDown || worker.intentionalStop) {
return;
}
+ try {
+ await this.assertRecoveryAllowed();
+ } catch (recoveryError) {
+ if (!isSupervisorGenerationStale(recoveryError)) {
+ this.deferWorkerRecovery(worker, error);
+ }
+ return;
+ }
+ if (!this.isWorkerRecoveryEligible(worker)) {
+ return;
+ }
worker.descriptor.lifecycle = "recovering";
worker.descriptor.lastError = error.message;
this.persistWorker(worker);
@@ -1370,6 +1591,61 @@ export class DaemonSupervisor {
void this.recoverWorker(worker);
}
+ private isWorkerRecoveryEligible(worker: ResidentWorker): boolean {
+ return this.isWorkerRecoveryCandidate(worker) && worker.recovery === undefined;
+ }
+
+ private isWorkerRecoveryCandidate(worker: ResidentWorker): boolean {
+ return (
+ !this.shuttingDown &&
+ !worker.intentionalStop &&
+ worker.descriptor.stopRequestedAt === undefined &&
+ this.workers.get(worker.descriptor.workerId) === worker &&
+ worker.client === undefined
+ );
+ }
+
+ private deferWorkerRecovery(worker: ResidentWorker, disconnectError: Error): void {
+ if (worker.deferredRecovery) {
+ return;
+ }
+ worker.deferredRecovery = this.resumeDeferredWorkerRecovery(worker, disconnectError).finally(() => {
+ worker.deferredRecovery = undefined;
+ });
+ }
+
+ private async resumeDeferredWorkerRecovery(worker: ResidentWorker, disconnectError: Error): Promise<void> {
+ while (true) {
+ await unrefDelay(DEFERRED_RECOVERY_RECHECK_MS);
+ if (!this.isWorkerRecoveryCandidate(worker)) {
+ return;
+ }
+ if (!this.isWorkerRecoveryEligible(worker)) {
+ continue;
+ }
+ try {
+ await this.assertRecoveryAllowed();
+ } catch (error) {
+ if (isSupervisorGenerationStale(error)) {
+ return;
+ }
+ continue;
+ }
+ if (!this.isWorkerRecoveryCandidate(worker)) {
+ return;
+ }
+ if (!this.isWorkerRecoveryEligible(worker)) {
+ continue;
+ }
+ worker.descriptor.lifecycle = "recovering";
+ worker.descriptor.lastError = disconnectError.message;
+ this.persistWorker(worker);
+ void this.syncAgentPeers().catch(() => undefined);
+ void this.recoverWorker(worker);
+ return;
+ }
+ }
+
private failWorkerSnapshotCache(
worker: ResidentWorker,
activeSessionId: string,
@@ -1436,6 +1712,7 @@ export class DaemonSupervisor {
return;
}
try {
+ await this.assertRecoveryAllowed();
const processAlive = isProcessAlive(worker.descriptor.pid);
const observedProcessStartId = processAlive ? getProcessStartId(worker.descriptor.pid) : undefined;
const processIdentityMatches =
@@ -1445,13 +1722,14 @@ export class DaemonSupervisor {
try {
await this.connectWorker(worker, 1500);
await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId);
- await this.refreshWorkerSummaries(worker);
+ await this.refreshWorkerSummaries(worker, true);
if (this.isWorkerRecoveryCancelled(worker)) {
return;
}
if (worker.descriptor.processStartId === undefined && observedProcessStartId) {
worker.descriptor.processStartId = observedProcessStartId;
}
+ await this.assertRecoveryAllowed();
worker.descriptor.lifecycle = "ready";
worker.descriptor.consecutiveFailures = 0;
this.persistWorker(worker);
@@ -1460,6 +1738,10 @@ export class DaemonSupervisor {
);
return;
} catch (error) {
+ if (isSupervisorRecoveryCancelled(error)) {
+ throw error;
+ }
+ await this.assertRecoveryAllowed();
worker.client?.close();
worker.client = undefined;
if (retryIndex < WORKER_RETRY_DELAYS_MS.length - 1) {
@@ -1484,7 +1766,12 @@ export class DaemonSupervisor {
await this.launchWorker(worker.descriptor.createCommand, worker);
return;
} catch (error) {
- if (this.isWorkerRecoveryCancelled(worker)) {
+ if (isSupervisorRecoveryCancelled(error) || this.isWorkerRecoveryCancelled(worker)) {
+ return;
+ }
+ try {
+ await this.assertRecoveryAllowed();
+ } catch {
return;
}
worker.client?.close();
@@ -1495,6 +1782,11 @@ export class DaemonSupervisor {
this.persistWorker(worker);
}
}
+ try {
+ await this.assertRecoveryAllowed();
+ } catch {
+ return;
+ }
worker.descriptor.lifecycle = "failed";
this.persistWorker(worker);
await this.syncAgentPeers().catch(() => undefined);
@@ -1515,6 +1807,7 @@ export class DaemonSupervisor {
}
private async recoverUncertainWorkerOperations(worker: ResidentWorker, killWorkerProcess = true): Promise<void> {
+ await this.assertRecoveryAllowed();
if (killWorkerProcess) {
signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");
}
@@ -1568,6 +1861,7 @@ export class DaemonSupervisor {
}
interrupted.operations.add(record.operation);
}
+ await this.assertRecoveryAllowed();
await Promise.all(
[...interruptedSessions.values()].map((interrupted) =>
this.catalog.markInterrupted(interrupted.sessionFile, interrupted.activeSessionId, [
@@ -1575,6 +1869,7 @@ export class DaemonSupervisor {
]),
),
);
+ await this.assertRecoveryAllowed();
for (const record of latest) {
journal.record({
activeSessionId: record.activeSessionId,
@@ -1591,7 +1886,7 @@ export class DaemonSupervisor {
);
}
- private async refreshWorkerSummaries(worker: ResidentWorker): Promise<void> {
+ private async refreshWorkerSummaries(worker: ResidentWorker, recovery = false): Promise<void> {
if (!worker.client) {
throw new Error("Session worker is not connected");
}
@@ -1608,6 +1903,9 @@ export class DaemonSupervisor {
}
const root = worker.summaries.get(worker.descriptor.rootActiveSessionId);
if (root) {
+ if (recovery) {
+ await this.assertRecoveryAllowed();
+ }
worker.descriptor.rootSessionId = root.sessionId;
worker.descriptor.sessionFile = root.sessionFile;
worker.descriptor.createCommand = {
@@ -2771,16 +3069,24 @@ export class DaemonSupervisor {
force = false,
archiveSession = false,
recoveryCleanup = false,
+ directChild?: { child: ChildProcess; closed: Promise<void> },
): Promise<void> {
if (!recoveryCleanup) {
worker.stopRevision++;
}
- if (removeDescriptor) {
- this.persistWorkerStopTombstone(worker, archiveSession);
- } else {
- worker.intentionalStop = true;
- worker.descriptor.lifecycle = "recovering";
- this.persistWorker(worker);
+ try {
+ if (removeDescriptor) {
+ this.persistWorkerStopTombstone(worker, archiveSession);
+ } else {
+ worker.intentionalStop = true;
+ worker.descriptor.lifecycle = "recovering";
+ this.persistWorker(worker);
+ }
+ } catch (error) {
+ if (!directChild) {
+ throw error;
+ }
+ this.reportCleanupFailure(`worker rollback state ${worker.descriptor.workerId}`, error);
}
this.dropPendingReplacementSnapshotsForWorker(worker);
const interruptedSnapshots = new Set([
@@ -2818,24 +3124,37 @@ export class DaemonSupervisor {
}
worker.client.close();
worker.client = undefined;
+ } else if (directChild) {
+ directChild.child.kill("SIGTERM");
} else if (isProcessAlive(worker.descriptor.pid)) {
signalProcessGroupOrProcess(worker.descriptor.pid, "SIGTERM");
}
+ const isWorkerProcessAlive = () =>
+ directChild
+ ? directChild.child.exitCode === null && directChild.child.signalCode === null
+ : isProcessAlive(worker.descriptor.pid);
const gracefulDeadline = Date.now() + (force ? 500 : 2000);
- while (isProcessAlive(worker.descriptor.pid) && Date.now() < gracefulDeadline) {
+ while (isWorkerProcessAlive() && Date.now() < gracefulDeadline) {
await delay(25);
}
- if (force && isProcessAlive(worker.descriptor.pid)) {
- signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");
+ if (force && isWorkerProcessAlive()) {
+ if (directChild) {
+ directChild.child.kill("SIGKILL");
+ } else {
+ signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");
+ }
const forceDeadline = Date.now() + 1000;
- while (isProcessAlive(worker.descriptor.pid) && Date.now() < forceDeadline) {
+ while (isWorkerProcessAlive() && Date.now() < forceDeadline) {
await delay(25);
}
}
- if (isProcessAlive(worker.descriptor.pid)) {
+ if (isWorkerProcessAlive()) {
worker.intentionalStop = worker.descriptor.stopRequestedAt !== undefined;
throw new Error(`Session worker ${worker.descriptor.workerId} did not stop${force ? " after SIGKILL" : ""}`);
}
+ if (directChild) {
+ await directChild.closed;
+ }
if (removeDescriptor && worker.descriptor.archiveOnStop) {
if (force) {
this.reclaimStoppedWorkerCronLock(worker);
diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts
index 6a54cd3f24..6adb2357e9 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts
@@ -13,6 +13,7 @@ type DistributiveOmit<T, K extends keyof T> = T extends unknown ? Omit<T, K> : n
type DaemonCommandBody = DistributiveOmit<DaemonCommand, "id">;
type DaemonWorkerWireCommandBody = DaemonCommandBody | DaemonWorkerCommandBody;
type DaemonWorkerWireCommand = DaemonCommand | DaemonWorkerCommand;
+type DaemonWorkerAuthentication = Omit<Extract<DaemonWorkerCommand, { type: "worker_auth" }>, "id" | "type" | "token">;
export type DaemonWorkerFrameListener = (frame: PrivateFrame<DaemonWorkerFrameHeader>) => void;
export type DaemonWorkerCloseListener = (error: Error) => void;
@@ -115,8 +116,8 @@ export class DaemonWorkerClient {
return this.requestWire(command, timeoutMs);
}
- async authenticateWorker(token: string, timeoutMs = 3000): Promise<void> {
- const response = await this.requestWorker({ type: "worker_auth", token }, timeoutMs);
+ async authenticateWorker(token: string, owner: DaemonWorkerAuthentication, timeoutMs = 3000): Promise<void> {
+ const response = await this.requestWorker({ type: "worker_auth", token, ...owner }, timeoutMs);
if (!response.success) {
throw new Error(response.error);
}
diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
index 0e8e0549ae..0ae2146ddf 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
@@ -1,3 +1,4 @@
+import { closeSync, readFileSync } from "node:fs";
import type {
AgentSessionMessageAgentSummary,
AgentSessionMessageDeliveryMode,
@@ -13,6 +14,8 @@ export const DAEMON_WORKER_TOKEN_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN
export const DAEMON_WORKER_ACTIVE_SESSION_ID_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID";
export const DAEMON_WORKER_SUPERVISOR_SOCKET_ENV = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET";
export const DAEMON_WORKER_RECOVERY_JOURNAL_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL";
+export const DAEMON_WORKER_STARTUP_GATE_FD_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD";
+export const DAEMON_WORKER_STARTUP_GATE_COMMIT = "start\n";
export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "failed";
export type DaemonWorkerFrameHeader =
@@ -34,7 +37,15 @@ export type DaemonWorkerFrameHeader =
export type DaemonCreateCommand = Extract<DaemonCommand, { type: "create" }>;
export type DaemonWorkerCommand =
- | { id?: string; type: "worker_auth"; token: string }
+ | {
+ id?: string;
+ type: "worker_auth";
+ token: string;
+ supervisorGeneration: string;
+ supervisorPid: number;
+ supervisorProcessStartId?: string;
+ supervisorSocketPath: string;
+ }
| {
id?: string;
type: "worker_subscribe";
@@ -93,6 +104,27 @@ export function isDaemonWorkerProcess(environment: NodeJS.ProcessEnv = process.e
return environment[DAEMON_WORKER_ROLE_ENV] === "1";
}
+export function waitForDaemonWorkerStartupGate(environment: NodeJS.ProcessEnv = process.env): void {
+ const rawFd = environment[DAEMON_WORKER_STARTUP_GATE_FD_ENV];
+ if (rawFd === undefined) {
+ return;
+ }
+ delete environment[DAEMON_WORKER_STARTUP_GATE_FD_ENV];
+ const fd = Number(rawFd);
+ if (!Number.isInteger(fd) || fd < 3) {
+ throw new Error("Daemon session worker has an invalid startup gate");
+ }
+ let marker: string;
+ try {
+ marker = readFileSync(fd, "utf8");
+ } finally {
+ closeSync(fd);
+ }
+ if (marker !== DAEMON_WORKER_STARTUP_GATE_COMMIT) {
+ throw new Error("Daemon session worker startup was cancelled");
+ }
+}
+
export function requireDaemonWorkerAuthenticationToken(environment: NodeJS.ProcessEnv = process.env): string {
const token = environment[DAEMON_WORKER_TOKEN_ENV];
if (!token) {
diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
index 7de450c045..01329ae217 100644
--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
@@ -1,4 +1,5 @@
-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import type { ChildProcess, SpawnOptions } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { AgentMessage } from "@earendil-works/pi-agent-core";
@@ -7,11 +8,100 @@ import { AgentDaemon } from "../src/modes/daemon/daemon-mode.js";
import type { DaemonAttachResult } from "../src/modes/daemon/daemon-protocol.js";
import type { SessionSummary } from "../src/modes/daemon/daemon-session-list.js";
import { DaemonSupervisor } from "../src/modes/daemon/daemon-supervisor.js";
+import { DAEMON_WORKER_STARTUP_GATE_COMMIT } from "../src/modes/daemon/daemon-worker-protocol.js";
import { WorkerRecoveryJournal } from "../src/modes/daemon/worker-recovery-journal.js";
+const workerLaunchTestState = vi.hoisted(() => ({
+ capture: false,
+ forceMissingProcessStartId: false,
+ fixtureMode: "worker" as "worker" | "close-gate" | "rollback-gate" | "successful-gate",
+ gateMarkerPath: "",
+ tsxCliPath: "",
+ cliEntrypoint: "",
+ spawned: [] as Array<{ child: ChildProcess; args: readonly string[] }>,
+}));
+
+vi.mock("node:child_process", async (importOriginal) => {
+ const actual = (await importOriginal()) as Record<string, unknown> & {
+ spawn(command: string, args: readonly string[], options: SpawnOptions): ChildProcess;
+ };
+ return {
+ ...actual,
+ spawn(command: string, args: readonly string[], options: SpawnOptions): ChildProcess {
+ const child = actual.spawn(command, args, options);
+ if (workerLaunchTestState.capture) {
+ workerLaunchTestState.spawned.push({ child, args });
+ }
+ return child;
+ },
+ };
+});
+
+vi.mock("../src/cli/subprocess-launch.js", async (importOriginal) => {
+ const actual = (await importOriginal()) as Record<string, unknown>;
+ return {
+ ...actual,
+ createCliSubprocessLaunchSpec(args: readonly string[]) {
+ if (!workerLaunchTestState.capture) {
+ return (actual.createCliSubprocessLaunchSpec as (args: readonly string[]) => unknown)(args);
+ }
+ if (workerLaunchTestState.fixtureMode === "rollback-gate") {
+ const markerPath = JSON.stringify(workerLaunchTestState.gateMarkerPath);
+ const commitMarker = JSON.stringify(DAEMON_WORKER_STARTUP_GATE_COMMIT);
+ return {
+ command: process.execPath,
+ args: [
+ "--eval",
+ `const fs = require("node:fs"); const marker = fs.readFileSync(3, "utf8"); if (marker === ${commitMarker}) { fs.writeFileSync(${markerPath}, marker); setInterval(() => {}, 1000); }`,
+ "--",
+ ...args,
+ ],
+ };
+ }
+ if (workerLaunchTestState.fixtureMode === "close-gate") {
+ return {
+ command: process.execPath,
+ args: ["--eval", 'require("node:fs").closeSync(3)'],
+ };
+ }
+ if (workerLaunchTestState.fixtureMode === "successful-gate") {
+ const markerPath = JSON.stringify(workerLaunchTestState.gateMarkerPath);
+ return {
+ command: process.execPath,
+ args: [
+ "--eval",
+ `const fs = require("node:fs"); const marker = fs.readFileSync(3, "utf8"); fs.writeFileSync(${markerPath}, marker); setInterval(() => {}, 1000);`,
+ ],
+ };
+ }
+ return {
+ command: process.execPath,
+ args: [workerLaunchTestState.tsxCliPath, workerLaunchTestState.cliEntrypoint, ...args],
+ };
+ },
+ };
+});
+
+vi.mock("../src/core/session-lease.js", async (importOriginal) => {
+ const actual = (await importOriginal()) as Record<string, unknown> & {
+ getProcessStartId(pid: number): string | undefined;
+ };
+ return {
+ ...actual,
+ getProcessStartId(pid: number): string | undefined {
+ return workerLaunchTestState.forceMissingProcessStartId ? undefined : actual.getProcessStartId(pid);
+ },
+ };
+});
+
+const supervisorRegistryDirEnv = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR";
+const previousSupervisorRegistryDir = process.env[supervisorRegistryDirEnv];
+const supervisorRegistryDirs = new Set<string>();
+
interface SupervisorMonitorHarness {
options: { worker: object };
clients: Set<{ authenticated: boolean }>;
+ supervisorClaims: Map<object, object>;
shuttingDown: boolean;
supervisorMonitorTimer?: ReturnType<typeof setTimeout>;
canConnectToSupervisor: (socketPath: string) => Promise<boolean>;
@@ -19,10 +109,143 @@ interface SupervisorMonitorHarness {
scheduleSupervisorAvailabilityCheck: (socketPath: string, delayMs: number) => void;
}
+interface DeferredRecoveryWorker {
+ descriptor: {
+ workerId: string;
+ pid: number;
+ rootActiveSessionId: string;
+ lifecycle: "ready" | "recovering";
+ lastError?: string;
+ stopRequestedAt?: string;
+ };
+ client?: object;
+ snapshotCache: Map<string, DaemonAttachResult>;
+ incomingTranscriptActiveSessionIds: Set<string>;
+ transcriptCaches: Map<string, { markFailed(error: Error): void }>;
+ duplicateIncomingTranscriptChunkIndexes: Map<string, number>;
+ snapshotTransferFrames: Map<string, never>;
+ recovery?: Promise<void>;
+ deferredRecovery?: Promise<void>;
+ intentionalStop: boolean;
+ stopRevision: number;
+}
+
+interface DeferredRecoveryHarness {
+ workers: Map<string, DeferredRecoveryWorker>;
+ shuttingDown: boolean;
+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;
+ persistWorker: ReturnType<typeof vi.fn>;
+ syncAgentPeers: ReturnType<typeof vi.fn>;
+ recoverWorker: ReturnType<typeof vi.fn>;
+ handleWorkerClose(worker: DeferredRecoveryWorker, client: object, error: Error): Promise<void>;
+ deferWorkerRecovery(worker: DeferredRecoveryWorker, error: Error): void;
+}
+
+function recoveryDeniedError(code: "supervisor_recovery_cancelled" | "supervisor_generation_stale"): Error {
+ return Object.assign(new Error(code), { code });
+}
+
+async function waitForCapturedChildClose(child: ChildProcess): Promise<void> {
+ if (child.exitCode !== null || child.signalCode !== null) {
+ return;
+ }
+ await new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));
+}
+
+async function waitForFile(path: string): Promise<void> {
+ const deadline = Date.now() + 1000;
+ while (!existsSync(path) && Date.now() < deadline) {
+ await new Promise((resolveDelay) => setTimeout(resolveDelay, 10));
+ }
+ if (!existsSync(path)) {
+ throw new Error(`Timed out waiting for ${path}`);
+ }
+}
+
+function createExistingLaunchWorker(root: string, descriptorDir: string) {
+ const workerId = "existing-worker";
+ const now = new Date().toISOString();
+ return {
+ descriptor: {
+ version: 1 as const,
+ workerId,
+ pid: 999_999,
+ socketPath: join(root, `${workerId}.sock`),
+ recoveryJournalPath: join(descriptorDir, `${workerId}.recovery.jsonl`),
+ orphanProcessJournalPath: join(descriptorDir, `${workerId}.orphans.jsonl`),
+ supervisorSocketPath: join(root, "supervisor.sock"),
+ authenticationToken: "existing-worker-token",
+ rootActiveSessionId: "existing-root-session",
+ createdAt: now,
+ updatedAt: now,
+ lifecycle: "recovering" as const,
+ stopRequestedAt: undefined as string | undefined,
+ createCommand: { type: "create" as const, config: { cwd: root, agentDir: root } },
+ consecutiveFailures: 0,
+ },
+ descriptorPath: join(descriptorDir, `${workerId}.json`),
+ summaries: new Map<string, SessionSummary>(),
+ snapshotCache: new Map<string, DaemonAttachResult>(),
+ transcriptCaches: new Map<string, never>(),
+ incomingTranscriptActiveSessionIds: new Set<string>(),
+ duplicateIncomingTranscriptChunkIndexes: new Map<string, number>(),
+ snapshotTransferFrames: new Map<string, never>(),
+ snapshotLoads: new Map<string, Promise<DaemonAttachResult>>(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+}
+
+function createSupervisorSnapshotState() {
+ return {
+ clients: new Set<object>(),
+ pendingReplacementSnapshots: new WeakMap<object, Map<string, unknown>>(),
+ };
+}
+
+const recoveryEligibilityInvalidations: Array<{
+ name: string;
+ invalidate(supervisor: DeferredRecoveryHarness, worker: DeferredRecoveryWorker): void;
+}> = [
+ {
+ name: "the worker reconnects",
+ invalidate: (_supervisor, worker) => {
+ worker.client = {};
+ },
+ },
+ {
+ name: "the worker is stopped",
+ invalidate: (_supervisor, worker) => {
+ worker.intentionalStop = true;
+ worker.descriptor.stopRequestedAt = new Date().toISOString();
+ },
+ },
+ {
+ name: "the worker is replaced",
+ invalidate: (supervisor, worker) => supervisor.workers.set(worker.descriptor.workerId, { ...worker }),
+ },
+ {
+ name: "supervisor cleanup begins",
+ invalidate: (supervisor) => {
+ supervisor.shuttingDown = true;
+ },
+ },
+ {
+ name: "another recovery begins",
+ invalidate: (_supervisor, worker) => {
+ worker.recovery = Promise.resolve();
+ },
+ },
+];
+
function createHarness(canConnect: () => Promise<boolean>): SupervisorMonitorHarness {
+ const registryDir = mkdtempSync(join(tmpdir(), "prime-supervisor-registry-test-"));
+ supervisorRegistryDirs.add(registryDir);
+ process.env[supervisorRegistryDirEnv] = registryDir;
return Object.assign(Object.create(AgentDaemon.prototype), {
options: { worker: {} },
clients: new Set<{ authenticated: boolean }>(),
+ supervisorClaims: new Map<object, object>(),
shuttingDown: false,
canConnectToSupervisor: vi.fn(canConnect),
launchReplacementSupervisor: vi.fn(async () => undefined),
@@ -30,8 +253,434 @@ function createHarness(canConnect: () => Promise<boolean>): SupervisorMonitorHar
}
describe("daemon worker supervisor monitoring", () => {
- afterEach(() => {
+ afterEach(async () => {
+ for (const { child } of workerLaunchTestState.spawned) {
+ if (child.exitCode === null && child.signalCode === null) {
+ const closed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));
+ child.kill("SIGKILL");
+ await closed;
+ }
+ }
+ workerLaunchTestState.capture = false;
+ workerLaunchTestState.forceMissingProcessStartId = false;
+ workerLaunchTestState.fixtureMode = "worker";
+ workerLaunchTestState.gateMarkerPath = "";
+ workerLaunchTestState.tsxCliPath = "";
+ workerLaunchTestState.cliEntrypoint = "";
+ workerLaunchTestState.spawned.length = 0;
vi.useRealTimers();
+ for (const registryDir of supervisorRegistryDirs) {
+ rmSync(registryDir, { recursive: true, force: true });
+ }
+ supervisorRegistryDirs.clear();
+ if (previousSupervisorRegistryDir === undefined) {
+ delete process.env[supervisorRegistryDirEnv];
+ } else {
+ process.env[supervisorRegistryDirEnv] = previousSupervisorRegistryDir;
+ }
+ });
+
+ it.each([
+ {
+ name: "first post-spawn ownership check",
+ assertionCall: 3,
+ error: recoveryDeniedError("supervisor_generation_stale"),
+ },
+ {
+ name: "pre-publication ownership check",
+ assertionCall: 4,
+ error: recoveryDeniedError("supervisor_generation_stale"),
+ },
+ { name: "descriptor persistence", persistFailure: true, error: new Error("descriptor persistence failed") },
+ ] as const)("keeps unidentifiable workers gated after $name fails", async (scenario) => {
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "rollback-gate";
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-launch-gate-test-"));
+ const gateMarkerPath = join(root, "committed-gate");
+ workerLaunchTestState.gateMarkerPath = gateMarkerPath;
+ const descriptorDir = join(root, "descriptors");
+ const registryDir = join(root, "registry");
+ mkdirSync(descriptorDir, { recursive: true });
+ mkdirSync(registryDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ process.env[supervisorRegistryDirEnv] = registryDir;
+ let assertionCount = 0;
+ const workers = new Map<string, unknown>();
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ assertRecoveryAllowed: vi.fn(async () => {
+ assertionCount++;
+ if ("assertionCall" in scenario && assertionCount === scenario.assertionCall) {
+ throw scenario.error;
+ }
+ }),
+ ...("persistFailure" in scenario
+ ? {
+ persistWorker: vi.fn(() => {
+ throw scenario.error;
+ }),
+ }
+ : {}),
+ log: vi.fn(),
+ }) as {
+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;
+ };
+
+ await expect(supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } })).rejects.toBe(
+ scenario.error,
+ );
+
+ expect(assertionCount).toBe("assertionCall" in scenario ? scenario.assertionCall : 4);
+ expect(workerLaunchTestState.spawned).toHaveLength(1);
+ const { child, args } = workerLaunchTestState.spawned[0]!;
+ const socketFlagIndex = args.indexOf("--daemon-socket");
+ expect(socketFlagIndex).toBeGreaterThanOrEqual(0);
+ const workerSocketPath = args[socketFlagIndex + 1];
+ expect(workerSocketPath).toBeDefined();
+
+ expect(child.exitCode !== null || child.signalCode !== null).toBe(true);
+ expect(existsSync(gateMarkerPath)).toBe(false);
+ expect(existsSync(workerSocketPath!)).toBe(false);
+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);
+ expect(readdirSync(registryDir)).toEqual([]);
+ expect(workers.size).toBe(0);
+ });
+
+ it("rolls back promptly when the child closes its startup gate before commit", async () => {
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-closed-gate-test-"));
+ const descriptorDir = join(root, "descriptors");
+ mkdirSync(descriptorDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "close-gate";
+ let assertionCount = 0;
+ const workers = new Map<string, unknown>();
+ const connectWorker = vi.fn();
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(async () => {
+ assertionCount++;
+ if (assertionCount === 3) {
+ const child = workerLaunchTestState.spawned.at(-1)?.child;
+ if (!child) {
+ throw new Error("Worker child was not captured");
+ }
+ await waitForCapturedChildClose(child);
+ }
+ }),
+ connectWorker,
+ syncAgentPeers: vi.fn(async () => undefined),
+ log: vi.fn(),
+ }) as {
+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;
+ };
+
+ const timeoutError = new Error("startup gate rejection timed out");
+ const result = await Promise.race([
+ supervisor
+ .launchWorker({ type: "create", config: { cwd: root, agentDir: root } })
+ .then(() => ({ value: "resolved" as const, error: undefined }))
+ .catch((error: unknown) => ({ value: "rejected" as const, error })),
+ new Promise<{ value: "timed-out"; error: Error }>((resolveTimeout) =>
+ setTimeout(() => resolveTimeout({ value: "timed-out", error: timeoutError }), 1000),
+ ),
+ ]);
+
+ expect(result.value).toBe("rejected");
+ expect(result.error).not.toBe(timeoutError);
+ expect(result.error).toBeInstanceOf(Error);
+ expect(connectWorker).not.toHaveBeenCalled();
+ expect(workers.size).toBe(0);
+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);
+ const child = workerLaunchTestState.spawned.at(-1)?.child;
+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);
+ });
+
+ it("commits the startup marker after durable worker publication", async () => {
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-committed-gate-test-"));
+ const descriptorDir = join(root, "descriptors");
+ const markerPath = join(root, "startup-marker");
+ mkdirSync(descriptorDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "successful-gate";
+ workerLaunchTestState.gateMarkerPath = markerPath;
+ const workers = new Map<string, unknown>();
+ const connectWorker = vi.fn(async (worker: { descriptor: { rootActiveSessionId: string } }) => {
+ await waitForFile(markerPath);
+ return {
+ request: vi.fn(async () => ({
+ success: true,
+ data: {
+ id: worker.descriptor.rootActiveSessionId,
+ activeSessionId: worker.descriptor.rootActiveSessionId,
+ sessionId: "session-committed-gate",
+ cwd: root,
+ },
+ })),
+ };
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(async () => undefined),
+ connectWorker,
+ subscribeWorker: vi.fn(async () => undefined),
+ refreshWorkerSummaries: vi.fn(async () => undefined),
+ syncAgentPeers: vi.fn(async () => undefined),
+ log: vi.fn(),
+ }) as {
+ launchWorker(command: {
+ type: "create";
+ config: { cwd: string; agentDir: string };
+ }): Promise<{ descriptor: { lifecycle: string } }>;
+ };
+
+ const worker = await supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } });
+
+ expect(readFileSync(markerPath, "utf8")).toBe("start\n");
+ expect(connectWorker).toHaveBeenCalledOnce();
+ expect(worker.descriptor.lifecycle).toBe("ready");
+ expect(workers.size).toBe(1);
+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toHaveLength(1);
+ const child = workerLaunchTestState.spawned.at(-1)?.child;
+ if (!child) {
+ throw new Error("Worker child was not captured");
+ }
+ const closed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));
+ child.kill("SIGKILL");
+ await closed;
+ });
+
+ it("rolls back a published worker when shutdown admission and rollback persistence fail", async () => {
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-cancelled-launch-test-"));
+ const descriptorDir = join(root, "descriptors");
+ const markerPath = join(root, "startup-marker");
+ mkdirSync(descriptorDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "successful-gate";
+ workerLaunchTestState.gateMarkerPath = markerPath;
+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");
+ const rollbackPersistenceError = new Error("rollback persistence failed");
+ const persistWorker = Reflect.get(DaemonSupervisor.prototype, "persistWorker");
+ if (typeof persistWorker !== "function") {
+ throw new Error("Could not access worker persistence");
+ }
+ let persistenceCalls = 0;
+ const workers = new Map<string, unknown>();
+ const connectWorker = vi.fn(async () => {
+ await waitForFile(markerPath);
+ throw cancellation;
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(async () => undefined),
+ connectWorker,
+ persistWorker: vi.fn(function (this: object, worker: object) {
+ persistenceCalls++;
+ if (persistenceCalls === 2) {
+ throw rollbackPersistenceError;
+ }
+ Reflect.apply(persistWorker, this, [worker]);
+ }),
+ syncAgentPeers: vi.fn(async () => undefined),
+ log: vi.fn(),
+ }) as {
+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;
+ };
+
+ await expect(supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } })).rejects.toBe(
+ cancellation,
+ );
+
+ expect(readFileSync(markerPath, "utf8")).toBe("start\n");
+ expect(connectWorker).toHaveBeenCalledOnce();
+ expect(persistenceCalls).toBe(2);
+ expect(workers.size).toBe(0);
+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);
+ const child = workerLaunchTestState.spawned.at(-1)?.child;
+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);
+ });
+
+ it("defers an eligible existing recovery when descriptor restoration fails", async () => {
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-existing-restore-test-"));
+ const descriptorDir = join(root, "descriptors");
+ const markerPath = join(root, "startup-marker");
+ mkdirSync(descriptorDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "successful-gate";
+ workerLaunchTestState.gateMarkerPath = markerPath;
+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");
+ const restorationError = new Error("descriptor restoration failed");
+ const persistWorker = Reflect.get(DaemonSupervisor.prototype, "persistWorker");
+ if (typeof persistWorker !== "function") {
+ throw new Error("Could not access worker persistence");
+ }
+ let persistenceCalls = 0;
+ const existing = createExistingLaunchWorker(root, descriptorDir);
+ const previousDescriptor = existing.descriptor;
+ const workers = new Map<string, object>([[existing.descriptor.workerId, existing]]);
+ const deferWorkerRecovery = vi.fn();
+ const connectWorker = vi.fn(async () => {
+ await waitForFile(markerPath);
+ throw cancellation;
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(async () => undefined),
+ connectWorker,
+ persistWorker: vi.fn(function (this: object, worker: object) {
+ persistenceCalls++;
+ if (persistenceCalls === 3) {
+ throw restorationError;
+ }
+ Reflect.apply(persistWorker, this, [worker]);
+ }),
+ deferWorkerRecovery,
+ syncAgentPeers: vi.fn(async () => undefined),
+ log: vi.fn(),
+ }) as {
+ launchWorker(
+ command: { type: "create"; config: { cwd: string; agentDir: string } },
+ existing: object,
+ ): Promise<unknown>;
+ };
+
+ await expect(
+ supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } }, existing),
+ ).rejects.toBe(cancellation);
+
+ expect(persistenceCalls).toBe(3);
+ expect(existing.descriptor).toBe(previousDescriptor);
+ expect(workers.get(existing.descriptor.workerId)).toBe(existing);
+ expect(deferWorkerRecovery).toHaveBeenCalledOnce();
+ expect(deferWorkerRecovery).toHaveBeenCalledWith(existing, cancellation);
+ const child = workerLaunchTestState.spawned.at(-1)?.child;
+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);
+ });
+
+ it("does not restore an existing recovery invalidated during rollback", async () => {
+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-existing-stop-race-test-"));
+ const descriptorDir = join(root, "descriptors");
+ const markerPath = join(root, "startup-marker");
+ mkdirSync(descriptorDir, { recursive: true });
+ supervisorRegistryDirs.add(root);
+ workerLaunchTestState.capture = true;
+ workerLaunchTestState.forceMissingProcessStartId = true;
+ workerLaunchTestState.fixtureMode = "successful-gate";
+ workerLaunchTestState.gateMarkerPath = markerPath;
+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");
+ const existing = createExistingLaunchWorker(root, descriptorDir);
+ const workers = new Map<string, object>([[existing.descriptor.workerId, existing]]);
+ const deferWorkerRecovery = vi.fn();
+ const stopWorker = Reflect.get(DaemonSupervisor.prototype, "stopWorker");
+ if (typeof stopWorker !== "function") {
+ throw new Error("Could not access worker shutdown");
+ }
+ let markRollbackStarted = () => {};
+ const rollbackStarted = new Promise<void>((resolveStarted) => {
+ markRollbackStarted = resolveStarted;
+ });
+ let releaseRollback = () => {};
+ const rollbackRelease = new Promise<void>((resolveRelease) => {
+ releaseRollback = resolveRelease;
+ });
+ const controlledStopWorker = vi.fn(async function (
+ this: object,
+ worker: object,
+ removeDescriptor: boolean,
+ force = false,
+ archiveSession = false,
+ recoveryCleanup = false,
+ directChild?: object,
+ ) {
+ if (recoveryCleanup) {
+ markRollbackStarted();
+ await rollbackRelease;
+ return;
+ }
+ await Reflect.apply(stopWorker, this, [
+ worker,
+ removeDescriptor,
+ force,
+ archiveSession,
+ recoveryCleanup,
+ directChild,
+ ]);
+ });
+ const connectWorker = vi.fn(async () => {
+ await waitForFile(markerPath);
+ throw cancellation;
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ defaultSessionConfig: { cwd: root, agentDir: root },
+ descriptorDir,
+ socketPath: join(root, "supervisor.sock"),
+ workers,
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(async () => undefined),
+ connectWorker,
+ stopWorker: controlledStopWorker,
+ deferWorkerRecovery,
+ syncAgentPeers: vi.fn(async () => undefined),
+ log: vi.fn(),
+ }) as {
+ shuttingDown: boolean;
+ launchWorker(
+ command: { type: "create"; config: { cwd: string; agentDir: string } },
+ existing: object,
+ ): Promise<unknown>;
+ stopWorker(worker: object, removeDescriptor: boolean, force: boolean): Promise<void>;
+ };
+
+ const launchResult = supervisor
+ .launchWorker({ type: "create", config: { cwd: root, agentDir: root } }, existing)
+ .then(
+ () => undefined,
+ (error: unknown) => error,
+ );
+ await rollbackStarted;
+ supervisor.shuttingDown = true;
+ await supervisor.stopWorker(existing, true, true);
+ releaseRollback();
+
+ expect(await launchResult).toBe(cancellation);
+ expect(existing.stopRevision).toBe(1);
+ expect(existing.descriptor.stopRequestedAt).toBeDefined();
+ expect(workers.size).toBe(0);
+ expect(existsSync(existing.descriptorPath)).toBe(false);
+ expect(deferWorkerRecovery).not.toHaveBeenCalled();
});
it("attempts every shutdown cleanup step before exiting", async () => {
@@ -82,10 +731,18 @@ describe("daemon worker supervisor monitoring", () => {
it("does not poll a healthy supervisor after the startup check", async () => {
vi.useFakeTimers();
- const daemon = createHarness(async () => true);
+ let resolveProbe: () => void = () => undefined;
+ const probeCompleted = new Promise<void>((resolve) => {
+ resolveProbe = resolve;
+ });
+ const daemon = createHarness(async () => {
+ resolveProbe();
+ return true;
+ });
daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 1500);
await vi.advanceTimersByTimeAsync(1500);
+ await probeCompleted;
expect(daemon.canConnectToSupervisor).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(60_000);
@@ -97,6 +754,7 @@ describe("daemon worker supervisor monitoring", () => {
vi.useFakeTimers();
const daemon = createHarness(async () => true);
daemon.clients.add({ authenticated: true });
+ daemon.supervisorClaims.set({}, {});
daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 0);
await vi.runAllTimersAsync();
@@ -104,6 +762,266 @@ describe("daemon worker supervisor monitoring", () => {
expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();
});
+ it("retries when shutdown admission lookup fails", async () => {
+ vi.useFakeTimers();
+ let resolveProbe: () => void = () => undefined;
+ const probeCompleted = new Promise<void>((resolve) => {
+ resolveProbe = resolve;
+ });
+ const daemon = createHarness(async () => {
+ resolveProbe();
+ return true;
+ });
+ const registryDir = process.env[supervisorRegistryDirEnv];
+ if (!registryDir) throw new Error("Supervisor registry test directory was not set");
+ rmSync(registryDir, { recursive: true, force: true });
+ writeFileSync(registryDir, "not a directory");
+
+ daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 0);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();
+ expect(daemon.supervisorMonitorTimer).toBeDefined();
+
+ rmSync(registryDir, { force: true });
+ mkdirSync(registryDir, { recursive: true });
+ await vi.advanceTimersByTimeAsync(5000);
+ await probeCompleted;
+ expect(daemon.canConnectToSupervisor).toHaveBeenCalledOnce();
+ });
+
+ it("recovers exactly once after shutdown admission clears", async () => {
+ vi.useFakeTimers();
+ const client = {};
+ const worker: DeferredRecoveryWorker = {
+ descriptor: {
+ workerId: "worker-deferred-recovery",
+ pid: process.pid,
+ rootActiveSessionId: "active-deferred-recovery",
+ lifecycle: "ready",
+ },
+ client,
+ snapshotCache: new Map(),
+ incomingTranscriptActiveSessionIds: new Set(),
+ transcriptCaches: new Map(),
+ duplicateIncomingTranscriptChunkIndexes: new Map(),
+ snapshotTransferFrames: new Map<string, never>(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+ let admissionActive = true;
+ const assertRecoveryAllowed = vi.fn(async () => {
+ if (admissionActive) {
+ throw recoveryDeniedError("supervisor_recovery_cancelled");
+ }
+ });
+ const persistWorker = vi.fn();
+ const recoverWorker = vi.fn(async () => undefined);
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ assertRecoveryAllowed,
+ persistWorker,
+ syncAgentPeers: vi.fn(async () => undefined),
+ recoverWorker,
+ }) as DeferredRecoveryHarness;
+
+ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));
+ const deferredRecovery = worker.deferredRecovery;
+ expect(deferredRecovery).toBeDefined();
+ supervisor.deferWorkerRecovery(worker, new Error("duplicate close"));
+ expect(worker.deferredRecovery).toBe(deferredRecovery);
+ expect(worker.descriptor.lifecycle).toBe("ready");
+ expect(persistWorker).not.toHaveBeenCalled();
+ expect(recoverWorker).not.toHaveBeenCalled();
+
+ admissionActive = false;
+ await vi.advanceTimersByTimeAsync(5000);
+ await deferredRecovery;
+
+ expect(worker.descriptor.lifecycle).toBe("recovering");
+ expect(worker.descriptor.lastError).toBe("worker disconnected");
+ expect(persistWorker).toHaveBeenCalledOnce();
+ expect(recoverWorker).toHaveBeenCalledOnce();
+ expect(worker.deferredRecovery).toBeUndefined();
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(recoverWorker).toHaveBeenCalledOnce();
+ });
+
+ it("resumes deferred recovery after a concurrent recovery is denied", async () => {
+ vi.useFakeTimers();
+ const client = {};
+ const worker: DeferredRecoveryWorker = {
+ descriptor: {
+ workerId: "worker-concurrent-recovery",
+ pid: process.pid,
+ rootActiveSessionId: "active-concurrent-recovery",
+ lifecycle: "ready",
+ },
+ client,
+ snapshotCache: new Map(),
+ incomingTranscriptActiveSessionIds: new Set(),
+ transcriptCaches: new Map(),
+ duplicateIncomingTranscriptChunkIndexes: new Map(),
+ snapshotTransferFrames: new Map<string, never>(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+ let admissionActive = true;
+ const assertRecoveryAllowed = vi.fn(async () => {
+ if (admissionActive) {
+ throw recoveryDeniedError("supervisor_recovery_cancelled");
+ }
+ });
+ const persistWorker = vi.fn();
+ const recoverWorker = vi.fn(async () => undefined);
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ assertRecoveryAllowed,
+ persistWorker,
+ syncAgentPeers: vi.fn(async () => undefined),
+ recoverWorker,
+ }) as DeferredRecoveryHarness;
+
+ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));
+ const deferredRecovery = worker.deferredRecovery;
+ expect(deferredRecovery).toBeDefined();
+ let startConcurrentRecovery: () => void = () => undefined;
+ const concurrentRecoveryBarrier = new Promise<void>((resolve) => {
+ startConcurrentRecovery = resolve;
+ });
+ const concurrentRecovery = (async () => {
+ await concurrentRecoveryBarrier;
+ await expect(assertRecoveryAllowed()).rejects.toMatchObject({ code: "supervisor_recovery_cancelled" });
+ })().finally(() => {
+ worker.recovery = undefined;
+ });
+ worker.recovery = concurrentRecovery;
+
+ await vi.advanceTimersByTimeAsync(5000);
+ expect(worker.deferredRecovery).toBe(deferredRecovery);
+ expect(worker.descriptor.lifecycle).toBe("ready");
+ expect(persistWorker).not.toHaveBeenCalled();
+ expect(recoverWorker).not.toHaveBeenCalled();
+
+ startConcurrentRecovery();
+ await concurrentRecovery;
+ expect(worker.recovery).toBeUndefined();
+ admissionActive = false;
+ await vi.advanceTimersByTimeAsync(5000);
+ await deferredRecovery;
+
+ expect(assertRecoveryAllowed).toHaveBeenCalledTimes(3);
+ expect(worker.descriptor.lifecycle).toBe("recovering");
+ expect(worker.descriptor.lastError).toBe("worker disconnected");
+ expect(persistWorker).toHaveBeenCalledOnce();
+ expect(recoverWorker).toHaveBeenCalledOnce();
+ expect(worker.deferredRecovery).toBeUndefined();
+ });
+
+ it("cancels deferred recovery permanently after ownership loss", async () => {
+ vi.useFakeTimers();
+ const client = {};
+ const worker: DeferredRecoveryWorker = {
+ descriptor: {
+ workerId: "worker-stale-recovery",
+ pid: process.pid,
+ rootActiveSessionId: "active-stale-recovery",
+ lifecycle: "ready",
+ },
+ client,
+ snapshotCache: new Map(),
+ incomingTranscriptActiveSessionIds: new Set(),
+ transcriptCaches: new Map(),
+ duplicateIncomingTranscriptChunkIndexes: new Map(),
+ snapshotTransferFrames: new Map<string, never>(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+ let stale = false;
+ const assertRecoveryAllowed = vi.fn(async () => {
+ throw recoveryDeniedError(stale ? "supervisor_generation_stale" : "supervisor_recovery_cancelled");
+ });
+ const persistWorker = vi.fn();
+ const recoverWorker = vi.fn(async () => undefined);
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ assertRecoveryAllowed,
+ persistWorker,
+ syncAgentPeers: vi.fn(async () => undefined),
+ recoverWorker,
+ }) as DeferredRecoveryHarness;
+
+ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));
+ const deferredRecovery = worker.deferredRecovery;
+ expect(deferredRecovery).toBeDefined();
+ stale = true;
+ await vi.advanceTimersByTimeAsync(5000);
+ await deferredRecovery;
+
+ expect(worker.descriptor.lifecycle).toBe("ready");
+ expect(worker.descriptor.lastError).toBeUndefined();
+ expect(persistWorker).not.toHaveBeenCalled();
+ expect(recoverWorker).not.toHaveBeenCalled();
+ expect(worker.deferredRecovery).toBeUndefined();
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(recoverWorker).not.toHaveBeenCalled();
+ });
+
+ it.each(recoveryEligibilityInvalidations)(
+ "does not recover when $name during the ownership assertion",
+ async ({ invalidate }) => {
+ const client = {};
+ const worker: DeferredRecoveryWorker = {
+ descriptor: {
+ workerId: "worker-eligibility-race",
+ pid: process.pid,
+ rootActiveSessionId: "active-eligibility-race",
+ lifecycle: "ready",
+ },
+ client,
+ snapshotCache: new Map(),
+ incomingTranscriptActiveSessionIds: new Set(),
+ transcriptCaches: new Map(),
+ duplicateIncomingTranscriptChunkIndexes: new Map(),
+ snapshotTransferFrames: new Map<string, never>(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+ let resolveAssertion: () => void = () => undefined;
+ const assertion = new Promise<void>((resolve) => {
+ resolveAssertion = resolve;
+ });
+ const persistWorker = vi.fn();
+ const syncAgentPeers = vi.fn(async () => undefined);
+ const recoverWorker = vi.fn(async () => undefined);
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ ...createSupervisorSnapshotState(),
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ assertRecoveryAllowed: vi.fn(() => assertion),
+ persistWorker,
+ syncAgentPeers,
+ recoverWorker,
+ }) as DeferredRecoveryHarness;
+
+ const handling = supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));
+ invalidate(supervisor, worker);
+ resolveAssertion();
+ await handling;
+
+ expect(worker.descriptor.lifecycle).toBe("ready");
+ expect(worker.descriptor.lastError).toBeUndefined();
+ expect(persistWorker).not.toHaveBeenCalled();
+ expect(syncAgentPeers).not.toHaveBeenCalled();
+ expect(recoverWorker).not.toHaveBeenCalled();
+ },
+ );
+
it("cancels an in-flight recovery after an intentional stop tombstone", async () => {
vi.useFakeTimers();
type RecoveryWorker = {
@@ -159,6 +1077,7 @@ describe("daemon worker supervisor monitoring", () => {
connectWorker: ReturnType<typeof vi.fn>;
recoverUncertainWorkerOperations: ReturnType<typeof vi.fn>;
launchWorker: ReturnType<typeof vi.fn>;
+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;
recoverWorker(worker: RecoveryWorker): Promise<void>;
};
const worker: RecoveryWorker = {
@@ -178,6 +1097,7 @@ describe("daemon worker supervisor monitoring", () => {
connectWorker: vi.fn(),
recoverUncertainWorkerOperations: vi.fn(async () => {}),
launchWorker: vi.fn(async () => worker),
+ assertRecoveryAllowed: vi.fn(async () => {}),
}) as RecoveryHarness;
const recovery = supervisor.recoverWorker(worker);
@@ -216,6 +1136,7 @@ describe("daemon worker supervisor monitoring", () => {
persistWorker: ReturnType<typeof vi.fn>;
syncAgentPeers: ReturnType<typeof vi.fn>;
log: ReturnType<typeof vi.fn>;
+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;
recoverWorker(worker: RecoveryWorker): Promise<void>;
};
const worker: RecoveryWorker = {
@@ -240,6 +1161,7 @@ describe("daemon worker supervisor monitoring", () => {
persistWorker: vi.fn(),
syncAgentPeers: vi.fn(async () => {}),
log: vi.fn(),
+ assertRecoveryAllowed: vi.fn(async () => {}),
}) as RecoveryHarness;
const recovery = supervisor.recoverWorker(worker);
@@ -279,6 +1201,7 @@ describe("daemon worker supervisor monitoring", () => {
persistWorker: ReturnType<typeof vi.fn>;
syncAgentPeers: ReturnType<typeof vi.fn>;
log: ReturnType<typeof vi.fn>;
+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;
recoverWorker(worker: RecoveryWorker): Promise<void>;
};
const worker: RecoveryWorker = {
@@ -305,6 +1228,7 @@ describe("daemon worker supervisor monitoring", () => {
throw new Error("peer unavailable");
}),
log: vi.fn(),
+ assertRecoveryAllowed: vi.fn(async () => {}),
}) as RecoveryHarness;
const recovery = supervisor.recoverWorker(worker);
@@ -552,6 +1476,7 @@ describe("daemon worker supervisor monitoring", () => {
const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
catalog: { markInterrupted },
log: vi.fn(),
+ assertRecoveryAllowed: vi.fn(async () => {}),
}) as {
recoverUncertainWorkerOperations(worker: RecoveryWorker, killWorkerProcess: boolean): Promise<void>;
};
diff --git a/packages/coding-agent/test/daemon-supervisor-side-question.test.ts b/packages/coding-agent/test/daemon-supervisor-side-question.test.ts
index a52c8e028a..7c68f0e1b3 100644
--- a/packages/coding-agent/test/daemon-supervisor-side-question.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-side-question.test.ts
@@ -20,6 +20,7 @@ describe("daemon supervisor side-question routing", () => {
const write = vi.fn();
const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
ready: Promise.resolve(),
+ ownership: { assertCurrent: vi.fn(async () => undefined) },
handleCommand,
write,
log: vi.fn(),
diff --git a/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts b/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts
index 43742d6130..11adf1bc85 100644
--- a/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts
+++ b/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts
@@ -1,10 +1,12 @@
import { existsSync, unlinkSync } from "node:fs";
+import { fileURLToPath } from "node:url";
import lockfile from "proper-lockfile";
+import { APP_NAME } from "../../src/config.js";
import { isDaemonCatalogProcess, runDaemonCatalogProcess } from "../../src/modes/daemon/daemon-catalog-process.js";
import { DaemonSupervisor } from "../../src/modes/daemon/daemon-supervisor.js";
import { acquireDaemonSupervisorOwnership } from "../../src/modes/daemon/daemon-supervisor-ownership.js";
-type ControlMessage = { type: "go" | "probe" | "release" | "shutdown" | "cleanup" };
+type ControlMessage = { type: "go" | "probe" | "release" | "release_runtime" | "shutdown" | "cleanup" };
function requiredEnvironment(name: string): string {
const value = process.env[name];
@@ -49,6 +51,8 @@ async function runOwnershipHolder(): Promise<never> {
}
async function runSupervisor(): Promise<never> {
+ process.argv[1] = fileURLToPath(new URL("../../src/cli.ts", import.meta.url));
+ process.title = APP_NAME;
const socketPath = requiredEnvironment("ENG_4600_SOCKET_PATH");
const agentDir = requiredEnvironment("ENG_4600_AGENT_DIR");
const descriptorDir = requiredEnvironment("ENG_4600_DESCRIPTOR_DIR");
@@ -66,6 +70,16 @@ async function runSupervisor(): Promise<never> {
try {
await supervisor.start();
send({ type: "ready" });
+ process.on("message", (message: unknown) => {
+ if (
+ !message ||
+ typeof message !== "object" ||
+ (message as Partial<ControlMessage>).type !== "release_runtime"
+ ) {
+ return;
+ }
+ void releaseSupervisorRuntime(supervisor);
+ });
return await new Promise<never>(() => {});
} catch (error) {
send({ type: "failed", error: error instanceof Error ? error.message : String(error) });
@@ -73,6 +87,20 @@ async function runSupervisor(): Promise<never> {
}
}
+async function releaseSupervisorRuntime(supervisor: DaemonSupervisor): Promise<void> {
+ const ownership = Reflect.get(supervisor, "ownership") as { release: () => Promise<void> } | undefined;
+ await ownership?.release();
+ Reflect.set(supervisor, "ownership", undefined);
+ const cleanupSocket = Reflect.get(supervisor, "cleanupSocket");
+ if (typeof cleanupSocket === "function") {
+ Reflect.apply(cleanupSocket, supervisor, []);
+ }
+ const lease = Reflect.get(supervisor, "socketLease") as { release: () => Promise<void> } | undefined;
+ await lease?.release();
+ Reflect.set(supervisor, "socketLease", undefined);
+ send({ type: "runtime_released" });
+}
+
async function runLegacyCleanup(): Promise<never> {
const socketPath = requiredEnvironment("ENG_4600_SOCKET_PATH");
send({ type: "ready" });
diff --git a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
index 3143f85c1a..eaec8fbc06 100644
--- a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
@@ -703,7 +703,7 @@ describe("ENG-4602 snapshot transfer containment", () => {
expect(worker.snapshotTransferFrames.size).toBe(0);
});
- it("rejects same-ID reentrant begins and mismatched duplicate metadata", () => {
+ it("rejects same-ID reentrant begins and mismatched duplicate metadata", async () => {
const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor-invalid.sock", {
defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },
descriptorDir: "/tmp/eng-4602-supervisor-invalid-state",
@@ -711,20 +711,26 @@ describe("ENG-4602 snapshot transfer containment", () => {
const recoverWorker = vi.fn(async () => {});
const persistWorker = vi.fn();
const syncAgentPeers = vi.fn(async () => {});
+ const assertRecoveryAllowed = vi.fn(async () => {});
const internals = supervisor as unknown as {
+ workers: Map<string, WorkerHarness>;
recoverWorker: typeof recoverWorker;
persistWorker: typeof persistWorker;
syncAgentPeers: typeof syncAgentPeers;
+ assertRecoveryAllowed: typeof assertRecoveryAllowed;
handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
};
internals.recoverWorker = recoverWorker;
internals.persistWorker = persistWorker;
internals.syncAgentPeers = syncAgentPeers;
+ internals.assertRecoveryAllowed = assertRecoveryAllowed;
const frames = snapshotFrames([{ role: "user", content: "stable", timestamp: 1 }]);
const reentrant = workerHarness();
+ internals.workers.set(reentrant.worker.descriptor.workerId, reentrant.worker);
internals.handleWorkerFrame(reentrant.worker, frame(frames.begin));
internals.handleWorkerFrame(reentrant.worker, frame(frames.begin));
+ await new Promise<void>((resolve) => setImmediate(resolve));
expect(reentrant.close).toHaveBeenCalledOnce();
expect(reentrant.worker.transcriptCaches.has(activeSessionId)).toBe(false);
expect(reentrant.worker.client).toBeUndefined();
diff --git a/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.ts b/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.ts
new file mode 100644
index 0000000000..c7d824ab61
--- /dev/null
+++ b/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.ts
@@ -0,0 +1,1071 @@
+import { type ChildProcess, spawn, spawnSync } from "node:child_process";
+import {
+ chmodSync,
+ existsSync,
+ linkSync,
+ mkdirSync,
+ readdirSync,
+ readFileSync,
+ renameSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { createConnection, type Socket } from "node:net";
+import { join, resolve } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { APP_NAME, ENV_AGENT_DIR } from "../../../src/config.js";
+import { getProcessStartId } from "../../../src/core/session-lease.js";
+import { DaemonAgentConnection } from "../../../src/modes/agent-connection/daemon-agent-connection.js";
+import { DaemonClient } from "../../../src/modes/daemon/daemon-client.js";
+import type { DaemonResponse } from "../../../src/modes/daemon/daemon-protocol.js";
+import type { SessionSummary } from "../../../src/modes/daemon/daemon-session-list.js";
+import {
+ acquireDaemonShutdownAdmission,
+ acquireDaemonSupervisorOwnership,
+} from "../../../src/modes/daemon/daemon-supervisor-ownership.js";
+import {
+ DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,
+ DAEMON_WORKER_ROLE_ENV,
+ DAEMON_WORKER_SUPERVISOR_SOCKET_ENV,
+ DAEMON_WORKER_TOKEN_ENV,
+ type DaemonWorkerDescriptor,
+ type DaemonWorkerFrameHeader,
+ isDaemonWorkerFrameHeader,
+} from "../../../src/modes/daemon/daemon-worker-protocol.js";
+import { serializeJsonLine } from "../../../src/modes/rpc/jsonl.js";
+import {
+ encodePrivateFrame,
+ type PrivateFrame,
+ PrivateFrameDecoder,
+} from "../../../src/modes/session-worker/private-framing.js";
+import { createHarness, type Harness } from "../harness.js";
+
+interface OwnerRecord {
+ token: string;
+ generation: string;
+ pid: number;
+ processStartId?: string;
+ socketPath: string;
+ descriptorDir: string;
+ agentDir: string;
+}
+
+type FixtureMessage =
+ | { type: "booted" }
+ | { type: "ready" }
+ | { type: "failed"; error: string }
+ | { type: "runtime_released" };
+
+interface ProcessHandle {
+ child: ChildProcess;
+ identity?: FixtureProcessIdentity;
+ role: FixtureProcessIdentity["role"];
+ stdout: string;
+ stderr: string;
+ messages: FixtureMessage[];
+ waiters: Array<{
+ predicate: (message: FixtureMessage) => boolean;
+ resolve: (message: FixtureMessage) => void;
+ timeout: ReturnType<typeof setTimeout>;
+ }>;
+}
+
+interface FixtureProcessIdentity {
+ pid: number;
+ processStartId: string;
+ role: "client" | "supervisor" | "worker";
+}
+
+interface FixtureProcessSnapshot {
+ ppid: number;
+ state: string;
+}
+
+interface TestPaths {
+ agentDir: string;
+ descriptorDir: string;
+ executablePath: string;
+ registryDir: string;
+ socketTmpDir: string;
+ socketPath: string;
+}
+
+const fixturePath = resolve(__dirname, "../../fixtures/eng-4600-supervisor-fixture.ts");
+const fauxExtensionPath = resolve(__dirname, "../../fixtures/eng-4600-faux-extension.ts");
+const cliPath = resolve(__dirname, "../../../src/cli.ts");
+const tsxPath = resolve(__dirname, "../../../../../node_modules/tsx/dist/cli.mjs");
+const tsconfigPath = resolve(__dirname, "../../../../../tsconfig.json");
+const supervisorRegistryDirEnv = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR";
+const handles = new Set<ProcessHandle>();
+const harnesses: Harness[] = [];
+const socketTempDirs = new Set<string>();
+const fixtureDescriptorDirs = new Set<string>();
+const fixtureProcesses = new Map<string, FixtureProcessIdentity>();
+const fixtureRegistryDirs = new Set<string>();
+// Covers the worker's five second supervisor-availability retry.
+const fixtureProcessQuietMs = 5500;
+
+afterEach(async () => {
+ registerFixtureOwnedProcesses();
+ await stopFixtureOwnedProcesses();
+ await Promise.all([...handles].map((handle) => waitForExit(handle)));
+ handles.clear();
+ fixtureDescriptorDirs.clear();
+ fixtureProcesses.clear();
+ fixtureRegistryDirs.clear();
+ while (harnesses.length > 0) {
+ harnesses.pop()?.cleanup();
+ }
+ for (const path of socketTempDirs) {
+ rmSync(path, { recursive: true, force: true, maxRetries: 50, retryDelay: 50 });
+ }
+ socketTempDirs.clear();
+}, 60_000);
+
+async function createPaths(): Promise<TestPaths> {
+ const harness = await createHarness();
+ harnesses.push(harness);
+ const executablePath = join(harness.tempDir, APP_NAME);
+ linkSync(process.execPath, executablePath);
+ const socketTmpDir = `/tmp/eng-4603-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+ mkdirSync(socketTmpDir, { recursive: true, mode: 0o700 });
+ socketTempDirs.add(socketTmpDir);
+ fixtureDescriptorDirs.add(join(harness.tempDir, "workers"));
+ fixtureRegistryDirs.add(join(harness.tempDir, "registry"));
+ return {
+ agentDir: harness.tempDir,
+ descriptorDir: join(harness.tempDir, "workers"),
+ executablePath,
+ registryDir: join(harness.tempDir, "registry"),
+ socketTmpDir,
+ socketPath:
+ process.platform === "win32"
+ ? `\\\\.\\pipe\\prime-agent-eng-4603-${process.pid}-${Date.now()}`
+ : join(harness.tempDir, "daemon.sock"),
+ };
+}
+
+function spawnSupervisor(paths: TestPaths): ProcessHandle {
+ return trackProcess(
+ spawn(paths.executablePath, [tsxPath, fixturePath], {
+ cwd: paths.agentDir,
+ env: {
+ ...process.env,
+ [supervisorRegistryDirEnv]: paths.registryDir,
+ [ENV_AGENT_DIR]: paths.agentDir,
+ ENG_4600_AGENT_DIR: paths.agentDir,
+ ENG_4600_DESCRIPTOR_DIR: paths.descriptorDir,
+ ENG_4600_FIXTURE_MODE: "supervisor",
+ ENG_4600_REGISTRY_DIR: paths.registryDir,
+ ENG_4600_SOCKET_PATH: paths.socketPath,
+ PI_OFFLINE: "1",
+ TMPDIR: paths.socketTmpDir,
+ TSX_TSCONFIG_PATH: tsconfigPath,
+ },
+ stdio: ["ignore", "pipe", "pipe", "ipc"],
+ }),
+ "supervisor",
+ );
+}
+
+function spawnStandaloneWorker(
+ paths: TestPaths,
+ workerSocketPath: string,
+ token: string,
+ extraEnv: NodeJS.ProcessEnv = {},
+): ProcessHandle {
+ return trackProcess(
+ spawn(
+ paths.executablePath,
+ [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", workerSocketPath, "--offline"],
+ {
+ cwd: paths.agentDir,
+ env: {
+ ...process.env,
+ ...extraEnv,
+ [supervisorRegistryDirEnv]: paths.registryDir,
+ [ENV_AGENT_DIR]: paths.agentDir,
+ [DAEMON_WORKER_ROLE_ENV]: "1",
+ [DAEMON_WORKER_TOKEN_ENV]: token,
+ [DAEMON_WORKER_ACTIVE_SESSION_ID_ENV]: "eng-4603-worker",
+ [DAEMON_WORKER_SUPERVISOR_SOCKET_ENV]: paths.socketPath,
+ PI_OFFLINE: "1",
+ TSX_TSCONFIG_PATH: tsconfigPath,
+ },
+ stdio: ["ignore", "pipe", "pipe"],
+ },
+ ),
+ "worker",
+ );
+}
+
+function trackProcess(child: ChildProcess, role: FixtureProcessIdentity["role"]): ProcessHandle {
+ const identity = registerFixtureProcess(child.pid, getProcessStartId(child.pid ?? -1), role);
+ const handle: ProcessHandle = { child, identity, role, stdout: "", stderr: "", messages: [], waiters: [] };
+ handles.add(handle);
+ child.stdout?.on("data", (chunk: Buffer) => {
+ handle.stdout += chunk.toString("utf8");
+ });
+ child.stderr?.on("data", (chunk: Buffer) => {
+ handle.stderr += chunk.toString("utf8");
+ });
+ child.on("message", (message: FixtureMessage) => {
+ const index = handle.waiters.findIndex((waiter) => waiter.predicate(message));
+ if (index === -1) {
+ handle.messages.push(message);
+ return;
+ }
+ const [waiter] = handle.waiters.splice(index, 1);
+ if (waiter) {
+ clearTimeout(waiter.timeout);
+ waiter.resolve(message);
+ }
+ });
+ return handle;
+}
+
+function fixtureProcessKey(identity: FixtureProcessIdentity): string {
+ return `${identity.role}:${identity.pid}:${identity.processStartId}`;
+}
+
+function registerFixtureProcess(
+ pid: number | undefined,
+ processStartId: string | undefined,
+ role: FixtureProcessIdentity["role"],
+): FixtureProcessIdentity | undefined {
+ if (pid === undefined) return undefined;
+ if (!Number.isSafeInteger(pid) || pid <= 0) {
+ throw new Error(`Invalid fixture process pid: ${String(pid)}`);
+ }
+ if (!processStartId) {
+ if (fixturePidIsAlive(pid)) {
+ throw new Error(`Live fixture process ${pid} has no exact start identity`);
+ }
+ return undefined;
+ }
+ const identity = { pid, processStartId, role };
+ fixtureProcesses.set(fixtureProcessKey(identity), identity);
+ return identity;
+}
+
+function registerFixtureOwnedProcesses(): void {
+ for (const handle of handles) {
+ if (!handle.identity && (handle.child.exitCode !== null || handle.child.signalCode !== null)) {
+ continue;
+ }
+ handle.identity ??= registerFixtureProcess(
+ handle.child.pid,
+ getProcessStartId(handle.child.pid ?? -1),
+ handle.role,
+ );
+ }
+ for (const registryDir of fixtureRegistryDirs) {
+ for (const name of readFixtureDirectory(registryDir).filter((entry) => entry.endsWith(".owner"))) {
+ const ownerPath = join(registryDir, name, "owner.json");
+ const owner = readFixtureJson<unknown>(ownerPath);
+ if (owner === undefined) continue;
+ registerFixtureRecord(owner, "supervisor", ownerPath);
+ }
+ }
+ for (const descriptorDir of fixtureDescriptorDirs) {
+ for (const name of readFixtureDirectory(descriptorDir).filter((entry) => entry.endsWith(".json"))) {
+ const descriptorPath = join(descriptorDir, name);
+ const descriptor = readFixtureJson<unknown>(descriptorPath);
+ if (descriptor === undefined) continue;
+ registerFixtureRecord(descriptor, "worker", descriptorPath);
+ }
+ }
+}
+
+function registerFixtureRecord(value: unknown, role: "supervisor" | "worker", path: string): FixtureProcessIdentity {
+ if (!value || typeof value !== "object") {
+ throw new Error(`Invalid fixture process record: ${path}`);
+ }
+ const record = value as { pid?: unknown; processStartId?: unknown };
+ if (
+ typeof record.pid !== "number" ||
+ !Number.isSafeInteger(record.pid) ||
+ record.pid <= 0 ||
+ typeof record.processStartId !== "string" ||
+ record.processStartId.length === 0
+ ) {
+ throw new Error(`Invalid fixture process identity: ${path}`);
+ }
+ return registerFixtureProcess(record.pid, record.processStartId, role)!;
+}
+
+function readFixtureProcessSnapshot(): Map<number, FixtureProcessSnapshot> {
+ const listing = spawnSync("ps", ["-axo", "pid=,ppid=,state="], { encoding: "utf8" });
+ if (listing.error) throw listing.error;
+ if (listing.status !== 0) {
+ throw new Error(`Could not enumerate fixture descendants: ${listing.stderr.trim()}`);
+ }
+ const processes = new Map<number, FixtureProcessSnapshot>();
+ for (const line of listing.stdout.split("\n")) {
+ const match = /^\s*(\d+)\s+(\d+)\s+(\S+)\s*$/.exec(line);
+ if (!match) continue;
+ processes.set(Number(match[1]), { ppid: Number(match[2]), state: match[3]! });
+ }
+ return processes;
+}
+
+function fixtureDescendantPids(rootPid: number, processes: Map<number, FixtureProcessSnapshot>): number[] {
+ const childrenByParent = new Map<number, number[]>();
+ for (const [pid, process] of processes) {
+ const children = childrenByParent.get(process.ppid) ?? [];
+ children.push(pid);
+ childrenByParent.set(process.ppid, children);
+ }
+ const descendants: number[] = [];
+ const pending = [rootPid];
+ while (pending.length > 0) {
+ const parentPid = pending.shift()!;
+ for (const childPid of childrenByParent.get(parentPid) ?? []) {
+ descendants.push(childPid);
+ pending.push(childPid);
+ }
+ }
+ return descendants;
+}
+
+function isFixtureDescendant(pid: number, rootPid: number, processes: Map<number, FixtureProcessSnapshot>): boolean {
+ const visited = new Set<number>();
+ let current = pid;
+ while (!visited.has(current)) {
+ visited.add(current);
+ const process = processes.get(current);
+ if (!process) return false;
+ if (process.ppid === rootPid) return true;
+ current = process.ppid;
+ }
+ return false;
+}
+
+function signalFixtureProcess(identity: FixtureProcessIdentity, signal: NodeJS.Signals): boolean {
+ const state = fixtureProcessState(identity);
+ if (state === "exited") return false;
+ if (state === "unverified") {
+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);
+ }
+ try {
+ process.kill(identity.pid, signal);
+ return true;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ESRCH") return false;
+ throw error;
+ }
+}
+
+async function terminateFixtureProcessTree(root: FixtureProcessIdentity): Promise<void> {
+ if (fixtureProcessState(root) === "exited") return;
+ if (process.platform === "win32") {
+ if (signalFixtureProcess(root, "SIGKILL")) await waitForFixtureProcessExit(root);
+ return;
+ }
+ if (!signalFixtureProcess(root, "SIGSTOP")) return;
+ if (!(await waitForFixtureProcessStopped(root))) {
+ throw new Error(`Fixture root ${root.pid}/${root.processStartId} exited before descendant discovery`);
+ }
+ const tree = new Map<string, FixtureProcessIdentity>([[`${root.pid}:${root.processStartId}`, root]]);
+ let unchangedScans = 0;
+ while (unchangedScans < 2) {
+ const processesBefore = readFixtureProcessSnapshot();
+ let discovered = 0;
+ for (const pid of fixtureDescendantPids(root.pid, processesBefore)) {
+ const processStartId = getProcessStartId(pid);
+ if (!processStartId) {
+ if (fixturePidIsAlive(pid)) throw new Error(`Live fixture descendant ${pid} has no exact start identity`);
+ continue;
+ }
+ if (
+ [...tree.values()].some((identity) => identity.pid === pid && identity.processStartId === processStartId)
+ ) {
+ continue;
+ }
+ const processesAfter = readFixtureProcessSnapshot();
+ const currentStartId = getProcessStartId(pid);
+ if (currentStartId !== processStartId) {
+ if (currentStartId === undefined && fixturePidIsAlive(pid)) {
+ throw new Error(`Could not revalidate fixture descendant ${pid}`);
+ }
+ continue;
+ }
+ if (!isFixtureDescendant(pid, root.pid, processesAfter)) {
+ throw new Error(`Fixture descendant ${pid}/${processStartId} changed ancestry during cleanup`);
+ }
+ const identity = registerFixtureProcess(pid, processStartId, "worker")!;
+ tree.set(`${identity.pid}:${identity.processStartId}`, identity);
+ if (signalFixtureProcess(identity, "SIGSTOP")) {
+ await waitForFixtureProcessStopped(identity);
+ }
+ discovered++;
+ }
+ unchangedScans = discovered === 0 ? unchangedScans + 1 : 0;
+ await delay(25);
+ }
+ for (const identity of [...tree.values()].reverse()) {
+ signalFixtureProcess(identity, "SIGKILL");
+ }
+ for (const identity of tree.values()) {
+ await waitForFixtureProcessExit(identity);
+ }
+}
+
+async function waitForFixtureProcessStopped(identity: FixtureProcessIdentity, timeoutMs = 5000): Promise<boolean> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ const state = fixtureProcessState(identity);
+ if (state === "exited") return false;
+ if (state === "unverified") {
+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);
+ }
+ const processStatus = readFixtureProcessSnapshot().get(identity.pid)?.state;
+ if (processStatus && ["T", "Z"].includes(processStatus[0]!) && fixtureProcessState(identity) === "matching") {
+ return true;
+ }
+ await delay(25);
+ }
+ throw new Error(`Timed out waiting for fixture process ${identity.pid}/${identity.processStartId} to stop`);
+}
+
+function readFixtureDirectory(path: string): string[] {
+ try {
+ return readdirSync(path);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
+ throw error;
+ }
+}
+
+function readFixtureJson<T>(path: string): T | undefined {
+ try {
+ return JSON.parse(readFileSync(path, "utf8")) as T;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+ throw error;
+ }
+}
+
+function fixtureProcessState(identity: FixtureProcessIdentity): "exited" | "matching" | "unverified" {
+ const processStartId = getProcessStartId(identity.pid);
+ if (processStartId === identity.processStartId) return "matching";
+ if (processStartId !== undefined) return "exited";
+ return fixturePidIsAlive(identity.pid) ? "unverified" : "exited";
+}
+
+function fixturePidIsAlive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code === "ESRCH") return false;
+ if (code === "EPERM") return true;
+ throw error;
+ }
+}
+
+async function stopFixtureOwnedProcesses(): Promise<void> {
+ const deadline = Date.now() + 30_000;
+ let quietSince: number | undefined;
+ while (Date.now() < deadline) {
+ for (const role of ["supervisor", "worker", "client"] as const) {
+ for (const identity of [...fixtureProcesses.values()].filter((process) => process.role === role)) {
+ const state = fixtureProcessState(identity);
+ if (state === "unverified") {
+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);
+ }
+ if (state === "matching") {
+ await terminateFixtureProcessTree(identity);
+ }
+ fixtureProcesses.delete(fixtureProcessKey(identity));
+ }
+ }
+ registerFixtureOwnedProcesses();
+ const remaining = [...fixtureProcesses.values()].filter((identity) => fixtureProcessState(identity) !== "exited");
+ if (remaining.length > 0) {
+ quietSince = undefined;
+ continue;
+ }
+ quietSince ??= Date.now();
+ if (Date.now() - quietSince >= fixtureProcessQuietMs) return;
+ await delay(25);
+ }
+ throw new Error("Timed out stopping ENG-4603 fixture processes");
+}
+
+async function terminateTrackedFixtureProcess(handle: ProcessHandle): Promise<void> {
+ if (!handle.identity && (handle.child.exitCode !== null || handle.child.signalCode !== null)) {
+ await waitForExit(handle);
+ return;
+ }
+ const identity =
+ handle.identity ??
+ registerFixtureProcess(handle.child.pid, getProcessStartId(handle.child.pid ?? -1), handle.role);
+ if (identity) await terminateFixtureProcessTree(identity);
+ await waitForExit(handle);
+}
+
+async function waitForFixtureProcessExit(identity: FixtureProcessIdentity): Promise<void> {
+ const deadline = Date.now() + 30_000;
+ while (fixtureProcessState(identity) !== "exited" && Date.now() < deadline) {
+ await delay(25);
+ }
+ const state = fixtureProcessState(identity);
+ if (state !== "exited") {
+ throw new Error(`Timed out waiting for fixture process ${identity.pid}/${identity.processStartId}`);
+ }
+}
+
+function waitForMessage(
+ handle: ProcessHandle,
+ predicate: (message: FixtureMessage) => boolean,
+ timeoutMs = 30_000,
+): Promise<FixtureMessage> {
+ const index = handle.messages.findIndex(predicate);
+ if (index !== -1) {
+ return Promise.resolve(handle.messages.splice(index, 1)[0]!);
+ }
+ return new Promise((resolveMessage, rejectMessage) => {
+ const timeout = setTimeout(() => {
+ handle.waiters = handle.waiters.filter((waiter) => waiter.timeout !== timeout);
+ rejectMessage(new Error(`Timed out waiting for fixture message\n${handle.stderr}`));
+ }, timeoutMs);
+ handle.waiters.push({ predicate, resolve: resolveMessage, timeout });
+ });
+}
+
+function waitForType<T extends FixtureMessage["type"]>(
+ handle: ProcessHandle,
+ type: T,
+ timeoutMs?: number,
+): Promise<Extract<FixtureMessage, { type: T }>> {
+ return waitForMessage(handle, (message) => message.type === type || message.type === "failed", timeoutMs)
+ .then((message) => {
+ if (message.type === "failed") throw new Error(message.error);
+ return message as Extract<FixtureMessage, { type: T }>;
+ })
+ .catch((error: unknown) => {
+ throw new Error(`Timed out waiting for fixture message ${type}: ${String(error)}`);
+ });
+}
+
+function waitForExit(handle: ProcessHandle, timeoutMs = 30_000): Promise<void> {
+ if (handle.child.exitCode !== null || handle.child.signalCode !== null) {
+ return Promise.resolve();
+ }
+ return new Promise((resolveExit, rejectExit) => {
+ const timeout = setTimeout(
+ () => rejectExit(new Error(`Timed out waiting for process exit\n${handle.stderr}`)),
+ timeoutMs,
+ );
+ handle.child.once("exit", () => {
+ clearTimeout(timeout);
+ resolveExit();
+ });
+ });
+}
+
+async function waitForPath(path: string, timeoutMs = 30_000): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (!existsSync(path) && Date.now() < deadline) {
+ await delay(25);
+ }
+ if (!existsSync(path)) {
+ throw new Error(`Timed out waiting for ${path}`);
+ }
+}
+
+async function connectEventually(socketPath: string): Promise<DaemonClient> {
+ const deadline = Date.now() + 30_000;
+ let lastError: unknown;
+ while (Date.now() < deadline) {
+ const client = new DaemonClient(socketPath);
+ try {
+ await client.connect(500);
+ await client.waitForHello(2000);
+ return client;
+ } catch (error) {
+ lastError = error;
+ client.close();
+ await delay(25);
+ }
+ }
+ throw new Error(`Timed out connecting to supervisor: ${String(lastError)}`);
+}
+
+function listOwnerRecords(registryDir: string): OwnerRecord[] {
+ if (!existsSync(registryDir)) {
+ return [];
+ }
+ const records = readdirSync(registryDir)
+ .filter((name) => name.endsWith(".owner"))
+ .map((name) => {
+ const path = join(registryDir, name, "owner.json");
+ const record = JSON.parse(readFileSync(path, "utf8")) as unknown;
+ registerFixtureRecord(record, "supervisor", path);
+ return record as OwnerRecord;
+ });
+ return records;
+}
+
+function readWorkerDescriptor(descriptorDir: string): DaemonWorkerDescriptor {
+ const descriptors = readdirSync(descriptorDir).filter((name) => name.endsWith(".json"));
+ expect(descriptors).toHaveLength(1);
+ const descriptor = JSON.parse(readFileSync(join(descriptorDir, descriptors[0]!), "utf8")) as DaemonWorkerDescriptor;
+ registerFixtureRecord(descriptor, "worker", join(descriptorDir, descriptors[0]!));
+ return descriptor;
+}
+
+function requireSummary(value: unknown): SessionSummary {
+ if (!value || typeof value !== "object" || typeof (value as Partial<SessionSummary>).id !== "string") {
+ throw new Error("Daemon returned an invalid session summary");
+ }
+ return value as SessionSummary;
+}
+
+function exactProcessIsAlive(pid: number, processStartId: string | undefined): boolean {
+ if (!processStartId) {
+ return false;
+ }
+ return getProcessStartId(pid) === processStartId;
+}
+
+async function waitForExactProcessExit(
+ pid: number,
+ processStartId: string | undefined,
+ timeoutMs = 30_000,
+): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (exactProcessIsAlive(pid, processStartId) && Date.now() < deadline) {
+ await delay(25);
+ }
+ if (exactProcessIsAlive(pid, processStartId)) {
+ throw new Error(`Timed out waiting for exact process ${pid}/${processStartId ?? "unknown"}`);
+ }
+}
+
+async function createResidentSession(client: DaemonClient, agentDir: string): Promise<SessionSummary> {
+ const response = await client.request(
+ {
+ type: "create",
+ config: {
+ agentDir,
+ apiKey: "faux-key",
+ cwd: agentDir,
+ extensions: [fauxExtensionPath],
+ model: "faux",
+ noContextFiles: true,
+ noExtensions: false,
+ noSkills: true,
+ noTools: true,
+ provider: "faux",
+ },
+ },
+ 60_000,
+ );
+ if (!response.success) {
+ throw new Error(response.error);
+ }
+ return requireSummary(response.data);
+}
+
+async function runCli(
+ paths: TestPaths,
+ args: string[],
+ timeoutMs = 60_000,
+ extraEnv: NodeJS.ProcessEnv = {},
+): Promise<{ code: number; stdout: string; stderr: string }> {
+ const handle = trackProcess(
+ spawn(process.execPath, [tsxPath, cliPath, ...args], {
+ cwd: paths.agentDir,
+ env: {
+ ...process.env,
+ ...extraEnv,
+ [supervisorRegistryDirEnv]: paths.registryDir,
+ [ENV_AGENT_DIR]: paths.agentDir,
+ PI_OFFLINE: "1",
+ TMPDIR: paths.socketTmpDir,
+ TSX_TSCONFIG_PATH: tsconfigPath,
+ },
+ stdio: ["ignore", "pipe", "pipe"],
+ }),
+ "client",
+ );
+ await waitForExit(handle, timeoutMs);
+ return { code: handle.child.exitCode ?? 0, stdout: handle.stdout, stderr: handle.stderr };
+}
+
+function createFrameReader(socket: Socket): {
+ waitFor: (
+ predicate: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => boolean,
+ timeoutMs?: number,
+ ) => Promise<PrivateFrame<DaemonWorkerFrameHeader>>;
+} {
+ const decoder = new PrivateFrameDecoder(isDaemonWorkerFrameHeader);
+ const frames: PrivateFrame<DaemonWorkerFrameHeader>[] = [];
+ const waiters: Array<{
+ predicate: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => boolean;
+ resolve: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => void;
+ timeout: ReturnType<typeof setTimeout>;
+ }> = [];
+ socket.on("data", (chunk: Buffer) => {
+ for (const frame of decoder.push(chunk)) {
+ const index = waiters.findIndex((waiter) => waiter.predicate(frame));
+ if (index === -1) {
+ frames.push(frame);
+ continue;
+ }
+ const waiter = waiters.splice(index, 1)[0];
+ if (waiter) {
+ clearTimeout(waiter.timeout);
+ waiter.resolve(frame);
+ }
+ }
+ });
+ return {
+ waitFor: (predicate, timeoutMs = 10_000) => {
+ const index = frames.findIndex(predicate);
+ if (index !== -1) {
+ return Promise.resolve(frames.splice(index, 1)[0]!);
+ }
+ return new Promise((resolveFrame, rejectFrame) => {
+ const timeout = setTimeout(() => {
+ const waiterIndex = waiters.findIndex((waiter) => waiter.timeout === timeout);
+ if (waiterIndex !== -1) waiters.splice(waiterIndex, 1);
+ rejectFrame(new Error("Timed out waiting for worker frame"));
+ }, timeoutMs);
+ waiters.push({ predicate, resolve: resolveFrame, timeout });
+ });
+ },
+ };
+}
+
+function decodeResponse(frame: PrivateFrame<DaemonWorkerFrameHeader>): DaemonResponse {
+ return JSON.parse(frame.payload.toString("utf8")) as DaemonResponse;
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
+}
+
+describe("ENG-4603 worker recovery convergence", () => {
+ it("allows only the current generation to replace a crashed resident worker", async () => {
+ if (process.platform === "win32") return;
+ const paths = await createPaths();
+ const predecessor = spawnSupervisor(paths);
+ await waitForType(predecessor, "booted");
+ predecessor.child.send({ type: "go" });
+ await waitForType(predecessor, "ready", 60_000);
+ const predecessorClient = await connectEventually(paths.socketPath);
+ const summary = await createResidentSession(predecessorClient, paths.agentDir);
+ const activeSessionId = summary.activeSessionId ?? summary.id;
+ const originalWorkerPid = summary.workerPid;
+ if (!originalWorkerPid) throw new Error("Resident worker did not expose its pid");
+ const originalWorkerStartId = getProcessStartId(originalWorkerPid);
+ const originalWorkerIdentity = registerFixtureProcess(originalWorkerPid, originalWorkerStartId, "worker")!;
+
+ predecessor.child.send({ type: "release_runtime" });
+ await waitForType(predecessor, "runtime_released");
+ await terminateFixtureProcessTree(originalWorkerIdentity);
+ const successor = spawnSupervisor(paths);
+ await waitForType(successor, "booted");
+ successor.child.send({ type: "go" });
+ await waitForType(successor, "ready", 60_000);
+ await waitForExactProcessExit(originalWorkerPid, originalWorkerStartId);
+
+ const successorClient = await connectEventually(paths.socketPath);
+ let replacement: DaemonWorkerDescriptor | undefined;
+ const deadline = Date.now() + 30_000;
+ while (Date.now() < deadline) {
+ const candidate = readWorkerDescriptor(paths.descriptorDir);
+ if (candidate.pid !== originalWorkerPid && candidate.lifecycle === "ready") {
+ replacement = candidate;
+ break;
+ }
+ await delay(25);
+ }
+ if (!replacement) throw new Error("Current supervisor did not publish a replacement worker");
+ expect(getProcessStartId(replacement.pid)).toBe(replacement.processStartId);
+ await delay(750);
+ expect(exactProcessIsAlive(replacement.pid, replacement.processStartId)).toBe(true);
+ expect(readdirSync(paths.descriptorDir).filter((name) => name.endsWith(".json"))).toHaveLength(1);
+ const listed = await successorClient.request({ type: "list" });
+ if (!listed.success) throw new Error(listed.error);
+ const sessions = (listed.data as { sessions: unknown[] }).sessions;
+ expect(sessions).toHaveLength(1);
+ expect(requireSummary(sessions[0]).workerPid).toBe(replacement.pid);
+
+ const connection = await DaemonAgentConnection.attach(successorClient, activeSessionId, {
+ recoverDaemon: async () => {},
+ });
+ await connection.getInitialSnapshot();
+ await connection.prompt("after recovery");
+ await connection.waitForIdle();
+ expect(await connection.getMessages()).toContainEqual(
+ expect.objectContaining({
+ role: "assistant",
+ content: [expect.objectContaining({ text: "upgrade response 1" })],
+ }),
+ );
+ await connection.dispose();
+ await successorClient.request({ type: "shutdown", force: true }, 10_000);
+ successorClient.close();
+ predecessorClient.close();
+ await waitForExit(successor);
+ await waitForExactProcessExit(replacement.pid, replacement.processStartId);
+ await terminateTrackedFixtureProcess(predecessor);
+ }, 120_000);
+
+ it("rejects stale commands at worker receipt and before public journal insertion", async () => {
+ if (process.platform === "win32") return;
+ const workerPaths = await createPaths();
+ const workerSocketPath = join(workerPaths.agentDir, "worker-command.sock");
+ const token = "eng-4603-token";
+ const psCountPath = join(workerPaths.agentDir, "ps-count");
+ const workerEnvironment: NodeJS.ProcessEnv = {};
+ if (process.platform === "darwin") {
+ const psWrapperPath = join(workerPaths.agentDir, "ps");
+ writeFileSync(psWrapperPath, '#!/bin/sh\nprintf x >> "$ENG_4603_PS_COUNT_PATH"\nexec /bin/ps "$@"\n', {
+ mode: 0o700,
+ });
+ chmodSync(psWrapperPath, 0o700);
+ workerEnvironment.ENG_4603_PS_COUNT_PATH = psCountPath;
+ workerEnvironment.PATH = `${workerPaths.agentDir}:${process.env.PATH ?? ""}`;
+ }
+ const oldOwner = await acquireDaemonSupervisorOwnership({
+ agentDir: workerPaths.agentDir,
+ appVersion: "test",
+ descriptorDir: workerPaths.descriptorDir,
+ generation: "old-generation",
+ registryDir: workerPaths.registryDir,
+ socketPath: workerPaths.socketPath,
+ });
+ const worker = spawnStandaloneWorker(workerPaths, workerSocketPath, token, workerEnvironment);
+ await waitForPath(workerSocketPath);
+ const socket = createConnection(workerSocketPath);
+ const frames = createFrameReader(socket);
+ await new Promise<void>((resolveConnect, rejectConnect) => {
+ socket.once("connect", resolveConnect);
+ socket.once("error", rejectConnect);
+ });
+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.outboundType === "daemon_hello");
+ const authId = "auth-old";
+ socket.write(
+ encodePrivateFrame<DaemonWorkerFrameHeader>(
+ { kind: "command", requestId: authId, commandType: "worker_auth" },
+ Buffer.from(
+ serializeJsonLine({
+ id: authId,
+ type: "worker_auth",
+ token,
+ supervisorGeneration: oldOwner.record.generation,
+ supervisorPid: oldOwner.record.pid,
+ supervisorProcessStartId: oldOwner.record.processStartId,
+ supervisorSocketPath: oldOwner.record.socketPath,
+ }),
+ ),
+ ),
+ );
+ expect(
+ decodeResponse(
+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.requestId === authId),
+ ).success,
+ ).toBe(true);
+ if (process.platform === "darwin") {
+ const countAfterAuthentication = readFileSync(psCountPath, "utf8").length;
+ await delay(750);
+ expect(readFileSync(psCountPath, "utf8").length).toBe(countAfterAuthentication);
+ const ownerPath = join(workerPaths.registryDir, `${oldOwner.record.generation}.owner`, "owner.json");
+ const ownerRecord = JSON.parse(readFileSync(ownerPath, "utf8")) as { updatedAt: string };
+ ownerRecord.updatedAt = new Date(Date.now() + 1000).toISOString();
+ const updatedOwnerPath = `${ownerPath}.updated`;
+ writeFileSync(updatedOwnerPath, `${JSON.stringify(ownerRecord, null, 2)}\n`);
+ renameSync(updatedOwnerPath, ownerPath);
+ await delay(500);
+ const countAfterOwnerChange = readFileSync(psCountPath, "utf8").length;
+ expect(countAfterOwnerChange).toBeGreaterThan(countAfterAuthentication);
+ await delay(750);
+ expect(readFileSync(psCountPath, "utf8").length).toBe(countAfterOwnerChange);
+ }
+ const commandId = "stale-list";
+ const commandFrame = encodePrivateFrame<DaemonWorkerFrameHeader>(
+ { kind: "command", requestId: commandId, commandType: "list" },
+ Buffer.from(serializeJsonLine({ id: commandId, type: "list" })),
+ );
+ socket.write(commandFrame.subarray(0, commandFrame.length - 1));
+ const ownerDirectory = join(workerPaths.registryDir, `${oldOwner.record.generation}.owner`);
+ const ownerPath = join(ownerDirectory, "owner.json");
+ const transitionedOwner = JSON.parse(readFileSync(ownerPath, "utf8")) as OwnerRecord & { updatedAt: string };
+ transitionedOwner.token = "successor-token";
+ transitionedOwner.pid = worker.child.pid!;
+ transitionedOwner.processStartId = getProcessStartId(worker.child.pid!);
+ transitionedOwner.updatedAt = new Date().toISOString();
+ const transitionedOwnerPath = `${ownerPath}.transitioned`;
+ writeFileSync(transitionedOwnerPath, `${JSON.stringify(transitionedOwner, null, 2)}\n`);
+ renameSync(transitionedOwnerPath, ownerPath);
+ socket.write(commandFrame.subarray(commandFrame.length - 1));
+ const staleResponse = decodeResponse(
+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.requestId === commandId),
+ );
+ expect(staleResponse).toMatchObject({ success: false, error: "supervisor_generation_stale" });
+ socket.destroy();
+ await oldOwner.release();
+ rmSync(ownerDirectory, { recursive: true, force: true });
+ await terminateTrackedFixtureProcess(worker);
+
+ const publicPaths = await createPaths();
+ const staleSupervisor = spawnSupervisor(publicPaths);
+ await waitForType(staleSupervisor, "booted");
+ staleSupervisor.child.send({ type: "go" });
+ await waitForType(staleSupervisor, "ready", 60_000);
+ const publicClient = await connectEventually(publicPaths.socketPath);
+ const [publishedOwner] = listOwnerRecords(publicPaths.registryDir);
+ if (!publishedOwner) throw new Error("Supervisor did not publish its owner");
+ const journalPath = join(publicPaths.descriptorDir, "command-journal.jsonl");
+ const journalBefore = existsSync(journalPath) ? readFileSync(journalPath, "utf8") : "";
+ const displacedOwnerDir = join(publicPaths.registryDir, `${publishedOwner.generation}.displaced`);
+ renameSync(join(publicPaths.registryDir, `${publishedOwner.generation}.owner`), displacedOwnerDir);
+ const replacementOwner = await acquireDaemonSupervisorOwnership({
+ agentDir: publicPaths.agentDir,
+ appVersion: "test",
+ descriptorDir: publicPaths.descriptorDir,
+ generation: "journal-successor",
+ registryDir: publicPaths.registryDir,
+ socketPath: publicPaths.socketPath,
+ });
+ const rejected = await publicClient.request({ type: "create" });
+ expect(rejected).toMatchObject({ success: false, error: expect.stringContaining("no longer owns") });
+ expect(existsSync(journalPath) ? readFileSync(journalPath, "utf8") : "").toBe(journalBefore);
+ publicClient.close();
+ await replacementOwner.release();
+ rmSync(displacedOwnerDir, { recursive: true, force: true });
+ await terminateTrackedFixtureProcess(staleSupervisor);
+ }, 90_000);
+
+ it("serializes shutdown admission and reclaims an unrenewed live lease", async () => {
+ const paths = await createPaths();
+ const previousRegistryDir = process.env[supervisorRegistryDirEnv];
+ process.env[supervisorRegistryDirEnv] = paths.registryDir;
+ try {
+ const first = await acquireDaemonShutdownAdmission();
+ const record = JSON.parse(readFileSync(join(paths.registryDir, "shutdown-admission.json"), "utf8")) as {
+ pid: number;
+ processStartId?: string;
+ };
+ expect(record.pid).toBe(process.pid);
+ expect(record.processStartId).toBe(getProcessStartId(process.pid));
+ const refreshTimer = Reflect.get(first, "refreshTimer") as ReturnType<typeof setInterval> | undefined;
+ if (!refreshTimer) throw new Error("Shutdown admission did not start its lease refresh");
+ clearInterval(refreshTimer);
+ let acquired = false;
+ const waiting = acquireDaemonShutdownAdmission().then((admission) => {
+ acquired = true;
+ return admission;
+ });
+ await delay(250);
+ expect(acquired).toBe(false);
+ const second = await waiting;
+ expect(acquired).toBe(true);
+ let destructivePasses = 0;
+ const destructivePass = async (admission: { assertOrRenew: () => Promise<void> }) => {
+ await admission.assertOrRenew();
+ destructivePasses++;
+ };
+ await expect(destructivePass(first)).rejects.toMatchObject({ code: "daemon_shutdown_in_progress" });
+ await destructivePass(second);
+ expect(destructivePasses).toBe(1);
+ await second.release();
+ await first.release();
+ } finally {
+ if (previousRegistryDir === undefined) {
+ delete process.env[supervisorRegistryDirEnv];
+ } else {
+ process.env[supervisorRegistryDirEnv] = previousRegistryDir;
+ }
+ }
+ }, 15_000);
+
+ it("shutdown --all removes hidden supervisors and workers without changing CLI contracts", async () => {
+ if (process.platform === "win32") return;
+ const paths = await createPaths();
+ const predecessor = spawnSupervisor(paths);
+ await waitForType(predecessor, "booted");
+ predecessor.child.send({ type: "go" });
+ await waitForType(predecessor, "ready", 60_000);
+ const client = await connectEventually(paths.socketPath);
+ const session = await createResidentSession(client, paths.agentDir);
+ const workerPid = session.workerPid;
+ if (!workerPid) throw new Error("Resident worker did not expose its pid");
+ const predecessorStartId = getProcessStartId(predecessor.child.pid!);
+ const workerStartId = getProcessStartId(workerPid);
+ registerFixtureProcess(workerPid, workerStartId, "worker");
+ predecessor.child.send({ type: "release_runtime" });
+ await waitForType(predecessor, "runtime_released");
+ const successor = spawnSupervisor(paths);
+ await waitForType(successor, "booted");
+ successor.child.send({ type: "go" });
+ await waitForType(successor, "ready", 60_000);
+ const successorStartId = getProcessStartId(successor.child.pid!);
+ client.close();
+ const systemLsofPath = spawnSync("which", ["lsof"], { encoding: "utf8" }).stdout.trim();
+ if (!systemLsofPath) throw new Error("Could not locate lsof for the shutdown regression");
+ const lsofPath = join(paths.agentDir, "lsof");
+ writeFileSync(lsofPath, '#!/bin/sh\nexec "$ENG_4603_SYSTEM_LSOF" -nP -F pn -U -a -p "$ENG_4603_LSOF_PIDS"\n', {
+ mode: 0o700,
+ });
+ const lsofEnvironment = {
+ ENG_4603_LSOF_PIDS: `${predecessor.child.pid},${successor.child.pid},${workerPid}`,
+ ENG_4603_SYSTEM_LSOF: systemLsofPath,
+ PATH: `${paths.agentDir}:${process.env.PATH ?? ""}`,
+ };
+ const listenersBeforeShutdown = spawnSync(lsofPath, [], {
+ encoding: "utf8",
+ env: { ...process.env, ...lsofEnvironment },
+ }).stdout;
+ expect(listenersBeforeShutdown).toContain(`p${predecessor.child.pid}`);
+ expect(listenersBeforeShutdown).toContain(`p${successor.child.pid}`);
+
+ const shutdown = await runCli(paths, ["daemon", "shutdown", "--all", "--json"], 60_000, lsofEnvironment);
+ expect(shutdown.code).toBe(0);
+ const shutdownResult = JSON.parse(shutdown.stdout) as { stopped: unknown[]; failed: unknown[] };
+ const survivingIdentities = [
+ { pid: predecessor.child.pid!, processStartId: predecessorStartId },
+ { pid: successor.child.pid!, processStartId: successorStartId },
+ { pid: workerPid, processStartId: workerStartId },
+ ].filter((identity) => exactProcessIsAlive(identity.pid, identity.processStartId));
+ if (survivingIdentities.length > 0) {
+ expect(shutdownResult.failed).not.toHaveLength(0);
+ }
+ expect(shutdownResult).toMatchObject({ stopped: expect.any(Array), failed: [] });
+ await waitForExactProcessExit(predecessor.child.pid!, predecessorStartId);
+ await waitForExactProcessExit(successor.child.pid!, successorStartId);
+ await waitForExactProcessExit(workerPid, workerStartId);
+ await delay(11_000);
+ expect(exactProcessIsAlive(predecessor.child.pid!, predecessorStartId)).toBe(false);
+ expect(exactProcessIsAlive(successor.child.pid!, successorStartId)).toBe(false);
+ expect(exactProcessIsAlive(workerPid, workerStartId)).toBe(false);
+
+ const contracts = [
+ { args: ["daemon", "ps", "--json"], json: [] },
+ { args: ["daemon", "ps", "--reap", "--json"], json: { reaped: [], skipped: [] } },
+ { args: ["daemon", "shutdown", "--all", "--json"], json: { stopped: [], failed: [] } },
+ ];
+ for (const contract of contracts) {
+ const result = await runCli(paths, contract.args, 60_000, lsofEnvironment);
+ if (result.code !== 0) {
+ throw new Error(`${contract.args.join(" ")} exited ${result.code}: ${result.stderr}`);
+ }
+ expect(JSON.parse(result.stdout)).toEqual(contract.json);
+ }
+ for (const args of [
+ ["daemon", "ps"],
+ ["daemon", "ps", "--reap"],
+ ["daemon", "shutdown", "--all"],
+ ]) {
+ const result = await runCli(paths, args, 60_000, lsofEnvironment);
+ expect(result.code).toBe(0);
+ expect(result.stdout).toBe("No daemons found.\n");
+ }
+ }, 150_000);
+});