A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
serialize worker recovery and daemon shutdown (#408)
kt16e97e6dea79
diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.mdindex 546faa9d40..18b0108fd4 100644--- a/packages/coding-agent/CHANGELOG.md+++ b/packages/coding-agent/CHANGELOG.md@@ -9,6 +9,7 @@- Changed tool expansion hints to appear only on the latest tool row instead of every tool call ([ENG-4583](https://linear.app/primeintellect/issue/ENG-4583/too-many-ctrlo-alerts)).- Changed IPython kernels to set `NO_COLOR=1`, preventing ANSI color escapes from inflating `%%bash` output.- Fixed update restarts starting concurrent daemon supervisors or unlinking a replacement supervisor's socket ([ENG-4600](https://linear.app/primeintellect/issue/ENG-4600/prevent-concurrent-daemon-supervisors-after-update-restart)).+- Fixed worker recovery races and made daemon shutdown-all converge across hidden supervisors ([ENG-4603](https://linear.app/primeintellect/issue/ENG-4603/serialize-worker-recovery-and-make-shutdown-all-converge)).- Changed provider, model, and MCP setup to use one tabbed configuration menu ([ENG-4539](https://linear.app/primeintellect/issue/ENG-4539/unify-providers-models-and-mcp-connections-menu)).- Changed the shared configuration menu to show prominent, responsive tabs with configurable navigation shortcuts ([ENG-4534](https://linear.app/primeintellect/issue/ENG-4534/make-login-tabs-more-obvious)).- Fixed IPython edit diffs replacing syntax highlighting with a single foreground color ([ENG-4616](https://linear.app/primeintellect/issue/ENG-4616/syntax-highlighting-is-overridden-in-diff-view)).diff --git a/packages/coding-agent/src/cli/daemon-ps.ts b/packages/coding-agent/src/cli/daemon-ps.tsindex c376bfc1b4..1b25850aab 100644--- a/packages/coding-agent/src/cli/daemon-ps.ts+++ b/packages/coding-agent/src/cli/daemon-ps.ts@@ -1,11 +1,13 @@import { spawnSync } from "node:child_process";import { existsSync, lstatSync, readdirSync, unlinkSync } from "node:fs";-import { join, resolve } from "node:path";+import { basename, dirname, join, resolve } from "node:path";import chalk from "chalk";import { APP_NAME, VERSION } from "../config.js";+import { getProcessStartId } from "../core/session-lease.js";import { DaemonClient } from "../modes/daemon/daemon-client.js";import { DAEMON_PROTOCOL_VERSION } from "../modes/daemon/daemon-protocol.js";import { defaultDaemonSocketDir, defaultDaemonSocketPath } from "../modes/daemon/daemon-socket.js";+import { acquireDaemonShutdownAdmission } from "../modes/daemon/daemon-supervisor-ownership.js";import { formatDaemonListTable } from "./daemon-ps-format.js";/**@@ -182,6 +184,7 @@ interface ProbeResult {version?: string;protocolVersion?: number;sessionCount?: number;+ supervisorPid?: number;reachable: boolean;}@@ -196,11 +199,13 @@ async function probeDaemon(socketPath: string): Promise<ProbeResult> {try {let version: string | undefined;let protocolVersion: number | undefined;+ let supervisorPid: number | undefined;let greeted = false;try {const hello = await client.waitForHello(1500);version = hello.appVersion;protocolVersion = hello.protocol.version;+ supervisorPid = hello.supervisorPid;greeted = true;} catch {// Connected but no recognizable greeting: an old/foreign daemon.@@ -217,7 +222,7 @@ async function probeDaemon(socketPath: string): Promise<ProbeResult> {} catch {// Leave sessionCount undefined when the daemon will not answer list.}- return { version, protocolVersion, sessionCount, reachable: true };+ return { version, protocolVersion, sessionCount, supervisorPid, reachable: true };} finally {client.close();}@@ -363,10 +368,22 @@ const SHUTDOWN_ALL_ACTION_ORDER: Record<ReapAction["kind"], number> = {};export async function runShutdownAll(json: boolean): Promise<void> {- const daemons = await discoverDaemons();+ const admission = await acquireDaemonShutdownAdmission();+ try {+ await runShutdownAllConverging(json, () => admission.assertOrRenew());+ } finally {+ await admission.release();+ }+}++async function runShutdownAllConverging(json: boolean, assertAdmission: () => Promise<void>): Promise<void> {const stopped: Array<{ socketPath: string; action: string }> = [];const failed: Array<{ socketPath: string; reason: string }> = [];const handledPids = new Set<number>();+ const reportedFailures = new Set<string>();++ await stopHiddenSupervisors(stopped, failed, handledPids, reportedFailures, assertAdmission);+ const daemons = (await discoverDaemons()).filter((daemon) => !isWorkerSocketPath(daemon.socketPath));const actions = [...planShutdownAll(daemons)].sort((left, right) => SHUTDOWN_ALL_ACTION_ORDER[left.kind] - SHUTDOWN_ALL_ACTION_ORDER[right.kind],@@ -375,6 +392,7 @@ export async function runShutdownAll(json: boolean): Promise<void> {for (const action of actions) {const { socketPath, pid } = action.daemon;if (pid !== undefined && handledPids.has(pid)) {+ await assertAdmission();removeSocketFile(socketPath);stopped.push({ socketPath, action: `already stopped (pid ${pid})` });continue;@@ -382,36 +400,59 @@ export async function runShutdownAll(json: boolean): Promise<void> {switch (action.kind) {case "remove-file": {if ((await probeDaemon(socketPath)).reachable) {- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);- } else if (removeSocketFile(socketPath)) {- stopped.push({ socketPath, action: "removed stale socket file" });+ apply(+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),+ socketPath,+ stopped,+ failed,+ );} else {- failed.push({ socketPath, reason: "could not remove socket file" });+ await assertAdmission();+ if (removeSocketFile(socketPath)) {+ stopped.push({ socketPath, action: "removed stale socket file" });+ } else {+ failed.push({ socketPath, reason: "could not remove socket file" });+ }}break;}case "kill": {if ((await probeDaemon(socketPath)).reachable) {- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);+ apply(+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),+ socketPath,+ stopped,+ failed,+ );} else if (isDaemonProcessListening(pid!, socketPath)) {+ await assertAdmission();await forceKillDaemon(pid!);handledPids.add(pid!);+ await assertAdmission();removeSocketFile(socketPath);stopped.push({ socketPath, action: `killed unreachable daemon (pid ${pid})` });} else {+ await assertAdmission();removeSocketFile(socketPath);stopped.push({ socketPath, action: "daemon already stopped" });}break;}case "shutdown":- apply(await stopDaemonForcefully(socketPath, pid, handledPids), socketPath, stopped, failed);+ apply(+ await stopDaemonForcefully(socketPath, pid, handledPids, assertAdmission),+ socketPath,+ stopped,+ failed,+ );break;case "skip":break;}}+ await terminateVerifiedResiduals(stopped, failed, handledPids, reportedFailures, assertAdmission);+if (json) {console.log(JSON.stringify({ stopped, failed }, null, 2));return;@@ -428,11 +469,187 @@ export async function runShutdownAll(json: boolean): Promise<void> {}}+async function stopHiddenSupervisors(+ stopped: Array<{ socketPath: string; action: string }>,+ failed: Array<{ socketPath: string; reason: string }>,+ handledPids: Set<number>,+ reportedFailures: Set<string>,+ assertAdmission: () => Promise<void>,+): Promise<void> {+ while (true) {+ const listeners = scanListeningDaemons().filter((listener) => !isWorkerSocketPath(listener.socketPath));+ const bySocket = new Map<string, DiscoveredDaemonProcess[]>();+ for (const listener of listeners) {+ const group = bySocket.get(listener.socketPath) ?? [];+ group.push(listener);+ bySocket.set(listener.socketPath, group);+ }+ const hidden: DiscoveredDaemonProcess[] = [];+ for (const [socketPath, group] of bySocket) {+ if (new Set(group.map((listener) => listener.pid)).size < 2) {+ continue;+ }+ const currentPid = (await probeDaemon(socketPath)).supervisorPid;+ if (currentPid === undefined || !group.some((listener) => listener.pid === currentPid)) {+ recordShutdownFailure(+ failed,+ reportedFailures,+ socketPath,+ "could not identify the current same-path daemon",+ );+ continue;+ }+ hidden.push(...group.filter((listener) => listener.pid !== currentPid));+ }+ if (hidden.length === 0) {+ return;+ }+ const before = daemonListenerSignature(hidden);+ for (const listener of hidden) {+ if (await terminateVerifiedListener(listener, failed, reportedFailures, assertAdmission)) {+ handledPids.add(listener.pid);+ stopped.push({ socketPath: listener.socketPath, action: `stopped hidden daemon (pid ${listener.pid})` });+ }+ }+ const afterHidden = scanListeningDaemons().filter(+ (listener) =>+ !isWorkerSocketPath(listener.socketPath) &&+ hidden.some((candidate) => candidate.pid === listener.pid && candidate.socketPath === listener.socketPath),+ );+ if (afterHidden.length === 0 || daemonListenerSignature(afterHidden) === before) {+ return;+ }+ }+}++async function terminateVerifiedResiduals(+ stopped: Array<{ socketPath: string; action: string }>,+ failed: Array<{ socketPath: string; reason: string }>,+ handledPids: Set<number>,+ reportedFailures: Set<string>,+ assertAdmission: () => Promise<void>,+): Promise<void> {+ let previousSignature: string | undefined;+ while (true) {+ const listeners = scanListeningDaemons();+ if (listeners.length === 0) {+ return;+ }+ const signature = daemonListenerSignature(listeners);+ if (signature === previousSignature) {+ for (const listener of listeners) {+ const processStartId = getProcessStartId(listener.pid);+ if (processStartId && getProcessStartId(listener.pid) === processStartId) {+ recordShutdownFailure(+ failed,+ reportedFailures,+ listener.socketPath,+ `daemon process remained after shutdown (pid ${listener.pid}, start ${processStartId})`,+ );+ }+ }+ return;+ }+ previousSignature = signature;+ const seenPids = new Set<number>();+ for (const listener of listeners) {+ if (seenPids.has(listener.pid)) {+ continue;+ }+ seenPids.add(listener.pid);+ const alreadyReported = handledPids.has(listener.pid);+ if (await terminateVerifiedListener(listener, failed, reportedFailures, assertAdmission)) {+ handledPids.add(listener.pid);+ if (!alreadyReported) {+ stopped.push({+ socketPath: listener.socketPath,+ action: `stopped residual daemon process (pid ${listener.pid})`,+ });+ }+ }+ }+ }+}++async function terminateVerifiedListener(+ listener: DiscoveredDaemonProcess,+ failed: Array<{ socketPath: string; reason: string }>,+ reportedFailures: Set<string>,+ assertAdmission: () => Promise<void>,+): Promise<boolean> {+ const processStartId = getProcessStartId(listener.pid);+ if (!processStartId) {+ recordShutdownFailure(+ failed,+ reportedFailures,+ listener.socketPath,+ `could not verify daemon process identity (pid ${listener.pid})`,+ );+ return false;+ }+ if (getProcessStartId(listener.pid) !== processStartId) {+ return false;+ }+ await assertAdmission();+ if (getProcessStartId(listener.pid) !== processStartId) {+ return false;+ }+ killDaemon(listener.pid);+ const deadline = Date.now() + 1000;+ while (getProcessStartId(listener.pid) === processStartId && Date.now() < deadline) {+ await delay(50);+ }+ if (getProcessStartId(listener.pid) === processStartId) {+ await assertAdmission();+ if (getProcessStartId(listener.pid) !== processStartId) {+ return false;+ }+ try {+ process.kill(listener.pid, "SIGKILL");+ } catch {+ // The verified process exited between the identity check and signal.+ }+ }+ return getProcessStartId(listener.pid) !== processStartId;+}++function daemonListenerSignature(listeners: readonly DiscoveredDaemonProcess[]): string {+ return listeners+ .map((listener) => `${listener.pid}:${getProcessStartId(listener.pid) ?? "unknown"}:${listener.socketPath}`)+ .sort()+ .join("\n");+}++function recordShutdownFailure(+ failed: Array<{ socketPath: string; reason: string }>,+ reportedFailures: Set<string>,+ socketPath: string,+ reason: string,+): void {+ const key = `${socketPath}\0${reason}`;+ if (reportedFailures.has(key)) {+ return;+ }+ reportedFailures.add(key);+ failed.push({ socketPath, reason });+}++function isWorkerSocketPath(socketPath: string): boolean {+ return (+ process.platform !== "win32" &&+ resolve(dirname(socketPath)) === resolve(defaultDaemonSocketDir()) &&+ basename(socketPath).startsWith("worker-") &&+ basename(socketPath).endsWith(".sock")+ );+}+async function stopDaemonForcefully(socketPath: string,pid: number | undefined,handledPids: Set<number>,+ assertAdmission: () => Promise<void>,): Promise<ReapOutcome> {+ await assertAdmission();if (await shutdownDaemon(socketPath)) {if (pid !== undefined) {handledPids.add(pid);@@ -440,14 +657,17 @@ async function stopDaemonForcefully(return { reaped: `stopped daemon${pid ? ` (pid ${pid})` : ""}` };}if (!(await canConnectToSocket(socketPath, 250))) {+ await assertAdmission();removeSocketFile(socketPath);return { reaped: "daemon already stopped" };}if (pid === undefined) {return { skipped: "still listening but no pid to kill" };}+ await assertAdmission();await forceKillDaemon(pid);handledPids.add(pid);+ await assertAdmission();removeSocketFile(socketPath);return { reaped: `force-killed unresponsive daemon (pid ${pid})` };}diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.tsindex 3b359fa36c..56bdfe9d91 100644--- a/packages/coding-agent/src/main.ts+++ b/packages/coding-agent/src/main.ts@@ -71,6 +71,7 @@ import {DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,isDaemonWorkerProcess,requireDaemonWorkerAuthenticationToken,+ waitForDaemonWorkerStartupGate,} from "./modes/daemon/daemon-worker-protocol.js";import {type AgentConnection,@@ -1091,6 +1092,9 @@ export interface MainOptions {export async function main(args: string[], options?: MainOptions) {resetTimings();+ if (isDaemonWorkerProcess()) {+ waitForDaemonWorkerStartupGate();+ }installFileLogSink();if (isDaemonCatalogProcess()) {await runDaemonCatalogProcess();diff --git a/packages/coding-agent/src/modes/daemon/daemon-client.ts b/packages/coding-agent/src/modes/daemon/daemon-client.tsindex b51a8de475..a23a065b78 100644--- a/packages/coding-agent/src/modes/daemon/daemon-client.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-client.ts@@ -288,7 +288,8 @@ export class DaemonClient {}async authenticateWorker(token: string, timeoutMs = 3000): Promise<void> {- const response = await this.requestWire({ type: "worker_auth", token }, timeoutMs, {}, false);+ const legacyAuthentication = { type: "worker_auth", token } as DaemonWorkerCommandBody;+ const response = await this.requestWire(legacyAuthentication, timeoutMs, {}, false);if (!response.success) {throw new Error(response.error);}diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.tsindex d5ac89d1c9..47f341a843 100644--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts@@ -144,6 +144,7 @@ import {prepareDaemonSocketPath,restrictDaemonSocketPath,} from "./daemon-socket.js";+import { assertDaemonSupervisorOwnerCurrent, isDaemonShutdownAdmissionActive } from "./daemon-supervisor-ownership.js";import {DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,DAEMON_WORKER_RECOVERY_JOURNAL_ENV,@@ -272,6 +273,7 @@ const DAEMON_SERVER_CAPABILITIES: readonly DaemonClientCapability[] = [const DAEMON_CLIENT_CAPABILITY_SET: ReadonlySet<string> = new Set(DAEMON_SERVER_CAPABILITIES);const UPDATE_RESTART_ABORT_BASH_TIMEOUT_MS = 5000;+const SUPERVISOR_FENCE_POLL_MS = 250;const UPDATE_RESTART_MARKER ="<prime_agent_update_interrupted>\n" +"Prime Agent was updated and intentionally interrupted this session. Continue from the saved transcript and restored tool/kernel state. Any running model, tool, bash, or child-agent work may have been partially completed.\n" +@@ -300,6 +302,12 @@ function delay(ms: number): Promise<void> {}type RuntimeOpenGuard = () => boolean | Promise<boolean>;+type SupervisorGenerationClaim = Omit<Extract<DaemonWorkerCommand, { type: "worker_auth" }>, "id" | "type" | "token">;++interface BoundSupervisorGenerationClaim {+ claim: SupervisorGenerationClaim;+ ownerFingerprint: string;+}class RuntimeOpenCancelledError extends Error {}@@ -342,7 +350,9 @@ export class AgentDaemon {private readonly bindingSessions = new Set<string>();private restoreActiveSessionId: string | undefined;private supervisorMonitorTimer?: ReturnType<typeof setTimeout>;+ private supervisorFenceTimer?: ReturnType<typeof setTimeout>;private supervisorLaunchInProgress = false;+ private readonly supervisorClaims = new Map<DaemonSocketClient, BoundSupervisorGenerationClaim>();private agentMessagesPaused = false;private readonly summarizer = new DaemonSessionSummarizer(() => [...this.sessions.values()],@@ -472,7 +482,11 @@ export class AgentDaemon {}this.supervisorMonitorTimer = setTimeout(() => {this.supervisorMonitorTimer = undefined;- void this.checkSupervisorAvailability(supervisorSocketPath);+ void this.checkSupervisorAvailability(supervisorSocketPath).catch(() => {+ if (!this.shuttingDown && !this.hasAuthenticatedSupervisorConnection()) {+ this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);+ }+ });}, delayMs);}@@ -480,6 +494,10 @@ export class AgentDaemon {if (this.shuttingDown || this.hasAuthenticatedSupervisorConnection()) {return;}+ if (await isDaemonShutdownAdmissionActive()) {+ this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);+ return;+ }if (await this.canConnectToSupervisor(supervisorSocketPath)) {return;}@@ -490,7 +508,7 @@ export class AgentDaemon {}private hasAuthenticatedSupervisorConnection(): boolean {- return [...this.clients].some((client) => client.authenticated === true);+ return this.supervisorClaims.size > 0;}private clearSupervisorAvailabilityCheck(): void {@@ -498,6 +516,49 @@ export class AgentDaemon {clearTimeout(this.supervisorMonitorTimer);this.supervisorMonitorTimer = undefined;}+ if (this.supervisorFenceTimer) {+ clearTimeout(this.supervisorFenceTimer);+ this.supervisorFenceTimer = undefined;+ }+ }++ private scheduleSupervisorFenceCheck(): void {+ if (this.shuttingDown || this.supervisorFenceTimer || this.supervisorClaims.size === 0) {+ return;+ }+ this.supervisorFenceTimer = setTimeout(() => {+ this.supervisorFenceTimer = undefined;+ void this.checkSupervisorFences();+ }, SUPERVISOR_FENCE_POLL_MS);+ }++ private async checkSupervisorFences(): Promise<void> {+ for (const [client, boundClaim] of this.supervisorClaims) {+ try {+ boundClaim.ownerFingerprint = await this.assertSupervisorClaimCurrent(+ boundClaim.claim,+ boundClaim.ownerFingerprint,+ );+ } catch {+ client.socket.end();+ }+ }+ this.scheduleSupervisorFenceCheck();+ }++ private assertSupervisorClaimCurrent(+ claim: SupervisorGenerationClaim,+ validatedFingerprint?: string,+ ): Promise<string> {+ return assertDaemonSupervisorOwnerCurrent(+ {+ generation: claim.supervisorGeneration,+ pid: claim.supervisorPid,+ ...(claim.supervisorProcessStartId ? { processStartId: claim.supervisorProcessStartId } : {}),+ socketPath: claim.supervisorSocketPath,+ },+ validatedFingerprint,+ );}private canConnectToSupervisor(socketPath: string): Promise<boolean> {@@ -570,6 +631,9 @@ export class AgentDaemon {if (await this.canConnectToSupervisor(supervisorSocketPath)) {return;}+ if (await isDaemonShutdownAdmissionActive()) {+ return;+ }const launch = createCliSubprocessLaunchSpec(["--mode", "daemon", "--daemon-socket", supervisorSocketPath]);const environment = { ...process.env };delete environment[DAEMON_WORKER_ROLE_ENV];@@ -1718,6 +1782,7 @@ export class AgentDaemon {this.detachClient(client);client.detachInput();this.clients.delete(client);+ this.supervisorClaims.delete(client);const supervisorSocketPath = process.env[DAEMON_WORKER_SUPERVISOR_SOCKET_ENV];if (this.options.worker && client.authenticated === true && supervisorSocketPath) {this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 100);@@ -1744,6 +1809,10 @@ export class AgentDaemon {id?: unknown;type?: unknown;token?: unknown;+ supervisorGeneration?: unknown;+ supervisorPid?: unknown;+ supervisorProcessStartId?: unknown;+ supervisorSocketPath?: unknown;activeSessionId?: unknown;capabilities?: unknown;supportsExtensionUi?: unknown;@@ -1754,16 +1823,79 @@ export class AgentDaemon {}if (this.options.worker && client.authenticated !== true) {const commandId = typeof parsed.id === "string" ? parsed.id : undefined;- if (parsed.type !== "worker_auth" || parsed.token !== this.options.worker.authenticationToken) {+ if (+ parsed.type !== "worker_auth" ||+ parsed.token !== this.options.worker.authenticationToken ||+ typeof parsed.supervisorGeneration !== "string" ||+ !Number.isInteger(parsed.supervisorPid) ||+ (parsed.supervisorPid as number) <= 0 ||+ (parsed.supervisorProcessStartId !== undefined && typeof parsed.supervisorProcessStartId !== "string") ||+ typeof parsed.supervisorSocketPath !== "string"+ ) {this.write(client, failure(commandId, "worker_auth", "Worker authentication failed"));client.socket.end();return;}+ const claim: SupervisorGenerationClaim = {+ supervisorGeneration: parsed.supervisorGeneration,+ supervisorPid: parsed.supervisorPid as number,+ ...(typeof parsed.supervisorProcessStartId === "string"+ ? { supervisorProcessStartId: parsed.supervisorProcessStartId }+ : {}),+ supervisorSocketPath: parsed.supervisorSocketPath,+ };+ let ownerFingerprint: string;+ try {+ ownerFingerprint = await this.assertSupervisorClaimCurrent(claim);+ } catch {+ this.write(client, failure(commandId, "worker_auth", "supervisor_generation_stale"));+ client.socket.end();+ return;+ }+ for (const previous of this.supervisorClaims.keys()) {+ if (previous !== client) {+ previous.socket.end();+ }+ }client.authenticated = true;+ this.supervisorClaims.set(client, { claim, ownerFingerprint });this.clearSupervisorAvailabilityCheck();+ this.scheduleSupervisorFenceCheck();this.write(client, { id: commandId, type: "response", command: "worker_auth", success: true });return;}+ if (this.options.worker) {+ const boundClaim = this.supervisorClaims.get(client);+ if (!boundClaim) {+ this.write(+ client,+ failure(+ typeof parsed.id === "string" ? parsed.id : undefined,+ "worker_auth",+ "supervisor_generation_stale",+ ),+ );+ client.socket.end();+ return;+ }+ try {+ boundClaim.ownerFingerprint = await this.assertSupervisorClaimCurrent(+ boundClaim.claim,+ boundClaim.ownerFingerprint,+ );+ } catch {+ this.write(+ client,+ failure(+ typeof parsed.id === "string" ? parsed.id : undefined,+ typeof parsed.type === "string" ? parsed.type : "worker_auth",+ "supervisor_generation_stale",+ ),+ );+ client.socket.end();+ return;+ }+ }if (this.options.worker && typeof parsed.type === "string" && parsed.type.startsWith("worker_")) {await this.handleWorkerCommand(client, parsed as DaemonWorkerCommand);return;@@ -4135,6 +4267,10 @@ export class AgentDaemon {clearTimeout(this.supervisorMonitorTimer);this.supervisorMonitorTimer = undefined;}+ if (this.supervisorFenceTimer) {+ clearTimeout(this.supervisorFenceTimer);+ this.supervisorFenceTimer = undefined;+ }this.log(`shutting down (exit ${exitCode}); closing ${this.sessions.size} active session(s)`);const closingReason: DaemonClosingReason = this.updateRestartPreparing ? "update" : "shutdown";for (const client of this.clients) {diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.tsindex 3e989e0fc9..84334a0222 100644--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor-ownership.ts@@ -22,6 +22,10 @@ const REGISTRY_LOCK_UPDATE_MS = 1000;const REGISTRY_LOCK_RETRIES = 500;const REGISTRY_LOCK_RETRY_MS = 10;const STARTUP_FENCE_POLL_MS = 250;+const SHUTDOWN_ADMISSION_FILE_NAME = "shutdown-admission.json";+const SHUTDOWN_ADMISSION_LEASE_MS = 5000;+const SHUTDOWN_ADMISSION_REFRESH_MS = 1000;+const SHUTDOWN_ADMISSION_WAIT_MS = 50;type DaemonSupervisorOwnerPhase = "starting" | "owner" | "stopping";@@ -44,6 +48,14 @@ interface DaemonSupervisorOwnerRecord extends ProcessIdentity {updatedAt: string;}+interface DaemonShutdownAdmissionRecord extends ProcessIdentity {+ version: 1;+ token: string;+ createdAt: string;+ updatedAt: string;+ expiresAt: string;+}+interface DaemonSupervisorOwnerScope {version: 1;role: "supervisor";@@ -88,6 +100,24 @@ class DaemonSupervisorAlreadyRunningError extends Error {}}+class DaemonSupervisorOwnershipLostError extends Error {+ readonly code = "supervisor_generation_stale" as const;++ constructor(generation: string) {+ super(`Daemon supervisor generation ${generation} no longer owns its registry entry`);+ this.name = "DaemonSupervisorOwnershipLostError";+ }+}++class DaemonShutdownAdmissionError extends Error {+ readonly code = "daemon_shutdown_in_progress" as const;++ constructor(message = "Daemon shutdown is in progress") {+ super(message);+ this.name = "DaemonShutdownAdmissionError";+ }+}+class DaemonSupervisorOwnership {private released = false;@@ -97,6 +127,16 @@ class DaemonSupervisorOwnership {private readonly ownerDirectory: string,) {}+ async assertCurrent(): Promise<void> {+ if (this.released) {+ throw new DaemonSupervisorOwnershipLostError(this.record.generation);+ }+ const current = readOwnerRecord(this.ownerDirectory);+ if (!current || !sameOwnerRecord(current, this.record)) {+ throw new DaemonSupervisorOwnershipLostError(this.record.generation);+ }+ }+async updatePhase(phase: DaemonSupervisorOwnerPhase): Promise<void> {if (this.released) {return;@@ -139,6 +179,73 @@ class DaemonSupervisorOwnership {}}+class DaemonShutdownAdmission {+ private released = false;+ private lost = false;+ private refreshPromise?: Promise<void>;+ private readonly refreshTimer: ReturnType<typeof setInterval>;++ constructor(+ private readonly record: DaemonShutdownAdmissionRecord,+ private readonly registryDir: string,+ ) {+ this.refreshTimer = setInterval(() => {+ this.refreshPromise ??= this.assertOrRenew()+ .catch(() => undefined)+ .finally(() => {+ this.refreshPromise = undefined;+ });+ }, SHUTDOWN_ADMISSION_REFRESH_MS);+ this.refreshTimer.unref();+ }++ async assertOrRenew(): Promise<void> {+ if (this.released || this.lost) {+ throw new DaemonShutdownAdmissionError("Daemon shutdown admission was lost");+ }+ try {+ await withDaemonSupervisorRegistryGuard(this.registryDir, () => {+ const path = shutdownAdmissionPath(this.registryDir);+ const current = readShutdownAdmission(path);+ if (+ !current ||+ current.token !== this.record.token ||+ current.pid !== this.record.pid ||+ current.processStartId !== this.record.processStartId ||+ Date.parse(current.expiresAt) <= Date.now() ||+ !matchesExactProcessIdentity(this.record)+ ) {+ throw new DaemonShutdownAdmissionError("Daemon shutdown admission was lost");+ }+ const now = Date.now();+ this.record.updatedAt = new Date(now).toISOString();+ this.record.expiresAt = new Date(now + SHUTDOWN_ADMISSION_LEASE_MS).toISOString();+ writeJsonAtomically(path, this.record);+ });+ } catch (error) {+ this.lost = true;+ clearInterval(this.refreshTimer);+ throw error;+ }+ }++ async release(): Promise<void> {+ if (this.released) {+ return;+ }+ this.released = true;+ clearInterval(this.refreshTimer);+ await this.refreshPromise;+ await withDaemonSupervisorRegistryGuard(this.registryDir, () => {+ const path = shutdownAdmissionPath(this.registryDir);+ const current = readShutdownAdmission(path);+ if (current?.token === this.record.token) {+ rmSync(path, { force: true });+ }+ });+ }+}+function defaultDaemonSupervisorRegistryDir(environment: NodeJS.ProcessEnv = process.env): string {return environment[DAEMON_SUPERVISOR_REGISTRY_DIR_ENV] ?? resolve(defaultDaemonSocketDir(), "supervisor-owners");}@@ -224,6 +331,9 @@ export async function acquireDaemonSupervisorOwnership(writeOwnerScope(candidateDirectory, record);writeOwnerRecord(candidateDirectory, record);await withDaemonSupervisorRegistryGuard(registryDir, () => {+ if (readActiveShutdownAdmission(registryDir)) {+ throw new DaemonShutdownAdmissionError();+ }for (const directory of listOwnerDirectories(registryDir)) {const owner = readOwnerRecordForScope(directory, (scope) => ownerConflicts(scope, record));if (!owner) {@@ -252,6 +362,66 @@ export async function acquireDaemonSupervisorOwnership(return new DaemonSupervisorOwnership(record, registryDir, ownerDirectory);}+export async function assertDaemonSupervisorOwnerCurrent(+ owner: {+ generation: string;+ pid: number;+ processStartId?: string;+ socketPath: string;+ },+ validatedFingerprint?: string,+): Promise<string> {+ const registryDir = defaultDaemonSupervisorRegistryDir();+ const current = readOwnerRecord(ownerDirectoryPath(registryDir, owner.generation));+ if (+ !current ||+ current.pid !== owner.pid ||+ current.processStartId !== owner.processStartId ||+ current.socketPath !== normalizeSocketPath(owner.socketPath) ||+ !isProcessAlive(current.pid)+ ) {+ throw new DaemonSupervisorOwnershipLostError(owner.generation);+ }+ const fingerprint = ownerRecordFingerprint(current);+ if (fingerprint !== validatedFingerprint && !isProcessIdentityAlive(current)) {+ throw new DaemonSupervisorOwnershipLostError(owner.generation);+ }+ return fingerprint;+}++export async function acquireDaemonShutdownAdmission(): Promise<DaemonShutdownAdmission> {+ const registryDir = defaultDaemonSupervisorRegistryDir();+ const processStartId = getProcessStartId(process.pid);+ while (true) {+ let acquired: DaemonShutdownAdmissionRecord | undefined;+ await withDaemonSupervisorRegistryGuard(registryDir, () => {+ if (readActiveShutdownAdmission(registryDir)) {+ return;+ }+ const now = Date.now();+ acquired = {+ version: OWNER_VERSION,+ token: randomUUID(),+ pid: process.pid,+ ...(processStartId ? { processStartId } : {}),+ createdAt: new Date(now).toISOString(),+ updatedAt: new Date(now).toISOString(),+ expiresAt: new Date(now + SHUTDOWN_ADMISSION_LEASE_MS).toISOString(),+ };+ writeJsonAtomically(shutdownAdmissionPath(registryDir), acquired);+ });+ if (acquired) {+ return new DaemonShutdownAdmission(acquired, registryDir);+ }+ await delay(SHUTDOWN_ADMISSION_WAIT_MS);+ }+}++export async function isDaemonShutdownAdmissionActive(): Promise<boolean> {+ const registryDir = defaultDaemonSupervisorRegistryDir();+ return withDaemonSupervisorRegistryGuard(registryDir, () => readActiveShutdownAdmission(registryDir) !== undefined);+}+export async function persistDaemonStartupFenceFromOwner(socketPath: string,hello: DaemonSupervisorHelloIdentity,@@ -349,10 +519,8 @@ export async function waitForDaemonStartupFence(}function isProcessIdentityAlive(identity: ProcessIdentity): boolean {- try {- process.kill(identity.pid, 0);- } catch (error) {- return (error as NodeJS.ErrnoException).code !== "ESRCH";+ if (!isProcessAlive(identity.pid)) {+ return false;}if (!identity.processStartId) {return true;@@ -361,6 +529,22 @@ function isProcessIdentityAlive(identity: ProcessIdentity): boolean {return observed === undefined || observed === identity.processStartId;}+function matchesExactProcessIdentity(identity: ProcessIdentity): boolean {+ if (!isProcessAlive(identity.pid)) {+ return false;+ }+ return identity.processStartId === undefined || getProcessStartId(identity.pid) === identity.processStartId;+}++function isProcessAlive(pid: number): boolean {+ try {+ process.kill(pid, 0);+ } catch (error) {+ return (error as NodeJS.ErrnoException).code !== "ESRCH";+ }+ return true;+}+function normalizeSocketPath(socketPath: string): string {if (process.platform === "win32") {return socketPath.toLowerCase();@@ -395,6 +579,20 @@ function ownerConflicts(left: DaemonSupervisorOwnerScope, right: DaemonSupervisoreturn left.socketPath === right.socketPath || left.descriptorDir === right.descriptorDir;}+function sameOwnerRecord(left: DaemonSupervisorOwnerRecord, right: DaemonSupervisorOwnerRecord): boolean {+ return (+ left.token === right.token &&+ left.generation === right.generation &&+ left.pid === right.pid &&+ left.processStartId === right.processStartId &&+ left.socketPath === right.socketPath+ );+}++function ownerRecordFingerprint(record: DaemonSupervisorOwnerRecord): string {+ return createHash("sha256").update(JSON.stringify(record)).digest("hex");+}+function listOwnerDirectories(registryDir: string): string[] {return readdirSync(registryDir).filter((name) => name.endsWith(".owner"))@@ -535,6 +733,48 @@ function readStartupFence(path: string): DaemonStartupFenceRecord | undefined {}}+function readActiveShutdownAdmission(registryDir: string): DaemonShutdownAdmissionRecord | undefined {+ const path = shutdownAdmissionPath(registryDir);+ const admission = readShutdownAdmission(path);+ if (!admission) {+ return undefined;+ }+ if (Date.parse(admission.expiresAt) > Date.now() && isProcessIdentityAlive(admission)) {+ return admission;+ }+ rmSync(path, { force: true });+ return undefined;+}++function readShutdownAdmission(path: string): DaemonShutdownAdmissionRecord | undefined {+ try {+ const value = JSON.parse(readFileSync(path, "utf8")) as unknown;+ if (!value || typeof value !== "object") {+ throw new Error(`Invalid daemon shutdown admission: ${path}`);+ }+ const admission = value as Partial<DaemonShutdownAdmissionRecord>;+ if (+ admission.version !== OWNER_VERSION ||+ typeof admission.token !== "string" ||+ !Number.isInteger(admission.pid) ||+ (admission.pid ?? 0) <= 0 ||+ (admission.processStartId !== undefined && typeof admission.processStartId !== "string") ||+ typeof admission.createdAt !== "string" ||+ typeof admission.updatedAt !== "string" ||+ typeof admission.expiresAt !== "string" ||+ !Number.isFinite(Date.parse(admission.expiresAt))+ ) {+ throw new Error(`Invalid daemon shutdown admission: ${path}`);+ }+ return admission as DaemonShutdownAdmissionRecord;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ENOENT") {+ return undefined;+ }+ throw error;+ }+}+function writeJsonAtomically(path: string, value: unknown): void {const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;try {@@ -551,6 +791,10 @@ function startupFencePath(directory: string, socketPath: string): string {return resolve(directory, `${key}.json`);}+function shutdownAdmissionPath(registryDir: string): string {+ return resolve(registryDir, SHUTDOWN_ADMISSION_FILE_NAME);+}+function delay(ms: number): Promise<void> {return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));}diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.tsindex 0b2cc45f96..831253f705 100644--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts@@ -3,6 +3,7 @@ import { createHash, randomBytes, randomUUID } from "node:crypto";import { chmodSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";import { createServer, type Server, type Socket } from "node:net";import { dirname, isAbsolute, join, resolve } from "node:path";+import { Writable } from "node:stream";import { getLogger } from "@earendil-works/pi-ai";import { createCliSubprocessLaunchSpec } from "../../cli/subprocess-launch.js";import {@@ -65,12 +66,18 @@ import {prepareDaemonSocketPath,restrictDaemonSocketPath,} from "./daemon-socket.js";-import { acquireDaemonSupervisorOwnership, waitForDaemonStartupFence } from "./daemon-supervisor-ownership.js";+import {+ acquireDaemonSupervisorOwnership,+ isDaemonShutdownAdmissionActive,+ waitForDaemonStartupFence,+} from "./daemon-supervisor-ownership.js";import { DaemonWorkerClient } from "./daemon-worker-client.js";import {DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,DAEMON_WORKER_RECOVERY_JOURNAL_ENV,DAEMON_WORKER_ROLE_ENV,+ DAEMON_WORKER_STARTUP_GATE_COMMIT,+ DAEMON_WORKER_STARTUP_GATE_FD_ENV,DAEMON_WORKER_SUPERVISOR_SOCKET_ENV,DAEMON_WORKER_TOKEN_ENV,type DaemonCreateCommand,@@ -89,7 +96,9 @@ const structuredLog = getLogger("coding-agent.daemon-supervisor");const WORKER_CONNECT_TIMEOUT_MS = 30_000;const WORKER_REQUEST_TIMEOUT_MS = 24 * 60 * 60 * 1000;const WORKER_RETRY_DELAYS_MS = [250, 1000, 5000] as const;+const DEFERRED_RECOVERY_RECHECK_MS = 5000;const SUPERVISOR_CONFIG_FILE_NAME = "supervisor-config";+const WORKER_STARTUP_GATE_FD = 3;const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set(["ack_result",@@ -193,6 +202,7 @@ interface ResidentWorker {>;snapshotLoads: Map<string, Promise<DaemonAttachResult>>;recovery?: Promise<void>;+ deferredRecovery?: Promise<void>;intentionalStop: boolean;stopRevision: number;}@@ -225,10 +235,62 @@ interface WorkerAttachData {worker: ResidentWorker;}+class SupervisorRecoveryCancelledError extends Error {+ readonly code = "supervisor_recovery_cancelled" as const;+}++function isSupervisorGenerationStale(error: unknown): boolean {+ return (+ typeof error === "object" &&+ error !== null &&+ "code" in error &&+ (error as { code?: unknown }).code === "supervisor_generation_stale"+ );+}++function isSupervisorRecoveryCancelled(error: unknown): boolean {+ return isSupervisorShutdownAdmissionCancelled(error) || isSupervisorGenerationStale(error);+}++function isSupervisorShutdownAdmissionCancelled(error: unknown): boolean {+ return (+ error instanceof SupervisorRecoveryCancelledError ||+ (typeof error === "object" &&+ error !== null &&+ "code" in error &&+ (error as { code?: unknown }).code === "supervisor_recovery_cancelled")+ );+}+function delay(ms: number): Promise<void> {return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));}+function unrefDelay(ms: number): Promise<void> {+ return new Promise((resolveDelay) => setTimeout(resolveDelay, ms).unref());+}++function commitWorkerStartupGate(gate: Writable): Promise<void> {+ return new Promise((resolveCommit, rejectCommit) => {+ let settled = false;+ const finish = (error?: Error | null) => {+ if (settled) {+ return;+ }+ settled = true;+ if (error) {+ rejectCommit(error);+ } else {+ resolveCommit();+ }+ };+ const onError = (error: Error) => finish(error);+ gate.on("error", onError);+ gate.once("close", () => gate.off("error", onError));+ gate.end(DAEMON_WORKER_STARTUP_GATE_COMMIT, (error?: Error | null) => finish(error));+ });+}+function withoutCommandId(command: DaemonCommand): DaemonCommandBody {const { id: _id, ...body } = command;return body as DaemonCommandBody;@@ -585,6 +647,41 @@ export class DaemonSupervisor {appendRotatingLog(getDaemonLogPath(this.socketPath), `[${new Date().toISOString()}] supervisor: ${message}`);}+ private async assertCurrentOwnership(): Promise<void> {+ const ownership = this.ownership;+ if (!ownership) {+ const error = new Error(`Daemon supervisor generation ${this.generation} no longer owns its registry entry`);+ Object.assign(error, { code: "supervisor_generation_stale" as const });+ throw error;+ }+ await ownership.assertCurrent();+ }++ private async assertRecoveryAllowed(): Promise<void> {+ await this.assertCurrentOwnership();+ if (await isDaemonShutdownAdmissionActive()) {+ throw new SupervisorRecoveryCancelledError("Daemon shutdown admission cancelled worker recovery");+ }+ }++ private supervisorAuthenticationClaim(): {+ supervisorGeneration: string;+ supervisorPid: number;+ supervisorProcessStartId?: string;+ supervisorSocketPath: string;+ } {+ const record = this.ownership?.record;+ if (!record) {+ throw new SupervisorRecoveryCancelledError("Daemon supervisor ownership is unavailable");+ }+ return {+ supervisorGeneration: this.generation,+ supervisorPid: record.pid,+ ...(record.processStartId ? { supervisorProcessStartId: record.processStartId } : {}),+ supervisorSocketPath: record.socketPath,+ };+ }+private loadWorkerDescriptors(): void {for (const name of readdirSync(this.descriptorDir)) {if (name === SUPERVISOR_CONFIG_FILE_NAME || !name.endsWith(".json")) {@@ -778,6 +875,13 @@ export class DaemonSupervisor {return;}+ try {+ await this.assertCurrentOwnership();+ } catch (error) {+ this.write(client, failure(command.id, command.type, error));+ return;+ }+const journalIdentity =envelopeClientId && command.id && isDaemonMutatingCommand(command)? { clientId: envelopeClientId, commandId: command.id }@@ -804,15 +908,21 @@ export class DaemonSupervisor {const response = await this.handleCommand(client, command);if (response) {if (journalIdentity) {+ await this.assertCurrentOwnership();this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);}this.write(client, response);}} catch (error) {this.log(`Supervisor command ${command.type} failed: ${error instanceof Error ? error.stack : String(error)}`);- const response = failure(command.id, command.type, error, serializeDaemonError(error));- if (journalIdentity) {- this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);+ let response = failure(command.id, command.type, error, serializeDaemonError(error));+ if (journalIdentity && !isSupervisorGenerationStale(error)) {+ try {+ await this.assertCurrentOwnership();+ this.commandJournal.recordResult(journalIdentity.clientId, journalIdentity.commandId, response);+ } catch (ownershipError) {+ response = failure(command.id, command.type, ownershipError, serializeDaemonError(ownershipError));+ }}this.write(client, response);}@@ -1130,6 +1240,7 @@ export class DaemonSupervisor {}private async launchWorker(command: DaemonCreateCommand, existing?: ResidentWorker): Promise<ResidentWorker> {+ await this.assertRecoveryAllowed();if (existing && this.isWorkerRecoveryCancelled(existing)) {throw new Error(`Session worker ${existing.descriptor.workerId} recovery was cancelled`);}@@ -1149,6 +1260,7 @@ export class DaemonSupervisor {const orphanProcessJournalPath =existing?.descriptor.orphanProcessJournalPath ?? join(this.descriptorDir, `${workerId}.orphans.jsonl`);const launch = createCliSubprocessLaunchSpec(["--mode", "daemon", "--daemon-socket", socketPath]);+ await this.assertRecoveryAllowed();const child: ChildProcess = spawn(launch.command, launch.args, {cwd: createCommand.config?.cwd ?? process.cwd(),detached: true,@@ -1159,59 +1271,104 @@ export class DaemonSupervisor {[DAEMON_WORKER_ACTIVE_SESSION_ID_ENV]: rootActiveSessionId,[DAEMON_WORKER_SUPERVISOR_SOCKET_ENV]: this.socketPath,[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]: recoveryJournalPath,+ [DAEMON_WORKER_STARTUP_GATE_FD_ENV]: String(WORKER_STARTUP_GATE_FD),[ORPHAN_PROCESS_JOURNAL_ENV]: orphanProcessJournalPath,[SESSION_LEASES_ENABLED_ENV]: "1",[SESSION_LEASE_OWNER_ID_ENV]: rootActiveSessionId,},- stdio: "ignore",+ stdio: ["ignore", "ignore", "ignore", "pipe"],});+ const childClosed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));child.on("error", (error) => {this.log(`Session worker ${workerId} process error: ${error instanceof Error ? error.message : String(error)}`,);});- if (!child.pid) {- throw new Error("Failed to obtain daemon session worker pid");+ const startupGate = child.stdio[WORKER_STARTUP_GATE_FD];+ const previousDescriptor = existing?.descriptor;+ const previousIntentionalStop = existing?.intentionalStop;+ let descriptorAssigned = false;+ let childPid: number;+ let childProcessStartId: string | undefined;+ let worker: ResidentWorker;+ try {+ if (!child.pid) {+ throw new Error("Failed to obtain daemon session worker pid");+ }+ if (!(startupGate instanceof Writable)) {+ throw new Error("Failed to create daemon session worker startup gate");+ }+ childPid = child.pid;+ childProcessStartId = getProcessStartId(childPid);+ await this.assertRecoveryAllowed();++ const descriptor: DaemonWorkerDescriptor = {+ version: 1,+ workerId,+ pid: childPid,+ ...(childProcessStartId ? { processStartId: childProcessStartId } : {}),+ socketPath,+ recoveryJournalPath,+ orphanProcessJournalPath,+ supervisorSocketPath: this.socketPath,+ authenticationToken: token,+ rootActiveSessionId,+ createdAt: existing?.descriptor.createdAt ?? now,+ updatedAt: now,+ lifecycle: "starting",+ createCommand: { ...createCommand, id: undefined },+ consecutiveFailures: existing?.descriptor.consecutiveFailures ?? 0,+ };+ worker = existing ?? {+ descriptor,+ descriptorPath,+ summaries: new Map(),+ snapshotCache: new Map(),+ transcriptCaches: new Map(),+ incomingTranscriptActiveSessionIds: new Set(),+ duplicateIncomingTranscriptChunkIndexes: new Map(),+ snapshotTransferFrames: new Map(),+ snapshotLoads: new Map(),+ intentionalStop: false,+ stopRevision: 0,+ };+ await this.assertRecoveryAllowed();+ worker.descriptor = descriptor;+ descriptorAssigned = true;+ this.persistWorker(worker);+ worker.intentionalStop = false;+ this.workers.set(workerId, worker);+ } catch (error) {+ if (startupGate instanceof Writable) {+ startupGate.destroy();+ }+ await childClosed;+ child.unref();+ try {+ rmSync(`${descriptorPath}.${process.pid}.tmp`, { force: true });+ } catch (cleanupError) {+ this.reportCleanupFailure(`worker launch temp ${workerId}`, cleanupError);+ }+ if (existing && descriptorAssigned && previousDescriptor) {+ try {+ existing.descriptor = previousDescriptor;+ } catch (cleanupError) {+ this.reportCleanupFailure(`worker launch descriptor ${workerId}`, cleanupError);+ }+ }+ throw error;}- const childProcessStartId = getProcessStartId(child.pid);- child.unref();-- const descriptor: DaemonWorkerDescriptor = {- version: 1,- workerId,- pid: child.pid,- ...(childProcessStartId ? { processStartId: childProcessStartId } : {}),- socketPath,- recoveryJournalPath,- orphanProcessJournalPath,- supervisorSocketPath: this.socketPath,- authenticationToken: token,- rootActiveSessionId,- createdAt: existing?.descriptor.createdAt ?? now,- updatedAt: now,- lifecycle: "starting",- createCommand: { ...createCommand, id: undefined },- consecutiveFailures: existing?.descriptor.consecutiveFailures ?? 0,- };- const worker = existing ?? {- descriptor,- descriptorPath,- summaries: new Map(),- snapshotCache: new Map(),- transcriptCaches: new Map(),- incomingTranscriptActiveSessionIds: new Set(),- duplicateIncomingTranscriptChunkIndexes: new Map(),- snapshotTransferFrames: new Map(),- snapshotLoads: new Map(),- intentionalStop: false,- stopRevision: 0,- };- worker.descriptor = descriptor;- worker.intentionalStop = false;- this.workers.set(workerId, worker);- this.persistWorker(worker);try {+ try {+ await commitWorkerStartupGate(startupGate);+ } catch (error) {+ startupGate.destroy();+ await childClosed;+ throw error;+ } finally {+ child.unref();+ }const client = await this.connectWorker(worker, WORKER_CONNECT_TIMEOUT_MS);const response = await client.request(withoutCommandId(createCommand), WORKER_REQUEST_TIMEOUT_MS);if (!response.success) {@@ -1228,10 +1385,11 @@ export class DaemonSupervisor {worker.descriptor.rootSessionId = summary.sessionId;worker.descriptor.sessionFile = summary.sessionFile;await this.subscribeWorker(worker, rootActiveSessionId);- await this.refreshWorkerSummaries(worker);+ await this.refreshWorkerSummaries(worker, true);if (existing && (this.isWorkerRecoveryCancelled(worker) || worker.stopRevision !== recoveryStopRevision)) {throw new Error(`Session worker ${workerId} recovery was cancelled`);}+ await this.assertRecoveryAllowed();worker.descriptor.lifecycle = "ready";worker.descriptor.consecutiveFailures = 0;worker.descriptor.lastError = undefined;@@ -1239,6 +1397,43 @@ export class DaemonSupervisor {await this.syncAgentPeers();return worker;} catch (error) {+ if (isSupervisorGenerationStale(error)) {+ throw error;+ }+ if (isSupervisorShutdownAdmissionCancelled(error)) {+ let rolledBack = false;+ try {+ await this.stopWorker(worker, existing === undefined, true, false, existing !== undefined, {+ child,+ closed: childClosed,+ });+ rolledBack = true;+ } catch (cleanupError) {+ this.reportCleanupFailure(`cancelled worker launch ${workerId}`, cleanupError);+ }+ const mappedWorker = this.workers.get(workerId);+ if (+ rolledBack &&+ existing &&+ previousDescriptor &&+ !this.shuttingDown &&+ existing.stopRevision === recoveryStopRevision &&+ existing.descriptor.stopRequestedAt === undefined &&+ (mappedWorker === undefined || mappedWorker === existing)+ ) {+ existing.descriptor = previousDescriptor;+ existing.intentionalStop = previousIntentionalStop ?? false;+ this.workers.set(workerId, existing);+ try {+ this.persistWorker(existing);+ } catch (cleanupError) {+ this.reportCleanupFailure(`cancelled worker recovery ${workerId}`, cleanupError);+ }+ this.deferWorkerRecovery(existing, error instanceof Error ? error : new Error(String(error)));+ }+ throw error;+ }+ await this.assertRecoveryAllowed();const shouldResumeRecovery =existing !== undefined &&!this.shuttingDown &&@@ -1253,6 +1448,7 @@ export class DaemonSupervisor {worker.descriptor.stopRequestedAt === undefined &&worker.stopRevision === recoveryStopRevision) {+ await this.assertRecoveryAllowed();worker.intentionalStop = false;worker.descriptor.lifecycle = "recovering";this.workers.set(workerId, worker);@@ -1266,19 +1462,28 @@ export class DaemonSupervisor {const deadline = Date.now() + timeoutMs;let lastError: unknown;while (Date.now() < deadline) {+ await this.assertRecoveryAllowed();const client = new DaemonWorkerClient(worker.descriptor.socketPath);try {await client.connect(Math.min(500, Math.max(50, deadline - Date.now())));await client.waitForHello(1000);- await client.authenticateWorker(worker.descriptor.authenticationToken, 1000);+ await client.authenticateWorker(+ worker.descriptor.authenticationToken,+ this.supervisorAuthenticationClaim(),+ 1000,+ );+ await this.assertRecoveryAllowed();client.onFrame((frame) => this.handleWorkerFrame(worker, frame));- client.onClose((error) => this.handleWorkerClose(worker, client, error));+ client.onClose((error) => void this.handleWorkerClose(worker, client, error));worker.client?.close();worker.client = client;return client;} catch (error) {lastError = error;client.close();+ if (isSupervisorRecoveryCancelled(error)) {+ throw error;+ }await delay(25);}}@@ -1306,6 +1511,7 @@ export class DaemonSupervisor {}private async adoptOrRecoverWorker(worker: ResidentWorker): Promise<void> {+ await this.assertRecoveryAllowed();if (worker.descriptor.stopRequestedAt) {try {// A tombstoned worker must not run long enough to elect another@@ -1328,20 +1534,24 @@ export class DaemonSupervisor {const observedProcessStartId = getProcessStartId(worker.descriptor.pid);await this.connectWorker(worker, 2000);await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId);- await this.refreshWorkerSummaries(worker);+ await this.refreshWorkerSummaries(worker, true);if (worker.descriptor.processStartId === undefined && observedProcessStartId) {worker.descriptor.processStartId = observedProcessStartId;}+ await this.assertRecoveryAllowed();worker.descriptor.lifecycle = "ready";worker.descriptor.consecutiveFailures = 0;this.persistWorker(worker);} catch (error) {+ if (isSupervisorRecoveryCancelled(error)) {+ return;+ }this.log(`Could not adopt worker ${worker.descriptor.workerId}: ${String(error)}`);await this.recoverWorker(worker);}}- private handleWorkerClose(worker: ResidentWorker, client: DaemonWorkerClient, error: Error): void {+ private async handleWorkerClose(worker: ResidentWorker, client: DaemonWorkerClient, error: Error): Promise<void> {if (worker.client !== client) {return;}@@ -1363,6 +1573,17 @@ export class DaemonSupervisor {if (this.shuttingDown || worker.intentionalStop) {return;}+ try {+ await this.assertRecoveryAllowed();+ } catch (recoveryError) {+ if (!isSupervisorGenerationStale(recoveryError)) {+ this.deferWorkerRecovery(worker, error);+ }+ return;+ }+ if (!this.isWorkerRecoveryEligible(worker)) {+ return;+ }worker.descriptor.lifecycle = "recovering";worker.descriptor.lastError = error.message;this.persistWorker(worker);@@ -1370,6 +1591,61 @@ export class DaemonSupervisor {void this.recoverWorker(worker);}+ private isWorkerRecoveryEligible(worker: ResidentWorker): boolean {+ return this.isWorkerRecoveryCandidate(worker) && worker.recovery === undefined;+ }++ private isWorkerRecoveryCandidate(worker: ResidentWorker): boolean {+ return (+ !this.shuttingDown &&+ !worker.intentionalStop &&+ worker.descriptor.stopRequestedAt === undefined &&+ this.workers.get(worker.descriptor.workerId) === worker &&+ worker.client === undefined+ );+ }++ private deferWorkerRecovery(worker: ResidentWorker, disconnectError: Error): void {+ if (worker.deferredRecovery) {+ return;+ }+ worker.deferredRecovery = this.resumeDeferredWorkerRecovery(worker, disconnectError).finally(() => {+ worker.deferredRecovery = undefined;+ });+ }++ private async resumeDeferredWorkerRecovery(worker: ResidentWorker, disconnectError: Error): Promise<void> {+ while (true) {+ await unrefDelay(DEFERRED_RECOVERY_RECHECK_MS);+ if (!this.isWorkerRecoveryCandidate(worker)) {+ return;+ }+ if (!this.isWorkerRecoveryEligible(worker)) {+ continue;+ }+ try {+ await this.assertRecoveryAllowed();+ } catch (error) {+ if (isSupervisorGenerationStale(error)) {+ return;+ }+ continue;+ }+ if (!this.isWorkerRecoveryCandidate(worker)) {+ return;+ }+ if (!this.isWorkerRecoveryEligible(worker)) {+ continue;+ }+ worker.descriptor.lifecycle = "recovering";+ worker.descriptor.lastError = disconnectError.message;+ this.persistWorker(worker);+ void this.syncAgentPeers().catch(() => undefined);+ void this.recoverWorker(worker);+ return;+ }+ }+private failWorkerSnapshotCache(worker: ResidentWorker,activeSessionId: string,@@ -1436,6 +1712,7 @@ export class DaemonSupervisor {return;}try {+ await this.assertRecoveryAllowed();const processAlive = isProcessAlive(worker.descriptor.pid);const observedProcessStartId = processAlive ? getProcessStartId(worker.descriptor.pid) : undefined;const processIdentityMatches =@@ -1445,13 +1722,14 @@ export class DaemonSupervisor {try {await this.connectWorker(worker, 1500);await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId);- await this.refreshWorkerSummaries(worker);+ await this.refreshWorkerSummaries(worker, true);if (this.isWorkerRecoveryCancelled(worker)) {return;}if (worker.descriptor.processStartId === undefined && observedProcessStartId) {worker.descriptor.processStartId = observedProcessStartId;}+ await this.assertRecoveryAllowed();worker.descriptor.lifecycle = "ready";worker.descriptor.consecutiveFailures = 0;this.persistWorker(worker);@@ -1460,6 +1738,10 @@ export class DaemonSupervisor {);return;} catch (error) {+ if (isSupervisorRecoveryCancelled(error)) {+ throw error;+ }+ await this.assertRecoveryAllowed();worker.client?.close();worker.client = undefined;if (retryIndex < WORKER_RETRY_DELAYS_MS.length - 1) {@@ -1484,7 +1766,12 @@ export class DaemonSupervisor {await this.launchWorker(worker.descriptor.createCommand, worker);return;} catch (error) {- if (this.isWorkerRecoveryCancelled(worker)) {+ if (isSupervisorRecoveryCancelled(error) || this.isWorkerRecoveryCancelled(worker)) {+ return;+ }+ try {+ await this.assertRecoveryAllowed();+ } catch {return;}worker.client?.close();@@ -1495,6 +1782,11 @@ export class DaemonSupervisor {this.persistWorker(worker);}}+ try {+ await this.assertRecoveryAllowed();+ } catch {+ return;+ }worker.descriptor.lifecycle = "failed";this.persistWorker(worker);await this.syncAgentPeers().catch(() => undefined);@@ -1515,6 +1807,7 @@ export class DaemonSupervisor {}private async recoverUncertainWorkerOperations(worker: ResidentWorker, killWorkerProcess = true): Promise<void> {+ await this.assertRecoveryAllowed();if (killWorkerProcess) {signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");}@@ -1568,6 +1861,7 @@ export class DaemonSupervisor {}interrupted.operations.add(record.operation);}+ await this.assertRecoveryAllowed();await Promise.all([...interruptedSessions.values()].map((interrupted) =>this.catalog.markInterrupted(interrupted.sessionFile, interrupted.activeSessionId, [@@ -1575,6 +1869,7 @@ export class DaemonSupervisor {]),),);+ await this.assertRecoveryAllowed();for (const record of latest) {journal.record({activeSessionId: record.activeSessionId,@@ -1591,7 +1886,7 @@ export class DaemonSupervisor {);}- private async refreshWorkerSummaries(worker: ResidentWorker): Promise<void> {+ private async refreshWorkerSummaries(worker: ResidentWorker, recovery = false): Promise<void> {if (!worker.client) {throw new Error("Session worker is not connected");}@@ -1608,6 +1903,9 @@ export class DaemonSupervisor {}const root = worker.summaries.get(worker.descriptor.rootActiveSessionId);if (root) {+ if (recovery) {+ await this.assertRecoveryAllowed();+ }worker.descriptor.rootSessionId = root.sessionId;worker.descriptor.sessionFile = root.sessionFile;worker.descriptor.createCommand = {@@ -2771,16 +3069,24 @@ export class DaemonSupervisor {force = false,archiveSession = false,recoveryCleanup = false,+ directChild?: { child: ChildProcess; closed: Promise<void> },): Promise<void> {if (!recoveryCleanup) {worker.stopRevision++;}- if (removeDescriptor) {- this.persistWorkerStopTombstone(worker, archiveSession);- } else {- worker.intentionalStop = true;- worker.descriptor.lifecycle = "recovering";- this.persistWorker(worker);+ try {+ if (removeDescriptor) {+ this.persistWorkerStopTombstone(worker, archiveSession);+ } else {+ worker.intentionalStop = true;+ worker.descriptor.lifecycle = "recovering";+ this.persistWorker(worker);+ }+ } catch (error) {+ if (!directChild) {+ throw error;+ }+ this.reportCleanupFailure(`worker rollback state ${worker.descriptor.workerId}`, error);}this.dropPendingReplacementSnapshotsForWorker(worker);const interruptedSnapshots = new Set([@@ -2818,24 +3124,37 @@ export class DaemonSupervisor {}worker.client.close();worker.client = undefined;+ } else if (directChild) {+ directChild.child.kill("SIGTERM");} else if (isProcessAlive(worker.descriptor.pid)) {signalProcessGroupOrProcess(worker.descriptor.pid, "SIGTERM");}+ const isWorkerProcessAlive = () =>+ directChild+ ? directChild.child.exitCode === null && directChild.child.signalCode === null+ : isProcessAlive(worker.descriptor.pid);const gracefulDeadline = Date.now() + (force ? 500 : 2000);- while (isProcessAlive(worker.descriptor.pid) && Date.now() < gracefulDeadline) {+ while (isWorkerProcessAlive() && Date.now() < gracefulDeadline) {await delay(25);}- if (force && isProcessAlive(worker.descriptor.pid)) {- signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");+ if (force && isWorkerProcessAlive()) {+ if (directChild) {+ directChild.child.kill("SIGKILL");+ } else {+ signalProcessGroupOrProcess(worker.descriptor.pid, "SIGKILL");+ }const forceDeadline = Date.now() + 1000;- while (isProcessAlive(worker.descriptor.pid) && Date.now() < forceDeadline) {+ while (isWorkerProcessAlive() && Date.now() < forceDeadline) {await delay(25);}}- if (isProcessAlive(worker.descriptor.pid)) {+ if (isWorkerProcessAlive()) {worker.intentionalStop = worker.descriptor.stopRequestedAt !== undefined;throw new Error(`Session worker ${worker.descriptor.workerId} did not stop${force ? " after SIGKILL" : ""}`);}+ if (directChild) {+ await directChild.closed;+ }if (removeDescriptor && worker.descriptor.archiveOnStop) {if (force) {this.reclaimStoppedWorkerCronLock(worker);diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-client.tsindex 6a54cd3f24..6adb2357e9 100644--- a/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-client.ts@@ -13,6 +13,7 @@ type DistributiveOmit<T, K extends keyof T> = T extends unknown ? Omit<T, K> : ntype DaemonCommandBody = DistributiveOmit<DaemonCommand, "id">;type DaemonWorkerWireCommandBody = DaemonCommandBody | DaemonWorkerCommandBody;type DaemonWorkerWireCommand = DaemonCommand | DaemonWorkerCommand;+type DaemonWorkerAuthentication = Omit<Extract<DaemonWorkerCommand, { type: "worker_auth" }>, "id" | "type" | "token">;export type DaemonWorkerFrameListener = (frame: PrivateFrame<DaemonWorkerFrameHeader>) => void;export type DaemonWorkerCloseListener = (error: Error) => void;@@ -115,8 +116,8 @@ export class DaemonWorkerClient {return this.requestWire(command, timeoutMs);}- async authenticateWorker(token: string, timeoutMs = 3000): Promise<void> {- const response = await this.requestWorker({ type: "worker_auth", token }, timeoutMs);+ async authenticateWorker(token: string, owner: DaemonWorkerAuthentication, timeoutMs = 3000): Promise<void> {+ const response = await this.requestWorker({ type: "worker_auth", token, ...owner }, timeoutMs);if (!response.success) {throw new Error(response.error);}diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.tsindex 0e8e0549ae..0ae2146ddf 100644--- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts@@ -1,3 +1,4 @@+import { closeSync, readFileSync } from "node:fs";import type {AgentSessionMessageAgentSummary,AgentSessionMessageDeliveryMode,@@ -13,6 +14,8 @@ export const DAEMON_WORKER_TOKEN_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKENexport const DAEMON_WORKER_ACTIVE_SESSION_ID_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID";export const DAEMON_WORKER_SUPERVISOR_SOCKET_ENV = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET";export const DAEMON_WORKER_RECOVERY_JOURNAL_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL";+export const DAEMON_WORKER_STARTUP_GATE_FD_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD";+export const DAEMON_WORKER_STARTUP_GATE_COMMIT = "start\n";export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "failed";export type DaemonWorkerFrameHeader =@@ -34,7 +37,15 @@ export type DaemonWorkerFrameHeader =export type DaemonCreateCommand = Extract<DaemonCommand, { type: "create" }>;export type DaemonWorkerCommand =- | { id?: string; type: "worker_auth"; token: string }+ | {+ id?: string;+ type: "worker_auth";+ token: string;+ supervisorGeneration: string;+ supervisorPid: number;+ supervisorProcessStartId?: string;+ supervisorSocketPath: string;+ }| {id?: string;type: "worker_subscribe";@@ -93,6 +104,27 @@ export function isDaemonWorkerProcess(environment: NodeJS.ProcessEnv = process.ereturn environment[DAEMON_WORKER_ROLE_ENV] === "1";}+export function waitForDaemonWorkerStartupGate(environment: NodeJS.ProcessEnv = process.env): void {+ const rawFd = environment[DAEMON_WORKER_STARTUP_GATE_FD_ENV];+ if (rawFd === undefined) {+ return;+ }+ delete environment[DAEMON_WORKER_STARTUP_GATE_FD_ENV];+ const fd = Number(rawFd);+ if (!Number.isInteger(fd) || fd < 3) {+ throw new Error("Daemon session worker has an invalid startup gate");+ }+ let marker: string;+ try {+ marker = readFileSync(fd, "utf8");+ } finally {+ closeSync(fd);+ }+ if (marker !== DAEMON_WORKER_STARTUP_GATE_COMMIT) {+ throw new Error("Daemon session worker startup was cancelled");+ }+}+export function requireDaemonWorkerAuthenticationToken(environment: NodeJS.ProcessEnv = process.env): string {const token = environment[DAEMON_WORKER_TOKEN_ENV];if (!token) {diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.tsindex 7de450c045..01329ae217 100644--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts@@ -1,4 +1,5 @@-import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import type { ChildProcess, SpawnOptions } from "node:child_process";+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import type { AgentMessage } from "@earendil-works/pi-agent-core";@@ -7,11 +8,100 @@ import { AgentDaemon } from "../src/modes/daemon/daemon-mode.js";import type { DaemonAttachResult } from "../src/modes/daemon/daemon-protocol.js";import type { SessionSummary } from "../src/modes/daemon/daemon-session-list.js";import { DaemonSupervisor } from "../src/modes/daemon/daemon-supervisor.js";+import { DAEMON_WORKER_STARTUP_GATE_COMMIT } from "../src/modes/daemon/daemon-worker-protocol.js";import { WorkerRecoveryJournal } from "../src/modes/daemon/worker-recovery-journal.js";+const workerLaunchTestState = vi.hoisted(() => ({+ capture: false,+ forceMissingProcessStartId: false,+ fixtureMode: "worker" as "worker" | "close-gate" | "rollback-gate" | "successful-gate",+ gateMarkerPath: "",+ tsxCliPath: "",+ cliEntrypoint: "",+ spawned: [] as Array<{ child: ChildProcess; args: readonly string[] }>,+}));++vi.mock("node:child_process", async (importOriginal) => {+ const actual = (await importOriginal()) as Record<string, unknown> & {+ spawn(command: string, args: readonly string[], options: SpawnOptions): ChildProcess;+ };+ return {+ ...actual,+ spawn(command: string, args: readonly string[], options: SpawnOptions): ChildProcess {+ const child = actual.spawn(command, args, options);+ if (workerLaunchTestState.capture) {+ workerLaunchTestState.spawned.push({ child, args });+ }+ return child;+ },+ };+});++vi.mock("../src/cli/subprocess-launch.js", async (importOriginal) => {+ const actual = (await importOriginal()) as Record<string, unknown>;+ return {+ ...actual,+ createCliSubprocessLaunchSpec(args: readonly string[]) {+ if (!workerLaunchTestState.capture) {+ return (actual.createCliSubprocessLaunchSpec as (args: readonly string[]) => unknown)(args);+ }+ if (workerLaunchTestState.fixtureMode === "rollback-gate") {+ const markerPath = JSON.stringify(workerLaunchTestState.gateMarkerPath);+ const commitMarker = JSON.stringify(DAEMON_WORKER_STARTUP_GATE_COMMIT);+ return {+ command: process.execPath,+ args: [+ "--eval",+ `const fs = require("node:fs"); const marker = fs.readFileSync(3, "utf8"); if (marker === ${commitMarker}) { fs.writeFileSync(${markerPath}, marker); setInterval(() => {}, 1000); }`,+ "--",+ ...args,+ ],+ };+ }+ if (workerLaunchTestState.fixtureMode === "close-gate") {+ return {+ command: process.execPath,+ args: ["--eval", 'require("node:fs").closeSync(3)'],+ };+ }+ if (workerLaunchTestState.fixtureMode === "successful-gate") {+ const markerPath = JSON.stringify(workerLaunchTestState.gateMarkerPath);+ return {+ command: process.execPath,+ args: [+ "--eval",+ `const fs = require("node:fs"); const marker = fs.readFileSync(3, "utf8"); fs.writeFileSync(${markerPath}, marker); setInterval(() => {}, 1000);`,+ ],+ };+ }+ return {+ command: process.execPath,+ args: [workerLaunchTestState.tsxCliPath, workerLaunchTestState.cliEntrypoint, ...args],+ };+ },+ };+});++vi.mock("../src/core/session-lease.js", async (importOriginal) => {+ const actual = (await importOriginal()) as Record<string, unknown> & {+ getProcessStartId(pid: number): string | undefined;+ };+ return {+ ...actual,+ getProcessStartId(pid: number): string | undefined {+ return workerLaunchTestState.forceMissingProcessStartId ? undefined : actual.getProcessStartId(pid);+ },+ };+});++const supervisorRegistryDirEnv = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR";+const previousSupervisorRegistryDir = process.env[supervisorRegistryDirEnv];+const supervisorRegistryDirs = new Set<string>();+interface SupervisorMonitorHarness {options: { worker: object };clients: Set<{ authenticated: boolean }>;+ supervisorClaims: Map<object, object>;shuttingDown: boolean;supervisorMonitorTimer?: ReturnType<typeof setTimeout>;canConnectToSupervisor: (socketPath: string) => Promise<boolean>;@@ -19,10 +109,143 @@ interface SupervisorMonitorHarness {scheduleSupervisorAvailabilityCheck: (socketPath: string, delayMs: number) => void;}+interface DeferredRecoveryWorker {+ descriptor: {+ workerId: string;+ pid: number;+ rootActiveSessionId: string;+ lifecycle: "ready" | "recovering";+ lastError?: string;+ stopRequestedAt?: string;+ };+ client?: object;+ snapshotCache: Map<string, DaemonAttachResult>;+ incomingTranscriptActiveSessionIds: Set<string>;+ transcriptCaches: Map<string, { markFailed(error: Error): void }>;+ duplicateIncomingTranscriptChunkIndexes: Map<string, number>;+ snapshotTransferFrames: Map<string, never>;+ recovery?: Promise<void>;+ deferredRecovery?: Promise<void>;+ intentionalStop: boolean;+ stopRevision: number;+}++interface DeferredRecoveryHarness {+ workers: Map<string, DeferredRecoveryWorker>;+ shuttingDown: boolean;+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;+ persistWorker: ReturnType<typeof vi.fn>;+ syncAgentPeers: ReturnType<typeof vi.fn>;+ recoverWorker: ReturnType<typeof vi.fn>;+ handleWorkerClose(worker: DeferredRecoveryWorker, client: object, error: Error): Promise<void>;+ deferWorkerRecovery(worker: DeferredRecoveryWorker, error: Error): void;+}++function recoveryDeniedError(code: "supervisor_recovery_cancelled" | "supervisor_generation_stale"): Error {+ return Object.assign(new Error(code), { code });+}++async function waitForCapturedChildClose(child: ChildProcess): Promise<void> {+ if (child.exitCode !== null || child.signalCode !== null) {+ return;+ }+ await new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));+}++async function waitForFile(path: string): Promise<void> {+ const deadline = Date.now() + 1000;+ while (!existsSync(path) && Date.now() < deadline) {+ await new Promise((resolveDelay) => setTimeout(resolveDelay, 10));+ }+ if (!existsSync(path)) {+ throw new Error(`Timed out waiting for ${path}`);+ }+}++function createExistingLaunchWorker(root: string, descriptorDir: string) {+ const workerId = "existing-worker";+ const now = new Date().toISOString();+ return {+ descriptor: {+ version: 1 as const,+ workerId,+ pid: 999_999,+ socketPath: join(root, `${workerId}.sock`),+ recoveryJournalPath: join(descriptorDir, `${workerId}.recovery.jsonl`),+ orphanProcessJournalPath: join(descriptorDir, `${workerId}.orphans.jsonl`),+ supervisorSocketPath: join(root, "supervisor.sock"),+ authenticationToken: "existing-worker-token",+ rootActiveSessionId: "existing-root-session",+ createdAt: now,+ updatedAt: now,+ lifecycle: "recovering" as const,+ stopRequestedAt: undefined as string | undefined,+ createCommand: { type: "create" as const, config: { cwd: root, agentDir: root } },+ consecutiveFailures: 0,+ },+ descriptorPath: join(descriptorDir, `${workerId}.json`),+ summaries: new Map<string, SessionSummary>(),+ snapshotCache: new Map<string, DaemonAttachResult>(),+ transcriptCaches: new Map<string, never>(),+ incomingTranscriptActiveSessionIds: new Set<string>(),+ duplicateIncomingTranscriptChunkIndexes: new Map<string, number>(),+ snapshotTransferFrames: new Map<string, never>(),+ snapshotLoads: new Map<string, Promise<DaemonAttachResult>>(),+ intentionalStop: false,+ stopRevision: 0,+ };+}++function createSupervisorSnapshotState() {+ return {+ clients: new Set<object>(),+ pendingReplacementSnapshots: new WeakMap<object, Map<string, unknown>>(),+ };+}++const recoveryEligibilityInvalidations: Array<{+ name: string;+ invalidate(supervisor: DeferredRecoveryHarness, worker: DeferredRecoveryWorker): void;+}> = [+ {+ name: "the worker reconnects",+ invalidate: (_supervisor, worker) => {+ worker.client = {};+ },+ },+ {+ name: "the worker is stopped",+ invalidate: (_supervisor, worker) => {+ worker.intentionalStop = true;+ worker.descriptor.stopRequestedAt = new Date().toISOString();+ },+ },+ {+ name: "the worker is replaced",+ invalidate: (supervisor, worker) => supervisor.workers.set(worker.descriptor.workerId, { ...worker }),+ },+ {+ name: "supervisor cleanup begins",+ invalidate: (supervisor) => {+ supervisor.shuttingDown = true;+ },+ },+ {+ name: "another recovery begins",+ invalidate: (_supervisor, worker) => {+ worker.recovery = Promise.resolve();+ },+ },+];+function createHarness(canConnect: () => Promise<boolean>): SupervisorMonitorHarness {+ const registryDir = mkdtempSync(join(tmpdir(), "prime-supervisor-registry-test-"));+ supervisorRegistryDirs.add(registryDir);+ process.env[supervisorRegistryDirEnv] = registryDir;return Object.assign(Object.create(AgentDaemon.prototype), {options: { worker: {} },clients: new Set<{ authenticated: boolean }>(),+ supervisorClaims: new Map<object, object>(),shuttingDown: false,canConnectToSupervisor: vi.fn(canConnect),launchReplacementSupervisor: vi.fn(async () => undefined),@@ -30,8 +253,434 @@ function createHarness(canConnect: () => Promise<boolean>): SupervisorMonitorHar}describe("daemon worker supervisor monitoring", () => {- afterEach(() => {+ afterEach(async () => {+ for (const { child } of workerLaunchTestState.spawned) {+ if (child.exitCode === null && child.signalCode === null) {+ const closed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));+ child.kill("SIGKILL");+ await closed;+ }+ }+ workerLaunchTestState.capture = false;+ workerLaunchTestState.forceMissingProcessStartId = false;+ workerLaunchTestState.fixtureMode = "worker";+ workerLaunchTestState.gateMarkerPath = "";+ workerLaunchTestState.tsxCliPath = "";+ workerLaunchTestState.cliEntrypoint = "";+ workerLaunchTestState.spawned.length = 0;vi.useRealTimers();+ for (const registryDir of supervisorRegistryDirs) {+ rmSync(registryDir, { recursive: true, force: true });+ }+ supervisorRegistryDirs.clear();+ if (previousSupervisorRegistryDir === undefined) {+ delete process.env[supervisorRegistryDirEnv];+ } else {+ process.env[supervisorRegistryDirEnv] = previousSupervisorRegistryDir;+ }+ });++ it.each([+ {+ name: "first post-spawn ownership check",+ assertionCall: 3,+ error: recoveryDeniedError("supervisor_generation_stale"),+ },+ {+ name: "pre-publication ownership check",+ assertionCall: 4,+ error: recoveryDeniedError("supervisor_generation_stale"),+ },+ { name: "descriptor persistence", persistFailure: true, error: new Error("descriptor persistence failed") },+ ] as const)("keeps unidentifiable workers gated after $name fails", async (scenario) => {+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "rollback-gate";+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-launch-gate-test-"));+ const gateMarkerPath = join(root, "committed-gate");+ workerLaunchTestState.gateMarkerPath = gateMarkerPath;+ const descriptorDir = join(root, "descriptors");+ const registryDir = join(root, "registry");+ mkdirSync(descriptorDir, { recursive: true });+ mkdirSync(registryDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ process.env[supervisorRegistryDirEnv] = registryDir;+ let assertionCount = 0;+ const workers = new Map<string, unknown>();+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ assertRecoveryAllowed: vi.fn(async () => {+ assertionCount++;+ if ("assertionCall" in scenario && assertionCount === scenario.assertionCall) {+ throw scenario.error;+ }+ }),+ ...("persistFailure" in scenario+ ? {+ persistWorker: vi.fn(() => {+ throw scenario.error;+ }),+ }+ : {}),+ log: vi.fn(),+ }) as {+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;+ };++ await expect(supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } })).rejects.toBe(+ scenario.error,+ );++ expect(assertionCount).toBe("assertionCall" in scenario ? scenario.assertionCall : 4);+ expect(workerLaunchTestState.spawned).toHaveLength(1);+ const { child, args } = workerLaunchTestState.spawned[0]!;+ const socketFlagIndex = args.indexOf("--daemon-socket");+ expect(socketFlagIndex).toBeGreaterThanOrEqual(0);+ const workerSocketPath = args[socketFlagIndex + 1];+ expect(workerSocketPath).toBeDefined();++ expect(child.exitCode !== null || child.signalCode !== null).toBe(true);+ expect(existsSync(gateMarkerPath)).toBe(false);+ expect(existsSync(workerSocketPath!)).toBe(false);+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);+ expect(readdirSync(registryDir)).toEqual([]);+ expect(workers.size).toBe(0);+ });++ it("rolls back promptly when the child closes its startup gate before commit", async () => {+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-closed-gate-test-"));+ const descriptorDir = join(root, "descriptors");+ mkdirSync(descriptorDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "close-gate";+ let assertionCount = 0;+ const workers = new Map<string, unknown>();+ const connectWorker = vi.fn();+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(async () => {+ assertionCount++;+ if (assertionCount === 3) {+ const child = workerLaunchTestState.spawned.at(-1)?.child;+ if (!child) {+ throw new Error("Worker child was not captured");+ }+ await waitForCapturedChildClose(child);+ }+ }),+ connectWorker,+ syncAgentPeers: vi.fn(async () => undefined),+ log: vi.fn(),+ }) as {+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;+ };++ const timeoutError = new Error("startup gate rejection timed out");+ const result = await Promise.race([+ supervisor+ .launchWorker({ type: "create", config: { cwd: root, agentDir: root } })+ .then(() => ({ value: "resolved" as const, error: undefined }))+ .catch((error: unknown) => ({ value: "rejected" as const, error })),+ new Promise<{ value: "timed-out"; error: Error }>((resolveTimeout) =>+ setTimeout(() => resolveTimeout({ value: "timed-out", error: timeoutError }), 1000),+ ),+ ]);++ expect(result.value).toBe("rejected");+ expect(result.error).not.toBe(timeoutError);+ expect(result.error).toBeInstanceOf(Error);+ expect(connectWorker).not.toHaveBeenCalled();+ expect(workers.size).toBe(0);+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);+ const child = workerLaunchTestState.spawned.at(-1)?.child;+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);+ });++ it("commits the startup marker after durable worker publication", async () => {+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-committed-gate-test-"));+ const descriptorDir = join(root, "descriptors");+ const markerPath = join(root, "startup-marker");+ mkdirSync(descriptorDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "successful-gate";+ workerLaunchTestState.gateMarkerPath = markerPath;+ const workers = new Map<string, unknown>();+ const connectWorker = vi.fn(async (worker: { descriptor: { rootActiveSessionId: string } }) => {+ await waitForFile(markerPath);+ return {+ request: vi.fn(async () => ({+ success: true,+ data: {+ id: worker.descriptor.rootActiveSessionId,+ activeSessionId: worker.descriptor.rootActiveSessionId,+ sessionId: "session-committed-gate",+ cwd: root,+ },+ })),+ };+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(async () => undefined),+ connectWorker,+ subscribeWorker: vi.fn(async () => undefined),+ refreshWorkerSummaries: vi.fn(async () => undefined),+ syncAgentPeers: vi.fn(async () => undefined),+ log: vi.fn(),+ }) as {+ launchWorker(command: {+ type: "create";+ config: { cwd: string; agentDir: string };+ }): Promise<{ descriptor: { lifecycle: string } }>;+ };++ const worker = await supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } });++ expect(readFileSync(markerPath, "utf8")).toBe("start\n");+ expect(connectWorker).toHaveBeenCalledOnce();+ expect(worker.descriptor.lifecycle).toBe("ready");+ expect(workers.size).toBe(1);+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toHaveLength(1);+ const child = workerLaunchTestState.spawned.at(-1)?.child;+ if (!child) {+ throw new Error("Worker child was not captured");+ }+ const closed = new Promise<void>((resolveClose) => child.once("close", () => resolveClose()));+ child.kill("SIGKILL");+ await closed;+ });++ it("rolls back a published worker when shutdown admission and rollback persistence fail", async () => {+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-cancelled-launch-test-"));+ const descriptorDir = join(root, "descriptors");+ const markerPath = join(root, "startup-marker");+ mkdirSync(descriptorDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "successful-gate";+ workerLaunchTestState.gateMarkerPath = markerPath;+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");+ const rollbackPersistenceError = new Error("rollback persistence failed");+ const persistWorker = Reflect.get(DaemonSupervisor.prototype, "persistWorker");+ if (typeof persistWorker !== "function") {+ throw new Error("Could not access worker persistence");+ }+ let persistenceCalls = 0;+ const workers = new Map<string, unknown>();+ const connectWorker = vi.fn(async () => {+ await waitForFile(markerPath);+ throw cancellation;+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(async () => undefined),+ connectWorker,+ persistWorker: vi.fn(function (this: object, worker: object) {+ persistenceCalls++;+ if (persistenceCalls === 2) {+ throw rollbackPersistenceError;+ }+ Reflect.apply(persistWorker, this, [worker]);+ }),+ syncAgentPeers: vi.fn(async () => undefined),+ log: vi.fn(),+ }) as {+ launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise<unknown>;+ };++ await expect(supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } })).rejects.toBe(+ cancellation,+ );++ expect(readFileSync(markerPath, "utf8")).toBe("start\n");+ expect(connectWorker).toHaveBeenCalledOnce();+ expect(persistenceCalls).toBe(2);+ expect(workers.size).toBe(0);+ expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]);+ const child = workerLaunchTestState.spawned.at(-1)?.child;+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);+ });++ it("defers an eligible existing recovery when descriptor restoration fails", async () => {+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-existing-restore-test-"));+ const descriptorDir = join(root, "descriptors");+ const markerPath = join(root, "startup-marker");+ mkdirSync(descriptorDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "successful-gate";+ workerLaunchTestState.gateMarkerPath = markerPath;+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");+ const restorationError = new Error("descriptor restoration failed");+ const persistWorker = Reflect.get(DaemonSupervisor.prototype, "persistWorker");+ if (typeof persistWorker !== "function") {+ throw new Error("Could not access worker persistence");+ }+ let persistenceCalls = 0;+ const existing = createExistingLaunchWorker(root, descriptorDir);+ const previousDescriptor = existing.descriptor;+ const workers = new Map<string, object>([[existing.descriptor.workerId, existing]]);+ const deferWorkerRecovery = vi.fn();+ const connectWorker = vi.fn(async () => {+ await waitForFile(markerPath);+ throw cancellation;+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(async () => undefined),+ connectWorker,+ persistWorker: vi.fn(function (this: object, worker: object) {+ persistenceCalls++;+ if (persistenceCalls === 3) {+ throw restorationError;+ }+ Reflect.apply(persistWorker, this, [worker]);+ }),+ deferWorkerRecovery,+ syncAgentPeers: vi.fn(async () => undefined),+ log: vi.fn(),+ }) as {+ launchWorker(+ command: { type: "create"; config: { cwd: string; agentDir: string } },+ existing: object,+ ): Promise<unknown>;+ };++ await expect(+ supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } }, existing),+ ).rejects.toBe(cancellation);++ expect(persistenceCalls).toBe(3);+ expect(existing.descriptor).toBe(previousDescriptor);+ expect(workers.get(existing.descriptor.workerId)).toBe(existing);+ expect(deferWorkerRecovery).toHaveBeenCalledOnce();+ expect(deferWorkerRecovery).toHaveBeenCalledWith(existing, cancellation);+ const child = workerLaunchTestState.spawned.at(-1)?.child;+ expect(child?.exitCode !== null || child?.signalCode !== null).toBe(true);+ });++ it("does not restore an existing recovery invalidated during rollback", async () => {+ const root = mkdtempSync(join(tmpdir(), "prime-supervisor-existing-stop-race-test-"));+ const descriptorDir = join(root, "descriptors");+ const markerPath = join(root, "startup-marker");+ mkdirSync(descriptorDir, { recursive: true });+ supervisorRegistryDirs.add(root);+ workerLaunchTestState.capture = true;+ workerLaunchTestState.forceMissingProcessStartId = true;+ workerLaunchTestState.fixtureMode = "successful-gate";+ workerLaunchTestState.gateMarkerPath = markerPath;+ const cancellation = recoveryDeniedError("supervisor_recovery_cancelled");+ const existing = createExistingLaunchWorker(root, descriptorDir);+ const workers = new Map<string, object>([[existing.descriptor.workerId, existing]]);+ const deferWorkerRecovery = vi.fn();+ const stopWorker = Reflect.get(DaemonSupervisor.prototype, "stopWorker");+ if (typeof stopWorker !== "function") {+ throw new Error("Could not access worker shutdown");+ }+ let markRollbackStarted = () => {};+ const rollbackStarted = new Promise<void>((resolveStarted) => {+ markRollbackStarted = resolveStarted;+ });+ let releaseRollback = () => {};+ const rollbackRelease = new Promise<void>((resolveRelease) => {+ releaseRollback = resolveRelease;+ });+ const controlledStopWorker = vi.fn(async function (+ this: object,+ worker: object,+ removeDescriptor: boolean,+ force = false,+ archiveSession = false,+ recoveryCleanup = false,+ directChild?: object,+ ) {+ if (recoveryCleanup) {+ markRollbackStarted();+ await rollbackRelease;+ return;+ }+ await Reflect.apply(stopWorker, this, [+ worker,+ removeDescriptor,+ force,+ archiveSession,+ recoveryCleanup,+ directChild,+ ]);+ });+ const connectWorker = vi.fn(async () => {+ await waitForFile(markerPath);+ throw cancellation;+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ defaultSessionConfig: { cwd: root, agentDir: root },+ descriptorDir,+ socketPath: join(root, "supervisor.sock"),+ workers,+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(async () => undefined),+ connectWorker,+ stopWorker: controlledStopWorker,+ deferWorkerRecovery,+ syncAgentPeers: vi.fn(async () => undefined),+ log: vi.fn(),+ }) as {+ shuttingDown: boolean;+ launchWorker(+ command: { type: "create"; config: { cwd: string; agentDir: string } },+ existing: object,+ ): Promise<unknown>;+ stopWorker(worker: object, removeDescriptor: boolean, force: boolean): Promise<void>;+ };++ const launchResult = supervisor+ .launchWorker({ type: "create", config: { cwd: root, agentDir: root } }, existing)+ .then(+ () => undefined,+ (error: unknown) => error,+ );+ await rollbackStarted;+ supervisor.shuttingDown = true;+ await supervisor.stopWorker(existing, true, true);+ releaseRollback();++ expect(await launchResult).toBe(cancellation);+ expect(existing.stopRevision).toBe(1);+ expect(existing.descriptor.stopRequestedAt).toBeDefined();+ expect(workers.size).toBe(0);+ expect(existsSync(existing.descriptorPath)).toBe(false);+ expect(deferWorkerRecovery).not.toHaveBeenCalled();});it("attempts every shutdown cleanup step before exiting", async () => {@@ -82,10 +731,18 @@ describe("daemon worker supervisor monitoring", () => {it("does not poll a healthy supervisor after the startup check", async () => {vi.useFakeTimers();- const daemon = createHarness(async () => true);+ let resolveProbe: () => void = () => undefined;+ const probeCompleted = new Promise<void>((resolve) => {+ resolveProbe = resolve;+ });+ const daemon = createHarness(async () => {+ resolveProbe();+ return true;+ });daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 1500);await vi.advanceTimersByTimeAsync(1500);+ await probeCompleted;expect(daemon.canConnectToSupervisor).toHaveBeenCalledOnce();await vi.advanceTimersByTimeAsync(60_000);@@ -97,6 +754,7 @@ describe("daemon worker supervisor monitoring", () => {vi.useFakeTimers();const daemon = createHarness(async () => true);daemon.clients.add({ authenticated: true });+ daemon.supervisorClaims.set({}, {});daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 0);await vi.runAllTimersAsync();@@ -104,6 +762,266 @@ describe("daemon worker supervisor monitoring", () => {expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();});+ it("retries when shutdown admission lookup fails", async () => {+ vi.useFakeTimers();+ let resolveProbe: () => void = () => undefined;+ const probeCompleted = new Promise<void>((resolve) => {+ resolveProbe = resolve;+ });+ const daemon = createHarness(async () => {+ resolveProbe();+ return true;+ });+ const registryDir = process.env[supervisorRegistryDirEnv];+ if (!registryDir) throw new Error("Supervisor registry test directory was not set");+ rmSync(registryDir, { recursive: true, force: true });+ writeFileSync(registryDir, "not a directory");++ daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 0);+ await vi.advanceTimersByTimeAsync(0);+ expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();+ expect(daemon.supervisorMonitorTimer).toBeDefined();++ rmSync(registryDir, { force: true });+ mkdirSync(registryDir, { recursive: true });+ await vi.advanceTimersByTimeAsync(5000);+ await probeCompleted;+ expect(daemon.canConnectToSupervisor).toHaveBeenCalledOnce();+ });++ it("recovers exactly once after shutdown admission clears", async () => {+ vi.useFakeTimers();+ const client = {};+ const worker: DeferredRecoveryWorker = {+ descriptor: {+ workerId: "worker-deferred-recovery",+ pid: process.pid,+ rootActiveSessionId: "active-deferred-recovery",+ lifecycle: "ready",+ },+ client,+ snapshotCache: new Map(),+ incomingTranscriptActiveSessionIds: new Set(),+ transcriptCaches: new Map(),+ duplicateIncomingTranscriptChunkIndexes: new Map(),+ snapshotTransferFrames: new Map<string, never>(),+ intentionalStop: false,+ stopRevision: 0,+ };+ let admissionActive = true;+ const assertRecoveryAllowed = vi.fn(async () => {+ if (admissionActive) {+ throw recoveryDeniedError("supervisor_recovery_cancelled");+ }+ });+ const persistWorker = vi.fn();+ const recoverWorker = vi.fn(async () => undefined);+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ assertRecoveryAllowed,+ persistWorker,+ syncAgentPeers: vi.fn(async () => undefined),+ recoverWorker,+ }) as DeferredRecoveryHarness;++ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));+ const deferredRecovery = worker.deferredRecovery;+ expect(deferredRecovery).toBeDefined();+ supervisor.deferWorkerRecovery(worker, new Error("duplicate close"));+ expect(worker.deferredRecovery).toBe(deferredRecovery);+ expect(worker.descriptor.lifecycle).toBe("ready");+ expect(persistWorker).not.toHaveBeenCalled();+ expect(recoverWorker).not.toHaveBeenCalled();++ admissionActive = false;+ await vi.advanceTimersByTimeAsync(5000);+ await deferredRecovery;++ expect(worker.descriptor.lifecycle).toBe("recovering");+ expect(worker.descriptor.lastError).toBe("worker disconnected");+ expect(persistWorker).toHaveBeenCalledOnce();+ expect(recoverWorker).toHaveBeenCalledOnce();+ expect(worker.deferredRecovery).toBeUndefined();+ await vi.advanceTimersByTimeAsync(10_000);+ expect(recoverWorker).toHaveBeenCalledOnce();+ });++ it("resumes deferred recovery after a concurrent recovery is denied", async () => {+ vi.useFakeTimers();+ const client = {};+ const worker: DeferredRecoveryWorker = {+ descriptor: {+ workerId: "worker-concurrent-recovery",+ pid: process.pid,+ rootActiveSessionId: "active-concurrent-recovery",+ lifecycle: "ready",+ },+ client,+ snapshotCache: new Map(),+ incomingTranscriptActiveSessionIds: new Set(),+ transcriptCaches: new Map(),+ duplicateIncomingTranscriptChunkIndexes: new Map(),+ snapshotTransferFrames: new Map<string, never>(),+ intentionalStop: false,+ stopRevision: 0,+ };+ let admissionActive = true;+ const assertRecoveryAllowed = vi.fn(async () => {+ if (admissionActive) {+ throw recoveryDeniedError("supervisor_recovery_cancelled");+ }+ });+ const persistWorker = vi.fn();+ const recoverWorker = vi.fn(async () => undefined);+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ assertRecoveryAllowed,+ persistWorker,+ syncAgentPeers: vi.fn(async () => undefined),+ recoverWorker,+ }) as DeferredRecoveryHarness;++ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));+ const deferredRecovery = worker.deferredRecovery;+ expect(deferredRecovery).toBeDefined();+ let startConcurrentRecovery: () => void = () => undefined;+ const concurrentRecoveryBarrier = new Promise<void>((resolve) => {+ startConcurrentRecovery = resolve;+ });+ const concurrentRecovery = (async () => {+ await concurrentRecoveryBarrier;+ await expect(assertRecoveryAllowed()).rejects.toMatchObject({ code: "supervisor_recovery_cancelled" });+ })().finally(() => {+ worker.recovery = undefined;+ });+ worker.recovery = concurrentRecovery;++ await vi.advanceTimersByTimeAsync(5000);+ expect(worker.deferredRecovery).toBe(deferredRecovery);+ expect(worker.descriptor.lifecycle).toBe("ready");+ expect(persistWorker).not.toHaveBeenCalled();+ expect(recoverWorker).not.toHaveBeenCalled();++ startConcurrentRecovery();+ await concurrentRecovery;+ expect(worker.recovery).toBeUndefined();+ admissionActive = false;+ await vi.advanceTimersByTimeAsync(5000);+ await deferredRecovery;++ expect(assertRecoveryAllowed).toHaveBeenCalledTimes(3);+ expect(worker.descriptor.lifecycle).toBe("recovering");+ expect(worker.descriptor.lastError).toBe("worker disconnected");+ expect(persistWorker).toHaveBeenCalledOnce();+ expect(recoverWorker).toHaveBeenCalledOnce();+ expect(worker.deferredRecovery).toBeUndefined();+ });++ it("cancels deferred recovery permanently after ownership loss", async () => {+ vi.useFakeTimers();+ const client = {};+ const worker: DeferredRecoveryWorker = {+ descriptor: {+ workerId: "worker-stale-recovery",+ pid: process.pid,+ rootActiveSessionId: "active-stale-recovery",+ lifecycle: "ready",+ },+ client,+ snapshotCache: new Map(),+ incomingTranscriptActiveSessionIds: new Set(),+ transcriptCaches: new Map(),+ duplicateIncomingTranscriptChunkIndexes: new Map(),+ snapshotTransferFrames: new Map<string, never>(),+ intentionalStop: false,+ stopRevision: 0,+ };+ let stale = false;+ const assertRecoveryAllowed = vi.fn(async () => {+ throw recoveryDeniedError(stale ? "supervisor_generation_stale" : "supervisor_recovery_cancelled");+ });+ const persistWorker = vi.fn();+ const recoverWorker = vi.fn(async () => undefined);+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ assertRecoveryAllowed,+ persistWorker,+ syncAgentPeers: vi.fn(async () => undefined),+ recoverWorker,+ }) as DeferredRecoveryHarness;++ await supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));+ const deferredRecovery = worker.deferredRecovery;+ expect(deferredRecovery).toBeDefined();+ stale = true;+ await vi.advanceTimersByTimeAsync(5000);+ await deferredRecovery;++ expect(worker.descriptor.lifecycle).toBe("ready");+ expect(worker.descriptor.lastError).toBeUndefined();+ expect(persistWorker).not.toHaveBeenCalled();+ expect(recoverWorker).not.toHaveBeenCalled();+ expect(worker.deferredRecovery).toBeUndefined();+ await vi.advanceTimersByTimeAsync(10_000);+ expect(recoverWorker).not.toHaveBeenCalled();+ });++ it.each(recoveryEligibilityInvalidations)(+ "does not recover when $name during the ownership assertion",+ async ({ invalidate }) => {+ const client = {};+ const worker: DeferredRecoveryWorker = {+ descriptor: {+ workerId: "worker-eligibility-race",+ pid: process.pid,+ rootActiveSessionId: "active-eligibility-race",+ lifecycle: "ready",+ },+ client,+ snapshotCache: new Map(),+ incomingTranscriptActiveSessionIds: new Set(),+ transcriptCaches: new Map(),+ duplicateIncomingTranscriptChunkIndexes: new Map(),+ snapshotTransferFrames: new Map<string, never>(),+ intentionalStop: false,+ stopRevision: 0,+ };+ let resolveAssertion: () => void = () => undefined;+ const assertion = new Promise<void>((resolve) => {+ resolveAssertion = resolve;+ });+ const persistWorker = vi.fn();+ const syncAgentPeers = vi.fn(async () => undefined);+ const recoverWorker = vi.fn(async () => undefined);+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ ...createSupervisorSnapshotState(),+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ assertRecoveryAllowed: vi.fn(() => assertion),+ persistWorker,+ syncAgentPeers,+ recoverWorker,+ }) as DeferredRecoveryHarness;++ const handling = supervisor.handleWorkerClose(worker, client, new Error("worker disconnected"));+ invalidate(supervisor, worker);+ resolveAssertion();+ await handling;++ expect(worker.descriptor.lifecycle).toBe("ready");+ expect(worker.descriptor.lastError).toBeUndefined();+ expect(persistWorker).not.toHaveBeenCalled();+ expect(syncAgentPeers).not.toHaveBeenCalled();+ expect(recoverWorker).not.toHaveBeenCalled();+ },+ );+it("cancels an in-flight recovery after an intentional stop tombstone", async () => {vi.useFakeTimers();type RecoveryWorker = {@@ -159,6 +1077,7 @@ describe("daemon worker supervisor monitoring", () => {connectWorker: ReturnType<typeof vi.fn>;recoverUncertainWorkerOperations: ReturnType<typeof vi.fn>;launchWorker: ReturnType<typeof vi.fn>;+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;recoverWorker(worker: RecoveryWorker): Promise<void>;};const worker: RecoveryWorker = {@@ -178,6 +1097,7 @@ describe("daemon worker supervisor monitoring", () => {connectWorker: vi.fn(),recoverUncertainWorkerOperations: vi.fn(async () => {}),launchWorker: vi.fn(async () => worker),+ assertRecoveryAllowed: vi.fn(async () => {}),}) as RecoveryHarness;const recovery = supervisor.recoverWorker(worker);@@ -216,6 +1136,7 @@ describe("daemon worker supervisor monitoring", () => {persistWorker: ReturnType<typeof vi.fn>;syncAgentPeers: ReturnType<typeof vi.fn>;log: ReturnType<typeof vi.fn>;+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;recoverWorker(worker: RecoveryWorker): Promise<void>;};const worker: RecoveryWorker = {@@ -240,6 +1161,7 @@ describe("daemon worker supervisor monitoring", () => {persistWorker: vi.fn(),syncAgentPeers: vi.fn(async () => {}),log: vi.fn(),+ assertRecoveryAllowed: vi.fn(async () => {}),}) as RecoveryHarness;const recovery = supervisor.recoverWorker(worker);@@ -279,6 +1201,7 @@ describe("daemon worker supervisor monitoring", () => {persistWorker: ReturnType<typeof vi.fn>;syncAgentPeers: ReturnType<typeof vi.fn>;log: ReturnType<typeof vi.fn>;+ assertRecoveryAllowed: ReturnType<typeof vi.fn>;recoverWorker(worker: RecoveryWorker): Promise<void>;};const worker: RecoveryWorker = {@@ -305,6 +1228,7 @@ describe("daemon worker supervisor monitoring", () => {throw new Error("peer unavailable");}),log: vi.fn(),+ assertRecoveryAllowed: vi.fn(async () => {}),}) as RecoveryHarness;const recovery = supervisor.recoverWorker(worker);@@ -552,6 +1476,7 @@ describe("daemon worker supervisor monitoring", () => {const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {catalog: { markInterrupted },log: vi.fn(),+ assertRecoveryAllowed: vi.fn(async () => {}),}) as {recoverUncertainWorkerOperations(worker: RecoveryWorker, killWorkerProcess: boolean): Promise<void>;};diff --git a/packages/coding-agent/test/daemon-supervisor-side-question.test.ts b/packages/coding-agent/test/daemon-supervisor-side-question.test.tsindex a52c8e028a..7c68f0e1b3 100644--- a/packages/coding-agent/test/daemon-supervisor-side-question.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-side-question.test.ts@@ -20,6 +20,7 @@ describe("daemon supervisor side-question routing", () => {const write = vi.fn();const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {ready: Promise.resolve(),+ ownership: { assertCurrent: vi.fn(async () => undefined) },handleCommand,write,log: vi.fn(),diff --git a/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts b/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.tsindex 43742d6130..11adf1bc85 100644--- a/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts+++ b/packages/coding-agent/test/fixtures/eng-4600-supervisor-fixture.ts@@ -1,10 +1,12 @@import { existsSync, unlinkSync } from "node:fs";+import { fileURLToPath } from "node:url";import lockfile from "proper-lockfile";+import { APP_NAME } from "../../src/config.js";import { isDaemonCatalogProcess, runDaemonCatalogProcess } from "../../src/modes/daemon/daemon-catalog-process.js";import { DaemonSupervisor } from "../../src/modes/daemon/daemon-supervisor.js";import { acquireDaemonSupervisorOwnership } from "../../src/modes/daemon/daemon-supervisor-ownership.js";-type ControlMessage = { type: "go" | "probe" | "release" | "shutdown" | "cleanup" };+type ControlMessage = { type: "go" | "probe" | "release" | "release_runtime" | "shutdown" | "cleanup" };function requiredEnvironment(name: string): string {const value = process.env[name];@@ -49,6 +51,8 @@ async function runOwnershipHolder(): Promise<never> {}async function runSupervisor(): Promise<never> {+ process.argv[1] = fileURLToPath(new URL("../../src/cli.ts", import.meta.url));+ process.title = APP_NAME;const socketPath = requiredEnvironment("ENG_4600_SOCKET_PATH");const agentDir = requiredEnvironment("ENG_4600_AGENT_DIR");const descriptorDir = requiredEnvironment("ENG_4600_DESCRIPTOR_DIR");@@ -66,6 +70,16 @@ async function runSupervisor(): Promise<never> {try {await supervisor.start();send({ type: "ready" });+ process.on("message", (message: unknown) => {+ if (+ !message ||+ typeof message !== "object" ||+ (message as Partial<ControlMessage>).type !== "release_runtime"+ ) {+ return;+ }+ void releaseSupervisorRuntime(supervisor);+ });return await new Promise<never>(() => {});} catch (error) {send({ type: "failed", error: error instanceof Error ? error.message : String(error) });@@ -73,6 +87,20 @@ async function runSupervisor(): Promise<never> {}}+async function releaseSupervisorRuntime(supervisor: DaemonSupervisor): Promise<void> {+ const ownership = Reflect.get(supervisor, "ownership") as { release: () => Promise<void> } | undefined;+ await ownership?.release();+ Reflect.set(supervisor, "ownership", undefined);+ const cleanupSocket = Reflect.get(supervisor, "cleanupSocket");+ if (typeof cleanupSocket === "function") {+ Reflect.apply(cleanupSocket, supervisor, []);+ }+ const lease = Reflect.get(supervisor, "socketLease") as { release: () => Promise<void> } | undefined;+ await lease?.release();+ Reflect.set(supervisor, "socketLease", undefined);+ send({ type: "runtime_released" });+}+async function runLegacyCleanup(): Promise<never> {const socketPath = requiredEnvironment("ENG_4600_SOCKET_PATH");send({ type: "ready" });diff --git a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.tsindex 3143f85c1a..eaec8fbc06 100644--- a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts+++ b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts@@ -703,7 +703,7 @@ describe("ENG-4602 snapshot transfer containment", () => {expect(worker.snapshotTransferFrames.size).toBe(0);});- it("rejects same-ID reentrant begins and mismatched duplicate metadata", () => {+ it("rejects same-ID reentrant begins and mismatched duplicate metadata", async () => {const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor-invalid.sock", {defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },descriptorDir: "/tmp/eng-4602-supervisor-invalid-state",@@ -711,20 +711,26 @@ describe("ENG-4602 snapshot transfer containment", () => {const recoverWorker = vi.fn(async () => {});const persistWorker = vi.fn();const syncAgentPeers = vi.fn(async () => {});+ const assertRecoveryAllowed = vi.fn(async () => {});const internals = supervisor as unknown as {+ workers: Map<string, WorkerHarness>;recoverWorker: typeof recoverWorker;persistWorker: typeof persistWorker;syncAgentPeers: typeof syncAgentPeers;+ assertRecoveryAllowed: typeof assertRecoveryAllowed;handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;};internals.recoverWorker = recoverWorker;internals.persistWorker = persistWorker;internals.syncAgentPeers = syncAgentPeers;+ internals.assertRecoveryAllowed = assertRecoveryAllowed;const frames = snapshotFrames([{ role: "user", content: "stable", timestamp: 1 }]);const reentrant = workerHarness();+ internals.workers.set(reentrant.worker.descriptor.workerId, reentrant.worker);internals.handleWorkerFrame(reentrant.worker, frame(frames.begin));internals.handleWorkerFrame(reentrant.worker, frame(frames.begin));+ await new Promise<void>((resolve) => setImmediate(resolve));expect(reentrant.close).toHaveBeenCalledOnce();expect(reentrant.worker.transcriptCaches.has(activeSessionId)).toBe(false);expect(reentrant.worker.client).toBeUndefined();diff --git a/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.ts b/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.tsnew file mode 100644index 0000000000..c7d824ab61--- /dev/null+++ b/packages/coding-agent/test/suite/regressions/4603-worker-recovery.test.ts@@ -0,0 +1,1071 @@+import { type ChildProcess, spawn, spawnSync } from "node:child_process";+import {+ chmodSync,+ existsSync,+ linkSync,+ mkdirSync,+ readdirSync,+ readFileSync,+ renameSync,+ rmSync,+ writeFileSync,+} from "node:fs";+import { createConnection, type Socket } from "node:net";+import { join, resolve } from "node:path";+import { afterEach, describe, expect, it } from "vitest";+import { APP_NAME, ENV_AGENT_DIR } from "../../../src/config.js";+import { getProcessStartId } from "../../../src/core/session-lease.js";+import { DaemonAgentConnection } from "../../../src/modes/agent-connection/daemon-agent-connection.js";+import { DaemonClient } from "../../../src/modes/daemon/daemon-client.js";+import type { DaemonResponse } from "../../../src/modes/daemon/daemon-protocol.js";+import type { SessionSummary } from "../../../src/modes/daemon/daemon-session-list.js";+import {+ acquireDaemonShutdownAdmission,+ acquireDaemonSupervisorOwnership,+} from "../../../src/modes/daemon/daemon-supervisor-ownership.js";+import {+ DAEMON_WORKER_ACTIVE_SESSION_ID_ENV,+ DAEMON_WORKER_ROLE_ENV,+ DAEMON_WORKER_SUPERVISOR_SOCKET_ENV,+ DAEMON_WORKER_TOKEN_ENV,+ type DaemonWorkerDescriptor,+ type DaemonWorkerFrameHeader,+ isDaemonWorkerFrameHeader,+} from "../../../src/modes/daemon/daemon-worker-protocol.js";+import { serializeJsonLine } from "../../../src/modes/rpc/jsonl.js";+import {+ encodePrivateFrame,+ type PrivateFrame,+ PrivateFrameDecoder,+} from "../../../src/modes/session-worker/private-framing.js";+import { createHarness, type Harness } from "../harness.js";++interface OwnerRecord {+ token: string;+ generation: string;+ pid: number;+ processStartId?: string;+ socketPath: string;+ descriptorDir: string;+ agentDir: string;+}++type FixtureMessage =+ | { type: "booted" }+ | { type: "ready" }+ | { type: "failed"; error: string }+ | { type: "runtime_released" };++interface ProcessHandle {+ child: ChildProcess;+ identity?: FixtureProcessIdentity;+ role: FixtureProcessIdentity["role"];+ stdout: string;+ stderr: string;+ messages: FixtureMessage[];+ waiters: Array<{+ predicate: (message: FixtureMessage) => boolean;+ resolve: (message: FixtureMessage) => void;+ timeout: ReturnType<typeof setTimeout>;+ }>;+}++interface FixtureProcessIdentity {+ pid: number;+ processStartId: string;+ role: "client" | "supervisor" | "worker";+}++interface FixtureProcessSnapshot {+ ppid: number;+ state: string;+}++interface TestPaths {+ agentDir: string;+ descriptorDir: string;+ executablePath: string;+ registryDir: string;+ socketTmpDir: string;+ socketPath: string;+}++const fixturePath = resolve(__dirname, "../../fixtures/eng-4600-supervisor-fixture.ts");+const fauxExtensionPath = resolve(__dirname, "../../fixtures/eng-4600-faux-extension.ts");+const cliPath = resolve(__dirname, "../../../src/cli.ts");+const tsxPath = resolve(__dirname, "../../../../../node_modules/tsx/dist/cli.mjs");+const tsconfigPath = resolve(__dirname, "../../../../../tsconfig.json");+const supervisorRegistryDirEnv = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR";+const handles = new Set<ProcessHandle>();+const harnesses: Harness[] = [];+const socketTempDirs = new Set<string>();+const fixtureDescriptorDirs = new Set<string>();+const fixtureProcesses = new Map<string, FixtureProcessIdentity>();+const fixtureRegistryDirs = new Set<string>();+// Covers the worker's five second supervisor-availability retry.+const fixtureProcessQuietMs = 5500;++afterEach(async () => {+ registerFixtureOwnedProcesses();+ await stopFixtureOwnedProcesses();+ await Promise.all([...handles].map((handle) => waitForExit(handle)));+ handles.clear();+ fixtureDescriptorDirs.clear();+ fixtureProcesses.clear();+ fixtureRegistryDirs.clear();+ while (harnesses.length > 0) {+ harnesses.pop()?.cleanup();+ }+ for (const path of socketTempDirs) {+ rmSync(path, { recursive: true, force: true, maxRetries: 50, retryDelay: 50 });+ }+ socketTempDirs.clear();+}, 60_000);++async function createPaths(): Promise<TestPaths> {+ const harness = await createHarness();+ harnesses.push(harness);+ const executablePath = join(harness.tempDir, APP_NAME);+ linkSync(process.execPath, executablePath);+ const socketTmpDir = `/tmp/eng-4603-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`;+ mkdirSync(socketTmpDir, { recursive: true, mode: 0o700 });+ socketTempDirs.add(socketTmpDir);+ fixtureDescriptorDirs.add(join(harness.tempDir, "workers"));+ fixtureRegistryDirs.add(join(harness.tempDir, "registry"));+ return {+ agentDir: harness.tempDir,+ descriptorDir: join(harness.tempDir, "workers"),+ executablePath,+ registryDir: join(harness.tempDir, "registry"),+ socketTmpDir,+ socketPath:+ process.platform === "win32"+ ? `\\\\.\\pipe\\prime-agent-eng-4603-${process.pid}-${Date.now()}`+ : join(harness.tempDir, "daemon.sock"),+ };+}++function spawnSupervisor(paths: TestPaths): ProcessHandle {+ return trackProcess(+ spawn(paths.executablePath, [tsxPath, fixturePath], {+ cwd: paths.agentDir,+ env: {+ ...process.env,+ [supervisorRegistryDirEnv]: paths.registryDir,+ [ENV_AGENT_DIR]: paths.agentDir,+ ENG_4600_AGENT_DIR: paths.agentDir,+ ENG_4600_DESCRIPTOR_DIR: paths.descriptorDir,+ ENG_4600_FIXTURE_MODE: "supervisor",+ ENG_4600_REGISTRY_DIR: paths.registryDir,+ ENG_4600_SOCKET_PATH: paths.socketPath,+ PI_OFFLINE: "1",+ TMPDIR: paths.socketTmpDir,+ TSX_TSCONFIG_PATH: tsconfigPath,+ },+ stdio: ["ignore", "pipe", "pipe", "ipc"],+ }),+ "supervisor",+ );+}++function spawnStandaloneWorker(+ paths: TestPaths,+ workerSocketPath: string,+ token: string,+ extraEnv: NodeJS.ProcessEnv = {},+): ProcessHandle {+ return trackProcess(+ spawn(+ paths.executablePath,+ [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", workerSocketPath, "--offline"],+ {+ cwd: paths.agentDir,+ env: {+ ...process.env,+ ...extraEnv,+ [supervisorRegistryDirEnv]: paths.registryDir,+ [ENV_AGENT_DIR]: paths.agentDir,+ [DAEMON_WORKER_ROLE_ENV]: "1",+ [DAEMON_WORKER_TOKEN_ENV]: token,+ [DAEMON_WORKER_ACTIVE_SESSION_ID_ENV]: "eng-4603-worker",+ [DAEMON_WORKER_SUPERVISOR_SOCKET_ENV]: paths.socketPath,+ PI_OFFLINE: "1",+ TSX_TSCONFIG_PATH: tsconfigPath,+ },+ stdio: ["ignore", "pipe", "pipe"],+ },+ ),+ "worker",+ );+}++function trackProcess(child: ChildProcess, role: FixtureProcessIdentity["role"]): ProcessHandle {+ const identity = registerFixtureProcess(child.pid, getProcessStartId(child.pid ?? -1), role);+ const handle: ProcessHandle = { child, identity, role, stdout: "", stderr: "", messages: [], waiters: [] };+ handles.add(handle);+ child.stdout?.on("data", (chunk: Buffer) => {+ handle.stdout += chunk.toString("utf8");+ });+ child.stderr?.on("data", (chunk: Buffer) => {+ handle.stderr += chunk.toString("utf8");+ });+ child.on("message", (message: FixtureMessage) => {+ const index = handle.waiters.findIndex((waiter) => waiter.predicate(message));+ if (index === -1) {+ handle.messages.push(message);+ return;+ }+ const [waiter] = handle.waiters.splice(index, 1);+ if (waiter) {+ clearTimeout(waiter.timeout);+ waiter.resolve(message);+ }+ });+ return handle;+}++function fixtureProcessKey(identity: FixtureProcessIdentity): string {+ return `${identity.role}:${identity.pid}:${identity.processStartId}`;+}++function registerFixtureProcess(+ pid: number | undefined,+ processStartId: string | undefined,+ role: FixtureProcessIdentity["role"],+): FixtureProcessIdentity | undefined {+ if (pid === undefined) return undefined;+ if (!Number.isSafeInteger(pid) || pid <= 0) {+ throw new Error(`Invalid fixture process pid: ${String(pid)}`);+ }+ if (!processStartId) {+ if (fixturePidIsAlive(pid)) {+ throw new Error(`Live fixture process ${pid} has no exact start identity`);+ }+ return undefined;+ }+ const identity = { pid, processStartId, role };+ fixtureProcesses.set(fixtureProcessKey(identity), identity);+ return identity;+}++function registerFixtureOwnedProcesses(): void {+ for (const handle of handles) {+ if (!handle.identity && (handle.child.exitCode !== null || handle.child.signalCode !== null)) {+ continue;+ }+ handle.identity ??= registerFixtureProcess(+ handle.child.pid,+ getProcessStartId(handle.child.pid ?? -1),+ handle.role,+ );+ }+ for (const registryDir of fixtureRegistryDirs) {+ for (const name of readFixtureDirectory(registryDir).filter((entry) => entry.endsWith(".owner"))) {+ const ownerPath = join(registryDir, name, "owner.json");+ const owner = readFixtureJson<unknown>(ownerPath);+ if (owner === undefined) continue;+ registerFixtureRecord(owner, "supervisor", ownerPath);+ }+ }+ for (const descriptorDir of fixtureDescriptorDirs) {+ for (const name of readFixtureDirectory(descriptorDir).filter((entry) => entry.endsWith(".json"))) {+ const descriptorPath = join(descriptorDir, name);+ const descriptor = readFixtureJson<unknown>(descriptorPath);+ if (descriptor === undefined) continue;+ registerFixtureRecord(descriptor, "worker", descriptorPath);+ }+ }+}++function registerFixtureRecord(value: unknown, role: "supervisor" | "worker", path: string): FixtureProcessIdentity {+ if (!value || typeof value !== "object") {+ throw new Error(`Invalid fixture process record: ${path}`);+ }+ const record = value as { pid?: unknown; processStartId?: unknown };+ if (+ typeof record.pid !== "number" ||+ !Number.isSafeInteger(record.pid) ||+ record.pid <= 0 ||+ typeof record.processStartId !== "string" ||+ record.processStartId.length === 0+ ) {+ throw new Error(`Invalid fixture process identity: ${path}`);+ }+ return registerFixtureProcess(record.pid, record.processStartId, role)!;+}++function readFixtureProcessSnapshot(): Map<number, FixtureProcessSnapshot> {+ const listing = spawnSync("ps", ["-axo", "pid=,ppid=,state="], { encoding: "utf8" });+ if (listing.error) throw listing.error;+ if (listing.status !== 0) {+ throw new Error(`Could not enumerate fixture descendants: ${listing.stderr.trim()}`);+ }+ const processes = new Map<number, FixtureProcessSnapshot>();+ for (const line of listing.stdout.split("\n")) {+ const match = /^\s*(\d+)\s+(\d+)\s+(\S+)\s*$/.exec(line);+ if (!match) continue;+ processes.set(Number(match[1]), { ppid: Number(match[2]), state: match[3]! });+ }+ return processes;+}++function fixtureDescendantPids(rootPid: number, processes: Map<number, FixtureProcessSnapshot>): number[] {+ const childrenByParent = new Map<number, number[]>();+ for (const [pid, process] of processes) {+ const children = childrenByParent.get(process.ppid) ?? [];+ children.push(pid);+ childrenByParent.set(process.ppid, children);+ }+ const descendants: number[] = [];+ const pending = [rootPid];+ while (pending.length > 0) {+ const parentPid = pending.shift()!;+ for (const childPid of childrenByParent.get(parentPid) ?? []) {+ descendants.push(childPid);+ pending.push(childPid);+ }+ }+ return descendants;+}++function isFixtureDescendant(pid: number, rootPid: number, processes: Map<number, FixtureProcessSnapshot>): boolean {+ const visited = new Set<number>();+ let current = pid;+ while (!visited.has(current)) {+ visited.add(current);+ const process = processes.get(current);+ if (!process) return false;+ if (process.ppid === rootPid) return true;+ current = process.ppid;+ }+ return false;+}++function signalFixtureProcess(identity: FixtureProcessIdentity, signal: NodeJS.Signals): boolean {+ const state = fixtureProcessState(identity);+ if (state === "exited") return false;+ if (state === "unverified") {+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);+ }+ try {+ process.kill(identity.pid, signal);+ return true;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ESRCH") return false;+ throw error;+ }+}++async function terminateFixtureProcessTree(root: FixtureProcessIdentity): Promise<void> {+ if (fixtureProcessState(root) === "exited") return;+ if (process.platform === "win32") {+ if (signalFixtureProcess(root, "SIGKILL")) await waitForFixtureProcessExit(root);+ return;+ }+ if (!signalFixtureProcess(root, "SIGSTOP")) return;+ if (!(await waitForFixtureProcessStopped(root))) {+ throw new Error(`Fixture root ${root.pid}/${root.processStartId} exited before descendant discovery`);+ }+ const tree = new Map<string, FixtureProcessIdentity>([[`${root.pid}:${root.processStartId}`, root]]);+ let unchangedScans = 0;+ while (unchangedScans < 2) {+ const processesBefore = readFixtureProcessSnapshot();+ let discovered = 0;+ for (const pid of fixtureDescendantPids(root.pid, processesBefore)) {+ const processStartId = getProcessStartId(pid);+ if (!processStartId) {+ if (fixturePidIsAlive(pid)) throw new Error(`Live fixture descendant ${pid} has no exact start identity`);+ continue;+ }+ if (+ [...tree.values()].some((identity) => identity.pid === pid && identity.processStartId === processStartId)+ ) {+ continue;+ }+ const processesAfter = readFixtureProcessSnapshot();+ const currentStartId = getProcessStartId(pid);+ if (currentStartId !== processStartId) {+ if (currentStartId === undefined && fixturePidIsAlive(pid)) {+ throw new Error(`Could not revalidate fixture descendant ${pid}`);+ }+ continue;+ }+ if (!isFixtureDescendant(pid, root.pid, processesAfter)) {+ throw new Error(`Fixture descendant ${pid}/${processStartId} changed ancestry during cleanup`);+ }+ const identity = registerFixtureProcess(pid, processStartId, "worker")!;+ tree.set(`${identity.pid}:${identity.processStartId}`, identity);+ if (signalFixtureProcess(identity, "SIGSTOP")) {+ await waitForFixtureProcessStopped(identity);+ }+ discovered++;+ }+ unchangedScans = discovered === 0 ? unchangedScans + 1 : 0;+ await delay(25);+ }+ for (const identity of [...tree.values()].reverse()) {+ signalFixtureProcess(identity, "SIGKILL");+ }+ for (const identity of tree.values()) {+ await waitForFixtureProcessExit(identity);+ }+}++async function waitForFixtureProcessStopped(identity: FixtureProcessIdentity, timeoutMs = 5000): Promise<boolean> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const state = fixtureProcessState(identity);+ if (state === "exited") return false;+ if (state === "unverified") {+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);+ }+ const processStatus = readFixtureProcessSnapshot().get(identity.pid)?.state;+ if (processStatus && ["T", "Z"].includes(processStatus[0]!) && fixtureProcessState(identity) === "matching") {+ return true;+ }+ await delay(25);+ }+ throw new Error(`Timed out waiting for fixture process ${identity.pid}/${identity.processStartId} to stop`);+}++function readFixtureDirectory(path: string): string[] {+ try {+ return readdirSync(path);+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];+ throw error;+ }+}++function readFixtureJson<T>(path: string): T | undefined {+ try {+ return JSON.parse(readFileSync(path, "utf8")) as T;+ } catch (error) {+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;+ throw error;+ }+}++function fixtureProcessState(identity: FixtureProcessIdentity): "exited" | "matching" | "unverified" {+ const processStartId = getProcessStartId(identity.pid);+ if (processStartId === identity.processStartId) return "matching";+ if (processStartId !== undefined) return "exited";+ return fixturePidIsAlive(identity.pid) ? "unverified" : "exited";+}++function fixturePidIsAlive(pid: number): boolean {+ try {+ process.kill(pid, 0);+ return true;+ } catch (error) {+ const code = (error as NodeJS.ErrnoException).code;+ if (code === "ESRCH") return false;+ if (code === "EPERM") return true;+ throw error;+ }+}++async function stopFixtureOwnedProcesses(): Promise<void> {+ const deadline = Date.now() + 30_000;+ let quietSince: number | undefined;+ while (Date.now() < deadline) {+ for (const role of ["supervisor", "worker", "client"] as const) {+ for (const identity of [...fixtureProcesses.values()].filter((process) => process.role === role)) {+ const state = fixtureProcessState(identity);+ if (state === "unverified") {+ throw new Error(`Could not verify fixture process ${identity.pid}/${identity.processStartId}`);+ }+ if (state === "matching") {+ await terminateFixtureProcessTree(identity);+ }+ fixtureProcesses.delete(fixtureProcessKey(identity));+ }+ }+ registerFixtureOwnedProcesses();+ const remaining = [...fixtureProcesses.values()].filter((identity) => fixtureProcessState(identity) !== "exited");+ if (remaining.length > 0) {+ quietSince = undefined;+ continue;+ }+ quietSince ??= Date.now();+ if (Date.now() - quietSince >= fixtureProcessQuietMs) return;+ await delay(25);+ }+ throw new Error("Timed out stopping ENG-4603 fixture processes");+}++async function terminateTrackedFixtureProcess(handle: ProcessHandle): Promise<void> {+ if (!handle.identity && (handle.child.exitCode !== null || handle.child.signalCode !== null)) {+ await waitForExit(handle);+ return;+ }+ const identity =+ handle.identity ??+ registerFixtureProcess(handle.child.pid, getProcessStartId(handle.child.pid ?? -1), handle.role);+ if (identity) await terminateFixtureProcessTree(identity);+ await waitForExit(handle);+}++async function waitForFixtureProcessExit(identity: FixtureProcessIdentity): Promise<void> {+ const deadline = Date.now() + 30_000;+ while (fixtureProcessState(identity) !== "exited" && Date.now() < deadline) {+ await delay(25);+ }+ const state = fixtureProcessState(identity);+ if (state !== "exited") {+ throw new Error(`Timed out waiting for fixture process ${identity.pid}/${identity.processStartId}`);+ }+}++function waitForMessage(+ handle: ProcessHandle,+ predicate: (message: FixtureMessage) => boolean,+ timeoutMs = 30_000,+): Promise<FixtureMessage> {+ const index = handle.messages.findIndex(predicate);+ if (index !== -1) {+ return Promise.resolve(handle.messages.splice(index, 1)[0]!);+ }+ return new Promise((resolveMessage, rejectMessage) => {+ const timeout = setTimeout(() => {+ handle.waiters = handle.waiters.filter((waiter) => waiter.timeout !== timeout);+ rejectMessage(new Error(`Timed out waiting for fixture message\n${handle.stderr}`));+ }, timeoutMs);+ handle.waiters.push({ predicate, resolve: resolveMessage, timeout });+ });+}++function waitForType<T extends FixtureMessage["type"]>(+ handle: ProcessHandle,+ type: T,+ timeoutMs?: number,+): Promise<Extract<FixtureMessage, { type: T }>> {+ return waitForMessage(handle, (message) => message.type === type || message.type === "failed", timeoutMs)+ .then((message) => {+ if (message.type === "failed") throw new Error(message.error);+ return message as Extract<FixtureMessage, { type: T }>;+ })+ .catch((error: unknown) => {+ throw new Error(`Timed out waiting for fixture message ${type}: ${String(error)}`);+ });+}++function waitForExit(handle: ProcessHandle, timeoutMs = 30_000): Promise<void> {+ if (handle.child.exitCode !== null || handle.child.signalCode !== null) {+ return Promise.resolve();+ }+ return new Promise((resolveExit, rejectExit) => {+ const timeout = setTimeout(+ () => rejectExit(new Error(`Timed out waiting for process exit\n${handle.stderr}`)),+ timeoutMs,+ );+ handle.child.once("exit", () => {+ clearTimeout(timeout);+ resolveExit();+ });+ });+}++async function waitForPath(path: string, timeoutMs = 30_000): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (!existsSync(path) && Date.now() < deadline) {+ await delay(25);+ }+ if (!existsSync(path)) {+ throw new Error(`Timed out waiting for ${path}`);+ }+}++async function connectEventually(socketPath: string): Promise<DaemonClient> {+ const deadline = Date.now() + 30_000;+ let lastError: unknown;+ while (Date.now() < deadline) {+ const client = new DaemonClient(socketPath);+ try {+ await client.connect(500);+ await client.waitForHello(2000);+ return client;+ } catch (error) {+ lastError = error;+ client.close();+ await delay(25);+ }+ }+ throw new Error(`Timed out connecting to supervisor: ${String(lastError)}`);+}++function listOwnerRecords(registryDir: string): OwnerRecord[] {+ if (!existsSync(registryDir)) {+ return [];+ }+ const records = readdirSync(registryDir)+ .filter((name) => name.endsWith(".owner"))+ .map((name) => {+ const path = join(registryDir, name, "owner.json");+ const record = JSON.parse(readFileSync(path, "utf8")) as unknown;+ registerFixtureRecord(record, "supervisor", path);+ return record as OwnerRecord;+ });+ return records;+}++function readWorkerDescriptor(descriptorDir: string): DaemonWorkerDescriptor {+ const descriptors = readdirSync(descriptorDir).filter((name) => name.endsWith(".json"));+ expect(descriptors).toHaveLength(1);+ const descriptor = JSON.parse(readFileSync(join(descriptorDir, descriptors[0]!), "utf8")) as DaemonWorkerDescriptor;+ registerFixtureRecord(descriptor, "worker", join(descriptorDir, descriptors[0]!));+ return descriptor;+}++function requireSummary(value: unknown): SessionSummary {+ if (!value || typeof value !== "object" || typeof (value as Partial<SessionSummary>).id !== "string") {+ throw new Error("Daemon returned an invalid session summary");+ }+ return value as SessionSummary;+}++function exactProcessIsAlive(pid: number, processStartId: string | undefined): boolean {+ if (!processStartId) {+ return false;+ }+ return getProcessStartId(pid) === processStartId;+}++async function waitForExactProcessExit(+ pid: number,+ processStartId: string | undefined,+ timeoutMs = 30_000,+): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (exactProcessIsAlive(pid, processStartId) && Date.now() < deadline) {+ await delay(25);+ }+ if (exactProcessIsAlive(pid, processStartId)) {+ throw new Error(`Timed out waiting for exact process ${pid}/${processStartId ?? "unknown"}`);+ }+}++async function createResidentSession(client: DaemonClient, agentDir: string): Promise<SessionSummary> {+ const response = await client.request(+ {+ type: "create",+ config: {+ agentDir,+ apiKey: "faux-key",+ cwd: agentDir,+ extensions: [fauxExtensionPath],+ model: "faux",+ noContextFiles: true,+ noExtensions: false,+ noSkills: true,+ noTools: true,+ provider: "faux",+ },+ },+ 60_000,+ );+ if (!response.success) {+ throw new Error(response.error);+ }+ return requireSummary(response.data);+}++async function runCli(+ paths: TestPaths,+ args: string[],+ timeoutMs = 60_000,+ extraEnv: NodeJS.ProcessEnv = {},+): Promise<{ code: number; stdout: string; stderr: string }> {+ const handle = trackProcess(+ spawn(process.execPath, [tsxPath, cliPath, ...args], {+ cwd: paths.agentDir,+ env: {+ ...process.env,+ ...extraEnv,+ [supervisorRegistryDirEnv]: paths.registryDir,+ [ENV_AGENT_DIR]: paths.agentDir,+ PI_OFFLINE: "1",+ TMPDIR: paths.socketTmpDir,+ TSX_TSCONFIG_PATH: tsconfigPath,+ },+ stdio: ["ignore", "pipe", "pipe"],+ }),+ "client",+ );+ await waitForExit(handle, timeoutMs);+ return { code: handle.child.exitCode ?? 0, stdout: handle.stdout, stderr: handle.stderr };+}++function createFrameReader(socket: Socket): {+ waitFor: (+ predicate: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => boolean,+ timeoutMs?: number,+ ) => Promise<PrivateFrame<DaemonWorkerFrameHeader>>;+} {+ const decoder = new PrivateFrameDecoder(isDaemonWorkerFrameHeader);+ const frames: PrivateFrame<DaemonWorkerFrameHeader>[] = [];+ const waiters: Array<{+ predicate: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => boolean;+ resolve: (frame: PrivateFrame<DaemonWorkerFrameHeader>) => void;+ timeout: ReturnType<typeof setTimeout>;+ }> = [];+ socket.on("data", (chunk: Buffer) => {+ for (const frame of decoder.push(chunk)) {+ const index = waiters.findIndex((waiter) => waiter.predicate(frame));+ if (index === -1) {+ frames.push(frame);+ continue;+ }+ const waiter = waiters.splice(index, 1)[0];+ if (waiter) {+ clearTimeout(waiter.timeout);+ waiter.resolve(frame);+ }+ }+ });+ return {+ waitFor: (predicate, timeoutMs = 10_000) => {+ const index = frames.findIndex(predicate);+ if (index !== -1) {+ return Promise.resolve(frames.splice(index, 1)[0]!);+ }+ return new Promise((resolveFrame, rejectFrame) => {+ const timeout = setTimeout(() => {+ const waiterIndex = waiters.findIndex((waiter) => waiter.timeout === timeout);+ if (waiterIndex !== -1) waiters.splice(waiterIndex, 1);+ rejectFrame(new Error("Timed out waiting for worker frame"));+ }, timeoutMs);+ waiters.push({ predicate, resolve: resolveFrame, timeout });+ });+ },+ };+}++function decodeResponse(frame: PrivateFrame<DaemonWorkerFrameHeader>): DaemonResponse {+ return JSON.parse(frame.payload.toString("utf8")) as DaemonResponse;+}++function delay(ms: number): Promise<void> {+ return new Promise((resolveDelay) => setTimeout(resolveDelay, ms));+}++describe("ENG-4603 worker recovery convergence", () => {+ it("allows only the current generation to replace a crashed resident worker", async () => {+ if (process.platform === "win32") return;+ const paths = await createPaths();+ const predecessor = spawnSupervisor(paths);+ await waitForType(predecessor, "booted");+ predecessor.child.send({ type: "go" });+ await waitForType(predecessor, "ready", 60_000);+ const predecessorClient = await connectEventually(paths.socketPath);+ const summary = await createResidentSession(predecessorClient, paths.agentDir);+ const activeSessionId = summary.activeSessionId ?? summary.id;+ const originalWorkerPid = summary.workerPid;+ if (!originalWorkerPid) throw new Error("Resident worker did not expose its pid");+ const originalWorkerStartId = getProcessStartId(originalWorkerPid);+ const originalWorkerIdentity = registerFixtureProcess(originalWorkerPid, originalWorkerStartId, "worker")!;++ predecessor.child.send({ type: "release_runtime" });+ await waitForType(predecessor, "runtime_released");+ await terminateFixtureProcessTree(originalWorkerIdentity);+ const successor = spawnSupervisor(paths);+ await waitForType(successor, "booted");+ successor.child.send({ type: "go" });+ await waitForType(successor, "ready", 60_000);+ await waitForExactProcessExit(originalWorkerPid, originalWorkerStartId);++ const successorClient = await connectEventually(paths.socketPath);+ let replacement: DaemonWorkerDescriptor | undefined;+ const deadline = Date.now() + 30_000;+ while (Date.now() < deadline) {+ const candidate = readWorkerDescriptor(paths.descriptorDir);+ if (candidate.pid !== originalWorkerPid && candidate.lifecycle === "ready") {+ replacement = candidate;+ break;+ }+ await delay(25);+ }+ if (!replacement) throw new Error("Current supervisor did not publish a replacement worker");+ expect(getProcessStartId(replacement.pid)).toBe(replacement.processStartId);+ await delay(750);+ expect(exactProcessIsAlive(replacement.pid, replacement.processStartId)).toBe(true);+ expect(readdirSync(paths.descriptorDir).filter((name) => name.endsWith(".json"))).toHaveLength(1);+ const listed = await successorClient.request({ type: "list" });+ if (!listed.success) throw new Error(listed.error);+ const sessions = (listed.data as { sessions: unknown[] }).sessions;+ expect(sessions).toHaveLength(1);+ expect(requireSummary(sessions[0]).workerPid).toBe(replacement.pid);++ const connection = await DaemonAgentConnection.attach(successorClient, activeSessionId, {+ recoverDaemon: async () => {},+ });+ await connection.getInitialSnapshot();+ await connection.prompt("after recovery");+ await connection.waitForIdle();+ expect(await connection.getMessages()).toContainEqual(+ expect.objectContaining({+ role: "assistant",+ content: [expect.objectContaining({ text: "upgrade response 1" })],+ }),+ );+ await connection.dispose();+ await successorClient.request({ type: "shutdown", force: true }, 10_000);+ successorClient.close();+ predecessorClient.close();+ await waitForExit(successor);+ await waitForExactProcessExit(replacement.pid, replacement.processStartId);+ await terminateTrackedFixtureProcess(predecessor);+ }, 120_000);++ it("rejects stale commands at worker receipt and before public journal insertion", async () => {+ if (process.platform === "win32") return;+ const workerPaths = await createPaths();+ const workerSocketPath = join(workerPaths.agentDir, "worker-command.sock");+ const token = "eng-4603-token";+ const psCountPath = join(workerPaths.agentDir, "ps-count");+ const workerEnvironment: NodeJS.ProcessEnv = {};+ if (process.platform === "darwin") {+ const psWrapperPath = join(workerPaths.agentDir, "ps");+ writeFileSync(psWrapperPath, '#!/bin/sh\nprintf x >> "$ENG_4603_PS_COUNT_PATH"\nexec /bin/ps "$@"\n', {+ mode: 0o700,+ });+ chmodSync(psWrapperPath, 0o700);+ workerEnvironment.ENG_4603_PS_COUNT_PATH = psCountPath;+ workerEnvironment.PATH = `${workerPaths.agentDir}:${process.env.PATH ?? ""}`;+ }+ const oldOwner = await acquireDaemonSupervisorOwnership({+ agentDir: workerPaths.agentDir,+ appVersion: "test",+ descriptorDir: workerPaths.descriptorDir,+ generation: "old-generation",+ registryDir: workerPaths.registryDir,+ socketPath: workerPaths.socketPath,+ });+ const worker = spawnStandaloneWorker(workerPaths, workerSocketPath, token, workerEnvironment);+ await waitForPath(workerSocketPath);+ const socket = createConnection(workerSocketPath);+ const frames = createFrameReader(socket);+ await new Promise<void>((resolveConnect, rejectConnect) => {+ socket.once("connect", resolveConnect);+ socket.once("error", rejectConnect);+ });+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.outboundType === "daemon_hello");+ const authId = "auth-old";+ socket.write(+ encodePrivateFrame<DaemonWorkerFrameHeader>(+ { kind: "command", requestId: authId, commandType: "worker_auth" },+ Buffer.from(+ serializeJsonLine({+ id: authId,+ type: "worker_auth",+ token,+ supervisorGeneration: oldOwner.record.generation,+ supervisorPid: oldOwner.record.pid,+ supervisorProcessStartId: oldOwner.record.processStartId,+ supervisorSocketPath: oldOwner.record.socketPath,+ }),+ ),+ ),+ );+ expect(+ decodeResponse(+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.requestId === authId),+ ).success,+ ).toBe(true);+ if (process.platform === "darwin") {+ const countAfterAuthentication = readFileSync(psCountPath, "utf8").length;+ await delay(750);+ expect(readFileSync(psCountPath, "utf8").length).toBe(countAfterAuthentication);+ const ownerPath = join(workerPaths.registryDir, `${oldOwner.record.generation}.owner`, "owner.json");+ const ownerRecord = JSON.parse(readFileSync(ownerPath, "utf8")) as { updatedAt: string };+ ownerRecord.updatedAt = new Date(Date.now() + 1000).toISOString();+ const updatedOwnerPath = `${ownerPath}.updated`;+ writeFileSync(updatedOwnerPath, `${JSON.stringify(ownerRecord, null, 2)}\n`);+ renameSync(updatedOwnerPath, ownerPath);+ await delay(500);+ const countAfterOwnerChange = readFileSync(psCountPath, "utf8").length;+ expect(countAfterOwnerChange).toBeGreaterThan(countAfterAuthentication);+ await delay(750);+ expect(readFileSync(psCountPath, "utf8").length).toBe(countAfterOwnerChange);+ }+ const commandId = "stale-list";+ const commandFrame = encodePrivateFrame<DaemonWorkerFrameHeader>(+ { kind: "command", requestId: commandId, commandType: "list" },+ Buffer.from(serializeJsonLine({ id: commandId, type: "list" })),+ );+ socket.write(commandFrame.subarray(0, commandFrame.length - 1));+ const ownerDirectory = join(workerPaths.registryDir, `${oldOwner.record.generation}.owner`);+ const ownerPath = join(ownerDirectory, "owner.json");+ const transitionedOwner = JSON.parse(readFileSync(ownerPath, "utf8")) as OwnerRecord & { updatedAt: string };+ transitionedOwner.token = "successor-token";+ transitionedOwner.pid = worker.child.pid!;+ transitionedOwner.processStartId = getProcessStartId(worker.child.pid!);+ transitionedOwner.updatedAt = new Date().toISOString();+ const transitionedOwnerPath = `${ownerPath}.transitioned`;+ writeFileSync(transitionedOwnerPath, `${JSON.stringify(transitionedOwner, null, 2)}\n`);+ renameSync(transitionedOwnerPath, ownerPath);+ socket.write(commandFrame.subarray(commandFrame.length - 1));+ const staleResponse = decodeResponse(+ await frames.waitFor((frame) => frame.header.kind === "outbound" && frame.header.requestId === commandId),+ );+ expect(staleResponse).toMatchObject({ success: false, error: "supervisor_generation_stale" });+ socket.destroy();+ await oldOwner.release();+ rmSync(ownerDirectory, { recursive: true, force: true });+ await terminateTrackedFixtureProcess(worker);++ const publicPaths = await createPaths();+ const staleSupervisor = spawnSupervisor(publicPaths);+ await waitForType(staleSupervisor, "booted");+ staleSupervisor.child.send({ type: "go" });+ await waitForType(staleSupervisor, "ready", 60_000);+ const publicClient = await connectEventually(publicPaths.socketPath);+ const [publishedOwner] = listOwnerRecords(publicPaths.registryDir);+ if (!publishedOwner) throw new Error("Supervisor did not publish its owner");+ const journalPath = join(publicPaths.descriptorDir, "command-journal.jsonl");+ const journalBefore = existsSync(journalPath) ? readFileSync(journalPath, "utf8") : "";+ const displacedOwnerDir = join(publicPaths.registryDir, `${publishedOwner.generation}.displaced`);+ renameSync(join(publicPaths.registryDir, `${publishedOwner.generation}.owner`), displacedOwnerDir);+ const replacementOwner = await acquireDaemonSupervisorOwnership({+ agentDir: publicPaths.agentDir,+ appVersion: "test",+ descriptorDir: publicPaths.descriptorDir,+ generation: "journal-successor",+ registryDir: publicPaths.registryDir,+ socketPath: publicPaths.socketPath,+ });+ const rejected = await publicClient.request({ type: "create" });+ expect(rejected).toMatchObject({ success: false, error: expect.stringContaining("no longer owns") });+ expect(existsSync(journalPath) ? readFileSync(journalPath, "utf8") : "").toBe(journalBefore);+ publicClient.close();+ await replacementOwner.release();+ rmSync(displacedOwnerDir, { recursive: true, force: true });+ await terminateTrackedFixtureProcess(staleSupervisor);+ }, 90_000);++ it("serializes shutdown admission and reclaims an unrenewed live lease", async () => {+ const paths = await createPaths();+ const previousRegistryDir = process.env[supervisorRegistryDirEnv];+ process.env[supervisorRegistryDirEnv] = paths.registryDir;+ try {+ const first = await acquireDaemonShutdownAdmission();+ const record = JSON.parse(readFileSync(join(paths.registryDir, "shutdown-admission.json"), "utf8")) as {+ pid: number;+ processStartId?: string;+ };+ expect(record.pid).toBe(process.pid);+ expect(record.processStartId).toBe(getProcessStartId(process.pid));+ const refreshTimer = Reflect.get(first, "refreshTimer") as ReturnType<typeof setInterval> | undefined;+ if (!refreshTimer) throw new Error("Shutdown admission did not start its lease refresh");+ clearInterval(refreshTimer);+ let acquired = false;+ const waiting = acquireDaemonShutdownAdmission().then((admission) => {+ acquired = true;+ return admission;+ });+ await delay(250);+ expect(acquired).toBe(false);+ const second = await waiting;+ expect(acquired).toBe(true);+ let destructivePasses = 0;+ const destructivePass = async (admission: { assertOrRenew: () => Promise<void> }) => {+ await admission.assertOrRenew();+ destructivePasses++;+ };+ await expect(destructivePass(first)).rejects.toMatchObject({ code: "daemon_shutdown_in_progress" });+ await destructivePass(second);+ expect(destructivePasses).toBe(1);+ await second.release();+ await first.release();+ } finally {+ if (previousRegistryDir === undefined) {+ delete process.env[supervisorRegistryDirEnv];+ } else {+ process.env[supervisorRegistryDirEnv] = previousRegistryDir;+ }+ }+ }, 15_000);++ it("shutdown --all removes hidden supervisors and workers without changing CLI contracts", async () => {+ if (process.platform === "win32") return;+ const paths = await createPaths();+ const predecessor = spawnSupervisor(paths);+ await waitForType(predecessor, "booted");+ predecessor.child.send({ type: "go" });+ await waitForType(predecessor, "ready", 60_000);+ const client = await connectEventually(paths.socketPath);+ const session = await createResidentSession(client, paths.agentDir);+ const workerPid = session.workerPid;+ if (!workerPid) throw new Error("Resident worker did not expose its pid");+ const predecessorStartId = getProcessStartId(predecessor.child.pid!);+ const workerStartId = getProcessStartId(workerPid);+ registerFixtureProcess(workerPid, workerStartId, "worker");+ predecessor.child.send({ type: "release_runtime" });+ await waitForType(predecessor, "runtime_released");+ const successor = spawnSupervisor(paths);+ await waitForType(successor, "booted");+ successor.child.send({ type: "go" });+ await waitForType(successor, "ready", 60_000);+ const successorStartId = getProcessStartId(successor.child.pid!);+ client.close();+ const systemLsofPath = spawnSync("which", ["lsof"], { encoding: "utf8" }).stdout.trim();+ if (!systemLsofPath) throw new Error("Could not locate lsof for the shutdown regression");+ const lsofPath = join(paths.agentDir, "lsof");+ writeFileSync(lsofPath, '#!/bin/sh\nexec "$ENG_4603_SYSTEM_LSOF" -nP -F pn -U -a -p "$ENG_4603_LSOF_PIDS"\n', {+ mode: 0o700,+ });+ const lsofEnvironment = {+ ENG_4603_LSOF_PIDS: `${predecessor.child.pid},${successor.child.pid},${workerPid}`,+ ENG_4603_SYSTEM_LSOF: systemLsofPath,+ PATH: `${paths.agentDir}:${process.env.PATH ?? ""}`,+ };+ const listenersBeforeShutdown = spawnSync(lsofPath, [], {+ encoding: "utf8",+ env: { ...process.env, ...lsofEnvironment },+ }).stdout;+ expect(listenersBeforeShutdown).toContain(`p${predecessor.child.pid}`);+ expect(listenersBeforeShutdown).toContain(`p${successor.child.pid}`);++ const shutdown = await runCli(paths, ["daemon", "shutdown", "--all", "--json"], 60_000, lsofEnvironment);+ expect(shutdown.code).toBe(0);+ const shutdownResult = JSON.parse(shutdown.stdout) as { stopped: unknown[]; failed: unknown[] };+ const survivingIdentities = [+ { pid: predecessor.child.pid!, processStartId: predecessorStartId },+ { pid: successor.child.pid!, processStartId: successorStartId },+ { pid: workerPid, processStartId: workerStartId },+ ].filter((identity) => exactProcessIsAlive(identity.pid, identity.processStartId));+ if (survivingIdentities.length > 0) {+ expect(shutdownResult.failed).not.toHaveLength(0);+ }+ expect(shutdownResult).toMatchObject({ stopped: expect.any(Array), failed: [] });+ await waitForExactProcessExit(predecessor.child.pid!, predecessorStartId);+ await waitForExactProcessExit(successor.child.pid!, successorStartId);+ await waitForExactProcessExit(workerPid, workerStartId);+ await delay(11_000);+ expect(exactProcessIsAlive(predecessor.child.pid!, predecessorStartId)).toBe(false);+ expect(exactProcessIsAlive(successor.child.pid!, successorStartId)).toBe(false);+ expect(exactProcessIsAlive(workerPid, workerStartId)).toBe(false);++ const contracts = [+ { args: ["daemon", "ps", "--json"], json: [] },+ { args: ["daemon", "ps", "--reap", "--json"], json: { reaped: [], skipped: [] } },+ { args: ["daemon", "shutdown", "--all", "--json"], json: { stopped: [], failed: [] } },+ ];+ for (const contract of contracts) {+ const result = await runCli(paths, contract.args, 60_000, lsofEnvironment);+ if (result.code !== 0) {+ throw new Error(`${contract.args.join(" ")} exited ${result.code}: ${result.stderr}`);+ }+ expect(JSON.parse(result.stdout)).toEqual(contract.json);+ }+ for (const args of [+ ["daemon", "ps"],+ ["daemon", "ps", "--reap"],+ ["daemon", "shutdown", "--all"],+ ]) {+ const result = await runCli(paths, args, 60_000, lsofEnvironment);+ expect(result.code).toBe(0);+ expect(result.stdout).toBe("No daemons found.\n");+ }+ }, 150_000);+});