Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

feat(coding-agent): garbage-collect orphaned session workers (ENG-6105) (#2246)

Seth Karten64a3d5205d3a
diff --git a/packages/coding-agent/.changes/orphaned-worker-gc.md b/packages/coding-agent/.changes/orphaned-worker-gc.md
new file mode 100644
index 0000000000..1f73d9d947
--- /dev/null
+++ b/packages/coding-agent/.changes/orphaned-worker-gc.md
@@ -0,0 +1 @@
+- Fixed orphaned session workers retrying supervisor resurrection forever when no replacement can come up: they now exit gracefully after a bounded supervisor-lost window, closing active sessions first.
diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
index 2bdfa8eee6..e8439d4496 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
@@ -267,6 +267,18 @@ const structuredLog = getLogger("coding-agent.daemon");
const WORKER_SNAPSHOT_TERMINAL_DRAIN_TIMEOUT_MS = 1_000;
const UPDATE_RESTART_PREPARE_TIMEOUT_MS = 90_000;
const MAX_SESSION_SNAPSHOT_STABILIZATION_RETRIES = 3;
+// Orphaned-worker garbage collection: a session worker whose supervisor stays
+// unreachable (no supervisor claim, no successful replacement launch) exits
+// after this window instead of retrying the resurrection loop forever.
+// Sessions persist on disk, and a later supervisor spawns fresh workers on
+// demand, so an unreachable-supervisor worker serves nothing by lingering.
+const WORKER_SUPERVISOR_LOST_EXIT_MS_ENV = "PRIME_AGENT_INTERNAL_WORKER_SUPERVISOR_LOST_EXIT_MS";
+const DEFAULT_WORKER_SUPERVISOR_LOST_EXIT_MS = 5 * 60_000;
+
+function workerSupervisorLostExitMs(): number {
+ const raw = Number(process.env[WORKER_SUPERVISOR_LOST_EXIT_MS_ENV]);
+ return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_WORKER_SUPERVISOR_LOST_EXIT_MS;
+}
const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set([
"ack_result",
@@ -564,6 +576,7 @@ export class AgentDaemon {
private supervisorMonitorTimer?: ReturnType<typeof setTimeout>;
private supervisorFenceTimer?: ReturnType<typeof setTimeout>;
private supervisorLaunchInProgress = false;
+ private supervisorAbsentSince?: number;
private readonly supervisorClaims = new Map<DaemonSocketClient, BoundSupervisorGenerationClaim>();
private readonly peerGrants = new Map<string, DaemonWorkerPeerGrant>();
private readonly peerClaims = new Map<DaemonSocketClient, DaemonWorkerPeerGrant>();
@@ -749,21 +762,76 @@ export class AgentDaemon {
private async checkSupervisorAvailability(supervisorSocketPath: string): Promise<void> {
if (this.shuttingDown || this.hasAuthenticatedSupervisorConnection()) {
+ this.supervisorAbsentSince = undefined;
return;
}
if (await isDaemonShutdownAdmissionActive()) {
+ this.supervisorAbsentSince = undefined;
this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);
return;
}
if (await this.canConnectToSupervisor(supervisorSocketPath)) {
+ this.supervisorAbsentSince = undefined;
return;
}
+ // The supervisor socket is unreachable; remember when the worker last
+ // saw it so the orphan window below stays bounded.
+ this.supervisorAbsentSince ??= Date.now();
await this.launchReplacementSupervisor(supervisorSocketPath);
+ if (await this.canConnectToSupervisor(supervisorSocketPath)) {
+ // A replacement came up during the launch: the orphan window must
+ // restart instead of exiting the worker. The monitor must stay
+ // armed, though — a replacement can bind and then exit before it
+ // ever claims the worker, and only an authenticated claim (or its
+ // later close) re-arms the monitor. Falling through reschedules
+ // the next availability check below.
+ this.supervisorAbsentSince = undefined;
+ }
+ await this.exitIfSupervisorOrphanedForTooLong(supervisorSocketPath);
if (!this.shuttingDown && !this.hasAuthenticatedSupervisorConnection()) {
this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);
}
}
+ /**
+ * Garbage-collect orphaned workers. If the supervisor has been unreachable
+ * for the whole bounded window — no supervisor claim, no successful
+ * replacement launch — the worker exits: retrying the resurrection loop
+ * forever only accumulates garbage. Sessions persist on disk, so the next
+ * supervisor to claim the socket spawns fresh workers on demand.
+ */
+ private async exitIfSupervisorOrphanedForTooLong(supervisorSocketPath: string): Promise<void> {
+ const absentSince = this.supervisorAbsentSince;
+ if (absentSince === undefined || Date.now() - absentSince < workerSupervisorLostExitMs()) {
+ return;
+ }
+ if (this.hasAuthenticatedSupervisorConnection()) {
+ this.supervisorAbsentSince = undefined;
+ return;
+ }
+ if (this.hasOngoingSessionWork()) {
+ // An active run owns the worker a little longer; its turn end
+ // lets the next availability check reconsider.
+ return;
+ }
+ this.log(
+ `supervisor ${supervisorSocketPath} unreachable for ${Math.round((Date.now() - absentSince) / 1000)}s; exiting orphaned worker`,
+ );
+ await this.shutdown(0);
+ }
+
+ private hasOngoingSessionWork(): boolean {
+ for (const state of this.sessions.values()) {
+ // Same predicate the daemon reports as isSessionActive: retrying,
+ // bash/kernel background work, refinement, compaction settlement,
+ // and queued actions must all keep holding the worker.
+ if (state.runtime.session.isSessionActive) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private hasAuthenticatedSupervisorConnection(): boolean {
return this.supervisorClaims.size > 0;
}
diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
index 50178543c4..ee408f9b5e 100644
--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
@@ -150,6 +150,7 @@ interface SupervisorMonitorHarness {
clients: Set<{ authenticated: boolean }>;
supervisorClaims: Map<object, object>;
shuttingDown: boolean;
+ supervisorAbsentSince?: number;
supervisorMonitorTimer?: ReturnType<typeof setTimeout>;
canConnectToSupervisor: (socketPath: string) => Promise<boolean>;
launchReplacementSupervisor: (socketPath: string) => Promise<void>;
@@ -1223,6 +1224,41 @@ describe("daemon worker supervisor monitoring", () => {
expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();
});
+ it("keeps the supervisor monitor armed after a replacement binds but exits before claiming", async () => {
+ vi.useFakeTimers();
+ // The supervisor socket is dead, comes up with the replacement launch,
+ // then dies again before the replacement ever claims the worker.
+ const probeResults = [false, true, false, false];
+ let probeCount = 0;
+ const daemon = createHarness(async () => {
+ const result = probeResults[Math.min(probeCount, probeResults.length - 1)];
+ probeCount += 1;
+ return result ?? false;
+ });
+ // Drive the fake clock until the expected number of probes have run;
+ // one advance alone does not flush the availability check chain.
+ const advanceUntilProbes = async (expected: number) => {
+ for (let step = 0; probeCount < expected && step < 200; step++) {
+ await vi.advanceTimersByTimeAsync(100);
+ }
+ expect(probeCount).toBe(expected);
+ };
+
+ daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 1500);
+ await advanceUntilProbes(2);
+ expect(daemon.launchReplacementSupervisor).toHaveBeenCalledOnce();
+ // The replacement binding mid-launch restarts the orphan window...
+ expect(daemon.supervisorAbsentSince).toBeUndefined();
+ // ...but a bind is not an authenticated claim: the monitor must stay armed
+ // instead of orphaning the worker if the replacement exits unclaimed.
+ expect(daemon.supervisorMonitorTimer).toBeDefined();
+
+ await advanceUntilProbes(4);
+ expect(daemon.launchReplacementSupervisor).toHaveBeenCalledTimes(2);
+ expect(daemon.canConnectToSupervisor).toHaveBeenCalledTimes(4);
+ expect(daemon.supervisorMonitorTimer).toBeDefined();
+ });
+
it("retries when shutdown admission lookup fails", async () => {
vi.useFakeTimers();
let resolveProbe: () => void = () => undefined;
diff --git a/packages/coding-agent/test/daemon-supervisor-process.test.ts b/packages/coding-agent/test/daemon-supervisor-process.test.ts
index 5ddedfc445..5423bed686 100644
--- a/packages/coding-agent/test/daemon-supervisor-process.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-process.test.ts
@@ -1,6 +1,15 @@
import { type ChildProcess, spawn } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ chmodSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
@@ -1724,4 +1733,95 @@ describe("daemon supervisor resident workers", () => {
await waitForProcessGone(summary.workerPid);
workerPids.delete(summary.workerPid);
});
+
+ it("exits an orphaned session worker when no replacement supervisor can come up", { timeout: 120_000 }, async () => {
+ const root = tempDir();
+ const agentDir = join(root, "agent");
+ const projectDir = join(root, "project");
+ const registryDir = join(root, "supervisor-registry");
+ // The socket path must stay short: macOS rejects listen() with EINVAL
+ // past the sun_path limit, and the per-test temp dir nests too deep.
+ const socketDir = mkdtempSync(join(tmpdir(), "prime-orphan-gc-"));
+ tempDirs.push(socketDir);
+ const sessionDir = join(agentDir, "sessions");
+ const socketPath = join(socketDir, "supervisor.sock");
+ mkdirSync(projectDir, { recursive: true });
+
+ const manager = SessionManager.create(projectDir, sessionDir);
+ manager.appendMessage({ role: "user", content: "orphan gc fixture", timestamp: 1 });
+ manager.flushNow();
+ const sessionFile = manager.getSessionFile();
+ if (!sessionFile) throw new Error("Missing fixture session path");
+
+ const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, [], {
+ PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR: registryDir,
+ PRIME_AGENT_INTERNAL_WORKER_SUPERVISOR_LOST_EXIT_MS: "3000",
+ });
+ // Cold tsx startup can exceed the shared helper's 15s readiness deadline.
+ let client: DaemonClient | undefined;
+ const readyDeadline = Date.now() + 60_000;
+ while (client === undefined && Date.now() < readyDeadline) {
+ if (supervisor.exitCode !== null || supervisor.signalCode !== null) {
+ const diagnostics = childDiagnostics.get(supervisor);
+ throw new Error(
+ `Supervisor exited before becoming ready\nstdout:\n${diagnostics?.stdout ?? ""}\nstderr:\n${diagnostics?.stderr ?? ""}`,
+ );
+ }
+ const candidate = new DaemonClient(socketPath);
+ try {
+ await candidate.connect(250);
+ await candidate.waitForHello(1000);
+ client = candidate;
+ } catch {
+ candidate.close();
+ await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));
+ }
+ }
+ if (!client) throw new Error("Supervisor did not become ready in time");
+ const created = await client.request({
+ type: "create",
+ sessionPath: sessionFile,
+ config: { cwd: projectDir, agentDir, sessionDir, noTools: true, noExtensions: true },
+ });
+ if (!created.success) throw new Error(created.error);
+ const summary = requireSummary(created.data);
+ if (!summary.workerPid) throw new Error("Session worker did not spawn");
+ workerPids.add(summary.workerPid);
+
+ // SIGKILL the registry-recorded supervisor pid: the spawned child can be
+ // a tsx wrapper, and killing the wrapper alone leaves the supervisor
+ // listening on its socket, so the worker would never orphan.
+ const ownerDirectories = readdirSync(registryDir).filter((name) => name.endsWith(".owner"));
+ expect(ownerDirectories).toHaveLength(1);
+ const owner = JSON.parse(readFileSync(join(registryDir, ownerDirectories[0], "owner.json"), "utf8")) as {
+ pid?: number;
+ };
+ if (!owner.pid) throw new Error("Supervisor owner record is missing its pid");
+ const supervisorPid = owner.pid;
+ const workerPid = summary.workerPid;
+ client.close();
+
+ // A read-only socket directory makes every replacement-launch attempt
+ // fail (lock/bind EACCES): the real-world orphan precondition where a
+ // worker cannot bring its supervisor back.
+ chmodSync(socketDir, 0o555);
+ try {
+ process.kill(supervisorPid, "SIGKILL");
+ await waitForCondition(
+ () => {
+ try {
+ process.kill(workerPid, 0);
+ return false;
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code === "ESRCH";
+ }
+ },
+ "Orphaned session worker did not exit after the supervisor-lost window",
+ 60_000,
+ );
+ } finally {
+ chmodSync(socketDir, 0o755);
+ }
+ workerPids.delete(workerPid);
+ });
});