A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
feat(coding-agent): garbage-collect orphaned session workers (ENG-6105) (#2246)
Seth Karten64a3d5205d3a
diff --git a/packages/coding-agent/.changes/orphaned-worker-gc.md b/packages/coding-agent/.changes/orphaned-worker-gc.mdnew file mode 100644index 0000000000..1f73d9d947--- /dev/null+++ b/packages/coding-agent/.changes/orphaned-worker-gc.md@@ -0,0 +1 @@+- Fixed orphaned session workers retrying supervisor resurrection forever when no replacement can come up: they now exit gracefully after a bounded supervisor-lost window, closing active sessions first.diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.tsindex 2bdfa8eee6..e8439d4496 100644--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts@@ -267,6 +267,18 @@ const structuredLog = getLogger("coding-agent.daemon");const WORKER_SNAPSHOT_TERMINAL_DRAIN_TIMEOUT_MS = 1_000;const UPDATE_RESTART_PREPARE_TIMEOUT_MS = 90_000;const MAX_SESSION_SNAPSHOT_STABILIZATION_RETRIES = 3;+// Orphaned-worker garbage collection: a session worker whose supervisor stays+// unreachable (no supervisor claim, no successful replacement launch) exits+// after this window instead of retrying the resurrection loop forever.+// Sessions persist on disk, and a later supervisor spawns fresh workers on+// demand, so an unreachable-supervisor worker serves nothing by lingering.+const WORKER_SUPERVISOR_LOST_EXIT_MS_ENV = "PRIME_AGENT_INTERNAL_WORKER_SUPERVISOR_LOST_EXIT_MS";+const DEFAULT_WORKER_SUPERVISOR_LOST_EXIT_MS = 5 * 60_000;++function workerSupervisorLostExitMs(): number {+ const raw = Number(process.env[WORKER_SUPERVISOR_LOST_EXIT_MS_ENV]);+ return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_WORKER_SUPERVISOR_LOST_EXIT_MS;+}const DAEMON_COMMAND_TYPES: ReadonlySet<string> = new Set(["ack_result",@@ -564,6 +576,7 @@ export class AgentDaemon {private supervisorMonitorTimer?: ReturnType<typeof setTimeout>;private supervisorFenceTimer?: ReturnType<typeof setTimeout>;private supervisorLaunchInProgress = false;+ private supervisorAbsentSince?: number;private readonly supervisorClaims = new Map<DaemonSocketClient, BoundSupervisorGenerationClaim>();private readonly peerGrants = new Map<string, DaemonWorkerPeerGrant>();private readonly peerClaims = new Map<DaemonSocketClient, DaemonWorkerPeerGrant>();@@ -749,21 +762,76 @@ export class AgentDaemon {private async checkSupervisorAvailability(supervisorSocketPath: string): Promise<void> {if (this.shuttingDown || this.hasAuthenticatedSupervisorConnection()) {+ this.supervisorAbsentSince = undefined;return;}if (await isDaemonShutdownAdmissionActive()) {+ this.supervisorAbsentSince = undefined;this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);return;}if (await this.canConnectToSupervisor(supervisorSocketPath)) {+ this.supervisorAbsentSince = undefined;return;}+ // The supervisor socket is unreachable; remember when the worker last+ // saw it so the orphan window below stays bounded.+ this.supervisorAbsentSince ??= Date.now();await this.launchReplacementSupervisor(supervisorSocketPath);+ if (await this.canConnectToSupervisor(supervisorSocketPath)) {+ // A replacement came up during the launch: the orphan window must+ // restart instead of exiting the worker. The monitor must stay+ // armed, though — a replacement can bind and then exit before it+ // ever claims the worker, and only an authenticated claim (or its+ // later close) re-arms the monitor. Falling through reschedules+ // the next availability check below.+ this.supervisorAbsentSince = undefined;+ }+ await this.exitIfSupervisorOrphanedForTooLong(supervisorSocketPath);if (!this.shuttingDown && !this.hasAuthenticatedSupervisorConnection()) {this.scheduleSupervisorAvailabilityCheck(supervisorSocketPath, 5000);}}+ /**+ * Garbage-collect orphaned workers. If the supervisor has been unreachable+ * for the whole bounded window — no supervisor claim, no successful+ * replacement launch — the worker exits: retrying the resurrection loop+ * forever only accumulates garbage. Sessions persist on disk, so the next+ * supervisor to claim the socket spawns fresh workers on demand.+ */+ private async exitIfSupervisorOrphanedForTooLong(supervisorSocketPath: string): Promise<void> {+ const absentSince = this.supervisorAbsentSince;+ if (absentSince === undefined || Date.now() - absentSince < workerSupervisorLostExitMs()) {+ return;+ }+ if (this.hasAuthenticatedSupervisorConnection()) {+ this.supervisorAbsentSince = undefined;+ return;+ }+ if (this.hasOngoingSessionWork()) {+ // An active run owns the worker a little longer; its turn end+ // lets the next availability check reconsider.+ return;+ }+ this.log(+ `supervisor ${supervisorSocketPath} unreachable for ${Math.round((Date.now() - absentSince) / 1000)}s; exiting orphaned worker`,+ );+ await this.shutdown(0);+ }++ private hasOngoingSessionWork(): boolean {+ for (const state of this.sessions.values()) {+ // Same predicate the daemon reports as isSessionActive: retrying,+ // bash/kernel background work, refinement, compaction settlement,+ // and queued actions must all keep holding the worker.+ if (state.runtime.session.isSessionActive) {+ return true;+ }+ }+ return false;+ }+private hasAuthenticatedSupervisorConnection(): boolean {return this.supervisorClaims.size > 0;}diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.tsindex 50178543c4..ee408f9b5e 100644--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts@@ -150,6 +150,7 @@ interface SupervisorMonitorHarness {clients: Set<{ authenticated: boolean }>;supervisorClaims: Map<object, object>;shuttingDown: boolean;+ supervisorAbsentSince?: number;supervisorMonitorTimer?: ReturnType<typeof setTimeout>;canConnectToSupervisor: (socketPath: string) => Promise<boolean>;launchReplacementSupervisor: (socketPath: string) => Promise<void>;@@ -1223,6 +1224,41 @@ describe("daemon worker supervisor monitoring", () => {expect(daemon.canConnectToSupervisor).not.toHaveBeenCalled();});+ it("keeps the supervisor monitor armed after a replacement binds but exits before claiming", async () => {+ vi.useFakeTimers();+ // The supervisor socket is dead, comes up with the replacement launch,+ // then dies again before the replacement ever claims the worker.+ const probeResults = [false, true, false, false];+ let probeCount = 0;+ const daemon = createHarness(async () => {+ const result = probeResults[Math.min(probeCount, probeResults.length - 1)];+ probeCount += 1;+ return result ?? false;+ });+ // Drive the fake clock until the expected number of probes have run;+ // one advance alone does not flush the availability check chain.+ const advanceUntilProbes = async (expected: number) => {+ for (let step = 0; probeCount < expected && step < 200; step++) {+ await vi.advanceTimersByTimeAsync(100);+ }+ expect(probeCount).toBe(expected);+ };++ daemon.scheduleSupervisorAvailabilityCheck("/tmp/supervisor.sock", 1500);+ await advanceUntilProbes(2);+ expect(daemon.launchReplacementSupervisor).toHaveBeenCalledOnce();+ // The replacement binding mid-launch restarts the orphan window...+ expect(daemon.supervisorAbsentSince).toBeUndefined();+ // ...but a bind is not an authenticated claim: the monitor must stay armed+ // instead of orphaning the worker if the replacement exits unclaimed.+ expect(daemon.supervisorMonitorTimer).toBeDefined();++ await advanceUntilProbes(4);+ expect(daemon.launchReplacementSupervisor).toHaveBeenCalledTimes(2);+ expect(daemon.canConnectToSupervisor).toHaveBeenCalledTimes(4);+ expect(daemon.supervisorMonitorTimer).toBeDefined();+ });+it("retries when shutdown admission lookup fails", async () => {vi.useFakeTimers();let resolveProbe: () => void = () => undefined;diff --git a/packages/coding-agent/test/daemon-supervisor-process.test.ts b/packages/coding-agent/test/daemon-supervisor-process.test.tsindex 5ddedfc445..5423bed686 100644--- a/packages/coding-agent/test/daemon-supervisor-process.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-process.test.ts@@ -1,6 +1,15 @@import { type ChildProcess, spawn } from "node:child_process";import { createHash, randomUUID } from "node:crypto";-import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import {+ chmodSync,+ existsSync,+ mkdirSync,+ mkdtempSync,+ readdirSync,+ readFileSync,+ rmSync,+ writeFileSync,+} from "node:fs";import { createServer } from "node:net";import { tmpdir } from "node:os";import { dirname, join, resolve } from "node:path";@@ -1724,4 +1733,95 @@ describe("daemon supervisor resident workers", () => {await waitForProcessGone(summary.workerPid);workerPids.delete(summary.workerPid);});++ it("exits an orphaned session worker when no replacement supervisor can come up", { timeout: 120_000 }, async () => {+ const root = tempDir();+ const agentDir = join(root, "agent");+ const projectDir = join(root, "project");+ const registryDir = join(root, "supervisor-registry");+ // The socket path must stay short: macOS rejects listen() with EINVAL+ // past the sun_path limit, and the per-test temp dir nests too deep.+ const socketDir = mkdtempSync(join(tmpdir(), "prime-orphan-gc-"));+ tempDirs.push(socketDir);+ const sessionDir = join(agentDir, "sessions");+ const socketPath = join(socketDir, "supervisor.sock");+ mkdirSync(projectDir, { recursive: true });++ const manager = SessionManager.create(projectDir, sessionDir);+ manager.appendMessage({ role: "user", content: "orphan gc fixture", timestamp: 1 });+ manager.flushNow();+ const sessionFile = manager.getSessionFile();+ if (!sessionFile) throw new Error("Missing fixture session path");++ const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, [], {+ PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_REGISTRY_DIR: registryDir,+ PRIME_AGENT_INTERNAL_WORKER_SUPERVISOR_LOST_EXIT_MS: "3000",+ });+ // Cold tsx startup can exceed the shared helper's 15s readiness deadline.+ let client: DaemonClient | undefined;+ const readyDeadline = Date.now() + 60_000;+ while (client === undefined && Date.now() < readyDeadline) {+ if (supervisor.exitCode !== null || supervisor.signalCode !== null) {+ const diagnostics = childDiagnostics.get(supervisor);+ throw new Error(+ `Supervisor exited before becoming ready\nstdout:\n${diagnostics?.stdout ?? ""}\nstderr:\n${diagnostics?.stderr ?? ""}`,+ );+ }+ const candidate = new DaemonClient(socketPath);+ try {+ await candidate.connect(250);+ await candidate.waitForHello(1000);+ client = candidate;+ } catch {+ candidate.close();+ await new Promise((resolveDelay) => setTimeout(resolveDelay, 250));+ }+ }+ if (!client) throw new Error("Supervisor did not become ready in time");+ const created = await client.request({+ type: "create",+ sessionPath: sessionFile,+ config: { cwd: projectDir, agentDir, sessionDir, noTools: true, noExtensions: true },+ });+ if (!created.success) throw new Error(created.error);+ const summary = requireSummary(created.data);+ if (!summary.workerPid) throw new Error("Session worker did not spawn");+ workerPids.add(summary.workerPid);++ // SIGKILL the registry-recorded supervisor pid: the spawned child can be+ // a tsx wrapper, and killing the wrapper alone leaves the supervisor+ // listening on its socket, so the worker would never orphan.+ const ownerDirectories = readdirSync(registryDir).filter((name) => name.endsWith(".owner"));+ expect(ownerDirectories).toHaveLength(1);+ const owner = JSON.parse(readFileSync(join(registryDir, ownerDirectories[0], "owner.json"), "utf8")) as {+ pid?: number;+ };+ if (!owner.pid) throw new Error("Supervisor owner record is missing its pid");+ const supervisorPid = owner.pid;+ const workerPid = summary.workerPid;+ client.close();++ // A read-only socket directory makes every replacement-launch attempt+ // fail (lock/bind EACCES): the real-world orphan precondition where a+ // worker cannot bring its supervisor back.+ chmodSync(socketDir, 0o555);+ try {+ process.kill(supervisorPid, "SIGKILL");+ await waitForCondition(+ () => {+ try {+ process.kill(workerPid, 0);+ return false;+ } catch (error) {+ return (error as NodeJS.ErrnoException).code === "ESRCH";+ }+ },+ "Orphaned session worker did not exit after the supervisor-lost window",+ 60_000,+ );+ } finally {+ chmodSync(socketDir, 0o755);+ }+ workerPids.delete(workerPid);+ });});