Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(coding-agent): report accurate worker lifecycle and hide stopping workers (#850)

Sebastian Müller2857e23469d0
diff --git a/packages/coding-agent/src/core/session-action-store.ts b/packages/coding-agent/src/core/session-action-store.ts
index 184841bdf3..312479986f 100644
--- a/packages/coding-agent/src/core/session-action-store.ts
+++ b/packages/coding-agent/src/core/session-action-store.ts
@@ -362,7 +362,7 @@ export interface SessionPassivationSnapshot extends SessionEvictionSnapshot {
}
export interface WorkerEvictionSnapshot {
- lifecycle: "starting" | "ready" | "recovering" | "failed";
+ lifecycle: "starting" | "ready" | "recovering" | "stopping" | "failed";
isConnected: boolean;
isStopping: boolean;
hasOwnerClient: boolean;
diff --git a/packages/coding-agent/src/modes/agents-view/agents-view-state.ts b/packages/coding-agent/src/modes/agents-view/agents-view-state.ts
index b7a70a2195..56501624b4 100644
--- a/packages/coding-agent/src/modes/agents-view/agents-view-state.ts
+++ b/packages/coding-agent/src/modes/agents-view/agents-view-state.ts
@@ -108,7 +108,7 @@ export function shouldShowAgentsViewSession(summary: SessionSummary, manuallyIna
if (manuallyInactive) {
return false;
}
- return summary.lifecycle === "live";
+ return summary.lifecycle === "live" && (summary.workerState === undefined || summary.workerState === "ready");
}
export function sectionTitle(section: AgentsViewSection): string {
diff --git a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
index 5ce29bf7ed..b26eb1bdc2 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-protocol.ts
@@ -59,8 +59,9 @@ export const DAEMON_COMMAND_ENVELOPE_MIN_PROTOCOL_VERSION = 7;
// Revision 13 narrows agent-origin reach and roster wire shapes to the nuclear family.
// Revision 14 carries the client's monotonic telemetry opt-out on attach and reattach.
// Revision 15 adds the mutate_queued_message command and queue_message_mutation capability.
-export const DAEMON_SCHEMA_REVISION = 15;
-export const DAEMON_SCHEMA_ID = "protocol-7-schema-15-1bcb9e7f1a49";
+// Revision 16 adds the "stopping" workerState and stops reporting disconnected workers as "ready".
+export const DAEMON_SCHEMA_REVISION = 16;
+export const DAEMON_SCHEMA_ID = "protocol-7-schema-16-1bcb9e7f1a49";
export type DaemonProtocolName = typeof DAEMON_PROTOCOL_NAME;
export type DaemonProtocolVersion = number;
diff --git a/packages/coding-agent/src/modes/daemon/daemon-session-list.ts b/packages/coding-agent/src/modes/daemon/daemon-session-list.ts
index edc1ca652a..73dc116ba5 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-session-list.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-session-list.ts
@@ -78,7 +78,7 @@ export interface SessionSummary {
/** Completion verdict for an idle session; absent while working or unjudged. */
taskState?: AgentTaskState;
/** Resident session-host process state, populated by the global supervisor. */
- workerState?: "starting" | "ready" | "recovering" | "failed";
+ workerState?: "starting" | "ready" | "recovering" | "stopping" | "failed";
/** Diagnostic process identity; clients must not use this as a stable session identifier. */
workerPid?: number;
}
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
index b308219ba2..c7bb17d1c7 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
@@ -113,6 +113,7 @@ import {
type DaemonCreateCommand,
type DaemonWorkerDescriptor,
type DaemonWorkerFrameHeader,
+ type DaemonWorkerLifecycle,
SESSION_LEASE_OWNER_ID_ENV,
SESSION_LEASES_ENABLED_ENV,
} from "./daemon-worker-protocol.js";
@@ -766,7 +767,7 @@ export class DaemonSupervisor {
return {
lifecycle: worker.descriptor.lifecycle,
isConnected: worker.client !== undefined,
- isStopping: worker.intentionalStop || worker.descriptor.stopRequestedAt !== undefined,
+ isStopping: this.isWorkerStopping(worker),
hasOwnerClient: worker.descriptor.ownerClientId !== undefined,
isPreparingUpdateRestart:
this.updateRestartPhase !== undefined || worker.updateRestartPrepareClient !== undefined,
@@ -1581,7 +1582,7 @@ export class DaemonSupervisor {
const match = await this.findWorkerForClient(client, command.activeSessionId);
return this.forwardToWorker(match.worker, command);
}
- const first = [...this.workers.values()].find((worker) => this.isVisibleWorker(worker) && worker.client);
+ const first = [...this.workers.values()].find((worker) => this.isLiveWorker(worker) && worker.client);
if (!first) {
return success(command.id, command.type, { paused: false, limits: {} });
}
@@ -1595,7 +1596,7 @@ export class DaemonSupervisor {
}
const responses = await Promise.all(
[...this.workers.values()]
- .filter((worker) => this.isVisibleWorker(worker) && worker.client)
+ .filter((worker) => this.isLiveWorker(worker) && worker.client)
.map((worker) => this.forwardToWorker(worker, command)),
);
const failed = responses.find((response) => !response.success);
@@ -1610,8 +1611,7 @@ export class DaemonSupervisor {
const responses = await Promise.all(
[...this.workers.values()]
.filter(
- (worker) =>
- this.isVisibleWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",
+ (worker) => this.isLiveWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",
)
.map((worker) =>
this.forwardToWorker(worker, command, 5000).catch((error: unknown) =>
@@ -1635,7 +1635,7 @@ export class DaemonSupervisor {
const match = await this.findWorkerForClient(client, command.activeSessionId);
return this.forwardToWorker(match.worker, command);
}
- const workers = [...this.workers.values()].filter((worker) => this.isVisibleWorker(worker));
+ const workers = [...this.workers.values()].filter((worker) => this.isLiveWorker(worker));
const heartbeats = new Map<string, AgentConnectionHeartbeat>();
const snapshots: Array<{ heartbeats?: AgentConnectionHeartbeat[]; response?: DaemonResponse }> =
await Promise.all(
@@ -1719,8 +1719,7 @@ export class DaemonSupervisor {
const listed = await Promise.all(
[...this.workers.values()]
.filter(
- (worker) =>
- this.isVisibleWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",
+ (worker) => this.isLiveWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",
)
.map(async (worker) => ({
worker,
@@ -1837,10 +1836,8 @@ export class DaemonSupervisor {
if ((source.summary.activeSessionId ?? source.summary.id) === targetActiveSessionId) {
throw new Error("Agent messaging cannot target the sending session");
}
- if (!target.worker.client) {
- throw new Error("Target session worker is not connected");
- }
- const response = await target.worker.client.requestWorker(
+ const targetClient = this.requireAvailableWorkerClient(target.worker);
+ const response = await targetClient.requestWorker(
{
type: "worker_deliver_message",
targetActiveSessionId,
@@ -1924,10 +1921,14 @@ export class DaemonSupervisor {
command: Extract<DaemonCommand, { type: "list" }>,
): Promise<DaemonResponse> {
await Promise.all(
- [...this.workers.values()].map((worker) => this.refreshWorkerSummaries(worker).catch(() => undefined)),
+ [...this.workers.values()]
+ .filter((worker) => !this.isWorkerStopping(worker))
+ .map((worker) => this.refreshWorkerSummaries(worker).catch(() => undefined)),
);
await this.syncAgentPeers().catch((error) => this.log(`Could not synchronize agent peers: ${String(error)}`));
const clientOwnedWorkers = [...this.workers.values()].filter((worker) => !this.isVisibleWorker(worker));
+ // Stopping workers stay listed (with an honest workerState) because this
+ // list also feeds busy-daemon safety checks in daemon-launch.
const active = [...this.workers.values()]
.filter(
(worker) =>
@@ -2906,6 +2907,9 @@ export class DaemonSupervisor {
}
private async refreshWorkerSummaries(worker: ResidentWorker, recovery = false): Promise<void> {
+ if (this.isWorkerStopping(worker)) {
+ throw new Error("Session worker is stopping");
+ }
if (!worker.client) {
throw new Error("Session worker is not connected");
}
@@ -3062,9 +3066,7 @@ export class DaemonSupervisor {
.then(async () => {
const readyWorkers = [...this.workers.values()].filter(
(worker): worker is ResidentWorker & { client: DaemonWorkerClient } =>
- this.isVisibleWorker(worker) &&
- worker.descriptor.lifecycle === "ready" &&
- worker.client !== undefined,
+ this.isLiveWorker(worker) && worker.descriptor.lifecycle === "ready" && worker.client !== undefined,
);
await Promise.all(
readyWorkers.map(async (worker) => {
@@ -3091,6 +3093,41 @@ export class DaemonSupervisor {
return worker.descriptor.ownerClientId === undefined;
}
+ /** A worker with a durable or in-memory stop intent is stopping, never live. */
+ private isWorkerStopping(worker: ResidentWorker): boolean {
+ return worker.intentionalStop || worker.descriptor.stopRequestedAt !== undefined;
+ }
+
+ /** Live workers are visible to all clients and not stopping. */
+ private isLiveWorker(worker: ResidentWorker): boolean {
+ return this.isVisibleWorker(worker) && !this.isWorkerStopping(worker);
+ }
+
+ /**
+ * The lifecycle reported to clients. A stop intent always wins, and a worker
+ * whose process connection is gone is never reported as "ready".
+ */
+ private effectiveWorkerState(worker: ResidentWorker): DaemonWorkerLifecycle {
+ if (this.isWorkerStopping(worker)) {
+ return "stopping";
+ }
+ if (worker.descriptor.lifecycle === "ready" && worker.client === undefined) {
+ return "recovering";
+ }
+ return worker.descriptor.lifecycle;
+ }
+
+ private requireAvailableWorkerClient(worker: ResidentWorker, allowStopping = false): DaemonWorkerClient {
+ if (
+ !worker.client ||
+ worker.descriptor.lifecycle !== "ready" ||
+ (!allowStopping && this.isWorkerStopping(worker))
+ ) {
+ throw new Error(`Session worker is ${this.effectiveWorkerState(worker)}`);
+ }
+ return worker.client;
+ }
+
private familyCatalogEntry(summary: SessionSummary): AgentFamilyCatalogEntry {
const depth = summary.rlmDepth ?? (summary.parentSessionPath ? 1 : 0);
return {
@@ -3135,7 +3172,7 @@ export class DaemonSupervisor {
...summary,
attachedClients: [...this.clients].filter((client) => client.attachedActiveSessionIds.has(activeSessionId))
.length,
- workerState: worker.descriptor.lifecycle,
+ workerState: this.effectiveWorkerState(worker),
workerPid: worker.descriptor.pid,
};
}
@@ -3240,10 +3277,8 @@ export class DaemonSupervisor {
command: DaemonCommand,
timeoutMs = WORKER_REQUEST_TIMEOUT_MS,
): Promise<DaemonResponse> {
- if (!worker.client || worker.descriptor.lifecycle !== "ready") {
- throw new Error(`Session worker is ${worker.descriptor.lifecycle}`);
- }
- const response = await worker.client.request(withoutCommandId(command), timeoutMs);
+ const client = this.requireAvailableWorkerClient(worker, command.type === "kill");
+ const response = await client.request(withoutCommandId(command), timeoutMs);
if (command.type === "get_state" && response.success && isSessionSummary(response.data)) {
return { ...response, id: command.id, data: this.publicSummary(worker, response.data) };
}
@@ -3285,6 +3320,7 @@ export class DaemonSupervisor {
}
const match = await this.findWorkerForClient(client, command.activeSessionId);
this.assertTelemetryAttachAllowed(match.worker, command.telemetryDisabled);
+ this.requireAvailableWorkerClient(match.worker);
const activeSessionId = match.summary.activeSessionId ?? match.summary.id;
const duplicateValidation = this.currentSnapshotGeneration(match.worker, activeSessionId)?.validation;
if (duplicateValidation) {
@@ -3314,10 +3350,8 @@ export class DaemonSupervisor {
match.worker.transcriptCaches.get(activeSessionId)?.snapshotId ??
match.worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id;
loading = (async () => {
- if (!match.worker.client) {
- throw new Error("Session worker is not connected");
- }
- const response = await match.worker.client.request({
+ const workerClient = this.requireAvailableWorkerClient(match.worker);
+ const response = await workerClient.request({
type: "attach",
activeSessionId,
capabilities: client.capabilities.has("chunked_snapshot")
@@ -3374,6 +3408,7 @@ export class DaemonSupervisor {
}
}
}
+ this.requireAvailableWorkerClient(match.worker);
const wasAttached = client.attachedActiveSessionIds.has(activeSessionId);
let transcript: SnapshotTranscriptCache | undefined;
if (client.capabilities.has("chunked_snapshot")) {
@@ -4333,11 +4368,12 @@ export class DaemonSupervisor {
private async prepareUpdateRestartFenced(deadline: number): Promise<DaemonUpdateRestartManifest> {
const residents = [...this.workers.values()];
const unavailable = residents.find(
- (worker) => worker.descriptor.lifecycle !== "ready" || worker.client === undefined,
+ (worker) =>
+ this.isWorkerStopping(worker) || worker.descriptor.lifecycle !== "ready" || worker.client === undefined,
);
if (unavailable) {
throw new Error(
- `Cannot prepare update restart while resident worker ${unavailable.descriptor.workerId} is ${unavailable.descriptor.lifecycle}${unavailable.client ? "" : " and disconnected"}`,
+ `Cannot prepare update restart while resident worker ${unavailable.descriptor.workerId} is ${this.effectiveWorkerState(unavailable)}${unavailable.client ? "" : " and disconnected"}`,
);
}
const workers = residents as Array<ResidentWorker & { client: DaemonWorkerClient }>;
diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
index c7f1495427..51dea75bd2 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
@@ -17,7 +17,7 @@ export const DAEMON_WORKER_SUPERVISOR_SOCKET_ENV = "PRIME_AGENT_INTERNAL_DAEMON_
export const DAEMON_WORKER_RECOVERY_JOURNAL_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL";
export const DAEMON_WORKER_STARTUP_GATE_FD_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD";
export const DAEMON_WORKER_STARTUP_GATE_COMMIT = "start\n";
-export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "failed";
+export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "stopping" | "failed";
export type DaemonWorkerFrameHeader =
| {
diff --git a/packages/coding-agent/test/daemon-protocol.test.ts b/packages/coding-agent/test/daemon-protocol.test.ts
index 9781b474c8..d7eb8acf4a 100644
--- a/packages/coding-agent/test/daemon-protocol.test.ts
+++ b/packages/coding-agent/test/daemon-protocol.test.ts
@@ -130,6 +130,14 @@ describe("daemon protocol helpers", () => {
expect(DAEMON_DEFAULT_SERVER_CAPABILITIES).toContain("prompt_admission_cancellation");
});
+ it("gates honest worker-state reporting at its introducing schema revision", () => {
+ // Revision 16 adds the "stopping" workerState and stops reporting
+ // disconnected workers as "ready". The field is optional and old clients
+ // ignore unknown values, so no capability gate is needed; the revision
+ // lets version probes distinguish daemons with the old semantics.
+ expect(DAEMON_SCHEMA_REVISION).toBeGreaterThanOrEqual(16);
+ });
+
it("keeps refine failure events backward-compatible on the existing session event channel", () => {
const event: DaemonOutbound = {
type: "session_event",
diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
index 036a05f4c4..112d75487a 100644
--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
@@ -1601,6 +1601,104 @@ describe("daemon worker supervisor monitoring", () => {
expect(worker.descriptor.consecutiveFailures).toBe(0);
});
+ it("reports a stop-tombstoned worker as stopping, not ready", () => {
+ const worker = {
+ descriptor: {
+ workerId: "worker-tombstoned",
+ pid: process.pid,
+ lifecycle: "ready" as const,
+ stopRequestedAt: new Date().toISOString(),
+ },
+ client: {},
+ intentionalStop: false,
+ };
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {
+ effectiveWorkerState(target: object): string;
+ };
+
+ expect(supervisor.effectiveWorkerState(worker)).toBe("stopping");
+ });
+
+ it("never reports a disconnected worker as ready", () => {
+ const worker = {
+ descriptor: { workerId: "worker-disconnected", pid: process.pid, lifecycle: "ready" as const },
+ client: undefined,
+ intentionalStop: false,
+ };
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {
+ effectiveWorkerState(target: object): string;
+ };
+
+ expect(supervisor.effectiveWorkerState(worker)).toBe("recovering");
+ });
+
+ it("reports a connected ready worker as ready", () => {
+ const worker = {
+ descriptor: { workerId: "worker-live", pid: process.pid, lifecycle: "ready" as const },
+ client: {},
+ intentionalStop: false,
+ };
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {
+ effectiveWorkerState(target: object): string;
+ };
+
+ expect(supervisor.effectiveWorkerState(worker)).toBe("ready");
+ });
+
+ it("keeps stopping workers listed with an honest state for busy-daemon checks", async () => {
+ const makeWorker = (workerId: string, stopRequestedAt?: string) => ({
+ descriptor: {
+ workerId,
+ pid: process.pid,
+ rootActiveSessionId: `${workerId}-active`,
+ lifecycle: "ready" as const,
+ ...(stopRequestedAt ? { stopRequestedAt } : {}),
+ },
+ client: {},
+ intentionalStop: false,
+ summaries: new Map([
+ [
+ `${workerId}-active`,
+ {
+ id: `${workerId}-active`,
+ activeSessionId: `${workerId}-active`,
+ sessionId: `${workerId}-session`,
+ cwd: "/tmp",
+ } as unknown as SessionSummary,
+ ],
+ ]),
+ });
+ const liveWorker = makeWorker("worker-live");
+ const stoppingWorker = makeWorker("worker-stopping", new Date().toISOString());
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ workers: new Map([
+ [liveWorker.descriptor.workerId, liveWorker],
+ [stoppingWorker.descriptor.workerId, stoppingWorker],
+ ]),
+ clients: new Set(),
+ refreshWorkerSummaries: vi.fn(async () => {}),
+ syncAgentPeers: vi.fn(async () => {}),
+ log: vi.fn(),
+ }) as {
+ handleList(
+ client: object,
+ command: { id: string; type: "list" },
+ ): Promise<{
+ success: boolean;
+ data?: { sessions: Array<{ activeSessionId?: string; id: string; workerState?: string }> };
+ }>;
+ };
+
+ const response = await supervisor.handleList({}, { id: "list-1", type: "list" });
+
+ expect(response.success).toBe(true);
+ const sessions = response.data?.sessions ?? [];
+ expect(sessions.map((session) => [session.activeSessionId ?? session.id, session.workerState]).sort()).toEqual([
+ ["worker-live-active", "ready"],
+ ["worker-stopping-active", "stopping"],
+ ]);
+ });
+
it("ignores malformed persisted worker descriptors", () => {
const descriptorDir = mkdtempSync(join(tmpdir(), "prime-supervisor-descriptor-test-"));
try {
@@ -1673,6 +1771,7 @@ describe("daemon worker supervisor monitoring", () => {
} as unknown as DaemonAttachResult;
const worker = {
descriptor: { workerId: "worker-1", lifecycle: "ready", pid: 1234 },
+ client: {},
summaries: new Map([[activeSessionId, summary]]),
snapshotCache: new Map([[activeSessionId, result]]),
snapshotTransferFrames: new Map(),
diff --git a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
index 195274e758..b982eea2b8 100644
--- a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
@@ -339,7 +339,6 @@ describe("ENG-4602 snapshot transfer containment", () => {
descriptorDir: "/tmp/eng-4602-supervisor-state",
});
const { close, worker } = workerHarness();
- worker.intentionalStop = true;
const client = socketClient("public", new PassThrough());
const streamSnapshot = vi.fn(async () => {});
const internals = supervisor as unknown as {
@@ -424,7 +423,6 @@ describe("ENG-4602 snapshot transfer containment", () => {
descriptorDir: "/tmp/eng-4602-supervisor-gate-state",
});
const { close, request, worker } = workerHarness();
- worker.intentionalStop = true;
const client = socketClient("catchup", new PassThrough());
const streamSnapshot = vi.fn(async () => {});
const internals = supervisor as unknown as {
diff --git a/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts
index a988a6d741..19258b6f41 100644
--- a/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts
@@ -133,6 +133,11 @@ function workerHarness(result: DaemonAttachResult, transcript: SnapshotTranscrip
lifecycle: "ready",
pid: 4677,
},
+ client: {
+ request: vi.fn(async () => {
+ throw new Error("unexpected snapshot reload");
+ }),
+ },
summaries: new Map([[activeSessionId, result.snapshot.summary]]),
snapshotCache: new Map([[activeSessionId, result]]),
transcriptCaches: new Map([[activeSessionId, transcript]]),