A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fix(coding-agent): report accurate worker lifecycle and hide stopping workers (#850)
Sebastian Müller2857e23469d0
diff --git a/packages/coding-agent/src/core/session-action-store.ts b/packages/coding-agent/src/core/session-action-store.tsindex 184841bdf3..312479986f 100644--- a/packages/coding-agent/src/core/session-action-store.ts+++ b/packages/coding-agent/src/core/session-action-store.ts@@ -362,7 +362,7 @@ export interface SessionPassivationSnapshot extends SessionEvictionSnapshot {}export interface WorkerEvictionSnapshot {- lifecycle: "starting" | "ready" | "recovering" | "failed";+ lifecycle: "starting" | "ready" | "recovering" | "stopping" | "failed";isConnected: boolean;isStopping: boolean;hasOwnerClient: boolean;diff --git a/packages/coding-agent/src/modes/agents-view/agents-view-state.ts b/packages/coding-agent/src/modes/agents-view/agents-view-state.tsindex b7a70a2195..56501624b4 100644--- a/packages/coding-agent/src/modes/agents-view/agents-view-state.ts+++ b/packages/coding-agent/src/modes/agents-view/agents-view-state.ts@@ -108,7 +108,7 @@ export function shouldShowAgentsViewSession(summary: SessionSummary, manuallyInaif (manuallyInactive) {return false;}- return summary.lifecycle === "live";+ return summary.lifecycle === "live" && (summary.workerState === undefined || summary.workerState === "ready");}export function sectionTitle(section: AgentsViewSection): string {diff --git a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-protocol.tsindex 5ce29bf7ed..b26eb1bdc2 100644--- a/packages/coding-agent/src/modes/daemon/daemon-protocol.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-protocol.ts@@ -59,8 +59,9 @@ export const DAEMON_COMMAND_ENVELOPE_MIN_PROTOCOL_VERSION = 7;// Revision 13 narrows agent-origin reach and roster wire shapes to the nuclear family.// Revision 14 carries the client's monotonic telemetry opt-out on attach and reattach.// Revision 15 adds the mutate_queued_message command and queue_message_mutation capability.-export const DAEMON_SCHEMA_REVISION = 15;-export const DAEMON_SCHEMA_ID = "protocol-7-schema-15-1bcb9e7f1a49";+// Revision 16 adds the "stopping" workerState and stops reporting disconnected workers as "ready".+export const DAEMON_SCHEMA_REVISION = 16;+export const DAEMON_SCHEMA_ID = "protocol-7-schema-16-1bcb9e7f1a49";export type DaemonProtocolName = typeof DAEMON_PROTOCOL_NAME;export type DaemonProtocolVersion = number;diff --git a/packages/coding-agent/src/modes/daemon/daemon-session-list.ts b/packages/coding-agent/src/modes/daemon/daemon-session-list.tsindex edc1ca652a..73dc116ba5 100644--- a/packages/coding-agent/src/modes/daemon/daemon-session-list.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-session-list.ts@@ -78,7 +78,7 @@ export interface SessionSummary {/** Completion verdict for an idle session; absent while working or unjudged. */taskState?: AgentTaskState;/** Resident session-host process state, populated by the global supervisor. */- workerState?: "starting" | "ready" | "recovering" | "failed";+ workerState?: "starting" | "ready" | "recovering" | "stopping" | "failed";/** Diagnostic process identity; clients must not use this as a stable session identifier. */workerPid?: number;}diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.tsindex b308219ba2..c7bb17d1c7 100644--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts@@ -113,6 +113,7 @@ import {type DaemonCreateCommand,type DaemonWorkerDescriptor,type DaemonWorkerFrameHeader,+ type DaemonWorkerLifecycle,SESSION_LEASE_OWNER_ID_ENV,SESSION_LEASES_ENABLED_ENV,} from "./daemon-worker-protocol.js";@@ -766,7 +767,7 @@ export class DaemonSupervisor {return {lifecycle: worker.descriptor.lifecycle,isConnected: worker.client !== undefined,- isStopping: worker.intentionalStop || worker.descriptor.stopRequestedAt !== undefined,+ isStopping: this.isWorkerStopping(worker),hasOwnerClient: worker.descriptor.ownerClientId !== undefined,isPreparingUpdateRestart:this.updateRestartPhase !== undefined || worker.updateRestartPrepareClient !== undefined,@@ -1581,7 +1582,7 @@ export class DaemonSupervisor {const match = await this.findWorkerForClient(client, command.activeSessionId);return this.forwardToWorker(match.worker, command);}- const first = [...this.workers.values()].find((worker) => this.isVisibleWorker(worker) && worker.client);+ const first = [...this.workers.values()].find((worker) => this.isLiveWorker(worker) && worker.client);if (!first) {return success(command.id, command.type, { paused: false, limits: {} });}@@ -1595,7 +1596,7 @@ export class DaemonSupervisor {}const responses = await Promise.all([...this.workers.values()]- .filter((worker) => this.isVisibleWorker(worker) && worker.client)+ .filter((worker) => this.isLiveWorker(worker) && worker.client).map((worker) => this.forwardToWorker(worker, command)),);const failed = responses.find((response) => !response.success);@@ -1610,8 +1611,7 @@ export class DaemonSupervisor {const responses = await Promise.all([...this.workers.values()].filter(- (worker) =>- this.isVisibleWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",+ (worker) => this.isLiveWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",).map((worker) =>this.forwardToWorker(worker, command, 5000).catch((error: unknown) =>@@ -1635,7 +1635,7 @@ export class DaemonSupervisor {const match = await this.findWorkerForClient(client, command.activeSessionId);return this.forwardToWorker(match.worker, command);}- const workers = [...this.workers.values()].filter((worker) => this.isVisibleWorker(worker));+ const workers = [...this.workers.values()].filter((worker) => this.isLiveWorker(worker));const heartbeats = new Map<string, AgentConnectionHeartbeat>();const snapshots: Array<{ heartbeats?: AgentConnectionHeartbeat[]; response?: DaemonResponse }> =await Promise.all(@@ -1719,8 +1719,7 @@ export class DaemonSupervisor {const listed = await Promise.all([...this.workers.values()].filter(- (worker) =>- this.isVisibleWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",+ (worker) => this.isLiveWorker(worker) && worker.client && worker.descriptor.lifecycle === "ready",).map(async (worker) => ({worker,@@ -1837,10 +1836,8 @@ export class DaemonSupervisor {if ((source.summary.activeSessionId ?? source.summary.id) === targetActiveSessionId) {throw new Error("Agent messaging cannot target the sending session");}- if (!target.worker.client) {- throw new Error("Target session worker is not connected");- }- const response = await target.worker.client.requestWorker(+ const targetClient = this.requireAvailableWorkerClient(target.worker);+ const response = await targetClient.requestWorker({type: "worker_deliver_message",targetActiveSessionId,@@ -1924,10 +1921,14 @@ export class DaemonSupervisor {command: Extract<DaemonCommand, { type: "list" }>,): Promise<DaemonResponse> {await Promise.all(- [...this.workers.values()].map((worker) => this.refreshWorkerSummaries(worker).catch(() => undefined)),+ [...this.workers.values()]+ .filter((worker) => !this.isWorkerStopping(worker))+ .map((worker) => this.refreshWorkerSummaries(worker).catch(() => undefined)),);await this.syncAgentPeers().catch((error) => this.log(`Could not synchronize agent peers: ${String(error)}`));const clientOwnedWorkers = [...this.workers.values()].filter((worker) => !this.isVisibleWorker(worker));+ // Stopping workers stay listed (with an honest workerState) because this+ // list also feeds busy-daemon safety checks in daemon-launch.const active = [...this.workers.values()].filter((worker) =>@@ -2906,6 +2907,9 @@ export class DaemonSupervisor {}private async refreshWorkerSummaries(worker: ResidentWorker, recovery = false): Promise<void> {+ if (this.isWorkerStopping(worker)) {+ throw new Error("Session worker is stopping");+ }if (!worker.client) {throw new Error("Session worker is not connected");}@@ -3062,9 +3066,7 @@ export class DaemonSupervisor {.then(async () => {const readyWorkers = [...this.workers.values()].filter((worker): worker is ResidentWorker & { client: DaemonWorkerClient } =>- this.isVisibleWorker(worker) &&- worker.descriptor.lifecycle === "ready" &&- worker.client !== undefined,+ this.isLiveWorker(worker) && worker.descriptor.lifecycle === "ready" && worker.client !== undefined,);await Promise.all(readyWorkers.map(async (worker) => {@@ -3091,6 +3093,41 @@ export class DaemonSupervisor {return worker.descriptor.ownerClientId === undefined;}+ /** A worker with a durable or in-memory stop intent is stopping, never live. */+ private isWorkerStopping(worker: ResidentWorker): boolean {+ return worker.intentionalStop || worker.descriptor.stopRequestedAt !== undefined;+ }++ /** Live workers are visible to all clients and not stopping. */+ private isLiveWorker(worker: ResidentWorker): boolean {+ return this.isVisibleWorker(worker) && !this.isWorkerStopping(worker);+ }++ /**+ * The lifecycle reported to clients. A stop intent always wins, and a worker+ * whose process connection is gone is never reported as "ready".+ */+ private effectiveWorkerState(worker: ResidentWorker): DaemonWorkerLifecycle {+ if (this.isWorkerStopping(worker)) {+ return "stopping";+ }+ if (worker.descriptor.lifecycle === "ready" && worker.client === undefined) {+ return "recovering";+ }+ return worker.descriptor.lifecycle;+ }++ private requireAvailableWorkerClient(worker: ResidentWorker, allowStopping = false): DaemonWorkerClient {+ if (+ !worker.client ||+ worker.descriptor.lifecycle !== "ready" ||+ (!allowStopping && this.isWorkerStopping(worker))+ ) {+ throw new Error(`Session worker is ${this.effectiveWorkerState(worker)}`);+ }+ return worker.client;+ }+private familyCatalogEntry(summary: SessionSummary): AgentFamilyCatalogEntry {const depth = summary.rlmDepth ?? (summary.parentSessionPath ? 1 : 0);return {@@ -3135,7 +3172,7 @@ export class DaemonSupervisor {...summary,attachedClients: [...this.clients].filter((client) => client.attachedActiveSessionIds.has(activeSessionId)).length,- workerState: worker.descriptor.lifecycle,+ workerState: this.effectiveWorkerState(worker),workerPid: worker.descriptor.pid,};}@@ -3240,10 +3277,8 @@ export class DaemonSupervisor {command: DaemonCommand,timeoutMs = WORKER_REQUEST_TIMEOUT_MS,): Promise<DaemonResponse> {- if (!worker.client || worker.descriptor.lifecycle !== "ready") {- throw new Error(`Session worker is ${worker.descriptor.lifecycle}`);- }- const response = await worker.client.request(withoutCommandId(command), timeoutMs);+ const client = this.requireAvailableWorkerClient(worker, command.type === "kill");+ const response = await client.request(withoutCommandId(command), timeoutMs);if (command.type === "get_state" && response.success && isSessionSummary(response.data)) {return { ...response, id: command.id, data: this.publicSummary(worker, response.data) };}@@ -3285,6 +3320,7 @@ export class DaemonSupervisor {}const match = await this.findWorkerForClient(client, command.activeSessionId);this.assertTelemetryAttachAllowed(match.worker, command.telemetryDisabled);+ this.requireAvailableWorkerClient(match.worker);const activeSessionId = match.summary.activeSessionId ?? match.summary.id;const duplicateValidation = this.currentSnapshotGeneration(match.worker, activeSessionId)?.validation;if (duplicateValidation) {@@ -3314,10 +3350,8 @@ export class DaemonSupervisor {match.worker.transcriptCaches.get(activeSessionId)?.snapshotId ??match.worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id;loading = (async () => {- if (!match.worker.client) {- throw new Error("Session worker is not connected");- }- const response = await match.worker.client.request({+ const workerClient = this.requireAvailableWorkerClient(match.worker);+ const response = await workerClient.request({type: "attach",activeSessionId,capabilities: client.capabilities.has("chunked_snapshot")@@ -3374,6 +3408,7 @@ export class DaemonSupervisor {}}}+ this.requireAvailableWorkerClient(match.worker);const wasAttached = client.attachedActiveSessionIds.has(activeSessionId);let transcript: SnapshotTranscriptCache | undefined;if (client.capabilities.has("chunked_snapshot")) {@@ -4333,11 +4368,12 @@ export class DaemonSupervisor {private async prepareUpdateRestartFenced(deadline: number): Promise<DaemonUpdateRestartManifest> {const residents = [...this.workers.values()];const unavailable = residents.find(- (worker) => worker.descriptor.lifecycle !== "ready" || worker.client === undefined,+ (worker) =>+ this.isWorkerStopping(worker) || worker.descriptor.lifecycle !== "ready" || worker.client === undefined,);if (unavailable) {throw new Error(- `Cannot prepare update restart while resident worker ${unavailable.descriptor.workerId} is ${unavailable.descriptor.lifecycle}${unavailable.client ? "" : " and disconnected"}`,+ `Cannot prepare update restart while resident worker ${unavailable.descriptor.workerId} is ${this.effectiveWorkerState(unavailable)}${unavailable.client ? "" : " and disconnected"}`,);}const workers = residents as Array<ResidentWorker & { client: DaemonWorkerClient }>;diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.tsindex c7f1495427..51dea75bd2 100644--- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts@@ -17,7 +17,7 @@ export const DAEMON_WORKER_SUPERVISOR_SOCKET_ENV = "PRIME_AGENT_INTERNAL_DAEMON_export const DAEMON_WORKER_RECOVERY_JOURNAL_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL";export const DAEMON_WORKER_STARTUP_GATE_FD_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD";export const DAEMON_WORKER_STARTUP_GATE_COMMIT = "start\n";-export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "failed";+export type DaemonWorkerLifecycle = "starting" | "ready" | "recovering" | "stopping" | "failed";export type DaemonWorkerFrameHeader =| {diff --git a/packages/coding-agent/test/daemon-protocol.test.ts b/packages/coding-agent/test/daemon-protocol.test.tsindex 9781b474c8..d7eb8acf4a 100644--- a/packages/coding-agent/test/daemon-protocol.test.ts+++ b/packages/coding-agent/test/daemon-protocol.test.ts@@ -130,6 +130,14 @@ describe("daemon protocol helpers", () => {expect(DAEMON_DEFAULT_SERVER_CAPABILITIES).toContain("prompt_admission_cancellation");});+ it("gates honest worker-state reporting at its introducing schema revision", () => {+ // Revision 16 adds the "stopping" workerState and stops reporting+ // disconnected workers as "ready". The field is optional and old clients+ // ignore unknown values, so no capability gate is needed; the revision+ // lets version probes distinguish daemons with the old semantics.+ expect(DAEMON_SCHEMA_REVISION).toBeGreaterThanOrEqual(16);+ });+it("keeps refine failure events backward-compatible on the existing session event channel", () => {const event: DaemonOutbound = {type: "session_event",diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.tsindex 036a05f4c4..112d75487a 100644--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts@@ -1601,6 +1601,104 @@ describe("daemon worker supervisor monitoring", () => {expect(worker.descriptor.consecutiveFailures).toBe(0);});+ it("reports a stop-tombstoned worker as stopping, not ready", () => {+ const worker = {+ descriptor: {+ workerId: "worker-tombstoned",+ pid: process.pid,+ lifecycle: "ready" as const,+ stopRequestedAt: new Date().toISOString(),+ },+ client: {},+ intentionalStop: false,+ };+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {+ effectiveWorkerState(target: object): string;+ };++ expect(supervisor.effectiveWorkerState(worker)).toBe("stopping");+ });++ it("never reports a disconnected worker as ready", () => {+ const worker = {+ descriptor: { workerId: "worker-disconnected", pid: process.pid, lifecycle: "ready" as const },+ client: undefined,+ intentionalStop: false,+ };+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {+ effectiveWorkerState(target: object): string;+ };++ expect(supervisor.effectiveWorkerState(worker)).toBe("recovering");+ });++ it("reports a connected ready worker as ready", () => {+ const worker = {+ descriptor: { workerId: "worker-live", pid: process.pid, lifecycle: "ready" as const },+ client: {},+ intentionalStop: false,+ };+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {}) as {+ effectiveWorkerState(target: object): string;+ };++ expect(supervisor.effectiveWorkerState(worker)).toBe("ready");+ });++ it("keeps stopping workers listed with an honest state for busy-daemon checks", async () => {+ const makeWorker = (workerId: string, stopRequestedAt?: string) => ({+ descriptor: {+ workerId,+ pid: process.pid,+ rootActiveSessionId: `${workerId}-active`,+ lifecycle: "ready" as const,+ ...(stopRequestedAt ? { stopRequestedAt } : {}),+ },+ client: {},+ intentionalStop: false,+ summaries: new Map([+ [+ `${workerId}-active`,+ {+ id: `${workerId}-active`,+ activeSessionId: `${workerId}-active`,+ sessionId: `${workerId}-session`,+ cwd: "/tmp",+ } as unknown as SessionSummary,+ ],+ ]),+ });+ const liveWorker = makeWorker("worker-live");+ const stoppingWorker = makeWorker("worker-stopping", new Date().toISOString());+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ workers: new Map([+ [liveWorker.descriptor.workerId, liveWorker],+ [stoppingWorker.descriptor.workerId, stoppingWorker],+ ]),+ clients: new Set(),+ refreshWorkerSummaries: vi.fn(async () => {}),+ syncAgentPeers: vi.fn(async () => {}),+ log: vi.fn(),+ }) as {+ handleList(+ client: object,+ command: { id: string; type: "list" },+ ): Promise<{+ success: boolean;+ data?: { sessions: Array<{ activeSessionId?: string; id: string; workerState?: string }> };+ }>;+ };++ const response = await supervisor.handleList({}, { id: "list-1", type: "list" });++ expect(response.success).toBe(true);+ const sessions = response.data?.sessions ?? [];+ expect(sessions.map((session) => [session.activeSessionId ?? session.id, session.workerState]).sort()).toEqual([+ ["worker-live-active", "ready"],+ ["worker-stopping-active", "stopping"],+ ]);+ });+it("ignores malformed persisted worker descriptors", () => {const descriptorDir = mkdtempSync(join(tmpdir(), "prime-supervisor-descriptor-test-"));try {@@ -1673,6 +1771,7 @@ describe("daemon worker supervisor monitoring", () => {} as unknown as DaemonAttachResult;const worker = {descriptor: { workerId: "worker-1", lifecycle: "ready", pid: 1234 },+ client: {},summaries: new Map([[activeSessionId, summary]]),snapshotCache: new Map([[activeSessionId, result]]),snapshotTransferFrames: new Map(),diff --git a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.tsindex 195274e758..b982eea2b8 100644--- a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts+++ b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts@@ -339,7 +339,6 @@ describe("ENG-4602 snapshot transfer containment", () => {descriptorDir: "/tmp/eng-4602-supervisor-state",});const { close, worker } = workerHarness();- worker.intentionalStop = true;const client = socketClient("public", new PassThrough());const streamSnapshot = vi.fn(async () => {});const internals = supervisor as unknown as {@@ -424,7 +423,6 @@ describe("ENG-4602 snapshot transfer containment", () => {descriptorDir: "/tmp/eng-4602-supervisor-gate-state",});const { close, request, worker } = workerHarness();- worker.intentionalStop = true;const client = socketClient("catchup", new PassThrough());const streamSnapshot = vi.fn(async () => {});const internals = supervisor as unknown as {diff --git a/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.tsindex a988a6d741..19258b6f41 100644--- a/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts+++ b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts@@ -133,6 +133,11 @@ function workerHarness(result: DaemonAttachResult, transcript: SnapshotTranscriplifecycle: "ready",pid: 4677,},+ client: {+ request: vi.fn(async () => {+ throw new Error("unexpected snapshot reload");+ }),+ },summaries: new Map([[activeSessionId, result.snapshot.summary]]),snapshotCache: new Map([[activeSessionId, result]]),transcriptCaches: new Map([[activeSessionId, transcript]]),