A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fix(coding-agent): wait for worker recovery before session reuse (#1756)
Sebastian Mülleree8fd699617e
diff --git a/packages/coding-agent/.changes/eng-5533-worker-recovery.md b/packages/coding-agent/.changes/eng-5533-worker-recovery.mdnew file mode 100644index 0000000000..7e607bb4a4--- /dev/null+++ b/packages/coding-agent/.changes/eng-5533-worker-recovery.md@@ -0,0 +1 @@+- Fixed saved-session resume when its resident worker is still recovering after a daemon restart.diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.tsindex 61bc39485a..d4e4d3e8bf 100644--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts@@ -2272,10 +2272,6 @@ export class DaemonSupervisor {? resolve(command.sessionPath): await this.catalog.resolve(command.sessionPath, config.cwd ?? process.cwd(), config.sessionDir);createCommand = { ...createCommand, sessionPath };- const existing = this.findWorkerBySessionFile(sessionPath);- if (existing && !(await this.reclaimStaleWorkerRegistration(existing, command.launchEnv !== undefined))) {- return this.reuseWorkerForCreate(existing, ownerClientId, sessionPath);- }}const key = createCommand.sessionPath? canonicalSessionPath(createCommand.sessionPath)@@ -2284,6 +2280,12 @@ export class DaemonSupervisor {if (pending) {return pending;}+ if (createCommand.sessionPath) {+ const existing = this.findWorkerBySessionFile(createCommand.sessionPath);+ if (existing && !(await this.reclaimStaleWorkerRegistration(existing, command.launchEnv !== undefined))) {+ return this.reuseWorkerForCreate(existing, ownerClientId, createCommand.sessionPath);+ }+ }const opening = (async () => {if (!createCommand.name) return this.launchWorker(createCommand, undefined, ownerClientId);const savedSiblings = createCommand.sessionPath ? await this.rlmLedgerSiblings(createCommand.sessionPath) : [];@@ -2311,20 +2313,52 @@ export class DaemonSupervisor {}}- private reuseWorkerForCreate(+ private async reuseWorkerForCreate(worker: ResidentWorker,ownerClientId: string | undefined,sessionPath: string,- ): ResidentWorker {+ ): Promise<ResidentWorker> {if (worker.descriptor.lifecycle === "failed") {throw new Error(`Session "${sessionPath}" is registered to a failed worker that could not be safely reclaimed`,);}- if (worker.descriptor.ownerClientId === ownerClientId) {- return worker;+ if (worker.descriptor.ownerClientId !== ownerClientId) {+ throw new SessionAlreadyActiveError(sessionPath, worker.descriptor.rootActiveSessionId);+ }+ if (!this.isWorkerReadyForCreate(worker)) {+ if (worker.recovery) {+ await worker.recovery;+ } else if (this.isWorkerRecoveryEligible(worker)) {+ await this.recoverWorker(worker);+ }}- throw new SessionAlreadyActiveError(sessionPath, worker.descriptor.rootActiveSessionId);+ const current = this.workers.get(worker.descriptor.workerId);+ if (!current) {+ throw new Error(`Session "${sessionPath}" worker recovery was interrupted; retry opening the session`);+ }+ if (current.descriptor.ownerClientId !== ownerClientId) {+ throw new SessionAlreadyActiveError(sessionPath, current.descriptor.rootActiveSessionId);+ }+ if (!this.isWorkerReadyForCreate(current)) {+ if (!current.summaries.has(current.descriptor.rootActiveSessionId)) {+ throw new Error(+ `Session "${sessionPath}" worker is unavailable for reuse: assigned root session is missing`,+ );+ }+ const detail = current.descriptor.lastError ? `: ${current.descriptor.lastError}` : "";+ throw new Error(`Session "${sessionPath}" worker is ${this.effectiveWorkerState(current)}${detail}`);+ }+ return current;+ }++ private isWorkerReadyForCreate(worker: ResidentWorker): boolean {+ return (+ worker.descriptor.lifecycle === "ready" &&+ worker.client !== undefined &&+ worker.summaries.has(worker.descriptor.rootActiveSessionId) &&+ !this.isWorkerStopping(worker)+ );}/**@@ -3251,7 +3285,12 @@ export class DaemonSupervisor {}const response = await worker.client.request({ type: "list" }, 5000);const summaries = sessionSummariesFromResponse(response);- worker.summaries = new Map(summaries.map((summary) => [summary.activeSessionId ?? summary.id, summary]));+ const nextSummaries = new Map(summaries.map((summary) => [summary.activeSessionId ?? summary.id, summary]));+ const root = nextSummaries.get(worker.descriptor.rootActiveSessionId);+ if (recovery && !root) {+ throw new Error(`Session worker omitted its root session during recovery`);+ }+ worker.summaries = nextSummaries;for (const summary of summaries) {const activeSessionId = summary.activeSessionId ?? summary.id;if (summary.streamingMessage?.role === "assistant") {@@ -3260,7 +3299,6 @@ export class DaemonSupervisor {this.streamReconstructor.clear(activeSessionId);}}- const root = worker.summaries.get(worker.descriptor.rootActiveSessionId);if (root) {if (recovery) {await this.assertRecoveryAllowed();diff --git a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.tsindex 3233a551da..665c13587a 100644--- a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts@@ -37,13 +37,13 @@ interface SupervisorInternals {interface WorkerFixture {descriptor: {workerId: string;- lifecycle: "ready";+ lifecycle: "ready" | "starting";rootActiveSessionId: string;rootSessionId: string;pid: number;authenticationToken: string;ownerClientId?: string;- createCommand: { config: { cwd: string } };+ createCommand: { config: { cwd: string }; sessionPath?: string };};client: {request: ReturnType<typeof vi.fn>;@@ -342,6 +342,10 @@ describe("daemon supervisor passive subagent topology", () => {});const first = supervisor.createOrReuseWorker("client", { type: "create", name: "named", sessionPath });+ const starting = worker("starting");+ starting.descriptor.lifecycle = "starting";+ starting.descriptor.createCommand = { config: { cwd: "/tmp/project" }, sessionPath };+ supervisor.workers.set(starting.descriptor.workerId, starting);const second = supervisor.createOrReuseWorker("client", { type: "create", sessionPath });releaseSiblings();expect(await Promise.all([first, second])).toEqual([resident, resident]);diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.tsindex 6a2c987547..5f5412e5d7 100644--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts@@ -1613,7 +1613,7 @@ describe("daemon worker supervisor monitoring", () => {expect(supervisor.persistWorker).toHaveBeenCalledWith(worker);});- it("rejects create reuse when a failed worker cannot be safely reclaimed", () => {+ it("rejects create reuse when a failed worker cannot be safely reclaimed", async () => {const worker = {descriptor: {workerId: "failed-unreclaimed",@@ -1622,10 +1622,173 @@ describe("daemon worker supervisor monitoring", () => {},};const supervisor = Object.create(DaemonSupervisor.prototype) as {- reuseWorkerForCreate(target: typeof worker, ownerClientId: undefined, sessionPath: string): typeof worker;+ reuseWorkerForCreate(+ target: typeof worker,+ ownerClientId: undefined,+ sessionPath: string,+ ): Promise<typeof worker>;};- expect(() => supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/failed.jsonl")).toThrow(/failed worker/);+ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/failed.jsonl")).rejects.toThrow(+ /failed worker/,+ );+ });++ it("waits for worker recovery before reusing a saved session", async () => {+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };+ const recovery = createDeferred<void>();+ const worker = {+ descriptor: {+ workerId: "recovering-worker",+ rootActiveSessionId: root.activeSessionId,+ lifecycle: "recovering",+ },+ client: undefined as object | undefined,+ summaries: new Map<string, SessionSummary>(),+ recovery: recovery.promise,+ intentionalStop: false,+ };+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ workers: new Map([[worker.descriptor.workerId, worker]]),+ }) as {+ reuseWorkerForCreate(+ target: typeof worker,+ ownerClientId: undefined,+ sessionPath: string,+ ): Promise<typeof worker>;+ };++ let settled = false;+ const reused = supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl").finally(() => {+ settled = true;+ });+ await Promise.resolve();+ expect(settled).toBe(false);++ worker.descriptor.lifecycle = "ready";+ worker.client = {};+ worker.summaries.set(root.activeSessionId, root as SessionSummary);+ recovery.resolve();++ await expect(reused).resolves.toBe(worker);+ });++ it("starts recovery before reusing a persisted recovering worker", async () => {+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };+ const worker = {+ descriptor: {+ workerId: "persisted-recovering-worker",+ rootActiveSessionId: root.activeSessionId,+ lifecycle: "recovering",+ },+ client: undefined as object | undefined,+ summaries: new Map<string, SessionSummary>(),+ intentionalStop: false,+ };+ const recoverWorker = vi.fn(async () => {+ worker.descriptor.lifecycle = "ready";+ worker.client = {};+ worker.summaries.set(root.activeSessionId, root as SessionSummary);+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ recoverWorker,+ }) as {+ reuseWorkerForCreate(+ target: typeof worker,+ ownerClientId: undefined,+ sessionPath: string,+ ): Promise<typeof worker>;+ };++ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).resolves.toBe(worker);+ expect(recoverWorker).toHaveBeenCalledOnce();+ });++ it("starts recovery for a disconnected worker still marked ready", async () => {+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };+ const worker = {+ descriptor: {+ workerId: "disconnected-ready-worker",+ rootActiveSessionId: root.activeSessionId,+ lifecycle: "ready",+ },+ client: undefined as object | undefined,+ summaries: new Map([[root.activeSessionId, root as SessionSummary]]),+ intentionalStop: false,+ };+ const recoverWorker = vi.fn(async () => {+ worker.client = {};+ });+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ workers: new Map([[worker.descriptor.workerId, worker]]),+ shuttingDown: false,+ recoverWorker,+ }) as {+ reuseWorkerForCreate(+ target: typeof worker,+ ownerClientId: undefined,+ sessionPath: string,+ ): Promise<typeof worker>;+ };++ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).resolves.toBe(worker);+ expect(recoverWorker).toHaveBeenCalledOnce();+ });++ it("rejects recovered workers whose assigned root is still missing", async () => {+ const worker = {+ descriptor: {+ workerId: "rootless-worker",+ rootActiveSessionId: "active-root",+ lifecycle: "ready",+ },+ client: {},+ summaries: new Map<string, SessionSummary>(),+ recovery: Promise.resolve(),+ intentionalStop: false,+ };+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {+ workers: new Map([[worker.descriptor.workerId, worker]]),+ }) as {+ reuseWorkerForCreate(+ target: typeof worker,+ ownerClientId: undefined,+ sessionPath: string,+ ): Promise<typeof worker>;+ };++ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).rejects.toThrow(+ "assigned root session is missing",+ );+ });++ it("preserves cached summaries when recovery omits the assigned root", async () => {+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };+ const worker = {+ descriptor: {+ workerId: "root-omitting-worker",+ rootActiveSessionId: root.activeSessionId,+ },+ client: {+ request: vi.fn(async () =>+ success(undefined, "list", {+ sessions: [{ id: "other", activeSessionId: "other", sessionId: "session-other", cwd: "/tmp" }],+ }),+ ),+ },+ summaries: new Map([[root.activeSessionId, root as SessionSummary]]),+ intentionalStop: false,+ };+ const supervisor = Object.create(DaemonSupervisor.prototype) as {+ refreshWorkerSummaries(target: typeof worker, recovery: boolean): Promise<void>;+ };++ await expect(supervisor.refreshWorkerSummaries(worker, true)).rejects.toThrow(+ "Session worker omitted its root session during recovery",+ );+ expect(worker.summaries.get(root.activeSessionId)).toBe(root);});it("ignores conflicting paths on workers unrelated to a session lookup", () => {