Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(coding-agent): wait for worker recovery before session reuse (#1756)

Sebastian Mülleree8fd699617e
diff --git a/packages/coding-agent/.changes/eng-5533-worker-recovery.md b/packages/coding-agent/.changes/eng-5533-worker-recovery.md
new file mode 100644
index 0000000000..7e607bb4a4
--- /dev/null
+++ b/packages/coding-agent/.changes/eng-5533-worker-recovery.md
@@ -0,0 +1 @@
+- Fixed saved-session resume when its resident worker is still recovering after a daemon restart.
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
index 61bc39485a..d4e4d3e8bf 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
@@ -2272,10 +2272,6 @@ export class DaemonSupervisor {
? resolve(command.sessionPath)
: await this.catalog.resolve(command.sessionPath, config.cwd ?? process.cwd(), config.sessionDir);
createCommand = { ...createCommand, sessionPath };
- const existing = this.findWorkerBySessionFile(sessionPath);
- if (existing && !(await this.reclaimStaleWorkerRegistration(existing, command.launchEnv !== undefined))) {
- return this.reuseWorkerForCreate(existing, ownerClientId, sessionPath);
- }
}
const key = createCommand.sessionPath
? canonicalSessionPath(createCommand.sessionPath)
@@ -2284,6 +2280,12 @@ export class DaemonSupervisor {
if (pending) {
return pending;
}
+ if (createCommand.sessionPath) {
+ const existing = this.findWorkerBySessionFile(createCommand.sessionPath);
+ if (existing && !(await this.reclaimStaleWorkerRegistration(existing, command.launchEnv !== undefined))) {
+ return this.reuseWorkerForCreate(existing, ownerClientId, createCommand.sessionPath);
+ }
+ }
const opening = (async () => {
if (!createCommand.name) return this.launchWorker(createCommand, undefined, ownerClientId);
const savedSiblings = createCommand.sessionPath ? await this.rlmLedgerSiblings(createCommand.sessionPath) : [];
@@ -2311,20 +2313,52 @@ export class DaemonSupervisor {
}
}
- private reuseWorkerForCreate(
+ private async reuseWorkerForCreate(
worker: ResidentWorker,
ownerClientId: string | undefined,
sessionPath: string,
- ): ResidentWorker {
+ ): Promise<ResidentWorker> {
if (worker.descriptor.lifecycle === "failed") {
throw new Error(
`Session "${sessionPath}" is registered to a failed worker that could not be safely reclaimed`,
);
}
- if (worker.descriptor.ownerClientId === ownerClientId) {
- return worker;
+ if (worker.descriptor.ownerClientId !== ownerClientId) {
+ throw new SessionAlreadyActiveError(sessionPath, worker.descriptor.rootActiveSessionId);
+ }
+ if (!this.isWorkerReadyForCreate(worker)) {
+ if (worker.recovery) {
+ await worker.recovery;
+ } else if (this.isWorkerRecoveryEligible(worker)) {
+ await this.recoverWorker(worker);
+ }
}
- throw new SessionAlreadyActiveError(sessionPath, worker.descriptor.rootActiveSessionId);
+ const current = this.workers.get(worker.descriptor.workerId);
+ if (!current) {
+ throw new Error(`Session "${sessionPath}" worker recovery was interrupted; retry opening the session`);
+ }
+ if (current.descriptor.ownerClientId !== ownerClientId) {
+ throw new SessionAlreadyActiveError(sessionPath, current.descriptor.rootActiveSessionId);
+ }
+ if (!this.isWorkerReadyForCreate(current)) {
+ if (!current.summaries.has(current.descriptor.rootActiveSessionId)) {
+ throw new Error(
+ `Session "${sessionPath}" worker is unavailable for reuse: assigned root session is missing`,
+ );
+ }
+ const detail = current.descriptor.lastError ? `: ${current.descriptor.lastError}` : "";
+ throw new Error(`Session "${sessionPath}" worker is ${this.effectiveWorkerState(current)}${detail}`);
+ }
+ return current;
+ }
+
+ private isWorkerReadyForCreate(worker: ResidentWorker): boolean {
+ return (
+ worker.descriptor.lifecycle === "ready" &&
+ worker.client !== undefined &&
+ worker.summaries.has(worker.descriptor.rootActiveSessionId) &&
+ !this.isWorkerStopping(worker)
+ );
}
/**
@@ -3251,7 +3285,12 @@ export class DaemonSupervisor {
}
const response = await worker.client.request({ type: "list" }, 5000);
const summaries = sessionSummariesFromResponse(response);
- worker.summaries = new Map(summaries.map((summary) => [summary.activeSessionId ?? summary.id, summary]));
+ const nextSummaries = new Map(summaries.map((summary) => [summary.activeSessionId ?? summary.id, summary]));
+ const root = nextSummaries.get(worker.descriptor.rootActiveSessionId);
+ if (recovery && !root) {
+ throw new Error(`Session worker omitted its root session during recovery`);
+ }
+ worker.summaries = nextSummaries;
for (const summary of summaries) {
const activeSessionId = summary.activeSessionId ?? summary.id;
if (summary.streamingMessage?.role === "assistant") {
@@ -3260,7 +3299,6 @@ export class DaemonSupervisor {
this.streamReconstructor.clear(activeSessionId);
}
}
- const root = worker.summaries.get(worker.descriptor.rootActiveSessionId);
if (root) {
if (recovery) {
await this.assertRecoveryAllowed();
diff --git a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts
index 3233a551da..665c13587a 100644
--- a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts
@@ -37,13 +37,13 @@ interface SupervisorInternals {
interface WorkerFixture {
descriptor: {
workerId: string;
- lifecycle: "ready";
+ lifecycle: "ready" | "starting";
rootActiveSessionId: string;
rootSessionId: string;
pid: number;
authenticationToken: string;
ownerClientId?: string;
- createCommand: { config: { cwd: string } };
+ createCommand: { config: { cwd: string }; sessionPath?: string };
};
client: {
request: ReturnType<typeof vi.fn>;
@@ -342,6 +342,10 @@ describe("daemon supervisor passive subagent topology", () => {
});
const first = supervisor.createOrReuseWorker("client", { type: "create", name: "named", sessionPath });
+ const starting = worker("starting");
+ starting.descriptor.lifecycle = "starting";
+ starting.descriptor.createCommand = { config: { cwd: "/tmp/project" }, sessionPath };
+ supervisor.workers.set(starting.descriptor.workerId, starting);
const second = supervisor.createOrReuseWorker("client", { type: "create", sessionPath });
releaseSiblings();
expect(await Promise.all([first, second])).toEqual([resident, resident]);
diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
index 6a2c987547..5f5412e5d7 100644
--- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
+++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts
@@ -1613,7 +1613,7 @@ describe("daemon worker supervisor monitoring", () => {
expect(supervisor.persistWorker).toHaveBeenCalledWith(worker);
});
- it("rejects create reuse when a failed worker cannot be safely reclaimed", () => {
+ it("rejects create reuse when a failed worker cannot be safely reclaimed", async () => {
const worker = {
descriptor: {
workerId: "failed-unreclaimed",
@@ -1622,10 +1622,173 @@ describe("daemon worker supervisor monitoring", () => {
},
};
const supervisor = Object.create(DaemonSupervisor.prototype) as {
- reuseWorkerForCreate(target: typeof worker, ownerClientId: undefined, sessionPath: string): typeof worker;
+ reuseWorkerForCreate(
+ target: typeof worker,
+ ownerClientId: undefined,
+ sessionPath: string,
+ ): Promise<typeof worker>;
};
- expect(() => supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/failed.jsonl")).toThrow(/failed worker/);
+ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/failed.jsonl")).rejects.toThrow(
+ /failed worker/,
+ );
+ });
+
+ it("waits for worker recovery before reusing a saved session", async () => {
+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };
+ const recovery = createDeferred<void>();
+ const worker = {
+ descriptor: {
+ workerId: "recovering-worker",
+ rootActiveSessionId: root.activeSessionId,
+ lifecycle: "recovering",
+ },
+ client: undefined as object | undefined,
+ summaries: new Map<string, SessionSummary>(),
+ recovery: recovery.promise,
+ intentionalStop: false,
+ };
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ }) as {
+ reuseWorkerForCreate(
+ target: typeof worker,
+ ownerClientId: undefined,
+ sessionPath: string,
+ ): Promise<typeof worker>;
+ };
+
+ let settled = false;
+ const reused = supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl").finally(() => {
+ settled = true;
+ });
+ await Promise.resolve();
+ expect(settled).toBe(false);
+
+ worker.descriptor.lifecycle = "ready";
+ worker.client = {};
+ worker.summaries.set(root.activeSessionId, root as SessionSummary);
+ recovery.resolve();
+
+ await expect(reused).resolves.toBe(worker);
+ });
+
+ it("starts recovery before reusing a persisted recovering worker", async () => {
+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };
+ const worker = {
+ descriptor: {
+ workerId: "persisted-recovering-worker",
+ rootActiveSessionId: root.activeSessionId,
+ lifecycle: "recovering",
+ },
+ client: undefined as object | undefined,
+ summaries: new Map<string, SessionSummary>(),
+ intentionalStop: false,
+ };
+ const recoverWorker = vi.fn(async () => {
+ worker.descriptor.lifecycle = "ready";
+ worker.client = {};
+ worker.summaries.set(root.activeSessionId, root as SessionSummary);
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ recoverWorker,
+ }) as {
+ reuseWorkerForCreate(
+ target: typeof worker,
+ ownerClientId: undefined,
+ sessionPath: string,
+ ): Promise<typeof worker>;
+ };
+
+ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).resolves.toBe(worker);
+ expect(recoverWorker).toHaveBeenCalledOnce();
+ });
+
+ it("starts recovery for a disconnected worker still marked ready", async () => {
+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };
+ const worker = {
+ descriptor: {
+ workerId: "disconnected-ready-worker",
+ rootActiveSessionId: root.activeSessionId,
+ lifecycle: "ready",
+ },
+ client: undefined as object | undefined,
+ summaries: new Map([[root.activeSessionId, root as SessionSummary]]),
+ intentionalStop: false,
+ };
+ const recoverWorker = vi.fn(async () => {
+ worker.client = {};
+ });
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ shuttingDown: false,
+ recoverWorker,
+ }) as {
+ reuseWorkerForCreate(
+ target: typeof worker,
+ ownerClientId: undefined,
+ sessionPath: string,
+ ): Promise<typeof worker>;
+ };
+
+ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).resolves.toBe(worker);
+ expect(recoverWorker).toHaveBeenCalledOnce();
+ });
+
+ it("rejects recovered workers whose assigned root is still missing", async () => {
+ const worker = {
+ descriptor: {
+ workerId: "rootless-worker",
+ rootActiveSessionId: "active-root",
+ lifecycle: "ready",
+ },
+ client: {},
+ summaries: new Map<string, SessionSummary>(),
+ recovery: Promise.resolve(),
+ intentionalStop: false,
+ };
+ const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), {
+ workers: new Map([[worker.descriptor.workerId, worker]]),
+ }) as {
+ reuseWorkerForCreate(
+ target: typeof worker,
+ ownerClientId: undefined,
+ sessionPath: string,
+ ): Promise<typeof worker>;
+ };
+
+ await expect(supervisor.reuseWorkerForCreate(worker, undefined, "/tmp/session.jsonl")).rejects.toThrow(
+ "assigned root session is missing",
+ );
+ });
+
+ it("preserves cached summaries when recovery omits the assigned root", async () => {
+ const root = { id: "active-root", activeSessionId: "active-root", sessionId: "session-root", cwd: "/tmp" };
+ const worker = {
+ descriptor: {
+ workerId: "root-omitting-worker",
+ rootActiveSessionId: root.activeSessionId,
+ },
+ client: {
+ request: vi.fn(async () =>
+ success(undefined, "list", {
+ sessions: [{ id: "other", activeSessionId: "other", sessionId: "session-other", cwd: "/tmp" }],
+ }),
+ ),
+ },
+ summaries: new Map([[root.activeSessionId, root as SessionSummary]]),
+ intentionalStop: false,
+ };
+ const supervisor = Object.create(DaemonSupervisor.prototype) as {
+ refreshWorkerSummaries(target: typeof worker, recovery: boolean): Promise<void>;
+ };
+
+ await expect(supervisor.refreshWorkerSummaries(worker, true)).rejects.toThrow(
+ "Session worker omitted its root session during recovery",
+ );
+ expect(worker.summaries.get(root.activeSessionId)).toBe(root);
});
it("ignores conflicting paths on workers unrelated to a session lookup", () => {