Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): drain terminal usage before shutdown completes (#956)
Genere482a619ec30
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.tsindex 82bae6d2a..8a1c29f4e 100644--- a/src/daemon/src/daemon/createDaemon.test.ts+++ b/src/daemon/src/daemon/createDaemon.test.ts@@ -4,6 +4,8 @@ import { EventEmitter } from "events";import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";+import { setImmediate as drainEventTasks } from "node:timers/promises";+import { DailyTokenUsageStore } from "../telemetry/dailyTokenUsage.js";import {createDaemon,createBuiltinDaemonSessionFactory,@@ -263,6 +265,7 @@ function daemonFakeSession(options: {let resolveClosed!: (result: AgentSessionResult) => void;const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });const emit = (payload: Omit<Event, "sequence" | "sessionInstanceId" | "at">) => {+ if (ended) return;const event = { ...payload, sequence: ++sequence, sessionInstanceId: "daemon-test", at: Date.now() } as Event;const waiter = waiters.shift();if (waiter) waiter({ done: false, value: event });@@ -278,7 +281,9 @@ function daemonFakeSession(options: {return {next: () => queued.length > 0? Promise.resolve({ done: false as const, value: queued.shift()! })- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),+ : ended+ ? Promise.resolve({ done: true as const, value: undefined })+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),};},},@@ -313,8 +318,10 @@ function daemonFakeSession(options: {async stop() {options.onStop?.();if (!ended) {- ended = true;const result: AgentSessionResult = { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: "released" } };+ emit({ type: "session_closed", result } as never);+ ended = true;+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });resolveClosed(result);}return { status: "accepted", requestId: "daemon-test-stop" };@@ -339,8 +346,12 @@ function daemonFakeSession(options: {emit(args[0] as never);} else if (event === "exit") {const result: AgentSessionResult = { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: "released" } };- ended = true;- resolveClosed(result);+ if (!ended) {+ emit({ type: "session_closed", result } as never);+ ended = true;+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });+ resolveClosed(result);+ }}await Promise.resolve();await Promise.resolve();@@ -1806,6 +1817,256 @@ for await (const line of createInterface({ input: process.stdin })) {});});+describe("createDaemon — terminal usage shutdown contract", () => {+ function gate<T>() {+ let resolve!: (value: T) => void;+ const promise = new Promise<T>((finish) => { resolve = finish; });+ return { promise, resolve };+ }++ async function setup(extra: Partial<Parameters<typeof createDaemon>[0]> = {}) {+ const sockets: FakeSocket[] = [];+ const sessions: DaemonFakeSession[] = [];+ const logger = stubLogger();+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "daemon-terminal-contract-"));+ startupSweepDirs.push(workingDirectoryBase);+ mkdirSync(join(workingDirectoryBase, "bot_1"), { recursive: true });+ mkdirSync(join(workingDirectoryBase, "bot_2"), { recursive: true });+ vi.stubGlobal("fetch", vi.fn(async (input: string | URL | Request) => {+ const url = String(input);+ if (url.includes("/enroll-agent")) return Response.json({ runnerKey: "runner_test" });+ if (url.includes("/daemon/bots")) return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }, { id: "bot_2", name: "Bot2", discriminator: "0002" }] });+ return Response.json({ attempted: 0 });+ }));+ const daemon = await createDaemon({+ machineKey: "cmk_terminal_contract",+ serverUrl: "http://server.invalid",+ serverWsUrl: "ws://x",+ webSocketFactory: factory(sockets) as never,+ runtimeReport: [{ id: "antigravity" }],+ driverFor: () => fullFakeDriver("antigravity"),+ sessionFactory: () => {+ const session = daemonFakeSession();+ sessions.push(session);+ return session;+ },+ capabilities: [],+ workingDirectoryBase,+ logger,+ ...extra,+ });+ sockets[0]!.emit("open");+ const wake = (agentId = "bot_1") => sockets[0]!.emit("message", JSON.stringify({+ type: "agent:wake",+ agentId,+ config: { version: 1, runtime: "antigravity", model: { kind: "default" }, mode: { kind: "default" } },+ launchId: "launch_terminal",+ unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 1 },+ }));+ return { daemon, sockets, sessions, logger, workingDirectoryBase, wake };+ }++ it.each([false, true])("closes WS early and settles accepted persistence despite failed stop=%s", async (failedStop) => {+ const persistenceStarted = gate<void>();+ const persist = gate<void>();+ const originalRecord = DailyTokenUsageStore.prototype.record;+ vi.spyOn(DailyTokenUsageStore.prototype, "record").mockImplementation(async function (...args) {+ if (args[1].input === 13) {+ persistenceStarted.resolve();+ await persist.promise;+ }+ return originalRecord.apply(this, args);+ });+ const ctx = await setup();+ ctx.wake();+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(1));+ const session = ctx.sessions[0]!;+ const stopping = gate<void>();+ const closeProducer = gate<void>();+ const stop = session.stop.bind(session);+ session.stop = vi.fn(async (input) => {+ stopping.resolve();+ await closeProducer.promise;+ const receipt = await stop(input);+ return failedStop+ ? { status: "failed" as const, error: { category: "process" as const, code: "stop_failed", message: "stop failed", retryable: false } }+ : receipt;+ });+ const closeWs = vi.spyOn(ctx.sockets[0]!, "close");+ const shutdown = ctx.daemon.stop();+ expect(ctx.daemon.stop()).toBe(shutdown);+ let settled = false;+ void shutdown.then(() => { settled = true; }, () => { settled = true; });+ const completed = failedStop+ ? expect(shutdown).rejects.toThrow("Daemon shutdown did not complete cleanly")+ : expect(shutdown).resolves.toBeUndefined();+ await stopping.promise;+ expect(closeWs).toHaveBeenCalledOnce();+ expect(ctx.daemon.isOpen()).toBe(false);+ const framesAtStop = ctx.sockets[0]!.sent.length;+ const identity = { source: "native:terminal", index: 0, occurredAt: new Date().toISOString() };+ const sample = { type: "token_usage", source: "native:test", usage: { input: 13, output: 5, cache: 2 }, identity };+ await session.fire("agent_event", sample);+ await persistenceStarted.promise;+ await session.fire("agent_event", sample); // Native replay must stay exactly once.+ await session.fire("agent_event", { type: "token_usage", source: "invalid", usage: { input: -1, output: 0, cache: 0 } });+ await session.fire("agent_event", { type: "token_usage", source: "later", usage: { input: 7, output: 3, cache: 1 } });+ closeProducer.resolve();+ await session.closed;+ await drainEventTasks();+ expect(settled).toBe(false);+ persist.resolve();+ await completed;+ expect(JSON.parse(readFileSync(join(ctx.workingDirectoryBase, ".telemetry", "daily-token-usage.json"), "utf8"))).toMatchObject({+ bots: { bot_1: [{ metrics: { input: 20, output: 8, cache: 3 } }] },+ });+ expect(ctx.logger.calls.warn.filter(([, message]) => message === "daily token usage persistence failed")).toHaveLength(1);+ expect(ctx.sockets[0]!.sent).toHaveLength(framesAtStop);+ expect(session.stop).toHaveBeenCalledOnce();+ rmSync(ctx.workingDirectoryBase, { recursive: true, force: true });+ await drainEventTasks();+ expect(ctx.logger.calls.warn.filter(([, message]) => message === "daily token usage persistence failed")).toHaveLength(1);+ });++ it("normal resync keeps current idle activity and drops an idle snapshot superseded by public root work", async () => {+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");+ const quotaStarted = gate<void>();+ const quota = gate<null>();+ let holdQuota = false;+ const ctx = await setup({ providerQuotaReader: async (backend) => {+ if (backend === "antigravity" && holdQuota) { quotaStarted.resolve(); return quota.promise; }+ return null;+ } });+ try {+ ctx.wake();+ ctx.wake("bot_2");+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(2));+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });+ await ctx.sessions[1]!.fire("runtime_event", { kind: "turn_end" });+ await drainEventTasks();+ holdQuota = true;+ const activities = resync.mock.calls[0]![0]().activities;+ await quotaStarted.promise;+ await ctx.sessions[0]!.fire("agent_event", { type: "internal_progress", turnId: "daemon-test-turn", source: "native", itemType: "root-work" });+ quota.resolve(null);+ const snapshot = await activities;+ expect(snapshot?.map((activity) => ({ agentId: activity.agentId, state: activity.state }))).toEqual([+ { agentId: "bot_2", state: "idle" },+ ]);+ expect(ctx.sockets[0]!.sent.map((frame) => JSON.parse(frame)).some((frame) =>+ frame.type === "agent_activity" && frame.agentId === "bot_1" && frame.state === "running",+ )).toBe(true);+ } finally {+ quota.resolve(null);+ await ctx.daemon.stop();+ }+ });++ it("joins an accepted resync root through its post-stop quota and usage-window reads", async () => {+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");+ const quotaStarted = gate<void>();+ const quota = gate<null>();+ let holdQuota = false;+ const ctx = await setup({ providerQuotaReader: async (backend) => {+ if (backend === "antigravity" && holdQuota) { quotaStarted.resolve(); return quota.promise; }+ return null;+ } });+ ctx.wake();+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(1));+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });+ await drainEventTasks();+ const readStarted = gate<void>();+ const read = gate<void>();+ const originalWindow = DailyTokenUsageStore.prototype.usageWindow;+ const windowSpy = vi.spyOn(DailyTokenUsageStore.prototype, "usageWindow").mockImplementation(async function (...args) {+ readStarted.resolve();+ await read.promise;+ return originalWindow.apply(this, args);+ });+ holdQuota = true;+ // A reconnect snapshot owns the whole enrichment root, including a later+ // usageWindow call after quota resolves. It bypasses activityReportTails.+ ctx.sockets[0]!.emit("open");+ await quotaStarted.promise;+ const shutdown = ctx.daemon.stop();+ let settled = false;+ void shutdown.then(() => { settled = true; });+ await ctx.sessions[0]!.closed;+ await drainEventTasks();+ expect(settled).toBe(false);+ expect(windowSpy).not.toHaveBeenCalled();+ const framesAtStop = ctx.sockets[0]!.sent.length;+ quota.resolve(null);+ await readStarted.promise;+ expect(settled).toBe(false);+ read.resolve();+ await shutdown;+ const reads = windowSpy.mock.calls.length;+ await resync.mock.calls[0]![0]().activities;+ await drainEventTasks();+ expect(windowSpy).toHaveBeenCalledTimes(reads);+ // The old snapshot cannot publish late activity after closure.+ expect(ctx.sockets[0]!.sent).toHaveLength(framesAtStop);+ });++ it("settles all accepted resync reads when one quota read fails", async () => {+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");+ const started = gate<void>();+ const quota = gate<null>();+ let reads = 0;+ let hold = false;+ const ctx = await setup({ providerQuotaReader: async (backend) => {+ if (backend !== "antigravity" || !hold) return null;+ if (++reads === 1) throw new Error("quota read failed");+ started.resolve();+ return quota.promise;+ } });+ ctx.wake();+ ctx.wake("bot_2");+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(2));+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });+ await ctx.sessions[1]!.fire("runtime_event", { kind: "turn_end" });+ await drainEventTasks();+ hold = true;+ const root = resync.mock.calls[0]![0]().activities as Promise<unknown>;+ const rootFailure = expect(root).rejects.toThrow("quota read failed");+ await started.promise;+ const shutdown = ctx.daemon.stop();+ let settled = false;+ void shutdown.then(() => { settled = true; });+ await Promise.all(ctx.sessions.map((session) => session.closed));+ await drainEventTasks();+ expect(settled).toBe(false);+ quota.resolve(null);+ await rootFailure;+ await shutdown;+ expect(reads).toBe(2);+ });++ it("rejects a router wake that resumes enrollment after shutdown without spawning", async () => {+ const ctx = await setup();+ const enrolling = gate<void>();+ const enroll = gate<void>();+ vi.stubGlobal("fetch", vi.fn(async (input: string | URL | Request) => {+ if (String(input).includes("/enroll-agent")) {+ enrolling.resolve();+ await enroll.promise;+ return Response.json({ runnerKey: "runner_test" });+ }+ return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }], attempted: 0 });+ }));+ const command = spyOnRouterCommandEntry();+ ctx.wake();+ await enrolling.promise;+ const commandDone = command.mock.results[0]!.value as Promise<void>;+ await ctx.daemon.stop();+ enroll.resolve();+ await commandDone;+ expect(ctx.sessions).toHaveLength(0);+ expect(ctx.daemon.isOpen()).toBe(false);+ });+});+describe("createDaemon — opt-in raw runtime trace (P0-1)", () => {const dirs: string[] = [];const expectSecureMode = (path: string) => {diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex 3e154a878..32289a586 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -403,6 +403,15 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDaconst workingDirectoryBase = opts.workingDirectoryBase ?? fallbackBase;const workdirFor = (agentId: string) => `${workingDirectoryBase}/${agentId}`;const dailyTokenUsage = new DailyTokenUsageStore(workingDirectoryBase);+ let stopping = false;+ let usageSealed = false;+ let stopPromise: Promise<void> | undefined;+ const telemetryTasks = new Set<Promise<unknown>>();+ const trackTelemetry = <T>(task: Promise<T>): Promise<T> => {+ telemetryTasks.add(task);+ void task.then(() => telemetryTasks.delete(task), () => telemetryTasks.delete(task));+ return task;+ };const providerQuotaReader = opts.providerQuotaReader?? (opts.sessionFactory ? async () => null : readBuiltinProviderQuota);const providerQuotaByBackend = new Map<ProviderQuotaSnapshot["agentBackendId"], ProviderQuotaSnapshot>();@@ -486,6 +495,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDavoid channelRef?.reportAgentActivity?.(info);};const reportTelemetryForIdleAgent = (agentId: string): void => {+ if (stopping) return;if (managerRef?.agentActivity(agentId) === "idle") {reportAgentActivity({ agentId, state: "idle" });}@@ -838,10 +848,13 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDachannelRef = channel;const activityReportTails = new Map<string, Promise<void>>();reportAgentActivity = (info): void => {+ if (stopping) return;const prior = activityReportTails.get(info.agentId) ?? Promise.resolve();const next = prior.then(async () => {- await channel.reportAgentActivity(await activityPayload(info));+ if (stopping) return;+ const payload = await activityPayload(info);+ if (!stopping) await channel.reportAgentActivity(payload);}).catch(() => {log.warn("agent activity telemetry report failed", { agentId: info.agentId, state: info.state });@@ -1001,6 +1014,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDatickIntervalMs: opts.tickIntervalMs ?? 2000,onAgentSession: (info) => { tokenUsageAvailability.delete(info.agentId); void channel.reportAgentSession(info); },onAgentActivity: (info) => {+ if (stopping) return;selfSleepScheduler?.observeAgentActivity(info.agentId, info.state === "running");reportAgentActivity(info);// Bot typing indicator (DM-only) — install / tear down the daemon-metered@@ -1022,11 +1036,12 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa},onTokenUsageStatus: ({ agentId, backendId, status }) => { tokenUsageAvailability.set(agentId, { backendId, status }); reportTelemetryForIdleAgent(agentId); },onTokenUsage: ({ agentId, usage, identity }) => {- void dailyTokenUsage.record(agentId, usage, identity)+ if (usageSealed) return;+ trackTelemetry(dailyTokenUsage.record(agentId, usage, identity).then(() => reportTelemetryForIdleAgent(agentId)).catch(() => {log.warn("daily token usage persistence failed", { agentId });- });+ }));},onProviderQuota: ({ agentId, backendId, quota }) => {if (backendId !== "claude" && backendId !== "codex" && backendId !== "grok" && backendId !== "antigravity") return;@@ -1106,16 +1121,22 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDadaemonVersion: opts.daemonVersion,timeZone: () => dailyTokenUsage.timeZone,providerQuotas: providerQuotaSnapshots,- resyncActivities: async () => {- const activities = await Promise.all(- manager.liveAgentActivities().map((info) => activityPayload(info)),- );- // Telemetry enrichment can await local I/O (and Claude quota refresh).- // Do not let an activity snapshot captured at connect overwrite a newer- // live state transition that was sent while that work was in flight.- return activities.filter(- (activity) => manager.agentActivity(activity.agentId) === activity.state,- );+ resyncActivities: () => {+ if (stopping) return Promise.resolve([]);+ return trackTelemetry((async () => {+ const results = await Promise.allSettled(+ manager.liveAgentActivities().map((info) => activityPayload(info)),+ );+ const failed = results.find((result): result is PromiseRejectedResult => result.status === "rejected");+ if (failed) throw failed.reason;+ const activities = results.map((result) => (result as PromiseFulfilledResult<HostAgentActivity>).value);+ // Telemetry enrichment can await local I/O (and Claude quota refresh).+ // Do not let an activity snapshot captured at connect overwrite a newer+ // live state transition that was sent while that work was in flight.+ return stopping ? [] : activities.filter(+ (activity) => manager.agentActivity(activity.agentId) === activity.state,+ );+ })());},typingTracker,logger: log.child("router"),@@ -1141,6 +1162,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa},});requestReadyQuotaResend = () => {+ if (stopping) return;if (router) channel.sendReady?.(router.buildReady());};@@ -1178,6 +1200,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa// after resync completes. Wake-resync runs alongside it so a message// dropped while this daemon was offline gets a second chance right away.channel.onOpen(() => {+ if (stopping) return;void coldStartWarmup();void resyncPendingWakes();void resyncPendingDiagnostics();@@ -1192,7 +1215,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDaselfSleepScheduler?.start();return {- isOpen: () => channel.status === "open",+ isOpen: () => !stopping && channel.status === "open",// `channel.connect()` already ran above, so a caller registering here is// racing the first open: if the socket won that race the hook would never// fire at all. Fire it once immediately in that case (on a microtask, so@@ -1203,18 +1226,33 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDaif (channel.status === "open") queueMicrotask(hook);},proxyUrl: proxy.url,- stop: async () => {+ stop: () => {+ if (stopPromise) return stopPromise;+ stopping = true;+ const failures: unknown[] = [];+ stopPromise = Promise.resolve().then(async () => {+ try {+ try { await proxy.close(); } catch (error) { failures.push(error); }+ try { await manager.stopAll(); } catch (error) { failures.push(error); }+ } finally {+ // stopAll joins factories, closed results and the unique consumers.+ // No accepted source can append usage after this sealing point.+ usageSealed = true;+ await Promise.allSettled([...telemetryTasks, ...activityReportTails.values()]);+ }+ if (failures.length) throw new AggregateError(failures, "Daemon shutdown did not complete cleanly");+ });+ manager.beginShutdown();selfSleepScheduler?.stop();reminderSchedulerRef?.clearAll();// Clear all bot-typing heartbeat intervals and emit final stops for// any outstanding scopes so no pill is left dangling.for (const agentId of [...typingHeartbeats.keys()]) {- emitTypingStopsAndClear(agentId);+ try { emitTypingStopsAndClear(agentId); } catch (error) { failures.push(error); }}if (statusTimer) clearInterval(statusTimer);- channel.close();- await proxy.close();- await manager.stopAll();+ try { channel.close(); } catch (error) { failures.push(error); }+ return stopPromise;},};}diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.tsindex 8eeda8d14..91452a09c 100644--- a/src/daemon/src/manager/managerRuntime.test.ts+++ b/src/daemon/src/manager/managerRuntime.test.ts@@ -6,6 +6,7 @@ import { PassThrough } from "stream";import * as fs from "node:fs";import * as os from "node:os";import * as path from "node:path";+import { setImmediate as drainEventTasks } from "node:timers/promises";import {AgentProcessManager,truncateThinking,@@ -102,9 +103,11 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {const queued: Event[] = [];const waiters: Array<(value: IteratorResult<Event>) => void> = [];let sequence = 0;+ let ended = false;let resolveClosed!: (result: AgentSessionResult) => void;const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });const push = (payload: Omit<Event, "sequence" | "sessionInstanceId" | "at">) => {+ if (ended) return;const event = {...payload,sequence: ++sequence,@@ -125,7 +128,9 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {return {next: () => queued.length > 0? Promise.resolve({ done: false as const, value: queued.shift()! })- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),+ : ended+ ? Promise.resolve({ done: true as const, value: undefined })+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),};},},@@ -156,7 +161,10 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {};},async interrupt() { return { status: "not_running" as const }; },- async stop() { return { status: "accepted" as const, requestId: "test-stop" }; },+ async stop(input) {+ if (input.reason === "shutdown") await s.fire("exit", { reason: "requested", code: 0 });+ return { status: "accepted" as const, requestId: "test-stop" };+ },snapshot() {return {sessionInstanceId,@@ -206,10 +214,18 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {: (info?.code ?? null) === 0 && (info?.signal ?? null) === null? { outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } }: { outcome: "crashed", requested: false, exitCode: info?.code ?? null, signal: info?.signal ?? null, cleanup: { status: "released" } };- resolveClosed(result);+ if (!ended) {+ push({ type: "session_closed", result } as never);+ ended = true;+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });+ resolveClosed(result);+ }}await Promise.resolve();await Promise.resolve();+ // Terminal handling joins producer closure and FIFO exhaustion. Yield the+ // event-loop phase so fire(exit) includes those consumer continuations.+ if (evt === "exit") await drainEventTasks();},} as unknown as FakeSession;return s;@@ -4549,9 +4565,10 @@ function b1Session(order: string[], name = "s", turnId = "test-turn", sessionInsvoid session.pushAgentEvent({ type: "turn_started", turnId, commandIds: [input.id] });return { status: "accepted" as const, delivery: "steer" as const, commandId: input.id, turnId };});- session.stop = vi.fn(async (opts?: { reason?: string }) => {- order.push(`${name}:stop:${opts?.reason ?? ""}`);- return { status: "accepted" as const, requestId: "test-stop" };+ const stop = session.stop.bind(session);+ session.stop = vi.fn(async (opts) => {+ order.push(`${name}:stop:${opts.reason}`);+ return stop(opts);});return session;}@@ -5043,6 +5060,364 @@ describe("B1 red gate — strict read-only trace sink", () => {});});+describe("AgentProcessManager — terminal usage shutdown contract", () => {+ function gate<T>() {+ let resolve!: (value: T) => void;+ const promise = new Promise<T>((finish) => { resolve = finish; });+ return { promise, resolve };+ }++ function managerFor(sessionFactory: SessionFactory, extra: Partial<ConstructorParameters<typeof AgentProcessManager>[0]> = {}) {+ const onTokenUsage = vi.fn();+ const mgr = new AgentProcessManager({+ driverFor: () => b1PersistentDriver(),+ baseContextFor: () => ({+ workingDirectory: "/tmp",+ agentId: "a1",+ standingPrompt: "",+ config: {} as LaunchContext["config"],+ credentialProxy: {} as LaunchContext["credentialProxy"],+ }),+ sessionFactory,+ onTokenUsage,+ ...extra,+ });+ mgr.register("a1");+ return { mgr, onTokenUsage };+ }++ function holdUsage(session: FakeSession) {+ const reached = gate<void>();+ const released = gate<void>();+ const events = session.events;+ Object.defineProperty(session, "events", { value: {+ maxBufferedBytes: events.maxBufferedBytes,+ [Symbol.asyncIterator]() {+ const iterator = events[Symbol.asyncIterator]();+ return {+ async next() {+ const item = await iterator.next();+ if (!item.done && item.value.type === "token_usage") {+ reached.resolve();+ await released.promise;+ }+ return item;+ },+ };+ },+ } });+ return { reached: reached.promise, release: () => released.resolve() };+ }++ const usage = { type: "token_usage", source: "native:test", usage: { input: 13, output: 5, cache: 2 } } as const;++ it.each(["accepted", "already_stopping", "closed"] as const)("joins buffered usage after closed with a %s stop receipt", async (status) => {+ const session = b1Session([]);+ const held = holdUsage(session);+ const stop = session.stop.bind(session);+ session.stop = vi.fn(async (input) => {+ await stop(input);+ return status === "closed"+ ? { status, result: await session.closed }+ : { status, requestId: "terminal-contract" };+ });+ const { mgr, onTokenUsage } = managerFor(() => session);+ mgr.deliver("a1", { id: "first", text: "first" });+ await session.fire("runtime_event", { kind: "session_init" });+ await session.pushAgentEvent(usage);+ await held.reached;++ const shutdown = mgr.stopAll();+ expect(mgr.stopAll()).toBe(shutdown);+ let settled = false;+ void shutdown.then(() => { settled = true; });+ await session.closed;+ await drainEventTasks();+ expect(settled).toBe(false);+ expect(onTokenUsage).not.toHaveBeenCalled();+ expect(mgr.deliver("a1", { text: "new work" })).toBe(false);+ expect(() => mgr.register("new")).toThrow("shutting down");+ await expect(mgr.resetSession("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "reset" })).rejects.toThrow("shutting down");++ held.release();+ await shutdown;+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ agentId: "a1", usage: usage.usage }));+ expect(session.stop).toHaveBeenCalledTimes(1);+ expect(session.send).not.toHaveBeenCalled();+ });++ it("waits for a late factory and closes its public stream without starting a prompt", async () => {+ const factory = gate<FakeSession>();+ const session = b1Session([]);+ const held = holdUsage(session);+ const stop = session.stop.bind(session);+ session.stop = vi.fn(async (input) => {+ await session.pushAgentEvent(usage);+ return stop(input);+ });+ const { mgr, onTokenUsage } = managerFor(() => factory.promise);+ mgr.deliver("a1", { text: "queued before shutdown" });+ const shutdown = mgr.stopAll();+ let settled = false;+ void shutdown.then(() => { settled = true; });+ await drainEventTasks();+ expect(settled).toBe(false);+ factory.resolve(session);+ await held.reached;+ await session.closed;+ expect(session.start).not.toHaveBeenCalled();+ expect(settled).toBe(false);+ held.release();+ await shutdown;+ expect(onTokenUsage).toHaveBeenCalledOnce();+ expect(session.stop).toHaveBeenCalledTimes(1);+ });++ it("does not execute queued business from terminal events during shutdown", async () => {+ const session = b1Session([]);+ const held = holdUsage(session);+ const factory = vi.fn(() => session);+ const { mgr, onTokenUsage } = managerFor(factory);+ mgr.deliver("a1", { id: "first", text: "first" });+ await session.pushAgentEvent(usage);+ await held.reached;+ mgr.enqueueRewake("a1", { id: "queued", text: "queued work" });+ await session.pushAgentEvent({ type: "turn_completed", turnId: "test-turn", commandIds: ["first"], result: { outcome: "success", backendSessionId: "test-session" } });+ await session.pushAgentEvent(usage);+ const shutdown = mgr.stopAll();+ await session.closed;+ held.release();+ await shutdown;+ expect(onTokenUsage).toHaveBeenCalledTimes(2);+ expect(factory).toHaveBeenCalledOnce();+ expect(session.start).toHaveBeenCalledOnce();+ expect(session.send).not.toHaveBeenCalled();+ });++ it("joins a retired owner while its late closure leaves the replacement intact", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const first = b1Session([], "old", "old-turn", "old-instance");+ const second = b1Session([], "new", "new-turn", "new-instance");+ const held = holdUsage(first);+ const sessions = [first, second];+ const factory = vi.fn(() => sessions.shift()!);+ const { mgr, onTokenUsage } = managerFor(factory, {+ now: () => now, tickIntervalMs: 5, resetStuckThresholdMs: 50, stoppingStuckThresholdMs: 50,+ });+ mgr.start();+ mgr.deliver("a1", { id: "first", text: "first" });+ await first.fire("runtime_event", { kind: "session_init", sessionId: "old-backend" });+ await first.pushAgentEvent(usage);+ await held.reached;+ await mgr.switchModel("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "replacement", rewakePrompt: "new work" });+ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ now = 200;+ await vi.advanceTimersByTimeAsync(5);+ expect(factory).toHaveBeenCalledTimes(2);+ await second.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });+ await second.pushAgentEvent(usage);+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));+ await first.fire("exit", { reason: "requested" });+ held.release();+ await drainEventTasks();+ expect(mgr.snapshot().agents.a1.execution.sessionInstanceId).toBe("new-instance");+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");+ expect(mgr.runningAgentCount()).toBe(1);+ expect(onTokenUsage).toHaveBeenCalledOnce();+ await mgr.stopAll();+ } finally {+ vi.useRealTimers();+ }+ });++ it("does not rearm handshake work when a pre-stop start receipt resolves during drainage", async () => {+ vi.useFakeTimers();+ try {+ const session = fakeSession();+ const held = holdUsage(session);+ const { mgr } = managerFor(() => session);+ mgr.deliver("a1", { text: "first" });+ await session.pushAgentEvent(usage);+ await held.reached;+ const shutdown = mgr.stopAll();+ await session.closed;+ session.startResolver!();+ await drainEventTasks();+ expect(vi.getTimerCount()).toBe(0);+ held.release();+ await shutdown;+ } finally {+ vi.useRealTimers();+ }+ });++ it("a late old factory rejection cannot exit or audit the replacement owner", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ let reject!: (error: unknown) => void;+ const old = new Promise<FakeSession>((_resolve, fail) => { reject = fail; });+ const replacement = b1Session([], "replacement", "new-turn", "new-instance");+ const factory = vi.fn<SessionFactory>().mockReturnValueOnce(old).mockReturnValue(replacement);+ const onRuntimeSpawnFailed = vi.fn();+ const { mgr } = managerFor(factory, {+ now: () => now, tickIntervalMs: 5, resetStuckThresholdMs: 50, stoppingStuckThresholdMs: 50,+ onRuntimeSpawnFailed,+ });+ mgr.start();+ mgr.deliver("a1", { text: "old factory" });+ await mgr.switchModel("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "replacement", rewakePrompt: "new work" });+ now = 100;+ await vi.advanceTimersByTimeAsync(5);+ now = 200;+ await vi.advanceTimersByTimeAsync(5);+ expect(factory).toHaveBeenCalledTimes(2);+ await replacement.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });+ const before = mgr.snapshot().agents.a1;+ reject(Object.assign(new Error("old factory rejected"), { code: "old_factory_failed" }));+ await drainEventTasks();+ expect(mgr.snapshot().agents.a1).toEqual(before);+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");+ expect(onRuntimeSpawnFailed).not.toHaveBeenCalledWith("codex", "old_factory_failed");+ await expect(mgr.stopAll()).rejects.toThrow("Agent shutdown did not complete cleanly");+ } finally {+ vi.useRealTimers();+ }+ });++ it("rejects mismatched instances in the public consumer before session or usage side effects", async () => {+ const session = b1Session([], "current", "test-turn", "current-instance");+ const events = session.events;+ Object.defineProperty(session, "events", { value: {+ maxBufferedBytes: events.maxBufferedBytes,+ async *[Symbol.asyncIterator]() {+ for await (const event of events) {+ const hostile = event.type === "session_started" && event.backendSessionId === "stale-backend"+ || event.type === "token_usage" && event.source === "stale-usage";+ yield hostile ? { ...event, sessionInstanceId: "wrong-instance" } : event;+ }+ },+ } });+ const setSession = vi.fn(() => true);+ const onAgentSession = vi.fn();+ const onRuntimeSessionEstablished = vi.fn();+ const { mgr, onTokenUsage } = managerFor(() => session, {+ timeline: { ...exactTimelineLifecycleStub(), setSession, resumeSessionId: () => null, forgetSession: () => true },+ onAgentSession,+ onRuntimeSessionEstablished,+ });+ mgr.deliver("a1", { text: "first" });+ await session.pushAgentEvent({ type: "session_started", backendSessionId: "stale-backend" });+ await session.pushAgentEvent({ ...usage, source: "stale-usage" });+ await drainEventTasks();+ expect(setSession).not.toHaveBeenCalled();+ expect(onAgentSession).not.toHaveBeenCalled();+ expect(onRuntimeSessionEstablished).not.toHaveBeenCalled();+ expect(onTokenUsage).not.toHaveBeenCalled();+ await session.fire("runtime_event", { kind: "session_init", sessionId: "current-backend" });+ await session.pushAgentEvent(usage);+ await drainEventTasks();+ expect(setSession).toHaveBeenCalledExactlyOnceWith("a1", "current-backend", "current-instance");+ expect(onRuntimeSessionEstablished).toHaveBeenCalledOnce();+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("current-backend");+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));+ await mgr.stopAll();+ });++ it("rejects public events from a still-draining old owner after replacement attachment", async () => {+ const old = b1Session([], "old", "old-turn", "old-instance");+ const replacement = b1Session([], "new", "new-turn", "new-instance");+ const held = holdUsage(old);+ let rejectClosed!: (error: unknown) => void;+ const closed = new Promise<AgentSessionResult>((_resolve, reject) => { rejectClosed = reject; });+ Object.defineProperty(old, "closed", { value: closed });+ const sessions = [old, replacement];+ const setSession = vi.fn(() => true);+ const onAgentSession = vi.fn();+ const { mgr, onTokenUsage } = managerFor(() => sessions.shift()!, {+ timeline: { ...exactTimelineLifecycleStub(), setSession, resumeSessionId: () => null, forgetSession: () => true },+ onAgentSession,+ });+ mgr.deliver("a1", { text: "first" });+ await old.fire("runtime_event", { kind: "session_init", sessionId: "old-backend" });+ await old.pushAgentEvent(usage);+ await held.reached;+ mgr.enqueueRewake("a1", { id: "replacement-command", text: "replacement work" });+ rejectClosed(new Error("old public closed rejected"));+ await drainEventTasks();+ await replacement.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });+ const before = mgr.snapshot().agents.a1;+ held.release();+ await old.pushAgentEvent({ type: "session_started", backendSessionId: "late-old-backend" });+ await drainEventTasks();+ expect(setSession.mock.calls.map((call) => call[1])).toEqual(["old-backend", "new-backend"]);+ expect(onAgentSession.mock.calls.map(([info]) => info.sessionId)).toEqual(["old-backend", "new-backend"]);+ expect(onTokenUsage).not.toHaveBeenCalled();+ expect(mgr.snapshot().agents.a1).toEqual(before);+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");+ await replacement.pushAgentEvent(usage);+ await drainEventTasks();+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));+ await expect(mgr.stopAll()).rejects.toThrow("Agent shutdown did not complete cleanly");+ });++ it("retains factory rejection as a failed shutdown", async () => {+ let reject!: (error: unknown) => void;+ const factory = new Promise<FakeSession>((_resolve, fail) => { reject = fail; });+ const { mgr } = managerFor(() => factory);+ mgr.deliver("a1", { text: "first" });+ const shutdown = mgr.stopAll();+ const rejected = expect(shutdown).rejects.toThrow("Agent shutdown did not complete cleanly");+ reject(new Error("factory rejected"));+ await rejected;+ });++ it.each(["event_stream", "closed", "stop", "cleanup", "timed_out", "forced"] as const)("keeps %s failure while another owner consumes valid tail usage", async (failure) => {+ const failed = b1Session([], "failed", "failed-turn", "failed-instance");+ if (failure === "event_stream") {+ Object.defineProperty(failed, "events", { value: {+ maxBufferedBytes: 4_194_304,+ [Symbol.asyncIterator]: async function* () { throw new Error("stream rejected"); },+ } });+ } else if (failure === "closed") {+ Object.defineProperty(failed, "closed", { value: Promise.reject(new Error("closed rejected")) });+ } else if (failure === "stop") {+ const stop = failed.stop.bind(failed);+ failed.stop = vi.fn(async (input) => {+ await stop(input);+ return { status: "failed", requestId: "failed-stop", error: { category: "process", code: "stop_failed", message: "stop failed", retryable: false } };+ });+ } else {+ const error = { category: "process" as const, code: "cleanup_failed", message: "cleanup failed", retryable: false };+ const result: AgentSessionResult = failure === "forced"+ ? { outcome: "forced", requested: true, error, cleanup: { status: "released" } }+ : { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: failure === "timed_out" ? "timed_out" : "failed", error } };+ Object.defineProperty(failed, "closed", { value: Promise.resolve(result) });+ }+ const other = b1Session([], "other", "other-turn", "other-instance");+ const held = holdUsage(other);+ const { mgr, onTokenUsage } = managerFor(({ agentId }) => agentId === "a1" ? failed : other);+ mgr.register("a2");+ mgr.deliver("a1", { text: "first" });+ mgr.deliver("a2", { text: "second" });+ await other.pushAgentEvent(usage);+ await held.reached;+ const shutdown = mgr.stopAll();+ const rejected = expect(shutdown).rejects.toThrow("Agent shutdown did not complete cleanly");+ let settled = false;+ void shutdown.catch(() => { settled = true; });+ await other.closed;+ await drainEventTasks();+ expect(settled).toBe(false);+ held.release();+ await rejected;+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ agentId: "a2", usage: usage.usage }));+ });+});+describe("B1 red gate — exact-once terminal matrix", () => {const requestedCases: Array<{name: string;@@ -5121,6 +5496,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {expect(session.stop).toHaveBeenCalledWith({ reason: "shutdown", forceAfterMs: 2_000 });expect(shutdownSettled).toBe(false);+ await session.fire("exit", { reason: "requested", code: 0 });finishCleanup({outcome: "stopped",requested: true,diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex e78fc723e..9b94c4811 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -95,6 +95,11 @@ interface ActiveSpawnState {timelineTurnOwner: TimelineTurnOwner | null;pendingDeliverySpans: Map<string, PendingDeliveryTrace>;}+interface ManagedSessionLifecycle {+ session: DaemonAgentSession;+ done: Promise<void>;+ stopping?: Promise<void>;+}interface TraceRecordBase {agentId: string;effects: string[];@@ -460,6 +465,10 @@ export class AgentProcessManager {private readonly liveSessions = new Map<string, string>();private readonly liveBackendIds = new Map<string, BuiltinBackendId>();private readonly activeSpawnState = new Map<string, ActiveSpawnState>();+ private readonly sessionLifecycles = new Map<DaemonAgentSession, ManagedSessionLifecycle>();+ private readonly sessionFactories = new Set<Promise<void>>();+ private shuttingDown = false;+ private shutdownPromise?: Promise<void>;private readonly publishedAgentActivity = new Map<string, AgentActivityState>();private readonly traceProcessNonce = randomUUID();private nextSpawnOrdinal = 1;@@ -542,6 +551,7 @@ export class AgentProcessManager {launchId?: string;applyRuntimeConfig?: boolean;}): void {+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");const runtimeConfigAcceptance = launch?.runtimeConfig? this.acceptRuntimeConfig(agentId, launch.runtimeConfig): undefined;@@ -562,6 +572,7 @@ export class AgentProcessManager {}async updateRuntimeConfig(agentId: string, config: RuntimeConfig): Promise<RuntimeConfigUpdateResult> {+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");const accepted = this.acceptRuntimeConfig(agentId, config);if (accepted === "stale" || accepted === "idempotent") return accepted;const session = this.sessions.get(agentId);@@ -630,7 +641,7 @@ export class AgentProcessManager {if (!session) return "saved_for_start";this.runtimeConfigApplyRunning.add(agentId);try {- while (this.sessions.get(agentId) === session) {+ while (!this.shuttingDown && this.sessions.get(agentId) === session) {const desired = this.pendingRuntimeConfigUpdates.get(agentId) ?? this.runtimeConfigs.get(agentId);if (!desired) return "idempotent";const desiredRevision = desired.runtimeConfigRevision ?? 0;@@ -692,6 +703,7 @@ export class AgentProcessManager {await this.stop(agentId);}deliver(agentId: string, message: AgentMsg): boolean {+ if (this.shuttingDown) return false;const normalized = message.id? message: {@@ -784,6 +796,7 @@ export class AgentProcessManager {opName: string;},): Promise<void> {+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");if (opts.forgetSession && !this.forgetSession(agentId, opts.barrierType ?? "reset_session")) {this.log.error("resume control transition failed; reset aborted", { agentId, barrierType: opts.barrierType });this.emitErrorAudit(agentId, "reset", "resume_control_update_failed", "Reset aborted because resume control could not be persisted");@@ -844,10 +857,19 @@ export class AgentProcessManager {});}start(): void {+ if (this.shuttingDown) return;if (this.tickTimer) return;this.tickTimer = setInterval(() => this.dispatch({ type: "tick", nowMs: this.now() }), this.opts.tickIntervalMs);this.tickTimer.unref?.();}+ beginShutdown(): void {+ // Fence router continuations before WS/proxy cleanup starts awaiting I/O.+ this.shuttingDown = true;+ if (this.tickTimer) {+ clearInterval(this.tickTimer);+ this.tickTimer = null;+ }+ }async stop(agentId: string): Promise<void> {const session = this.sessions.get(agentId);if (!session) return;@@ -875,18 +897,41 @@ export class AgentProcessManager {this.log.debug("audit emit failed (turn interrupt)", { agentId, err: String(err) });}}- async stopAll(): Promise<void> {- if (this.tickTimer) {- clearInterval(this.tickTimer);- this.tickTimer = null;+ stopAll(): Promise<void> {+ if (this.shutdownPromise) return this.shutdownPromise;+ this.beginShutdown();+ this.shutdownPromise = Promise.resolve().then(async () => {+ const failures: unknown[] = [];+ for (const agentId of this.activeSpawnState.keys()) {+ try { this.abortCurrentTurn(agentId, "shutdown"); } catch (error) { failures.push(error); }+ }+ for (const lifecycle of this.sessionLifecycles.values()) void this.stopManagedSession(lifecycle);+ // The synchronous business fence makes this factory set stable. Late+ // results attach a consumer and stop, without ever starting a prompt.+ const factories = await Promise.allSettled([...this.sessionFactories]);+ const lifecycles = [...this.sessionLifecycles.values()];+ const completed = await Promise.allSettled(lifecycles.flatMap((lifecycle) => [+ this.stopManagedSession(lifecycle),+ lifecycle.done,+ ]));+ this.sessions.clear();+ this.sessionLifecycles.clear();+ failures.push(...[...factories, ...completed]+ .filter((result): result is PromiseRejectedResult => result.status === "rejected")+ .map((result) => result.reason));+ if (failures.length) throw new AggregateError(failures, "Agent shutdown did not complete cleanly");+ });+ return this.shutdownPromise;+ }+ private stopManagedSession(lifecycle: ManagedSessionLifecycle): Promise<void> {+ if (!lifecycle.stopping) {+ lifecycle.stopping = Promise.resolve().then(async () => {+ const receipt = await lifecycle.session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });+ if (receipt.status === "failed") throw new Error("Agent session stop failed", { cause: receipt.error });+ });+ void lifecycle.stopping.catch(() => {});}- const entries = [...this.sessions.entries()];- for (const [agentId] of entries) this.abortCurrentTurn(agentId, "shutdown");- await Promise.all(entries.map(async ([, session]) => {- const receipt = await session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });- if (receipt.status === "already_stopping") await session.closed;- }));- this.sessions.clear();+ return lifecycle.stopping;}snapshot(): ManagerState {return this.state;@@ -1305,6 +1350,7 @@ export class AgentProcessManager {return pending;}private applyEffect(effect: ManagerEffect): void {+ if (this.shuttingDown && (effect.type === "spawn" || effect.type === "send")) return;switch (effect.type) {case "spawn":this.doSpawn(effect.agentId, effect.messages, effect.resumeSessionId);@@ -1559,6 +1605,7 @@ export class AgentProcessManager {this.log.info("agent session ended", { agentId, sessionId, reason });}private doSpawn(agentId: string, messages: AgentMsg[], resumeSessionId: string | null): void {+ if (this.shuttingDown) return;const [initial, ...pending] = messages;if (!initial) throw new Error(`AgentProcessManager: spawn for ${agentId} has no command`);this.dispatch({ type: "queue_delivery", agentId, messages: pending });@@ -1670,6 +1717,7 @@ export class AgentProcessManager {abortCause: options?.scope === "handshake_timeout" ? "handshake_timeout" : "spawn_failure",});this.log.warn("spawn failed", { agentId, runtime: driver.id, reason });+ if (this.activeSpawnState.get(agentId) !== state) return;this.opts.onRuntimeSpawnFailed?.(driver.id, reason);this.emitErrorAudit(agentId,@@ -1682,6 +1730,11 @@ export class AgentProcessManager {clearHandshakeTimer();if (state.torndown) return;state.torndown = true;+ if (this.activeSpawnState.get(agentId) !== state) {+ if (state.session && this.sessions.get(agentId) === state.session) this.sessions.delete(agentId);+ this.closeTurn(state, state.activeSpan, { event: "turn_abort", abortCause: "physical_exit" });+ return;+ }const failedStart = result.outcome === "failed_to_start";if (failedStart) reportSpawnFailure(result.error.code || "failed_to_start");if (!state.hasEstablished && !failedStart) reportSpawnFailure("pre_handshake_exit");@@ -1728,6 +1781,11 @@ export class AgentProcessManager {this.log.warn("ignored event from discarded backend session owner", { agentId, event: event.type });return;}+ if (event.sessionInstanceId !== state.sessionInstanceId+ || (state.superseded && this.state.agents[agentId]?.execution.sessionInstanceId !== state.sessionInstanceId)) {+ this.onAgentEvent(agentId, event, driver.id, state);+ return;+ }if (event.type === "session_failed" && !state.hasEstablished) {reportSpawnFailure(event.error.code || "failed_to_start", { message: event.error.message });}@@ -1752,12 +1810,45 @@ export class AgentProcessManager {this.onAgentEvent(agentId, event, driver.id, state);};const attach = (session: DaemonAgentSession) => {- if (state.torndown || this.activeSpawnState.get(agentId) !== state) {- void session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });- return;- }state.session = session;state.sessionInstanceId = session.sessionInstanceId;+ const eventsDone = (async () => {+ try {+ for await (const event of session.events) onEvent(event as ManagedEvent);+ } catch (error) {+ reportSpawnFailure("event_stream_failed", { message: String(error) });+ throw error;+ }+ })();+ const closed = session.closed.catch((error) => {+ reportSpawnFailure("session_closed_rejected", { message: String(error) });+ if (this.activeSpawnState.get(agentId) === state) {+ this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);+ }+ throw error;+ });+ // closed seals the producer; only the consumer can prove FIFO exhaustion.+ // Retain this owner until both settle, including explicit failed outcomes.+ const done = Promise.allSettled([closed, eventsDone]).then(([closedResult, eventsResult]) => {+ if (closedResult.status === "fulfilled") teardown(closedResult.value);+ const failures = [closedResult, eventsResult]+ .filter((result): result is PromiseRejectedResult => result.status === "rejected")+ .map((result) => result.reason);+ if (closedResult.status === "fulfilled" && (closedResult.value.cleanup.status === "failed" || closedResult.value.cleanup.status === "timed_out")) {+ failures.push(closedResult.value.cleanup.error);+ }+ if (closedResult.status === "fulfilled" && closedResult.value.outcome === "forced") failures.push(closedResult.value.error);+ if (failures.length) throw new AggregateError(failures, "Agent session did not finish cleanly");+ });+ const lifecycle: ManagedSessionLifecycle = { session, done };+ this.sessionLifecycles.set(session, lifecycle);+ void done.then(() => {+ if (!this.shuttingDown) this.sessionLifecycles.delete(session);+ }, () => {});+ if (state.torndown || this.activeSpawnState.get(agentId) !== state || this.shuttingDown) {+ void this.stopManagedSession(lifecycle);+ return;+ }this.sessions.set(agentId, session);this.appliedRuntimeConfigs.set(agentId, runtimeConfig);if (@@ -1774,17 +1865,6 @@ export class AgentProcessManager {turnSilence: session.snapshot().diagnostics?.turnSilence,}, state);previousOwner?.pendingDeliverySpans.clear();- void (async () => {- try {- for await (const event of session.events) onEvent(event as ManagedEvent);- } catch (error) {- reportSpawnFailure("event_stream_failed", { message: String(error) });- }- })();- void session.closed.then(teardown, (error) => {- reportSpawnFailure("session_closed_rejected", { message: String(error) });- this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);- });const startedSpan = this.openTurn(state);const commandId = first.id!;this.beginPendingDelivery(state, commandId, first, "idle", startedSpan, false);@@ -1807,6 +1887,7 @@ export class AgentProcessManager {void session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });return;}+ if (this.shuttingDown) return;const driverState = session.snapshot().state;if (this.state.agents[agentId]?.status !== "starting" || driverState === "stopping" || driverState === "closed") return;try {@@ -1839,8 +1920,10 @@ export class AgentProcessManager {(error as { code?: string } | undefined)?.code ?? "open_failed",{ message: String(error) },);- if (this.activeSpawnState.get(agentId) === state) this.activeSpawnState.delete(agentId);- this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);+ if (this.activeSpawnState.get(agentId) === state) {+ this.activeSpawnState.delete(agentId);+ this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);+ }};let created: ReturnType<SessionFactory>;try {@@ -1854,7 +1937,12 @@ export class AgentProcessManager {throw error;}if (created && typeof (created as Promise<DaemonAgentSession>).then === "function") {- void Promise.resolve(created).then(attach, failedOpen);+ const factory = Promise.resolve(created).then(attach).catch((error) => {+ failedOpen(error);+ throw error;+ });+ this.sessionFactories.add(factory);+ void factory.then(() => this.sessionFactories.delete(factory), () => {});} else {try {attach(created as DaemonAgentSession);diff --git a/src/daemon/src/manager/managerSwitchModel.test.ts b/src/daemon/src/manager/managerSwitchModel.test.tsindex 45fd684e4..242140971 100644--- a/src/daemon/src/manager/managerSwitchModel.test.ts+++ b/src/daemon/src/manager/managerSwitchModel.test.ts@@ -1,4 +1,5 @@import { describe, it, expect, vi } from "vitest";+import { setImmediate as drainEventTasks } from "node:timers/promises";import { AgentProcessManager, type DaemonAgentSession, type SessionFactory, type TimelineRecorder } from "./managerRuntime.js";import type { AgentBackend as Driver } from "../drivers/index.js";import type { HostLaunchContext as LaunchContext } from "./hostContext.js";@@ -40,7 +41,9 @@ function fakeSession(): FakeSession {type Event = AgentEvent<BuiltinBackendSpecs, "codex">;const queued: Event[] = [];const waiters: Array<(value: IteratorResult<Event>) => void> = [];+ let ended = false;const publish = (event: Event) => {+ if (ended) return;const waiter = waiters.shift();if (waiter) waiter({ done: false, value: event });else queued.push(event);@@ -58,7 +61,9 @@ function fakeSession(): FakeSession {return {next: () => queued.length > 0? Promise.resolve({ done: false as const, value: queued.shift()! })- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),+ : ended+ ? Promise.resolve({ done: true as const, value: undefined })+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),};},},@@ -70,7 +75,10 @@ function fakeSession(): FakeSession {return { status: "accepted", delivery: "steer", commandId: message.id, turnId: "test-turn" };},async interrupt() { return { status: "not_running" }; },- async stop() { return { status: "accepted", requestId: "test-stop" }; },+ async stop(input) {+ if (input.reason === "shutdown") await s.fire("exit");+ return { status: "accepted", requestId: "test-stop" };+ },snapshot() {return { sessionInstanceId: "switch-model-test", state: "working", queuedCommands: [], lastEventSequence: sequence };},@@ -101,10 +109,16 @@ function fakeSession(): FakeSession {signal: null,cleanup: { status: "released" },};- resolveClosed(result);+ if (!ended) {+ publish({ type: "session_closed", result, sequence: ++sequence, sessionInstanceId: "switch-model-test", at: Date.now() });+ ended = true;+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });+ resolveClosed(result);+ }}await Promise.resolve();await Promise.resolve();+ if (evt === "exit") await drainEventTasks();},};return s;diff --git a/src/daemon/src/manager/managerTimeline.test.ts b/src/daemon/src/manager/managerTimeline.test.tsindex 5e0e865b2..bb3dda424 100644--- a/src/daemon/src/manager/managerTimeline.test.ts+++ b/src/daemon/src/manager/managerTimeline.test.ts@@ -1,4 +1,5 @@import { describe, it, expect } from "vitest";+import { setImmediate as drainEventTasks } from "node:timers/promises";import { AgentProcessManager, type TimelineRecorder } from "./managerRuntime";import type { HostLaunchContext as LaunchContext } from "./hostContext.js";import type { AgentEvent, AgentSessionResult, BuiltinBackendSpecs } from "@alook/agent-driver";@@ -26,7 +27,9 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {let resolveClosed!: (result: AgentSessionResult) => void;const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });let sequence = 0;+ let ended = false;const publish = (event: Event) => {+ if (ended) return;const waiter = waiters.shift();if (waiter) waiter({ done: false, value: event });else queued.push(event);@@ -47,7 +50,9 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {return {next: () => queued.length > 0? Promise.resolve({ done: false as const, value: queued.shift()! })- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),+ : ended+ ? Promise.resolve({ done: true as const, value: undefined })+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),};},},@@ -67,7 +72,10 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {return { status: "accepted" as const, delivery: "steer" as const, commandId: message.id, turnId: "test-turn" };},async interrupt() { return { status: "not_running" as const }; },- async stop() { return { status: "accepted" as const, requestId: "test-stop" }; },+ async stop(input) {+ if (input.reason === "shutdown") await emit("exit");+ return { status: "accepted" as const, requestId: "test-stop" };+ },snapshot() {return { sessionInstanceId: "timeline-test", state: "working" as const, queuedCommands: [], lastEventSequence: sequence };},@@ -89,10 +97,17 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {publish({ ...base, type: "assistant_message_completed", turnId: "test-turn", text: event.text ?? "", truncated: false });}} else if (ev === "exit") {- resolveClosed({ outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } });+ if (!ended) {+ const result: AgentSessionResult = { outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } };+ publish({ type: "session_closed", result, sequence: ++sequence, sessionInstanceId: "timeline-test", at: Date.now() });+ ended = true;+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });+ resolveClosed(result);+ }}await Promise.resolve();await Promise.resolve();+ if (ev === "exit") await drainEventTasks();};return { mgr, emit };}@@ -200,15 +215,19 @@ describe("manager.resetSession", () => {sessionFactory: ({ ctx }) => {let finish!: (result: AgentSessionResult) => void;const closed = new Promise<AgentSessionResult>((resolve) => { finish = resolve; });+ const sessionInstanceId = `reset-test-${sessionsCreated.length + 1}`;const entry: (typeof sessionsCreated)[number] = { ctx, finish: (result) => finish(result), stopCalled: false, sends: [] };sessionsCreated.push(entry);return {backend: "codex" as const,capabilities: {} as never,- sessionInstanceId: `reset-test-${sessionsCreated.length}`,+ sessionInstanceId,events: {maxBufferedBytes: 4_194_304 as const,- async *[Symbol.asyncIterator]() { await new Promise(() => {}); },+ async *[Symbol.asyncIterator]() {+ const result = await closed;+ yield { type: "session_closed" as const, result, sequence: 1, sessionInstanceId, at: Date.now() };+ },},closed,async start(m: { id: string }) {@@ -264,8 +283,7 @@ describe("manager.resetSession", () => {// Fire the session exit → onExit drains rewake into a fresh spawn.first.finish({ outcome: "stopped", requested: true, exitCode: 0, signal: null, cleanup: { status: "released" } });- await Promise.resolve();- await Promise.resolve();+ await drainEventTasks();expect(sessionsCreated).toHaveLength(2);const second = sessionsCreated[1]!;