Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): drain terminal usage before shutdown completes (#956)

Genere482a619ec30
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index 82bae6d2a..8a1c29f4e 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -4,6 +4,8 @@ import { EventEmitter } from "events";
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
+import { setImmediate as drainEventTasks } from "node:timers/promises";
+import { DailyTokenUsageStore } from "../telemetry/dailyTokenUsage.js";
import {
createDaemon,
createBuiltinDaemonSessionFactory,
@@ -263,6 +265,7 @@ function daemonFakeSession(options: {
let resolveClosed!: (result: AgentSessionResult) => void;
const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });
const emit = (payload: Omit<Event, "sequence" | "sessionInstanceId" | "at">) => {
+ if (ended) return;
const event = { ...payload, sequence: ++sequence, sessionInstanceId: "daemon-test", at: Date.now() } as Event;
const waiter = waiters.shift();
if (waiter) waiter({ done: false, value: event });
@@ -278,7 +281,9 @@ function daemonFakeSession(options: {
return {
next: () => queued.length > 0
? Promise.resolve({ done: false as const, value: queued.shift()! })
- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
+ : ended
+ ? Promise.resolve({ done: true as const, value: undefined })
+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
};
},
},
@@ -313,8 +318,10 @@ function daemonFakeSession(options: {
async stop() {
options.onStop?.();
if (!ended) {
- ended = true;
const result: AgentSessionResult = { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: "released" } };
+ emit({ type: "session_closed", result } as never);
+ ended = true;
+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });
resolveClosed(result);
}
return { status: "accepted", requestId: "daemon-test-stop" };
@@ -339,8 +346,12 @@ function daemonFakeSession(options: {
emit(args[0] as never);
} else if (event === "exit") {
const result: AgentSessionResult = { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: "released" } };
- ended = true;
- resolveClosed(result);
+ if (!ended) {
+ emit({ type: "session_closed", result } as never);
+ ended = true;
+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });
+ resolveClosed(result);
+ }
}
await Promise.resolve();
await Promise.resolve();
@@ -1806,6 +1817,256 @@ for await (const line of createInterface({ input: process.stdin })) {
});
});
+describe("createDaemon — terminal usage shutdown contract", () => {
+ function gate<T>() {
+ let resolve!: (value: T) => void;
+ const promise = new Promise<T>((finish) => { resolve = finish; });
+ return { promise, resolve };
+ }
+
+ async function setup(extra: Partial<Parameters<typeof createDaemon>[0]> = {}) {
+ const sockets: FakeSocket[] = [];
+ const sessions: DaemonFakeSession[] = [];
+ const logger = stubLogger();
+ const workingDirectoryBase = mkdtempSync(join(tmpdir(), "daemon-terminal-contract-"));
+ startupSweepDirs.push(workingDirectoryBase);
+ mkdirSync(join(workingDirectoryBase, "bot_1"), { recursive: true });
+ mkdirSync(join(workingDirectoryBase, "bot_2"), { recursive: true });
+ vi.stubGlobal("fetch", vi.fn(async (input: string | URL | Request) => {
+ const url = String(input);
+ if (url.includes("/enroll-agent")) return Response.json({ runnerKey: "runner_test" });
+ if (url.includes("/daemon/bots")) return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }, { id: "bot_2", name: "Bot2", discriminator: "0002" }] });
+ return Response.json({ attempted: 0 });
+ }));
+ const daemon = await createDaemon({
+ machineKey: "cmk_terminal_contract",
+ serverUrl: "http://server.invalid",
+ serverWsUrl: "ws://x",
+ webSocketFactory: factory(sockets) as never,
+ runtimeReport: [{ id: "antigravity" }],
+ driverFor: () => fullFakeDriver("antigravity"),
+ sessionFactory: () => {
+ const session = daemonFakeSession();
+ sessions.push(session);
+ return session;
+ },
+ capabilities: [],
+ workingDirectoryBase,
+ logger,
+ ...extra,
+ });
+ sockets[0]!.emit("open");
+ const wake = (agentId = "bot_1") => sockets[0]!.emit("message", JSON.stringify({
+ type: "agent:wake",
+ agentId,
+ config: { version: 1, runtime: "antigravity", model: { kind: "default" }, mode: { kind: "default" } },
+ launchId: "launch_terminal",
+ unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 1 },
+ }));
+ return { daemon, sockets, sessions, logger, workingDirectoryBase, wake };
+ }
+
+ it.each([false, true])("closes WS early and settles accepted persistence despite failed stop=%s", async (failedStop) => {
+ const persistenceStarted = gate<void>();
+ const persist = gate<void>();
+ const originalRecord = DailyTokenUsageStore.prototype.record;
+ vi.spyOn(DailyTokenUsageStore.prototype, "record").mockImplementation(async function (...args) {
+ if (args[1].input === 13) {
+ persistenceStarted.resolve();
+ await persist.promise;
+ }
+ return originalRecord.apply(this, args);
+ });
+ const ctx = await setup();
+ ctx.wake();
+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(1));
+ const session = ctx.sessions[0]!;
+ const stopping = gate<void>();
+ const closeProducer = gate<void>();
+ const stop = session.stop.bind(session);
+ session.stop = vi.fn(async (input) => {
+ stopping.resolve();
+ await closeProducer.promise;
+ const receipt = await stop(input);
+ return failedStop
+ ? { status: "failed" as const, error: { category: "process" as const, code: "stop_failed", message: "stop failed", retryable: false } }
+ : receipt;
+ });
+ const closeWs = vi.spyOn(ctx.sockets[0]!, "close");
+ const shutdown = ctx.daemon.stop();
+ expect(ctx.daemon.stop()).toBe(shutdown);
+ let settled = false;
+ void shutdown.then(() => { settled = true; }, () => { settled = true; });
+ const completed = failedStop
+ ? expect(shutdown).rejects.toThrow("Daemon shutdown did not complete cleanly")
+ : expect(shutdown).resolves.toBeUndefined();
+ await stopping.promise;
+ expect(closeWs).toHaveBeenCalledOnce();
+ expect(ctx.daemon.isOpen()).toBe(false);
+ const framesAtStop = ctx.sockets[0]!.sent.length;
+ const identity = { source: "native:terminal", index: 0, occurredAt: new Date().toISOString() };
+ const sample = { type: "token_usage", source: "native:test", usage: { input: 13, output: 5, cache: 2 }, identity };
+ await session.fire("agent_event", sample);
+ await persistenceStarted.promise;
+ await session.fire("agent_event", sample); // Native replay must stay exactly once.
+ await session.fire("agent_event", { type: "token_usage", source: "invalid", usage: { input: -1, output: 0, cache: 0 } });
+ await session.fire("agent_event", { type: "token_usage", source: "later", usage: { input: 7, output: 3, cache: 1 } });
+ closeProducer.resolve();
+ await session.closed;
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ persist.resolve();
+ await completed;
+ expect(JSON.parse(readFileSync(join(ctx.workingDirectoryBase, ".telemetry", "daily-token-usage.json"), "utf8"))).toMatchObject({
+ bots: { bot_1: [{ metrics: { input: 20, output: 8, cache: 3 } }] },
+ });
+ expect(ctx.logger.calls.warn.filter(([, message]) => message === "daily token usage persistence failed")).toHaveLength(1);
+ expect(ctx.sockets[0]!.sent).toHaveLength(framesAtStop);
+ expect(session.stop).toHaveBeenCalledOnce();
+ rmSync(ctx.workingDirectoryBase, { recursive: true, force: true });
+ await drainEventTasks();
+ expect(ctx.logger.calls.warn.filter(([, message]) => message === "daily token usage persistence failed")).toHaveLength(1);
+ });
+
+ it("normal resync keeps current idle activity and drops an idle snapshot superseded by public root work", async () => {
+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");
+ const quotaStarted = gate<void>();
+ const quota = gate<null>();
+ let holdQuota = false;
+ const ctx = await setup({ providerQuotaReader: async (backend) => {
+ if (backend === "antigravity" && holdQuota) { quotaStarted.resolve(); return quota.promise; }
+ return null;
+ } });
+ try {
+ ctx.wake();
+ ctx.wake("bot_2");
+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(2));
+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });
+ await ctx.sessions[1]!.fire("runtime_event", { kind: "turn_end" });
+ await drainEventTasks();
+ holdQuota = true;
+ const activities = resync.mock.calls[0]![0]().activities;
+ await quotaStarted.promise;
+ await ctx.sessions[0]!.fire("agent_event", { type: "internal_progress", turnId: "daemon-test-turn", source: "native", itemType: "root-work" });
+ quota.resolve(null);
+ const snapshot = await activities;
+ expect(snapshot?.map((activity) => ({ agentId: activity.agentId, state: activity.state }))).toEqual([
+ { agentId: "bot_2", state: "idle" },
+ ]);
+ expect(ctx.sockets[0]!.sent.map((frame) => JSON.parse(frame)).some((frame) =>
+ frame.type === "agent_activity" && frame.agentId === "bot_1" && frame.state === "running",
+ )).toBe(true);
+ } finally {
+ quota.resolve(null);
+ await ctx.daemon.stop();
+ }
+ });
+
+ it("joins an accepted resync root through its post-stop quota and usage-window reads", async () => {
+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");
+ const quotaStarted = gate<void>();
+ const quota = gate<null>();
+ let holdQuota = false;
+ const ctx = await setup({ providerQuotaReader: async (backend) => {
+ if (backend === "antigravity" && holdQuota) { quotaStarted.resolve(); return quota.promise; }
+ return null;
+ } });
+ ctx.wake();
+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(1));
+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });
+ await drainEventTasks();
+ const readStarted = gate<void>();
+ const read = gate<void>();
+ const originalWindow = DailyTokenUsageStore.prototype.usageWindow;
+ const windowSpy = vi.spyOn(DailyTokenUsageStore.prototype, "usageWindow").mockImplementation(async function (...args) {
+ readStarted.resolve();
+ await read.promise;
+ return originalWindow.apply(this, args);
+ });
+ holdQuota = true;
+ // A reconnect snapshot owns the whole enrichment root, including a later
+ // usageWindow call after quota resolves. It bypasses activityReportTails.
+ ctx.sockets[0]!.emit("open");
+ await quotaStarted.promise;
+ const shutdown = ctx.daemon.stop();
+ let settled = false;
+ void shutdown.then(() => { settled = true; });
+ await ctx.sessions[0]!.closed;
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ expect(windowSpy).not.toHaveBeenCalled();
+ const framesAtStop = ctx.sockets[0]!.sent.length;
+ quota.resolve(null);
+ await readStarted.promise;
+ expect(settled).toBe(false);
+ read.resolve();
+ await shutdown;
+ const reads = windowSpy.mock.calls.length;
+ await resync.mock.calls[0]![0]().activities;
+ await drainEventTasks();
+ expect(windowSpy).toHaveBeenCalledTimes(reads);
+ // The old snapshot cannot publish late activity after closure.
+ expect(ctx.sockets[0]!.sent).toHaveLength(framesAtStop);
+ });
+
+ it("settles all accepted resync reads when one quota read fails", async () => {
+ const resync = vi.spyOn(WsControlChannel.prototype, "onResync");
+ const started = gate<void>();
+ const quota = gate<null>();
+ let reads = 0;
+ let hold = false;
+ const ctx = await setup({ providerQuotaReader: async (backend) => {
+ if (backend !== "antigravity" || !hold) return null;
+ if (++reads === 1) throw new Error("quota read failed");
+ started.resolve();
+ return quota.promise;
+ } });
+ ctx.wake();
+ ctx.wake("bot_2");
+ await vi.waitFor(() => expect(ctx.sessions).toHaveLength(2));
+ await ctx.sessions[0]!.fire("runtime_event", { kind: "turn_end" });
+ await ctx.sessions[1]!.fire("runtime_event", { kind: "turn_end" });
+ await drainEventTasks();
+ hold = true;
+ const root = resync.mock.calls[0]![0]().activities as Promise<unknown>;
+ const rootFailure = expect(root).rejects.toThrow("quota read failed");
+ await started.promise;
+ const shutdown = ctx.daemon.stop();
+ let settled = false;
+ void shutdown.then(() => { settled = true; });
+ await Promise.all(ctx.sessions.map((session) => session.closed));
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ quota.resolve(null);
+ await rootFailure;
+ await shutdown;
+ expect(reads).toBe(2);
+ });
+
+ it("rejects a router wake that resumes enrollment after shutdown without spawning", async () => {
+ const ctx = await setup();
+ const enrolling = gate<void>();
+ const enroll = gate<void>();
+ vi.stubGlobal("fetch", vi.fn(async (input: string | URL | Request) => {
+ if (String(input).includes("/enroll-agent")) {
+ enrolling.resolve();
+ await enroll.promise;
+ return Response.json({ runnerKey: "runner_test" });
+ }
+ return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }], attempted: 0 });
+ }));
+ const command = spyOnRouterCommandEntry();
+ ctx.wake();
+ await enrolling.promise;
+ const commandDone = command.mock.results[0]!.value as Promise<void>;
+ await ctx.daemon.stop();
+ enroll.resolve();
+ await commandDone;
+ expect(ctx.sessions).toHaveLength(0);
+ expect(ctx.daemon.isOpen()).toBe(false);
+ });
+});
+
describe("createDaemon — opt-in raw runtime trace (P0-1)", () => {
const dirs: string[] = [];
const expectSecureMode = (path: string) => {
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index 3e154a878..32289a586 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -403,6 +403,15 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
const workingDirectoryBase = opts.workingDirectoryBase ?? fallbackBase;
const workdirFor = (agentId: string) => `${workingDirectoryBase}/${agentId}`;
const dailyTokenUsage = new DailyTokenUsageStore(workingDirectoryBase);
+ let stopping = false;
+ let usageSealed = false;
+ let stopPromise: Promise<void> | undefined;
+ const telemetryTasks = new Set<Promise<unknown>>();
+ const trackTelemetry = <T>(task: Promise<T>): Promise<T> => {
+ telemetryTasks.add(task);
+ void task.then(() => telemetryTasks.delete(task), () => telemetryTasks.delete(task));
+ return task;
+ };
const providerQuotaReader = opts.providerQuotaReader
?? (opts.sessionFactory ? async () => null : readBuiltinProviderQuota);
const providerQuotaByBackend = new Map<ProviderQuotaSnapshot["agentBackendId"], ProviderQuotaSnapshot>();
@@ -486,6 +495,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
void channelRef?.reportAgentActivity?.(info);
};
const reportTelemetryForIdleAgent = (agentId: string): void => {
+ if (stopping) return;
if (managerRef?.agentActivity(agentId) === "idle") {
reportAgentActivity({ agentId, state: "idle" });
}
@@ -838,10 +848,13 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
channelRef = channel;
const activityReportTails = new Map<string, Promise<void>>();
reportAgentActivity = (info): void => {
+ if (stopping) return;
const prior = activityReportTails.get(info.agentId) ?? Promise.resolve();
const next = prior
.then(async () => {
- await channel.reportAgentActivity(await activityPayload(info));
+ if (stopping) return;
+ const payload = await activityPayload(info);
+ if (!stopping) await channel.reportAgentActivity(payload);
})
.catch(() => {
log.warn("agent activity telemetry report failed", { agentId: info.agentId, state: info.state });
@@ -1001,6 +1014,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
tickIntervalMs: opts.tickIntervalMs ?? 2000,
onAgentSession: (info) => { tokenUsageAvailability.delete(info.agentId); void channel.reportAgentSession(info); },
onAgentActivity: (info) => {
+ if (stopping) return;
selfSleepScheduler?.observeAgentActivity(info.agentId, info.state === "running");
reportAgentActivity(info);
// Bot typing indicator (DM-only) — install / tear down the daemon-metered
@@ -1022,11 +1036,12 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
},
onTokenUsageStatus: ({ agentId, backendId, status }) => { tokenUsageAvailability.set(agentId, { backendId, status }); reportTelemetryForIdleAgent(agentId); },
onTokenUsage: ({ agentId, usage, identity }) => {
- void dailyTokenUsage.record(agentId, usage, identity)
+ if (usageSealed) return;
+ trackTelemetry(dailyTokenUsage.record(agentId, usage, identity)
.then(() => reportTelemetryForIdleAgent(agentId))
.catch(() => {
log.warn("daily token usage persistence failed", { agentId });
- });
+ }));
},
onProviderQuota: ({ agentId, backendId, quota }) => {
if (backendId !== "claude" && backendId !== "codex" && backendId !== "grok" && backendId !== "antigravity") return;
@@ -1106,16 +1121,22 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
daemonVersion: opts.daemonVersion,
timeZone: () => dailyTokenUsage.timeZone,
providerQuotas: providerQuotaSnapshots,
- resyncActivities: async () => {
- const activities = await Promise.all(
- manager.liveAgentActivities().map((info) => activityPayload(info)),
- );
- // Telemetry enrichment can await local I/O (and Claude quota refresh).
- // Do not let an activity snapshot captured at connect overwrite a newer
- // live state transition that was sent while that work was in flight.
- return activities.filter(
- (activity) => manager.agentActivity(activity.agentId) === activity.state,
- );
+ resyncActivities: () => {
+ if (stopping) return Promise.resolve([]);
+ return trackTelemetry((async () => {
+ const results = await Promise.allSettled(
+ manager.liveAgentActivities().map((info) => activityPayload(info)),
+ );
+ const failed = results.find((result): result is PromiseRejectedResult => result.status === "rejected");
+ if (failed) throw failed.reason;
+ const activities = results.map((result) => (result as PromiseFulfilledResult<HostAgentActivity>).value);
+ // Telemetry enrichment can await local I/O (and Claude quota refresh).
+ // Do not let an activity snapshot captured at connect overwrite a newer
+ // live state transition that was sent while that work was in flight.
+ return stopping ? [] : activities.filter(
+ (activity) => manager.agentActivity(activity.agentId) === activity.state,
+ );
+ })());
},
typingTracker,
logger: log.child("router"),
@@ -1141,6 +1162,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
},
});
requestReadyQuotaResend = () => {
+ if (stopping) return;
if (router) channel.sendReady?.(router.buildReady());
};
@@ -1178,6 +1200,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
// after resync completes. Wake-resync runs alongside it so a message
// dropped while this daemon was offline gets a second chance right away.
channel.onOpen(() => {
+ if (stopping) return;
void coldStartWarmup();
void resyncPendingWakes();
void resyncPendingDiagnostics();
@@ -1192,7 +1215,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
selfSleepScheduler?.start();
return {
- isOpen: () => channel.status === "open",
+ isOpen: () => !stopping && channel.status === "open",
// `channel.connect()` already ran above, so a caller registering here is
// racing the first open: if the socket won that race the hook would never
// fire at all. Fire it once immediately in that case (on a microtask, so
@@ -1203,18 +1226,33 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
if (channel.status === "open") queueMicrotask(hook);
},
proxyUrl: proxy.url,
- stop: async () => {
+ stop: () => {
+ if (stopPromise) return stopPromise;
+ stopping = true;
+ const failures: unknown[] = [];
+ stopPromise = Promise.resolve().then(async () => {
+ try {
+ try { await proxy.close(); } catch (error) { failures.push(error); }
+ try { await manager.stopAll(); } catch (error) { failures.push(error); }
+ } finally {
+ // stopAll joins factories, closed results and the unique consumers.
+ // No accepted source can append usage after this sealing point.
+ usageSealed = true;
+ await Promise.allSettled([...telemetryTasks, ...activityReportTails.values()]);
+ }
+ if (failures.length) throw new AggregateError(failures, "Daemon shutdown did not complete cleanly");
+ });
+ manager.beginShutdown();
selfSleepScheduler?.stop();
reminderSchedulerRef?.clearAll();
// Clear all bot-typing heartbeat intervals and emit final stops for
// any outstanding scopes so no pill is left dangling.
for (const agentId of [...typingHeartbeats.keys()]) {
- emitTypingStopsAndClear(agentId);
+ try { emitTypingStopsAndClear(agentId); } catch (error) { failures.push(error); }
}
if (statusTimer) clearInterval(statusTimer);
- channel.close();
- await proxy.close();
- await manager.stopAll();
+ try { channel.close(); } catch (error) { failures.push(error); }
+ return stopPromise;
},
};
}
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index 8eeda8d14..91452a09c 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -6,6 +6,7 @@ import { PassThrough } from "stream";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
+import { setImmediate as drainEventTasks } from "node:timers/promises";
import {
AgentProcessManager,
truncateThinking,
@@ -102,9 +103,11 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {
const queued: Event[] = [];
const waiters: Array<(value: IteratorResult<Event>) => void> = [];
let sequence = 0;
+ let ended = false;
let resolveClosed!: (result: AgentSessionResult) => void;
const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });
const push = (payload: Omit<Event, "sequence" | "sessionInstanceId" | "at">) => {
+ if (ended) return;
const event = {
...payload,
sequence: ++sequence,
@@ -125,7 +128,9 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {
return {
next: () => queued.length > 0
? Promise.resolve({ done: false as const, value: queued.shift()! })
- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
+ : ended
+ ? Promise.resolve({ done: true as const, value: undefined })
+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
};
},
},
@@ -156,7 +161,10 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {
};
},
async interrupt() { return { status: "not_running" as const }; },
- async stop() { return { status: "accepted" as const, requestId: "test-stop" }; },
+ async stop(input) {
+ if (input.reason === "shutdown") await s.fire("exit", { reason: "requested", code: 0 });
+ return { status: "accepted" as const, requestId: "test-stop" };
+ },
snapshot() {
return {
sessionInstanceId,
@@ -206,10 +214,18 @@ function fakeSession(sessionInstanceId = "test-instance"): FakeSession {
: (info?.code ?? null) === 0 && (info?.signal ?? null) === null
? { outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } }
: { outcome: "crashed", requested: false, exitCode: info?.code ?? null, signal: info?.signal ?? null, cleanup: { status: "released" } };
- resolveClosed(result);
+ if (!ended) {
+ push({ type: "session_closed", result } as never);
+ ended = true;
+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });
+ resolveClosed(result);
+ }
}
await Promise.resolve();
await Promise.resolve();
+ // Terminal handling joins producer closure and FIFO exhaustion. Yield the
+ // event-loop phase so fire(exit) includes those consumer continuations.
+ if (evt === "exit") await drainEventTasks();
},
} as unknown as FakeSession;
return s;
@@ -4549,9 +4565,10 @@ function b1Session(order: string[], name = "s", turnId = "test-turn", sessionIns
void session.pushAgentEvent({ type: "turn_started", turnId, commandIds: [input.id] });
return { status: "accepted" as const, delivery: "steer" as const, commandId: input.id, turnId };
});
- session.stop = vi.fn(async (opts?: { reason?: string }) => {
- order.push(`${name}:stop:${opts?.reason ?? ""}`);
- return { status: "accepted" as const, requestId: "test-stop" };
+ const stop = session.stop.bind(session);
+ session.stop = vi.fn(async (opts) => {
+ order.push(`${name}:stop:${opts.reason}`);
+ return stop(opts);
});
return session;
}
@@ -5043,6 +5060,364 @@ describe("B1 red gate — strict read-only trace sink", () => {
});
});
+describe("AgentProcessManager — terminal usage shutdown contract", () => {
+ function gate<T>() {
+ let resolve!: (value: T) => void;
+ const promise = new Promise<T>((finish) => { resolve = finish; });
+ return { promise, resolve };
+ }
+
+ function managerFor(sessionFactory: SessionFactory, extra: Partial<ConstructorParameters<typeof AgentProcessManager>[0]> = {}) {
+ const onTokenUsage = vi.fn();
+ const mgr = new AgentProcessManager({
+ driverFor: () => b1PersistentDriver(),
+ baseContextFor: () => ({
+ workingDirectory: "/tmp",
+ agentId: "a1",
+ standingPrompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ }),
+ sessionFactory,
+ onTokenUsage,
+ ...extra,
+ });
+ mgr.register("a1");
+ return { mgr, onTokenUsage };
+ }
+
+ function holdUsage(session: FakeSession) {
+ const reached = gate<void>();
+ const released = gate<void>();
+ const events = session.events;
+ Object.defineProperty(session, "events", { value: {
+ maxBufferedBytes: events.maxBufferedBytes,
+ [Symbol.asyncIterator]() {
+ const iterator = events[Symbol.asyncIterator]();
+ return {
+ async next() {
+ const item = await iterator.next();
+ if (!item.done && item.value.type === "token_usage") {
+ reached.resolve();
+ await released.promise;
+ }
+ return item;
+ },
+ };
+ },
+ } });
+ return { reached: reached.promise, release: () => released.resolve() };
+ }
+
+ const usage = { type: "token_usage", source: "native:test", usage: { input: 13, output: 5, cache: 2 } } as const;
+
+ it.each(["accepted", "already_stopping", "closed"] as const)("joins buffered usage after closed with a %s stop receipt", async (status) => {
+ const session = b1Session([]);
+ const held = holdUsage(session);
+ const stop = session.stop.bind(session);
+ session.stop = vi.fn(async (input) => {
+ await stop(input);
+ return status === "closed"
+ ? { status, result: await session.closed }
+ : { status, requestId: "terminal-contract" };
+ });
+ const { mgr, onTokenUsage } = managerFor(() => session);
+ mgr.deliver("a1", { id: "first", text: "first" });
+ await session.fire("runtime_event", { kind: "session_init" });
+ await session.pushAgentEvent(usage);
+ await held.reached;
+
+ const shutdown = mgr.stopAll();
+ expect(mgr.stopAll()).toBe(shutdown);
+ let settled = false;
+ void shutdown.then(() => { settled = true; });
+ await session.closed;
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ expect(onTokenUsage).not.toHaveBeenCalled();
+ expect(mgr.deliver("a1", { text: "new work" })).toBe(false);
+ expect(() => mgr.register("new")).toThrow("shutting down");
+ await expect(mgr.resetSession("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "reset" })).rejects.toThrow("shutting down");
+
+ held.release();
+ await shutdown;
+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ agentId: "a1", usage: usage.usage }));
+ expect(session.stop).toHaveBeenCalledTimes(1);
+ expect(session.send).not.toHaveBeenCalled();
+ });
+
+ it("waits for a late factory and closes its public stream without starting a prompt", async () => {
+ const factory = gate<FakeSession>();
+ const session = b1Session([]);
+ const held = holdUsage(session);
+ const stop = session.stop.bind(session);
+ session.stop = vi.fn(async (input) => {
+ await session.pushAgentEvent(usage);
+ return stop(input);
+ });
+ const { mgr, onTokenUsage } = managerFor(() => factory.promise);
+ mgr.deliver("a1", { text: "queued before shutdown" });
+ const shutdown = mgr.stopAll();
+ let settled = false;
+ void shutdown.then(() => { settled = true; });
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ factory.resolve(session);
+ await held.reached;
+ await session.closed;
+ expect(session.start).not.toHaveBeenCalled();
+ expect(settled).toBe(false);
+ held.release();
+ await shutdown;
+ expect(onTokenUsage).toHaveBeenCalledOnce();
+ expect(session.stop).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not execute queued business from terminal events during shutdown", async () => {
+ const session = b1Session([]);
+ const held = holdUsage(session);
+ const factory = vi.fn(() => session);
+ const { mgr, onTokenUsage } = managerFor(factory);
+ mgr.deliver("a1", { id: "first", text: "first" });
+ await session.pushAgentEvent(usage);
+ await held.reached;
+ mgr.enqueueRewake("a1", { id: "queued", text: "queued work" });
+ await session.pushAgentEvent({ type: "turn_completed", turnId: "test-turn", commandIds: ["first"], result: { outcome: "success", backendSessionId: "test-session" } });
+ await session.pushAgentEvent(usage);
+ const shutdown = mgr.stopAll();
+ await session.closed;
+ held.release();
+ await shutdown;
+ expect(onTokenUsage).toHaveBeenCalledTimes(2);
+ expect(factory).toHaveBeenCalledOnce();
+ expect(session.start).toHaveBeenCalledOnce();
+ expect(session.send).not.toHaveBeenCalled();
+ });
+
+ it("joins a retired owner while its late closure leaves the replacement intact", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const first = b1Session([], "old", "old-turn", "old-instance");
+ const second = b1Session([], "new", "new-turn", "new-instance");
+ const held = holdUsage(first);
+ const sessions = [first, second];
+ const factory = vi.fn(() => sessions.shift()!);
+ const { mgr, onTokenUsage } = managerFor(factory, {
+ now: () => now, tickIntervalMs: 5, resetStuckThresholdMs: 50, stoppingStuckThresholdMs: 50,
+ });
+ mgr.start();
+ mgr.deliver("a1", { id: "first", text: "first" });
+ await first.fire("runtime_event", { kind: "session_init", sessionId: "old-backend" });
+ await first.pushAgentEvent(usage);
+ await held.reached;
+ await mgr.switchModel("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "replacement", rewakePrompt: "new work" });
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ now = 200;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(factory).toHaveBeenCalledTimes(2);
+ await second.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });
+ await second.pushAgentEvent(usage);
+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));
+ await first.fire("exit", { reason: "requested" });
+ held.release();
+ await drainEventTasks();
+ expect(mgr.snapshot().agents.a1.execution.sessionInstanceId).toBe("new-instance");
+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");
+ expect(mgr.runningAgentCount()).toBe(1);
+ expect(onTokenUsage).toHaveBeenCalledOnce();
+ await mgr.stopAll();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("does not rearm handshake work when a pre-stop start receipt resolves during drainage", async () => {
+ vi.useFakeTimers();
+ try {
+ const session = fakeSession();
+ const held = holdUsage(session);
+ const { mgr } = managerFor(() => session);
+ mgr.deliver("a1", { text: "first" });
+ await session.pushAgentEvent(usage);
+ await held.reached;
+ const shutdown = mgr.stopAll();
+ await session.closed;
+ session.startResolver!();
+ await drainEventTasks();
+ expect(vi.getTimerCount()).toBe(0);
+ held.release();
+ await shutdown;
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("a late old factory rejection cannot exit or audit the replacement owner", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ let reject!: (error: unknown) => void;
+ const old = new Promise<FakeSession>((_resolve, fail) => { reject = fail; });
+ const replacement = b1Session([], "replacement", "new-turn", "new-instance");
+ const factory = vi.fn<SessionFactory>().mockReturnValueOnce(old).mockReturnValue(replacement);
+ const onRuntimeSpawnFailed = vi.fn();
+ const { mgr } = managerFor(factory, {
+ now: () => now, tickIntervalMs: 5, resetStuckThresholdMs: 50, stoppingStuckThresholdMs: 50,
+ onRuntimeSpawnFailed,
+ });
+ mgr.start();
+ mgr.deliver("a1", { text: "old factory" });
+ await mgr.switchModel("a1", { runtimeConfig: B1_RUNTIME_CONFIG, launchId: "replacement", rewakePrompt: "new work" });
+ now = 100;
+ await vi.advanceTimersByTimeAsync(5);
+ now = 200;
+ await vi.advanceTimersByTimeAsync(5);
+ expect(factory).toHaveBeenCalledTimes(2);
+ await replacement.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });
+ const before = mgr.snapshot().agents.a1;
+ reject(Object.assign(new Error("old factory rejected"), { code: "old_factory_failed" }));
+ await drainEventTasks();
+ expect(mgr.snapshot().agents.a1).toEqual(before);
+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");
+ expect(onRuntimeSpawnFailed).not.toHaveBeenCalledWith("codex", "old_factory_failed");
+ await expect(mgr.stopAll()).rejects.toThrow("Agent shutdown did not complete cleanly");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("rejects mismatched instances in the public consumer before session or usage side effects", async () => {
+ const session = b1Session([], "current", "test-turn", "current-instance");
+ const events = session.events;
+ Object.defineProperty(session, "events", { value: {
+ maxBufferedBytes: events.maxBufferedBytes,
+ async *[Symbol.asyncIterator]() {
+ for await (const event of events) {
+ const hostile = event.type === "session_started" && event.backendSessionId === "stale-backend"
+ || event.type === "token_usage" && event.source === "stale-usage";
+ yield hostile ? { ...event, sessionInstanceId: "wrong-instance" } : event;
+ }
+ },
+ } });
+ const setSession = vi.fn(() => true);
+ const onAgentSession = vi.fn();
+ const onRuntimeSessionEstablished = vi.fn();
+ const { mgr, onTokenUsage } = managerFor(() => session, {
+ timeline: { ...exactTimelineLifecycleStub(), setSession, resumeSessionId: () => null, forgetSession: () => true },
+ onAgentSession,
+ onRuntimeSessionEstablished,
+ });
+ mgr.deliver("a1", { text: "first" });
+ await session.pushAgentEvent({ type: "session_started", backendSessionId: "stale-backend" });
+ await session.pushAgentEvent({ ...usage, source: "stale-usage" });
+ await drainEventTasks();
+ expect(setSession).not.toHaveBeenCalled();
+ expect(onAgentSession).not.toHaveBeenCalled();
+ expect(onRuntimeSessionEstablished).not.toHaveBeenCalled();
+ expect(onTokenUsage).not.toHaveBeenCalled();
+ await session.fire("runtime_event", { kind: "session_init", sessionId: "current-backend" });
+ await session.pushAgentEvent(usage);
+ await drainEventTasks();
+ expect(setSession).toHaveBeenCalledExactlyOnceWith("a1", "current-backend", "current-instance");
+ expect(onRuntimeSessionEstablished).toHaveBeenCalledOnce();
+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("current-backend");
+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));
+ await mgr.stopAll();
+ });
+
+ it("rejects public events from a still-draining old owner after replacement attachment", async () => {
+ const old = b1Session([], "old", "old-turn", "old-instance");
+ const replacement = b1Session([], "new", "new-turn", "new-instance");
+ const held = holdUsage(old);
+ let rejectClosed!: (error: unknown) => void;
+ const closed = new Promise<AgentSessionResult>((_resolve, reject) => { rejectClosed = reject; });
+ Object.defineProperty(old, "closed", { value: closed });
+ const sessions = [old, replacement];
+ const setSession = vi.fn(() => true);
+ const onAgentSession = vi.fn();
+ const { mgr, onTokenUsage } = managerFor(() => sessions.shift()!, {
+ timeline: { ...exactTimelineLifecycleStub(), setSession, resumeSessionId: () => null, forgetSession: () => true },
+ onAgentSession,
+ });
+ mgr.deliver("a1", { text: "first" });
+ await old.fire("runtime_event", { kind: "session_init", sessionId: "old-backend" });
+ await old.pushAgentEvent(usage);
+ await held.reached;
+ mgr.enqueueRewake("a1", { id: "replacement-command", text: "replacement work" });
+ rejectClosed(new Error("old public closed rejected"));
+ await drainEventTasks();
+ await replacement.fire("runtime_event", { kind: "session_init", sessionId: "new-backend" });
+ const before = mgr.snapshot().agents.a1;
+ held.release();
+ await old.pushAgentEvent({ type: "session_started", backendSessionId: "late-old-backend" });
+ await drainEventTasks();
+ expect(setSession.mock.calls.map((call) => call[1])).toEqual(["old-backend", "new-backend"]);
+ expect(onAgentSession.mock.calls.map(([info]) => info.sessionId)).toEqual(["old-backend", "new-backend"]);
+ expect(onTokenUsage).not.toHaveBeenCalled();
+ expect(mgr.snapshot().agents.a1).toEqual(before);
+ expect(mgr.liveSessionReports()[0]?.sessionId).toBe("new-backend");
+ await replacement.pushAgentEvent(usage);
+ await drainEventTasks();
+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ usage: usage.usage }));
+ await expect(mgr.stopAll()).rejects.toThrow("Agent shutdown did not complete cleanly");
+ });
+
+ it("retains factory rejection as a failed shutdown", async () => {
+ let reject!: (error: unknown) => void;
+ const factory = new Promise<FakeSession>((_resolve, fail) => { reject = fail; });
+ const { mgr } = managerFor(() => factory);
+ mgr.deliver("a1", { text: "first" });
+ const shutdown = mgr.stopAll();
+ const rejected = expect(shutdown).rejects.toThrow("Agent shutdown did not complete cleanly");
+ reject(new Error("factory rejected"));
+ await rejected;
+ });
+
+ it.each(["event_stream", "closed", "stop", "cleanup", "timed_out", "forced"] as const)("keeps %s failure while another owner consumes valid tail usage", async (failure) => {
+ const failed = b1Session([], "failed", "failed-turn", "failed-instance");
+ if (failure === "event_stream") {
+ Object.defineProperty(failed, "events", { value: {
+ maxBufferedBytes: 4_194_304,
+ [Symbol.asyncIterator]: async function* () { throw new Error("stream rejected"); },
+ } });
+ } else if (failure === "closed") {
+ Object.defineProperty(failed, "closed", { value: Promise.reject(new Error("closed rejected")) });
+ } else if (failure === "stop") {
+ const stop = failed.stop.bind(failed);
+ failed.stop = vi.fn(async (input) => {
+ await stop(input);
+ return { status: "failed", requestId: "failed-stop", error: { category: "process", code: "stop_failed", message: "stop failed", retryable: false } };
+ });
+ } else {
+ const error = { category: "process" as const, code: "cleanup_failed", message: "cleanup failed", retryable: false };
+ const result: AgentSessionResult = failure === "forced"
+ ? { outcome: "forced", requested: true, error, cleanup: { status: "released" } }
+ : { outcome: "stopped", requested: true, exitCode: null, signal: null, cleanup: { status: failure === "timed_out" ? "timed_out" : "failed", error } };
+ Object.defineProperty(failed, "closed", { value: Promise.resolve(result) });
+ }
+ const other = b1Session([], "other", "other-turn", "other-instance");
+ const held = holdUsage(other);
+ const { mgr, onTokenUsage } = managerFor(({ agentId }) => agentId === "a1" ? failed : other);
+ mgr.register("a2");
+ mgr.deliver("a1", { text: "first" });
+ mgr.deliver("a2", { text: "second" });
+ await other.pushAgentEvent(usage);
+ await held.reached;
+ const shutdown = mgr.stopAll();
+ const rejected = expect(shutdown).rejects.toThrow("Agent shutdown did not complete cleanly");
+ let settled = false;
+ void shutdown.catch(() => { settled = true; });
+ await other.closed;
+ await drainEventTasks();
+ expect(settled).toBe(false);
+ held.release();
+ await rejected;
+ expect(onTokenUsage).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ agentId: "a2", usage: usage.usage }));
+ });
+});
+
describe("B1 red gate — exact-once terminal matrix", () => {
const requestedCases: Array<{
name: string;
@@ -5121,6 +5496,7 @@ describe("B1 red gate — exact-once terminal matrix", () => {
expect(session.stop).toHaveBeenCalledWith({ reason: "shutdown", forceAfterMs: 2_000 });
expect(shutdownSettled).toBe(false);
+ await session.fire("exit", { reason: "requested", code: 0 });
finishCleanup({
outcome: "stopped",
requested: true,
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index e78fc723e..9b94c4811 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -95,6 +95,11 @@ interface ActiveSpawnState {
timelineTurnOwner: TimelineTurnOwner | null;
pendingDeliverySpans: Map<string, PendingDeliveryTrace>;
}
+interface ManagedSessionLifecycle {
+ session: DaemonAgentSession;
+ done: Promise<void>;
+ stopping?: Promise<void>;
+}
interface TraceRecordBase {
agentId: string;
effects: string[];
@@ -460,6 +465,10 @@ export class AgentProcessManager {
private readonly liveSessions = new Map<string, string>();
private readonly liveBackendIds = new Map<string, BuiltinBackendId>();
private readonly activeSpawnState = new Map<string, ActiveSpawnState>();
+ private readonly sessionLifecycles = new Map<DaemonAgentSession, ManagedSessionLifecycle>();
+ private readonly sessionFactories = new Set<Promise<void>>();
+ private shuttingDown = false;
+ private shutdownPromise?: Promise<void>;
private readonly publishedAgentActivity = new Map<string, AgentActivityState>();
private readonly traceProcessNonce = randomUUID();
private nextSpawnOrdinal = 1;
@@ -542,6 +551,7 @@ export class AgentProcessManager {
launchId?: string;
applyRuntimeConfig?: boolean;
}): void {
+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");
const runtimeConfigAcceptance = launch?.runtimeConfig
? this.acceptRuntimeConfig(agentId, launch.runtimeConfig)
: undefined;
@@ -562,6 +572,7 @@ export class AgentProcessManager {
}
async updateRuntimeConfig(agentId: string, config: RuntimeConfig): Promise<RuntimeConfigUpdateResult> {
+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");
const accepted = this.acceptRuntimeConfig(agentId, config);
if (accepted === "stale" || accepted === "idempotent") return accepted;
const session = this.sessions.get(agentId);
@@ -630,7 +641,7 @@ export class AgentProcessManager {
if (!session) return "saved_for_start";
this.runtimeConfigApplyRunning.add(agentId);
try {
- while (this.sessions.get(agentId) === session) {
+ while (!this.shuttingDown && this.sessions.get(agentId) === session) {
const desired = this.pendingRuntimeConfigUpdates.get(agentId) ?? this.runtimeConfigs.get(agentId);
if (!desired) return "idempotent";
const desiredRevision = desired.runtimeConfigRevision ?? 0;
@@ -692,6 +703,7 @@ export class AgentProcessManager {
await this.stop(agentId);
}
deliver(agentId: string, message: AgentMsg): boolean {
+ if (this.shuttingDown) return false;
const normalized = message.id
? message
: {
@@ -784,6 +796,7 @@ export class AgentProcessManager {
opName: string;
},
): Promise<void> {
+ if (this.shuttingDown) throw new Error("Agent manager is shutting down");
if (opts.forgetSession && !this.forgetSession(agentId, opts.barrierType ?? "reset_session")) {
this.log.error("resume control transition failed; reset aborted", { agentId, barrierType: opts.barrierType });
this.emitErrorAudit(agentId, "reset", "resume_control_update_failed", "Reset aborted because resume control could not be persisted");
@@ -844,10 +857,19 @@ export class AgentProcessManager {
});
}
start(): void {
+ if (this.shuttingDown) return;
if (this.tickTimer) return;
this.tickTimer = setInterval(() => this.dispatch({ type: "tick", nowMs: this.now() }), this.opts.tickIntervalMs);
this.tickTimer.unref?.();
}
+ beginShutdown(): void {
+ // Fence router continuations before WS/proxy cleanup starts awaiting I/O.
+ this.shuttingDown = true;
+ if (this.tickTimer) {
+ clearInterval(this.tickTimer);
+ this.tickTimer = null;
+ }
+ }
async stop(agentId: string): Promise<void> {
const session = this.sessions.get(agentId);
if (!session) return;
@@ -875,18 +897,41 @@ export class AgentProcessManager {
this.log.debug("audit emit failed (turn interrupt)", { agentId, err: String(err) });
}
}
- async stopAll(): Promise<void> {
- if (this.tickTimer) {
- clearInterval(this.tickTimer);
- this.tickTimer = null;
+ stopAll(): Promise<void> {
+ if (this.shutdownPromise) return this.shutdownPromise;
+ this.beginShutdown();
+ this.shutdownPromise = Promise.resolve().then(async () => {
+ const failures: unknown[] = [];
+ for (const agentId of this.activeSpawnState.keys()) {
+ try { this.abortCurrentTurn(agentId, "shutdown"); } catch (error) { failures.push(error); }
+ }
+ for (const lifecycle of this.sessionLifecycles.values()) void this.stopManagedSession(lifecycle);
+ // The synchronous business fence makes this factory set stable. Late
+ // results attach a consumer and stop, without ever starting a prompt.
+ const factories = await Promise.allSettled([...this.sessionFactories]);
+ const lifecycles = [...this.sessionLifecycles.values()];
+ const completed = await Promise.allSettled(lifecycles.flatMap((lifecycle) => [
+ this.stopManagedSession(lifecycle),
+ lifecycle.done,
+ ]));
+ this.sessions.clear();
+ this.sessionLifecycles.clear();
+ failures.push(...[...factories, ...completed]
+ .filter((result): result is PromiseRejectedResult => result.status === "rejected")
+ .map((result) => result.reason));
+ if (failures.length) throw new AggregateError(failures, "Agent shutdown did not complete cleanly");
+ });
+ return this.shutdownPromise;
+ }
+ private stopManagedSession(lifecycle: ManagedSessionLifecycle): Promise<void> {
+ if (!lifecycle.stopping) {
+ lifecycle.stopping = Promise.resolve().then(async () => {
+ const receipt = await lifecycle.session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });
+ if (receipt.status === "failed") throw new Error("Agent session stop failed", { cause: receipt.error });
+ });
+ void lifecycle.stopping.catch(() => {});
}
- const entries = [...this.sessions.entries()];
- for (const [agentId] of entries) this.abortCurrentTurn(agentId, "shutdown");
- await Promise.all(entries.map(async ([, session]) => {
- const receipt = await session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });
- if (receipt.status === "already_stopping") await session.closed;
- }));
- this.sessions.clear();
+ return lifecycle.stopping;
}
snapshot(): ManagerState {
return this.state;
@@ -1305,6 +1350,7 @@ export class AgentProcessManager {
return pending;
}
private applyEffect(effect: ManagerEffect): void {
+ if (this.shuttingDown && (effect.type === "spawn" || effect.type === "send")) return;
switch (effect.type) {
case "spawn":
this.doSpawn(effect.agentId, effect.messages, effect.resumeSessionId);
@@ -1559,6 +1605,7 @@ export class AgentProcessManager {
this.log.info("agent session ended", { agentId, sessionId, reason });
}
private doSpawn(agentId: string, messages: AgentMsg[], resumeSessionId: string | null): void {
+ if (this.shuttingDown) return;
const [initial, ...pending] = messages;
if (!initial) throw new Error(`AgentProcessManager: spawn for ${agentId} has no command`);
this.dispatch({ type: "queue_delivery", agentId, messages: pending });
@@ -1670,6 +1717,7 @@ export class AgentProcessManager {
abortCause: options?.scope === "handshake_timeout" ? "handshake_timeout" : "spawn_failure",
});
this.log.warn("spawn failed", { agentId, runtime: driver.id, reason });
+ if (this.activeSpawnState.get(agentId) !== state) return;
this.opts.onRuntimeSpawnFailed?.(driver.id, reason);
this.emitErrorAudit(
agentId,
@@ -1682,6 +1730,11 @@ export class AgentProcessManager {
clearHandshakeTimer();
if (state.torndown) return;
state.torndown = true;
+ if (this.activeSpawnState.get(agentId) !== state) {
+ if (state.session && this.sessions.get(agentId) === state.session) this.sessions.delete(agentId);
+ this.closeTurn(state, state.activeSpan, { event: "turn_abort", abortCause: "physical_exit" });
+ return;
+ }
const failedStart = result.outcome === "failed_to_start";
if (failedStart) reportSpawnFailure(result.error.code || "failed_to_start");
if (!state.hasEstablished && !failedStart) reportSpawnFailure("pre_handshake_exit");
@@ -1728,6 +1781,11 @@ export class AgentProcessManager {
this.log.warn("ignored event from discarded backend session owner", { agentId, event: event.type });
return;
}
+ if (event.sessionInstanceId !== state.sessionInstanceId
+ || (state.superseded && this.state.agents[agentId]?.execution.sessionInstanceId !== state.sessionInstanceId)) {
+ this.onAgentEvent(agentId, event, driver.id, state);
+ return;
+ }
if (event.type === "session_failed" && !state.hasEstablished) {
reportSpawnFailure(event.error.code || "failed_to_start", { message: event.error.message });
}
@@ -1752,12 +1810,45 @@ export class AgentProcessManager {
this.onAgentEvent(agentId, event, driver.id, state);
};
const attach = (session: DaemonAgentSession) => {
- if (state.torndown || this.activeSpawnState.get(agentId) !== state) {
- void session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });
- return;
- }
state.session = session;
state.sessionInstanceId = session.sessionInstanceId;
+ const eventsDone = (async () => {
+ try {
+ for await (const event of session.events) onEvent(event as ManagedEvent);
+ } catch (error) {
+ reportSpawnFailure("event_stream_failed", { message: String(error) });
+ throw error;
+ }
+ })();
+ const closed = session.closed.catch((error) => {
+ reportSpawnFailure("session_closed_rejected", { message: String(error) });
+ if (this.activeSpawnState.get(agentId) === state) {
+ this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);
+ }
+ throw error;
+ });
+ // closed seals the producer; only the consumer can prove FIFO exhaustion.
+ // Retain this owner until both settle, including explicit failed outcomes.
+ const done = Promise.allSettled([closed, eventsDone]).then(([closedResult, eventsResult]) => {
+ if (closedResult.status === "fulfilled") teardown(closedResult.value);
+ const failures = [closedResult, eventsResult]
+ .filter((result): result is PromiseRejectedResult => result.status === "rejected")
+ .map((result) => result.reason);
+ if (closedResult.status === "fulfilled" && (closedResult.value.cleanup.status === "failed" || closedResult.value.cleanup.status === "timed_out")) {
+ failures.push(closedResult.value.cleanup.error);
+ }
+ if (closedResult.status === "fulfilled" && closedResult.value.outcome === "forced") failures.push(closedResult.value.error);
+ if (failures.length) throw new AggregateError(failures, "Agent session did not finish cleanly");
+ });
+ const lifecycle: ManagedSessionLifecycle = { session, done };
+ this.sessionLifecycles.set(session, lifecycle);
+ void done.then(() => {
+ if (!this.shuttingDown) this.sessionLifecycles.delete(session);
+ }, () => {});
+ if (state.torndown || this.activeSpawnState.get(agentId) !== state || this.shuttingDown) {
+ void this.stopManagedSession(lifecycle);
+ return;
+ }
this.sessions.set(agentId, session);
this.appliedRuntimeConfigs.set(agentId, runtimeConfig);
if (
@@ -1774,17 +1865,6 @@ export class AgentProcessManager {
turnSilence: session.snapshot().diagnostics?.turnSilence,
}, state);
previousOwner?.pendingDeliverySpans.clear();
- void (async () => {
- try {
- for await (const event of session.events) onEvent(event as ManagedEvent);
- } catch (error) {
- reportSpawnFailure("event_stream_failed", { message: String(error) });
- }
- })();
- void session.closed.then(teardown, (error) => {
- reportSpawnFailure("session_closed_rejected", { message: String(error) });
- this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);
- });
const startedSpan = this.openTurn(state);
const commandId = first.id!;
this.beginPendingDelivery(state, commandId, first, "idle", startedSpan, false);
@@ -1807,6 +1887,7 @@ export class AgentProcessManager {
void session.stop({ reason: "shutdown", forceAfterMs: SESSION_STOP_GRACE_MS });
return;
}
+ if (this.shuttingDown) return;
const driverState = session.snapshot().state;
if (this.state.agents[agentId]?.status !== "starting" || driverState === "stopping" || driverState === "closed") return;
try {
@@ -1839,8 +1920,10 @@ export class AgentProcessManager {
(error as { code?: string } | undefined)?.code ?? "open_failed",
{ message: String(error) },
);
- if (this.activeSpawnState.get(agentId) === state) this.activeSpawnState.delete(agentId);
- this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);
+ if (this.activeSpawnState.get(agentId) === state) {
+ this.activeSpawnState.delete(agentId);
+ this.dispatch({ type: "exit", agentId, spawnFailureReason: state.spawnFailureReason }, state);
+ }
};
let created: ReturnType<SessionFactory>;
try {
@@ -1854,7 +1937,12 @@ export class AgentProcessManager {
throw error;
}
if (created && typeof (created as Promise<DaemonAgentSession>).then === "function") {
- void Promise.resolve(created).then(attach, failedOpen);
+ const factory = Promise.resolve(created).then(attach).catch((error) => {
+ failedOpen(error);
+ throw error;
+ });
+ this.sessionFactories.add(factory);
+ void factory.then(() => this.sessionFactories.delete(factory), () => {});
} else {
try {
attach(created as DaemonAgentSession);
diff --git a/src/daemon/src/manager/managerSwitchModel.test.ts b/src/daemon/src/manager/managerSwitchModel.test.ts
index 45fd684e4..242140971 100644
--- a/src/daemon/src/manager/managerSwitchModel.test.ts
+++ b/src/daemon/src/manager/managerSwitchModel.test.ts
@@ -1,4 +1,5 @@
import { describe, it, expect, vi } from "vitest";
+import { setImmediate as drainEventTasks } from "node:timers/promises";
import { AgentProcessManager, type DaemonAgentSession, type SessionFactory, type TimelineRecorder } from "./managerRuntime.js";
import type { AgentBackend as Driver } from "../drivers/index.js";
import type { HostLaunchContext as LaunchContext } from "./hostContext.js";
@@ -40,7 +41,9 @@ function fakeSession(): FakeSession {
type Event = AgentEvent<BuiltinBackendSpecs, "codex">;
const queued: Event[] = [];
const waiters: Array<(value: IteratorResult<Event>) => void> = [];
+ let ended = false;
const publish = (event: Event) => {
+ if (ended) return;
const waiter = waiters.shift();
if (waiter) waiter({ done: false, value: event });
else queued.push(event);
@@ -58,7 +61,9 @@ function fakeSession(): FakeSession {
return {
next: () => queued.length > 0
? Promise.resolve({ done: false as const, value: queued.shift()! })
- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
+ : ended
+ ? Promise.resolve({ done: true as const, value: undefined })
+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
};
},
},
@@ -70,7 +75,10 @@ function fakeSession(): FakeSession {
return { status: "accepted", delivery: "steer", commandId: message.id, turnId: "test-turn" };
},
async interrupt() { return { status: "not_running" }; },
- async stop() { return { status: "accepted", requestId: "test-stop" }; },
+ async stop(input) {
+ if (input.reason === "shutdown") await s.fire("exit");
+ return { status: "accepted", requestId: "test-stop" };
+ },
snapshot() {
return { sessionInstanceId: "switch-model-test", state: "working", queuedCommands: [], lastEventSequence: sequence };
},
@@ -101,10 +109,16 @@ function fakeSession(): FakeSession {
signal: null,
cleanup: { status: "released" },
};
- resolveClosed(result);
+ if (!ended) {
+ publish({ type: "session_closed", result, sequence: ++sequence, sessionInstanceId: "switch-model-test", at: Date.now() });
+ ended = true;
+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });
+ resolveClosed(result);
+ }
}
await Promise.resolve();
await Promise.resolve();
+ if (evt === "exit") await drainEventTasks();
},
};
return s;
diff --git a/src/daemon/src/manager/managerTimeline.test.ts b/src/daemon/src/manager/managerTimeline.test.ts
index 5e0e865b2..bb3dda424 100644
--- a/src/daemon/src/manager/managerTimeline.test.ts
+++ b/src/daemon/src/manager/managerTimeline.test.ts
@@ -1,4 +1,5 @@
import { describe, it, expect } from "vitest";
+import { setImmediate as drainEventTasks } from "node:timers/promises";
import { AgentProcessManager, type TimelineRecorder } from "./managerRuntime";
import type { HostLaunchContext as LaunchContext } from "./hostContext.js";
import type { AgentEvent, AgentSessionResult, BuiltinBackendSpecs } from "@alook/agent-driver";
@@ -26,7 +27,9 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {
let resolveClosed!: (result: AgentSessionResult) => void;
const closed = new Promise<AgentSessionResult>((resolve) => { resolveClosed = resolve; });
let sequence = 0;
+ let ended = false;
const publish = (event: Event) => {
+ if (ended) return;
const waiter = waiters.shift();
if (waiter) waiter({ done: false, value: event });
else queued.push(event);
@@ -47,7 +50,9 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {
return {
next: () => queued.length > 0
? Promise.resolve({ done: false as const, value: queued.shift()! })
- : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
+ : ended
+ ? Promise.resolve({ done: true as const, value: undefined })
+ : new Promise<IteratorResult<Event>>((resolve) => waiters.push(resolve)),
};
},
},
@@ -67,7 +72,10 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {
return { status: "accepted" as const, delivery: "steer" as const, commandId: message.id, turnId: "test-turn" };
},
async interrupt() { return { status: "not_running" as const }; },
- async stop() { return { status: "accepted" as const, requestId: "test-stop" }; },
+ async stop(input) {
+ if (input.reason === "shutdown") await emit("exit");
+ return { status: "accepted" as const, requestId: "test-stop" };
+ },
snapshot() {
return { sessionInstanceId: "timeline-test", state: "working" as const, queuedCommands: [], lastEventSequence: sequence };
},
@@ -89,10 +97,17 @@ function manager(rec: TimelineRecorder, capture: { ctx?: LaunchContext }) {
publish({ ...base, type: "assistant_message_completed", turnId: "test-turn", text: event.text ?? "", truncated: false });
}
} else if (ev === "exit") {
- resolveClosed({ outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } });
+ if (!ended) {
+ const result: AgentSessionResult = { outcome: "crashed", requested: false, exitCode: 0, signal: null, cleanup: { status: "released" } };
+ publish({ type: "session_closed", result, sequence: ++sequence, sessionInstanceId: "timeline-test", at: Date.now() });
+ ended = true;
+ for (const waiter of waiters.splice(0)) waiter({ done: true, value: undefined });
+ resolveClosed(result);
+ }
}
await Promise.resolve();
await Promise.resolve();
+ if (ev === "exit") await drainEventTasks();
};
return { mgr, emit };
}
@@ -200,15 +215,19 @@ describe("manager.resetSession", () => {
sessionFactory: ({ ctx }) => {
let finish!: (result: AgentSessionResult) => void;
const closed = new Promise<AgentSessionResult>((resolve) => { finish = resolve; });
+ const sessionInstanceId = `reset-test-${sessionsCreated.length + 1}`;
const entry: (typeof sessionsCreated)[number] = { ctx, finish: (result) => finish(result), stopCalled: false, sends: [] };
sessionsCreated.push(entry);
return {
backend: "codex" as const,
capabilities: {} as never,
- sessionInstanceId: `reset-test-${sessionsCreated.length}`,
+ sessionInstanceId,
events: {
maxBufferedBytes: 4_194_304 as const,
- async *[Symbol.asyncIterator]() { await new Promise(() => {}); },
+ async *[Symbol.asyncIterator]() {
+ const result = await closed;
+ yield { type: "session_closed" as const, result, sequence: 1, sessionInstanceId, at: Date.now() };
+ },
},
closed,
async start(m: { id: string }) {
@@ -264,8 +283,7 @@ describe("manager.resetSession", () => {
// Fire the session exit → onExit drains rewake into a fresh spawn.
first.finish({ outcome: "stopped", requested: true, exitCode: 0, signal: null, cleanup: { status: "released" } });
- await Promise.resolve();
- await Promise.resolve();
+ await drainEventTasks();
expect(sessionsCreated).toHaveLength(2);
const second = sessionsCreated[1]!;