Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(team): end every run's record before shutdownAllTeamRuns resolves

Jack Rudenkof34725adc869
diff --git a/packages/cli/src/team-orchestrator.ts b/packages/cli/src/team-orchestrator.ts
index 8fed5cb4..9df8277e 100644
--- a/packages/cli/src/team-orchestrator.ts
+++ b/packages/cli/src/team-orchestrator.ts
@@ -314,8 +314,19 @@ interface LiveTeamRun {
/** the in-memory object `updateModelStatus` serialises to status.json */
status: TeamStatus;
slots: Map<string, SlotEntry>;
+ /** resolves once `done` has ended the run's record (`onSettled`), whatever ended it */
+ settled: Promise<void>;
}
+/** A `startModels` still before its registry entry (credentials, the staggered spawn loop). */
+interface StartFlight {
+ /** set by shutdown: the loop starts no further pane and unwinds through its `catch` */
+ aborted: boolean;
+}
+
+/** Every `startModels` in progress, to the promise it returns (§20.3 item 8). */
+const startsInFlight = new Map<StartFlight, Promise<unknown>>();
+
/**
* Runs this process started, keyed by run_id (CA-13). An ACTIVE run stays until it
* settles; a SETTLED run stays listable and capturable for SETTLED_RUN_RETENTION_MS,
@@ -669,21 +680,29 @@ export function teamLiveMaps(
return { idle, activity, liveBytes, endRecordMissing };
}
+function cancelEverySlot(): void {
+ for (const run of teamRuns.values()) for (const e of run.slots.values()) e.session?.cancel();
+}
+
/**
- * Settle every live team run CANCELLED. Process shutdown only: the records end
- * `cancelled` because each run's `done` resolves on these transitions; the pane registry's
- * `reapAllPanes` does the killing.
+ * Settle every team run CANCELLED and wait until each one's record has ENDED (§20.3
+ * item 8). Process shutdown only; the pane registry's `reapAllPanes` does the killing.
+ *
+ * A run still in its spawn loop is not in `teamRuns` yet: it is told to stop spawning,
+ * unwinds through its own `catch` (its handler writes `start-failed`), and is awaited. A
+ * run whose loop finished meanwhile registers, so the cancel pass repeats until no start
+ * is in flight. Then every run's `settled` — the end of `done`, after `onSettled` — is
+ * awaited, so no record is left without its end on a clean shutdown.
*/
export async function shutdownAllTeamRuns(): Promise<void> {
- const pending: Promise<unknown>[] = [];
- for (const run of teamRuns.values()) {
- for (const e of run.slots.values()) {
- if (!e.session) continue;
- e.session.cancel();
- pending.push(e.session.terminal.catch(() => undefined));
- }
+ for (const f of startsInFlight.keys()) f.aborted = true;
+ for (;;) {
+ cancelEverySlot();
+ if (startsInFlight.size === 0) break;
+ await Promise.allSettled([...startsInFlight.values()]);
}
- await Promise.all(pending);
+ cancelEverySlot();
+ await Promise.allSettled([...teamRuns.values()].map((r) => r.settled));
}
/** @internal tests: forget every retained run. */
@@ -1107,9 +1126,27 @@ const CANCELLED_DETAIL =
* Each slot receives input.md (typed when it is one plain line, else as a task file it
* is told to Read) and its answer is written to response-{ID}.md.
*/
-export async function startModels(
+export function startModels(sessionPath: string, opts: TeamRunOptions = {}): Promise<TeamHandle> {
+ const flight: StartFlight = { aborted: false };
+ const p = startModelsIn(flight, sessionPath, opts);
+ startsInFlight.set(flight, p);
+ const forget = () => {
+ startsInFlight.delete(flight);
+ };
+ p.then(forget, forget);
+ return p;
+}
+
+/** The stagger before slot `i` (X-L8), then the shutdown check: no pane starts after it. */
+async function beforeSpawn(i: number, flight: StartFlight): Promise<void> {
+ if (i > 0) await new Promise((r) => setTimeout(r, BOOT_STAGGER_MS));
+ if (flight.aborted) throw new Error("cancelled: the server is shutting down");
+}
+
+async function startModelsIn(
+ flight: StartFlight,
sessionPath: string,
- opts: TeamRunOptions = {}
+ opts: TeamRunOptions
): Promise<TeamHandle> {
assertValidRequirePattern(opts.requirePattern);
const claudeFlags = opts.claudeFlags ?? [];
@@ -1391,7 +1428,7 @@ export async function startModels(
try {
let i = 0;
for (const [id, entry] of Object.entries(manifest.models)) {
- if (i++ > 0) await new Promise((r) => setTimeout(r, BOOT_STAGGER_MS));
+ await beforeSpawn(i++, flight);
// Spawn with the parent-resolved explicit spec when there is one, so the child
// skips routing entirely and finds its key in the inherited env. ABSENT from the
// map means "spawn it bare". The manifest keeps `entry.model` (the user's string)
@@ -1531,6 +1568,10 @@ export async function startModels(
// The registry entry exists from here, before the D9 `ready` await, so a run is
// listable and cancellable once all of its panes exist (§3.2, §20.3).
+ let resolveSettled: () => void = () => {};
+ const settled = new Promise<void>((r) => {
+ resolveSettled = r;
+ });
run = {
runId,
path,
@@ -1539,6 +1580,7 @@ export async function startModels(
finishedAt: null,
status: statusCache,
slots: entries,
+ settled,
};
teamRuns.set(runId, run);
newestRunByPath.set(path, runId);
@@ -1581,6 +1623,7 @@ export async function startModels(
} catch {
// A settle consumer must never be able to fail the run.
}
+ resolveSettled();
}
return statusCache;
})();
diff --git a/packages/cli/src/team-shutdown.test.ts b/packages/cli/src/team-shutdown.test.ts
new file mode 100644
index 00000000..82b7716b
--- /dev/null
+++ b/packages/cli/src/team-shutdown.test.ts
@@ -0,0 +1,106 @@
+/**
+ * Process shutdown with team runs in flight (§20.3 item 8): `shutdownAllTeamRuns` ends
+ * every run's RECORD before it resolves — including a run still in its D9 window and one
+ * still in its staggered spawn loop, which is not in the registry yet.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { livePaneCount } from "./pane/pane-registry.js";
+import {
+ type TeamStatus,
+ getStatus,
+ setupSession,
+ shutdownAllTeamRuns,
+ startModels,
+ teamRunRow,
+} from "./team-orchestrator.js";
+import {
+ MAGMUX,
+ NO_MAGMUX_MESSAGE,
+ type PaneTestEnv,
+ finishPaneTest,
+ makePaneTestEnv,
+ paneRunOptions,
+ teamDirOf,
+ waitUntil,
+} from "./test-helpers/team-pane.js";
+
+if (!MAGMUX) console.warn(NO_MAGMUX_MESSAGE);
+
+const T = 30_000;
+let t: PaneTestEnv;
+
+beforeEach(() => {
+ t = makePaneTestEnv();
+});
+
+afterEach(async () => {
+ await shutdownAllTeamRuns();
+ const report = await finishPaneTest(t);
+ expect(report).toEqual({ processes: [], files: [] });
+});
+
+describe.skipIf(!MAGMUX)("shutdownAllTeamRuns", () => {
+ it(
+ "a run in its D9 window: every slot CANCELLED and onSettled has run before shutdown resolves",
+ async () => {
+ const sessionPath = teamDirOf(t);
+ setupSession(sessionPath, ["fake-slow_boot-1", "fake-slow_boot-2"], "Answer.");
+ let settled: TeamStatus | null = null;
+ const start = startModels(
+ sessionPath,
+ paneRunOptions(
+ t,
+ {
+ onSettled: (s) => {
+ settled = structuredClone(s);
+ },
+ },
+ { FAKE_GAP_MS_BOOT: "8000" }
+ )
+ );
+ // registered (both panes exist) but still STARTING: start has not returned yet (D9)
+ await waitUntil(() => teamRunRow(sessionPath) !== null, "the run registered", 10_000);
+ await shutdownAllTeamRuns();
+ expect(settled).not.toBeNull();
+ const s = settled as unknown as TeamStatus;
+ for (const m of Object.values(s.models)) {
+ expect(m.state).toBe("CANCELLED");
+ expect(m.error?.reason).toBe("cancelled");
+ }
+ await start;
+ },
+ T
+ );
+
+ it(
+ "a run still in its spawn loop: no further pane starts, the start rejects, nothing is left",
+ async () => {
+ const sessionPath = teamDirOf(t);
+ const models = ["fake-hang-1", "fake-hang-2", "fake-hang-3", "fake-hang-4"];
+ setupSession(sessionPath, models, "Answer.");
+ const start = startModels(sessionPath, paneRunOptions(t));
+ const outcome = start.then(
+ () => "resolved",
+ (e: unknown) => String(e)
+ );
+ // the first slot is up; the loop is staggering towards the second
+ await waitUntil(
+ () => Object.values(getStatus(sessionPath).models).some((m) => m.startedAt !== null),
+ "the first slot spawned",
+ 10_000
+ );
+ await shutdownAllTeamRuns();
+ expect(await outcome).toMatch(/cancelled: the server is shutting down/);
+ expect(teamRunRow(sessionPath)).toBeNull();
+ // a slot that was spawned has a startedAt; the unwind leaves its row as it was
+ const started = Object.values(getStatus(sessionPath).models).filter(
+ (m) => m.startedAt !== null
+ ).length;
+ expect(started).toBeGreaterThan(0);
+ expect(started).toBeLessThan(models.length);
+ await waitUntil(() => livePaneCount(t.sockRoot) === 0, "every started pane reaped", 10_000);
+ },
+ T
+ );
+});