Multiplayer agent harness for work.
Abort inline turns before shutdown hands back their leases (#2051)
Regan Bellea2013405f1a
diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.tsindex 5decc942e..0cadd7ad3 100644--- a/src/api/app-helpers.ts+++ b/src/api/app-helpers.ts@@ -211,14 +211,21 @@ export function createAppHelpers(deps: AppDeps, app: App) {run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },);}+ const shutdown = deps.shutdown?.();+ if (shutdown?.aborted) return { status: "queued", sessionId: run.sessionId, runId };const claimed = await deps.runs.claimForSession(run.sessionId, "inline", deps.leaseTtlMs);if (claimed) {- const result = processRun(- { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },- claimed,- );- if (claimed.id === runId) return withAdminLink(await result);- await result.catch((error: unknown) => swallow("inline predecessor run failed", error));+ try {+ const result = await processRun(+ { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },+ claimed,+ { shutdown },+ );+ if (claimed.id === runId && !shutdown?.aborted) return withAdminLink(result);+ } catch (error) {+ if (claimed.id === runId && !shutdown?.aborted) throw error;+ swallow("inline run did not complete", error);+ }continue;}const remaining = deadline - performance.now();diff --git a/src/api/app-types.ts b/src/api/app-types.tsindex 59320b8c5..b8d28ee58 100644--- a/src/api/app-types.ts+++ b/src/api/app-types.ts@@ -618,6 +618,7 @@ export interface App {export interface AppDeps {externalSlackPolicies?: ExternalSlackPolicies;admittedWork?: AdmittedWork;+ shutdown?: () => AbortSignal;resourceSearch?: ResourceSearchStore;swarms?: SwarmService;identity: IdentityService;diff --git a/src/config.ts b/src/config.tsindex 60597cf04..ce381c279 100644--- a/src/config.ts+++ b/src/config.ts@@ -934,7 +934,7 @@ export const CONFIG_DEFAULTS = {leaseTtlMs: 120_000,heartbeatIntervalMs: 10_000,reaperIntervalMs: 15_000,- shutdownDrainMs: 10_000,+ shutdownDrainMs: 30_000,maxAttempts: 3,maxClaims: 8,processReaperIntervalMs: 30_000,diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex 72b63b376..15a027afd 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -69,6 +69,7 @@ export async function processRun(clearInterval(beat);};try {+ if (opts?.shutdown?.aborted) return { status: "queued", sessionId: run.sessionId, runId: run.id };if (run.request.swarm) {const { turnMs } = resolveSwarmSettings({ turnMs: run.request.turnWallClockMs });workDeadline = setTimeout(() => cancel.abort(), turnMs);diff --git a/src/wiring.ts b/src/wiring.tsindex 10a4fcc02..3f31ee3c4 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -581,6 +581,7 @@ export function buildApp(canStart: () => !config.backgroundDeploymentId || backgroundAdmission(),onAdmitted: () => noteAdmitted(),});+ let inlineShutdown = new AbortController();const createSweeper: typeof createUntrackedSweeper = (work, interval, options) =>createUntrackedSweeper(() => admittedWork.run(work), interval, options);if (config.databaseUrl && !config.connectorSecretKey) {@@ -2102,6 +2103,7 @@ export function buildApp(const app = createApp({externalSlackPolicies: config.externalSlackPolicies,admittedWork,+ shutdown: () => inlineShutdown.signal,...(pgArtifactMap ? { resourceSearch: createPostgresResourceSearch(pgArtifactMap.pool) } : {}),swarms,identity,@@ -2734,8 +2736,18 @@ export function buildApp(backgroundStopping = draining;return draining;}+ async function releaseInFlightRuns(): Promise<void> {+ inlineShutdown.abort();+ await Promise.all([+ withTimeout(() => admittedWork.drained(), 3_000, "inline turn handback").catch(+ swallowAs("wiring: inline turn handback failed", undefined),+ ),+ ...workers.map((w) => w.releaseInFlight()),+ ]);+ }const runtime: Runtime = {start() {+ if (inlineShutdown.signal.aborted) inlineShutdown = new AbortController();flyTunnel?.monitor();drain.start();if (config.backgroundWorkEnabled && !config.backgroundDeploymentId) startBackground();@@ -2753,9 +2765,7 @@ export function buildApp(await backgroundStopping;await Promise.all([admittedWork.drained(), ...workers.map((worker) => worker.drained())]);},- async releaseInFlightRuns() {- await Promise.all(workers.map((w) => w.releaseInFlight()));- },+ releaseInFlightRuns,async stop() {await stopBackground();await Promise.all([@@ -2764,7 +2774,7 @@ export function buildApp(),...workers.map((w) => w.stop(config.shutdownDrainMs)),]).catch(swallowAs("wiring: worker drain failed", undefined));- await Promise.all(workers.map((w) => w.releaseInFlight()));+ await releaseInFlightRuns();await drain.stop();runs.close?.();void runSignals.close?.();diff --git a/test/process-run.test.ts b/test/process-run.test.tsindex 3fb074244..0dfcf1eb6 100644--- a/test/process-run.test.ts+++ b/test/process-run.test.ts@@ -489,3 +489,21 @@ test("failed shutdown handback retains the lease for expiry without charging anassert.equal(retained.errorAttempts, 0);assert.equal(await runs.claim("replacement", 5_000), null);});++test("a claim that lands after shutdown began is handed back without starting a turn", async () => {+ const { runs } = createMemoryRunStore();+ const enq = (await runs.enqueue({ sessionId: "s1", request: turn, maxAttempts: 1 })).run;+ const run = await runs.claim("old", 5_000);+ const shutdown = new AbortController();+ shutdown.abort();+ const orchestrator = fakeOrchestrator(async () => {+ throw new Error("must not be reached");+ });+ const result = await processRun({ runs, orchestrator, leaseTtlMs: 5_000 }, run!, { shutdown: shutdown.signal });+ assert.equal(result.status, "queued");+ const handedBack = (await runs.get(enq.id))!;+ assert.equal(handedBack.status, "pending");+ assert.equal(handedBack.errorAttempts, 0);+ assert.equal(handedBack.leaseToken, null);+ assert.ok(await runs.claim("replacement", 5_000));+});diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.tsindex 9319d3823..0e377c7fa 100644--- a/test/worker-reaper.test.ts+++ b/test/worker-reaper.test.ts@@ -14,6 +14,7 @@ import { buildApp } from "../src/wiring.ts";import type { LeaderLease } from "../src/persistence/leader-lease.ts";import type { OrchestratorInput } from "../src/core/orchestrator.ts";import type { Principal } from "../src/types.ts";+import type { RunStore } from "../src/runs/run-store.ts";import { replayableRequest } from "../src/core/orchestrator/turn-helpers.ts";import { testConfig } from "./support/test-config.ts";@@ -329,6 +330,53 @@ test("an uncooperative turn keeps both leases until it actually exits", async ()assert.ok((await sessions.acquireLease(session.id)).lease);});+test("shutdown aborts the blocked step before the handback, and the next worker resumes it exactly once", async () => {+ const store = createMemoryRunStore();+ const events: string[] = [];+ const runs: RunStore = {+ ...store.runs,+ async releaseLease(runId, token) {+ events.push("release");+ return store.runs.releaseLease(runId, token);+ },+ };+ const blocked = Promise.withResolvers<void>();+ const orchestrator = {+ async handleTurn(input: OrchestratorInput) {+ if (input.attempt === 1) {+ events.push("step1");+ blocked.resolve();+ await new Promise<void>((resolve) => input.cancel!.addEventListener("abort", () => resolve(), { once: true }));+ events.push("abort");+ return { status: "silent" as const, stopped: true };+ }+ events.push(`resume@${input.attempt}`);+ events.push("step2");+ return { status: "ok" as const, reply: "finished" };+ },+ } as unknown as Orchestrator;+ const enq = (await store.runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 1 })).run;+ const old = createWorker({ runs, orchestrator, leaseTtlMs: 5_000, pollMs: 5, workerId: "old" });+ old.start();+ await blocked.promise;+ await old.stop(1);+ await old.releaseInFlight();+ assert.deepEqual(events, ["step1", "abort", "release"]);+ const handedBack = (await runs.get(enq.id))!;+ assert.equal(handedBack.status, "pending");+ assert.equal(handedBack.errorAttempts, 0);+ assert.equal(handedBack.leaseToken, null);++ const fresh = createWorker({ runs, orchestrator, leaseTtlMs: 5_000, pollMs: 5, workerId: "fresh" });+ fresh.start();+ const done = await runs.waitFor(enq.id, 2_000);+ await fresh.stop();+ assert.equal(done.status, "done");+ assert.equal(done.attempts, 2);+ assert.deepEqual(done.result, { status: "ok", reply: "finished" });+ assert.deepEqual(events, ["step1", "abort", "release", "resume@2", "step2"]);+});+test("a thrown claim neither kills the worker loop nor blocks the drain handback", async () => {const store = createMemoryRunStore();let explode = 2;@@ -785,3 +833,54 @@ test("web admission and replay preserve analytics exclusions", async (t) => {await built.runtime.stop();}});++test("final shutdown holds an inline turn's lease until the step unwinds, then hands the run back", async () => {+ const built = buildApp(+ testConfig({ dataDir: mkdtempSync(join(tmpdir(), "inline-shutdown-")), workers: 1, shutdownDrainMs: 200 }),+ );+ const blocked = Promise.withResolvers<void>();+ const gate = Promise.withResolvers<void>();+ const append = built.sessions.append.bind(built.sessions);+ let first = true;+ built.sessions.append = async (lease, entry) => {+ if (first && entry.type === "user") {+ first = false;+ blocked.resolve();+ await gate.promise;+ }+ return append(lease, entry);+ };+ const inline = built.app.turn({+ surface: "test",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "inline-shutdown" },+ text: "hello",+ });+ try {+ await blocked.promise;+ const running = (await built.runs.list()).find((run) => run.status === "running")!;+ assert.ok(running);+ let stopped = false;+ const stopping = built.runtime.stop().then(() => {+ stopped = true;+ });+ await sleep(300);+ assert.equal(stopped, false, "stop waits for the inline step to unwind");+ assert.equal((await built.runs.get(running.id))?.leaseToken, running.leaseToken);+ assert.equal(await built.runs.claim("replacement", 5_000), null);+ gate.resolve();+ const result = await inline;+ await stopping;+ assert.equal(result.status, "queued");+ assert.equal(result.runId, running.id);+ const handedBack = (await built.runs.get(running.id))!;+ assert.equal(handedBack.status, "pending");+ assert.equal(handedBack.leaseToken, null);+ assert.equal(handedBack.errorAttempts, 0);+ assert.equal(handedBack.attempts, 1);+ assert.equal((await built.runs.claim("replacement", 5_000))?.id, running.id);+ } finally {+ gate.resolve();+ await inline.catch(() => {});+ }+});