Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Abort inline turns before shutdown hands back their leases (#2051)

Regan Bellea2013405f1a
diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.ts
index 5decc942e..0cadd7ad3 100644
--- a/src/api/app-helpers.ts
+++ b/src/api/app-helpers.ts
@@ -211,14 +211,21 @@ export function createAppHelpers(deps: AppDeps, app: App) {
run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },
);
}
+ const shutdown = deps.shutdown?.();
+ if (shutdown?.aborted) return { status: "queued", sessionId: run.sessionId, runId };
const claimed = await deps.runs.claimForSession(run.sessionId, "inline", deps.leaseTtlMs);
if (claimed) {
- const result = processRun(
- { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },
- claimed,
- );
- if (claimed.id === runId) return withAdminLink(await result);
- await result.catch((error: unknown) => swallow("inline predecessor run failed", error));
+ try {
+ const result = await processRun(
+ { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },
+ claimed,
+ { shutdown },
+ );
+ if (claimed.id === runId && !shutdown?.aborted) return withAdminLink(result);
+ } catch (error) {
+ if (claimed.id === runId && !shutdown?.aborted) throw error;
+ swallow("inline run did not complete", error);
+ }
continue;
}
const remaining = deadline - performance.now();
diff --git a/src/api/app-types.ts b/src/api/app-types.ts
index 59320b8c5..b8d28ee58 100644
--- a/src/api/app-types.ts
+++ b/src/api/app-types.ts
@@ -618,6 +618,7 @@ export interface App {
export interface AppDeps {
externalSlackPolicies?: ExternalSlackPolicies;
admittedWork?: AdmittedWork;
+ shutdown?: () => AbortSignal;
resourceSearch?: ResourceSearchStore;
swarms?: SwarmService;
identity: IdentityService;
diff --git a/src/config.ts b/src/config.ts
index 60597cf04..ce381c279 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -934,7 +934,7 @@ export const CONFIG_DEFAULTS = {
leaseTtlMs: 120_000,
heartbeatIntervalMs: 10_000,
reaperIntervalMs: 15_000,
- shutdownDrainMs: 10_000,
+ shutdownDrainMs: 30_000,
maxAttempts: 3,
maxClaims: 8,
processReaperIntervalMs: 30_000,
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index 72b63b376..15a027afd 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -69,6 +69,7 @@ export async function processRun(
clearInterval(beat);
};
try {
+ if (opts?.shutdown?.aborted) return { status: "queued", sessionId: run.sessionId, runId: run.id };
if (run.request.swarm) {
const { turnMs } = resolveSwarmSettings({ turnMs: run.request.turnWallClockMs });
workDeadline = setTimeout(() => cancel.abort(), turnMs);
diff --git a/src/wiring.ts b/src/wiring.ts
index 10a4fcc02..3f31ee3c4 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -581,6 +581,7 @@ export function buildApp(
canStart: () => !config.backgroundDeploymentId || backgroundAdmission(),
onAdmitted: () => noteAdmitted(),
});
+ let inlineShutdown = new AbortController();
const createSweeper: typeof createUntrackedSweeper = (work, interval, options) =>
createUntrackedSweeper(() => admittedWork.run(work), interval, options);
if (config.databaseUrl && !config.connectorSecretKey) {
@@ -2102,6 +2103,7 @@ export function buildApp(
const app = createApp({
externalSlackPolicies: config.externalSlackPolicies,
admittedWork,
+ shutdown: () => inlineShutdown.signal,
...(pgArtifactMap ? { resourceSearch: createPostgresResourceSearch(pgArtifactMap.pool) } : {}),
swarms,
identity,
@@ -2734,8 +2736,18 @@ export function buildApp(
backgroundStopping = draining;
return draining;
}
+ async function releaseInFlightRuns(): Promise<void> {
+ inlineShutdown.abort();
+ await Promise.all([
+ withTimeout(() => admittedWork.drained(), 3_000, "inline turn handback").catch(
+ swallowAs("wiring: inline turn handback failed", undefined),
+ ),
+ ...workers.map((w) => w.releaseInFlight()),
+ ]);
+ }
const runtime: Runtime = {
start() {
+ if (inlineShutdown.signal.aborted) inlineShutdown = new AbortController();
flyTunnel?.monitor();
drain.start();
if (config.backgroundWorkEnabled && !config.backgroundDeploymentId) startBackground();
@@ -2753,9 +2765,7 @@ export function buildApp(
await backgroundStopping;
await Promise.all([admittedWork.drained(), ...workers.map((worker) => worker.drained())]);
},
- async releaseInFlightRuns() {
- await Promise.all(workers.map((w) => w.releaseInFlight()));
- },
+ releaseInFlightRuns,
async stop() {
await stopBackground();
await Promise.all([
@@ -2764,7 +2774,7 @@ export function buildApp(
),
...workers.map((w) => w.stop(config.shutdownDrainMs)),
]).catch(swallowAs("wiring: worker drain failed", undefined));
- await Promise.all(workers.map((w) => w.releaseInFlight()));
+ await releaseInFlightRuns();
await drain.stop();
runs.close?.();
void runSignals.close?.();
diff --git a/test/process-run.test.ts b/test/process-run.test.ts
index 3fb074244..0dfcf1eb6 100644
--- a/test/process-run.test.ts
+++ b/test/process-run.test.ts
@@ -489,3 +489,21 @@ test("failed shutdown handback retains the lease for expiry without charging an
assert.equal(retained.errorAttempts, 0);
assert.equal(await runs.claim("replacement", 5_000), null);
});
+
+test("a claim that lands after shutdown began is handed back without starting a turn", async () => {
+ const { runs } = createMemoryRunStore();
+ const enq = (await runs.enqueue({ sessionId: "s1", request: turn, maxAttempts: 1 })).run;
+ const run = await runs.claim("old", 5_000);
+ const shutdown = new AbortController();
+ shutdown.abort();
+ const orchestrator = fakeOrchestrator(async () => {
+ throw new Error("must not be reached");
+ });
+ const result = await processRun({ runs, orchestrator, leaseTtlMs: 5_000 }, run!, { shutdown: shutdown.signal });
+ assert.equal(result.status, "queued");
+ const handedBack = (await runs.get(enq.id))!;
+ assert.equal(handedBack.status, "pending");
+ assert.equal(handedBack.errorAttempts, 0);
+ assert.equal(handedBack.leaseToken, null);
+ assert.ok(await runs.claim("replacement", 5_000));
+});
diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.ts
index 9319d3823..0e377c7fa 100644
--- a/test/worker-reaper.test.ts
+++ b/test/worker-reaper.test.ts
@@ -14,6 +14,7 @@ import { buildApp } from "../src/wiring.ts";
import type { LeaderLease } from "../src/persistence/leader-lease.ts";
import type { OrchestratorInput } from "../src/core/orchestrator.ts";
import type { Principal } from "../src/types.ts";
+import type { RunStore } from "../src/runs/run-store.ts";
import { replayableRequest } from "../src/core/orchestrator/turn-helpers.ts";
import { testConfig } from "./support/test-config.ts";
@@ -329,6 +330,53 @@ test("an uncooperative turn keeps both leases until it actually exits", async ()
assert.ok((await sessions.acquireLease(session.id)).lease);
});
+test("shutdown aborts the blocked step before the handback, and the next worker resumes it exactly once", async () => {
+ const store = createMemoryRunStore();
+ const events: string[] = [];
+ const runs: RunStore = {
+ ...store.runs,
+ async releaseLease(runId, token) {
+ events.push("release");
+ return store.runs.releaseLease(runId, token);
+ },
+ };
+ const blocked = Promise.withResolvers<void>();
+ const orchestrator = {
+ async handleTurn(input: OrchestratorInput) {
+ if (input.attempt === 1) {
+ events.push("step1");
+ blocked.resolve();
+ await new Promise<void>((resolve) => input.cancel!.addEventListener("abort", () => resolve(), { once: true }));
+ events.push("abort");
+ return { status: "silent" as const, stopped: true };
+ }
+ events.push(`resume@${input.attempt}`);
+ events.push("step2");
+ return { status: "ok" as const, reply: "finished" };
+ },
+ } as unknown as Orchestrator;
+ const enq = (await store.runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 1 })).run;
+ const old = createWorker({ runs, orchestrator, leaseTtlMs: 5_000, pollMs: 5, workerId: "old" });
+ old.start();
+ await blocked.promise;
+ await old.stop(1);
+ await old.releaseInFlight();
+ assert.deepEqual(events, ["step1", "abort", "release"]);
+ const handedBack = (await runs.get(enq.id))!;
+ assert.equal(handedBack.status, "pending");
+ assert.equal(handedBack.errorAttempts, 0);
+ assert.equal(handedBack.leaseToken, null);
+
+ const fresh = createWorker({ runs, orchestrator, leaseTtlMs: 5_000, pollMs: 5, workerId: "fresh" });
+ fresh.start();
+ const done = await runs.waitFor(enq.id, 2_000);
+ await fresh.stop();
+ assert.equal(done.status, "done");
+ assert.equal(done.attempts, 2);
+ assert.deepEqual(done.result, { status: "ok", reply: "finished" });
+ assert.deepEqual(events, ["step1", "abort", "release", "resume@2", "step2"]);
+});
+
test("a thrown claim neither kills the worker loop nor blocks the drain handback", async () => {
const store = createMemoryRunStore();
let explode = 2;
@@ -785,3 +833,54 @@ test("web admission and replay preserve analytics exclusions", async (t) => {
await built.runtime.stop();
}
});
+
+test("final shutdown holds an inline turn's lease until the step unwinds, then hands the run back", async () => {
+ const built = buildApp(
+ testConfig({ dataDir: mkdtempSync(join(tmpdir(), "inline-shutdown-")), workers: 1, shutdownDrainMs: 200 }),
+ );
+ const blocked = Promise.withResolvers<void>();
+ const gate = Promise.withResolvers<void>();
+ const append = built.sessions.append.bind(built.sessions);
+ let first = true;
+ built.sessions.append = async (lease, entry) => {
+ if (first && entry.type === "user") {
+ first = false;
+ blocked.resolve();
+ await gate.promise;
+ }
+ return append(lease, entry);
+ };
+ const inline = built.app.turn({
+ surface: "test",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "inline-shutdown" },
+ text: "hello",
+ });
+ try {
+ await blocked.promise;
+ const running = (await built.runs.list()).find((run) => run.status === "running")!;
+ assert.ok(running);
+ let stopped = false;
+ const stopping = built.runtime.stop().then(() => {
+ stopped = true;
+ });
+ await sleep(300);
+ assert.equal(stopped, false, "stop waits for the inline step to unwind");
+ assert.equal((await built.runs.get(running.id))?.leaseToken, running.leaseToken);
+ assert.equal(await built.runs.claim("replacement", 5_000), null);
+ gate.resolve();
+ const result = await inline;
+ await stopping;
+ assert.equal(result.status, "queued");
+ assert.equal(result.runId, running.id);
+ const handedBack = (await built.runs.get(running.id))!;
+ assert.equal(handedBack.status, "pending");
+ assert.equal(handedBack.leaseToken, null);
+ assert.equal(handedBack.errorAttempts, 0);
+ assert.equal(handedBack.attempts, 1);
+ assert.equal((await built.runs.claim("replacement", 5_000))?.id, running.id);
+ } finally {
+ gate.resolve();
+ await inline.catch(() => {});
+ }
+});