Multiplayer agent harness for work.
Separate worker claim relinquishment from full drain (#1285)
Regan Bellf0ac0afb78b1
diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex b89d55cbb..4c7ddc9e3 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -117,6 +117,8 @@ export interface WorkerDeps extends ProcessDeps {export interface Worker {start(): void;+ stopClaims(): Promise<void>;+ drained(): Promise<void>;stop(drainMs?: number): Promise<void>;releaseInFlight(): Promise<void>;busy(): boolean;@@ -150,6 +152,7 @@ export function createWorker(deps: WorkerDeps): Worker {}let stopped = false;let loopDone: Promise<void> | null = null;+ let claimDone: Promise<void> | null = null;let inFlight: { runId: string; leaseToken: string; threadRef: string } | null = null;let releasedLeaseToken: string | null = null;let releasing: Promise<void> | null = null;@@ -163,10 +166,16 @@ export function createWorker(deps: WorkerDeps): Worker {}const observed = generation;let run: Run | null;+ let claimed!: () => void;+ claimDone = new Promise<void>((resolve) => {+ claimed = resolve;+ });try {run = await deps.runs.claim(workerId, deps.leaseTtlMs);claimFailures = 0;} catch (e) {+ claimed();+ claimDone = null;claimFailures += 1;if (claimFailures >= CLAIM_FAIL_CRASH_CONSECUTIVE) throw e;swallow("worker: claim failed (transient, retrying)", e);@@ -174,6 +183,8 @@ export function createWorker(deps: WorkerDeps): Worker {continue;}if (!run) {+ claimed();+ claimDone = null;await waitForWork(observed);continue;}@@ -182,10 +193,14 @@ export function createWorker(deps: WorkerDeps): Worker {await deps.runs.releaseLease(run.id, run.leaseToken).catch((e) => swallow("worker: post-stop claim handback failed", e));+ claimed();+ claimDone = null;break;}inFlight =run.leaseToken !== null ? { runId: run.id, leaseToken: run.leaseToken, threadRef: run.sessionId } : null;+ claimed();+ claimDone = null;deps.onClaimed?.();try {await processRun(deps, run, { background: true });@@ -197,6 +212,14 @@ export function createWorker(deps: WorkerDeps): Worker {}}+ function stopClaims(): Promise<void> {+ stopped = true;+ unsubscribe?.();+ unsubscribe = undefined;+ notify();+ return claimDone ?? Promise.resolve();+ }+return {start() {if (loopDone) return;@@ -205,7 +228,9 @@ export function createWorker(deps: WorkerDeps): Worker {pollMs,onResync: notify,});- loopDone = loop();+ loopDone = loop().finally(() => {+ loopDone = null;+ });},busy() {return inFlight !== null;@@ -230,13 +255,11 @@ export function createWorker(deps: WorkerDeps): Worker {})();return releasing;},+ stopClaims,+ drained: () => loopDone ?? Promise.resolve(),async stop(drainMs = STOP_DRAIN_MS) {- stopped = true;- unsubscribe?.();- unsubscribe = undefined;- notify();+ void stopClaims();await Promise.race([loopDone, sleep(drainMs, { unref: true })]);- loopDone = null;},};}diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.tsindex 0fe07a82f..7ecb4dae3 100644--- a/test/worker-reaper.test.ts+++ b/test/worker-reaper.test.ts@@ -532,3 +532,89 @@ test("runtime.start() leaves queued runs idle when background work is disabled",await built.runtime.stop();}});++test("timed-out stop cannot resurrect a worker while its previous turn drains", async () => {+ const { runs } = createMemoryRunStore();+ const sessions = createMemorySessionStore();+ const first = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;+ const { orchestrator, started, unblock } = gatedOrchestrator();+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, pollMs: 5 });+ worker.start();+ await started;+ await worker.stop(5);+ const second = (await runs.enqueue({ sessionId: "t2", request: turn, maxAttempts: 3 })).run;+ worker.start();+ await sleep(30);+ assert.equal((await runs.get(second.id))?.status, "pending");+ unblock();+ await worker.drained();+ assert.equal((await runs.get(first.id))?.status, "done");+ assert.equal((await runs.get(second.id))?.status, "pending");+ worker.start();+ await sleep(30);+ await worker.stop();+ assert.equal((await runs.get(second.id))?.status, "done");+});++test("stopClaims relinquishes new work while keeping an active turn and its heartbeats alive", async () => {+ const { runs } = createMemoryRunStore();+ const sessions = createMemorySessionStore();+ const first = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;+ const { orchestrator, started, unblock } = gatedOrchestrator();+ let beats = 0;+ const heartbeat = runs.heartbeat.bind(runs);+ runs.heartbeat = (...args) => {+ beats++;+ return heartbeat(...args);+ };+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, heartbeatIntervalMs: 5, pollMs: 5 });+ worker.start();+ await started;+ await worker.stopClaims();+ let drained = false;+ const draining = worker.drained().then(() => {+ drained = true;+ });+ await sleep(30);+ assert.equal(drained, false);+ assert.ok(beats > 0);+ assert.equal((await runs.get(first.id))?.status, "running");+ unblock();+ await draining;+ assert.equal((await runs.get(first.id))?.status, "done");+});++test("stopClaims waits for an outstanding claim to be handed back before acknowledging", async () => {+ const { runs } = createMemoryRunStore();+ const sessions = createMemorySessionStore();+ const pending = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;+ const claim = runs.claim.bind(runs);+ const gate = Promise.withResolvers<void>();+ const claiming = Promise.withResolvers<void>();+ runs.claim = async (...args) => {+ claiming.resolve();+ await gate.promise;+ return claim(...args);+ };+ let turns = 0;+ const orchestrator = {+ handleTurn: async () => {+ turns++;+ return { status: "ok", reply: "unexpected" };+ },+ } as unknown as Orchestrator;+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, pollMs: 5 });+ worker.start();+ await claiming.promise;+ let relinquished = false;+ const stopping = worker.stopClaims().then(() => {+ relinquished = true;+ });+ await sleep(10);+ assert.equal(relinquished, false);+ gate.resolve();+ await stopping;+ await worker.drained();+ assert.equal(turns, 0);+ assert.equal((await runs.get(pending.id))?.status, "pending");+});