Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Separate worker claim relinquishment from full drain (#1285)

Regan Bellf0ac0afb78b1
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index b89d55cbb..4c7ddc9e3 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -117,6 +117,8 @@ export interface WorkerDeps extends ProcessDeps {
export interface Worker {
start(): void;
+ stopClaims(): Promise<void>;
+ drained(): Promise<void>;
stop(drainMs?: number): Promise<void>;
releaseInFlight(): Promise<void>;
busy(): boolean;
@@ -150,6 +152,7 @@ export function createWorker(deps: WorkerDeps): Worker {
}
let stopped = false;
let loopDone: Promise<void> | null = null;
+ let claimDone: Promise<void> | null = null;
let inFlight: { runId: string; leaseToken: string; threadRef: string } | null = null;
let releasedLeaseToken: string | null = null;
let releasing: Promise<void> | null = null;
@@ -163,10 +166,16 @@ export function createWorker(deps: WorkerDeps): Worker {
}
const observed = generation;
let run: Run | null;
+ let claimed!: () => void;
+ claimDone = new Promise<void>((resolve) => {
+ claimed = resolve;
+ });
try {
run = await deps.runs.claim(workerId, deps.leaseTtlMs);
claimFailures = 0;
} catch (e) {
+ claimed();
+ claimDone = null;
claimFailures += 1;
if (claimFailures >= CLAIM_FAIL_CRASH_CONSECUTIVE) throw e;
swallow("worker: claim failed (transient, retrying)", e);
@@ -174,6 +183,8 @@ export function createWorker(deps: WorkerDeps): Worker {
continue;
}
if (!run) {
+ claimed();
+ claimDone = null;
await waitForWork(observed);
continue;
}
@@ -182,10 +193,14 @@ export function createWorker(deps: WorkerDeps): Worker {
await deps.runs
.releaseLease(run.id, run.leaseToken)
.catch((e) => swallow("worker: post-stop claim handback failed", e));
+ claimed();
+ claimDone = null;
break;
}
inFlight =
run.leaseToken !== null ? { runId: run.id, leaseToken: run.leaseToken, threadRef: run.sessionId } : null;
+ claimed();
+ claimDone = null;
deps.onClaimed?.();
try {
await processRun(deps, run, { background: true });
@@ -197,6 +212,14 @@ export function createWorker(deps: WorkerDeps): Worker {
}
}
+ function stopClaims(): Promise<void> {
+ stopped = true;
+ unsubscribe?.();
+ unsubscribe = undefined;
+ notify();
+ return claimDone ?? Promise.resolve();
+ }
+
return {
start() {
if (loopDone) return;
@@ -205,7 +228,9 @@ export function createWorker(deps: WorkerDeps): Worker {
pollMs,
onResync: notify,
});
- loopDone = loop();
+ loopDone = loop().finally(() => {
+ loopDone = null;
+ });
},
busy() {
return inFlight !== null;
@@ -230,13 +255,11 @@ export function createWorker(deps: WorkerDeps): Worker {
})();
return releasing;
},
+ stopClaims,
+ drained: () => loopDone ?? Promise.resolve(),
async stop(drainMs = STOP_DRAIN_MS) {
- stopped = true;
- unsubscribe?.();
- unsubscribe = undefined;
- notify();
+ void stopClaims();
await Promise.race([loopDone, sleep(drainMs, { unref: true })]);
- loopDone = null;
},
};
}
diff --git a/test/worker-reaper.test.ts b/test/worker-reaper.test.ts
index 0fe07a82f..7ecb4dae3 100644
--- a/test/worker-reaper.test.ts
+++ b/test/worker-reaper.test.ts
@@ -532,3 +532,89 @@ test("runtime.start() leaves queued runs idle when background work is disabled",
await built.runtime.stop();
}
});
+
+test("timed-out stop cannot resurrect a worker while its previous turn drains", async () => {
+ const { runs } = createMemoryRunStore();
+ const sessions = createMemorySessionStore();
+ const first = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;
+ const { orchestrator, started, unblock } = gatedOrchestrator();
+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, pollMs: 5 });
+ worker.start();
+ await started;
+ await worker.stop(5);
+ const second = (await runs.enqueue({ sessionId: "t2", request: turn, maxAttempts: 3 })).run;
+ worker.start();
+ await sleep(30);
+ assert.equal((await runs.get(second.id))?.status, "pending");
+ unblock();
+ await worker.drained();
+ assert.equal((await runs.get(first.id))?.status, "done");
+ assert.equal((await runs.get(second.id))?.status, "pending");
+ worker.start();
+ await sleep(30);
+ await worker.stop();
+ assert.equal((await runs.get(second.id))?.status, "done");
+});
+
+test("stopClaims relinquishes new work while keeping an active turn and its heartbeats alive", async () => {
+ const { runs } = createMemoryRunStore();
+ const sessions = createMemorySessionStore();
+ const first = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;
+ const { orchestrator, started, unblock } = gatedOrchestrator();
+ let beats = 0;
+ const heartbeat = runs.heartbeat.bind(runs);
+ runs.heartbeat = (...args) => {
+ beats++;
+ return heartbeat(...args);
+ };
+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, heartbeatIntervalMs: 5, pollMs: 5 });
+ worker.start();
+ await started;
+ await worker.stopClaims();
+ let drained = false;
+ const draining = worker.drained().then(() => {
+ drained = true;
+ });
+ await sleep(30);
+ assert.equal(drained, false);
+ assert.ok(beats > 0);
+ assert.equal((await runs.get(first.id))?.status, "running");
+ unblock();
+ await draining;
+ assert.equal((await runs.get(first.id))?.status, "done");
+});
+
+test("stopClaims waits for an outstanding claim to be handed back before acknowledging", async () => {
+ const { runs } = createMemoryRunStore();
+ const sessions = createMemorySessionStore();
+ const pending = (await runs.enqueue({ sessionId: "t1", request: turn, maxAttempts: 3 })).run;
+ const claim = runs.claim.bind(runs);
+ const gate = Promise.withResolvers<void>();
+ const claiming = Promise.withResolvers<void>();
+ runs.claim = async (...args) => {
+ claiming.resolve();
+ await gate.promise;
+ return claim(...args);
+ };
+ let turns = 0;
+ const orchestrator = {
+ handleTurn: async () => {
+ turns++;
+ return { status: "ok", reply: "unexpected" };
+ },
+ } as unknown as Orchestrator;
+ const worker = createWorker({ runs, sessions, orchestrator, leaseTtlMs: 5_000, pollMs: 5 });
+ worker.start();
+ await claiming.promise;
+ let relinquished = false;
+ const stopping = worker.stopClaims().then(() => {
+ relinquished = true;
+ });
+ await sleep(10);
+ assert.equal(relinquished, false);
+ gate.resolve();
+ await stopping;
+ await worker.drained();
+ assert.equal(turns, 0);
+ assert.equal((await runs.get(pending.id))?.status, "pending");
+});