Multiplayer agent harness for work.
Revert the wake-one-worker half of the run-claim change, keep OFFSET 0 (#2526)
Regan Bellf5d40e73f397
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex a931a69fe..1f3dcaaea 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -197,19 +197,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla],);- const availabilityListeners = new Map<() => boolean | void, number>();- let availabilityOff: (() => void) | undefined;- let wakeCursor = 0;- function wakeOne(): void {- const listeners = [...availabilityListeners.keys()];- for (let i = 0; i < listeners.length; i++) {- const at = (wakeCursor + i) % listeners.length;- if (listeners[at]!() === true) {- wakeCursor = at + 1;- return;- }- }- }+ const availabilityListeners = new Map<() => void, number>();let availabilityTimer: ReturnType<typeof setTimeout> | undefined;let availabilityProbe: Promise<void> | null = null;let availabilityClosed = false;@@ -230,7 +218,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla[Date.now()],).then(({ rows }) => {- if (rows[0]?.available) wakeOne();+ if (rows[0]?.available) for (const listener of availabilityListeners.keys()) listener();}).catch((error: unknown) => swallow("run availability probe", error)).finally(() => {@@ -317,9 +305,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla) RETURNING *`,[token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],);- if (!rows[0]) return null;- if (runId === undefined && sessionId === undefined) wakeOne();- return rowToRun(rows[0]);+ return rows[0] ? rowToRun(rows[0]) : null;} catch (err) {if (isUniqueViolation(err)) return null;throw err;@@ -331,15 +317,12 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaavailabilityListeners.set(listener, options?.pollMs ?? 50);clearTimeout(availabilityTimer);availabilityTimer = undefined;- availabilityOff ??= available.subscribe(wakeOne);- const off = available.subscribe(() => {}, options);+ const off = available.subscribe(listener, options);watchAvailability();return () => {off();availabilityListeners.delete(listener);if (availabilityListeners.size === 0) {- availabilityOff?.();- availabilityOff = undefined;clearTimeout(availabilityTimer);availabilityTimer = undefined;}@@ -682,8 +665,6 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaavailabilityClosed = true;clearTimeout(availabilityTimer);availabilityListeners.clear();- availabilityOff?.();- availabilityOff = undefined;await availabilityProbe;await available.close?.();await closePool();diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 39f5b2887..4a8b82748 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -91,7 +91,7 @@ export interface EnqueueResult {export interface RunStore {readonly maxClaims?: number;- subscribeAvailable?(listener: () => boolean | void, options?: SubscribeOptions & { pollMs?: number }): () => void;+ subscribeAvailable?(listener: () => void, options?: SubscribeOptions & { pollMs?: number }): () => void;enqueue(input: EnqueueInput): Promise<EnqueueResult>;getByDedupKey(dedupKey: string): Promise<Run | null>;diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex d612fd849..677877356 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -189,11 +189,9 @@ export function createWorker(deps: WorkerDeps): Worker {let generation = 0;let wake: (() => void) | null = null;let unsubscribe: (() => void) | undefined;- const notify = (): boolean => {+ const notify = (): void => {generation++;- if (!wake) return false;- wake();- return true;+ wake?.();};async function waitForWork(observed: number): Promise<void> {if (stopped || observed !== generation) return;diff --git a/test/run-availability.ts b/test/run-availability.tsindex d479b2ca6..d3c68421b 100644--- a/test/run-availability.ts+++ b/test/run-availability.ts@@ -134,15 +134,10 @@ test(const reader = createPostgresRunStore(url);let notifications = 0;let resyncs = 0;- const off = reader.runs.subscribeAvailable!(- () => {- notifications++;- },- {- pollMs: 60_000,- onResync: () => resyncs++,- },- );+ const off = reader.runs.subscribeAvailable!(() => notifications++, {+ pollMs: 60_000,+ onResync: () => resyncs++,+ });const ids: string[] = [];try {await writer.runs.list();@@ -358,110 +353,3 @@ test(}},);--test(- "Postgres wakes one idle worker per notification and a backlog still fans out",- { skip: !process.env.DATABASE_URL },- async () => {- const { url, cleanup } = await isolatedPostgres();- const writer = createPostgresRunStore(url);- const reader = createPostgresRunStore(url);- const direct = new pg.Pool({ connectionString: url });- const claim = reader.runs.claim.bind(reader.runs);- let claims = 0;- let claiming = 0;- reader.runs.claim = async (...args) => {- claims++;- claiming++;- try {- return await claim(...args);- } finally {- claiming--;- }- };- let resyncs = 0;- const subscribe = reader.runs.subscribeAvailable!.bind(reader.runs);- reader.runs.subscribeAvailable = (listener, options) =>- subscribe(listener, {- ...options,- onResync: () => {- resyncs++;- options?.onResync?.();- },- });- const handled: string[] = [];- let releaseHeld!: () => void;- const held = new Promise<void>((resolve) => {- releaseHeld = resolve;- });- const workers = Array.from({ length: 12 }, () =>- createWorker({- runs: reader.runs,- orchestrator: {- handleTurn: async (input: OrchestratorInput) => {- handled.push(input.conversation.threadRef);- if (input.text === "hold") await held;- return { status: "ok", sessionId: input.conversation.threadRef };- },- } as unknown as Orchestrator,- leaseTtlMs: 5_000,- pollMs: 60_000,- }),- );- const enqueue = (sessionId: string, text = "go") =>- writer.runs.enqueue({- sessionId,- request: { ...request, text, conversation: { ...request.conversation, threadRef: sessionId } },- });- try {- await writer.runs.list();- for (const worker of workers) worker.start();- await until(() => resyncs >= workers.length);- await until(() => claiming === 0);- await sleep(100);- let before = claims;- await enqueue("single");- await until(() => handled.includes("single"));- await sleep(200);- assert.ok(- claims - before <= 4,- `one run costs a handful of claims, not one per idle worker (${claims - before})`,- );-- before = claims;- const sessions = Array.from({ length: 6 }, (_, i) => `backlog-${i}`);- const startedAt = Date.now();- for (const sessionId of sessions)- await direct.query(- "INSERT INTO runs(id, session_id, status, request, created_at) VALUES($1,$2,'pending',$3,$4)",- [- randomUUID(),- sessionId,- JSON.stringify({- ...request,- text: "hold",- conversation: { ...request.conversation, threadRef: sessionId },- }),- Date.now(),- ],- );- await direct.query("SELECT pg_notify('qm_run_available', 'null')");- await until(() => sessions.every((sessionId) => handled.includes(sessionId)));- assert.ok(- Date.now() - startedAt < 2_000,- "one signal drains a backlog across idle workers without recovery polls",- );- const { rows } = await direct.query(- "SELECT count(DISTINCT worker_id)::int AS n FROM runs WHERE status='running'",- );- assert.equal(rows[0].n, sessions.length, "each backlog run is held by its own worker");- } finally {- releaseHeld();- await Promise.all(workers.map((worker) => worker.stop()));- await reader.close();- await writer.close();- await direct.end();- await cleanup();- }- },-);