Multiplayer agent harness for work.
Wake run workers through PostgreSQL notifications (#1255)
Regan Bellbf8f3264ea7a
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 252ffd178..bfc85e2a6 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -1,3 +1,4 @@+import { createPostgresNotifyBus } from "../persistence/postgres-notify-bus.ts";import { randomUUID } from "node:crypto";import { EventEmitter } from "node:events";import { createPgPool } from "../persistence/pg-pool.ts";@@ -7,7 +8,7 @@ import type { OrchestratorInput } from "../core/orchestrator.ts";import { resolveTurnOrigin } from "../core/turn-origin.ts";import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";import { isTerminal, releasesDedupKey } from "./run-store.ts";-import { errMessage } from "../util/errors.ts";+import { errMessage, swallow } from "../util/errors.ts";import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";export interface PostgresRuntime {@@ -46,6 +47,7 @@ function rowToRun(r: Record<string, unknown>): Run {const FENCE_HOLD_MS = 600_000;export function createPostgresRunStore(connectionString: string, opts?: { maxClaims?: number }): PostgresRuntime {+ const available = createPostgresNotifyBus<null>(connectionString, "qm_run_available", "run availability");const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;const events = new EventEmitter();events.setMaxListeners(0);@@ -144,6 +146,40 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla],);+ const availabilityListeners = new Map<() => void, number>();+ let availabilityTimer: ReturnType<typeof setTimeout> | undefined;+ let availabilityProbe: Promise<void> | null = null;+ let availabilityClosed = false;++ function watchAvailability(): void {+ if (availabilityClosed || availabilityListeners.size === 0 || availabilityTimer || availabilityProbe) return;+ availabilityTimer = setTimeout(+ () => {+ availabilityTimer = undefined;+ availabilityProbe = q(+ `SELECT EXISTS (+ SELECT 1 FROM runs r WHERE r.status='pending' AND r.retry_after <= $1+ AND NOT EXISTS (+ SELECT 1 FROM runs blocked WHERE blocked.session_id=r.session_id+ AND (blocked.status='running' OR (blocked.status='pending' AND blocked.retry_after > $1))+ )+ ) AS available`,+ [Date.now()],+ )+ .then(({ rows }) => {+ if (rows[0]?.available) for (const listener of availabilityListeners.keys()) listener();+ })+ .catch((error: unknown) => swallow("run availability probe", error))+ .finally(() => {+ availabilityProbe = null;+ watchAvailability();+ });+ },+ Math.min(...availabilityListeners.values()),+ );+ availabilityTimer.unref();+ }+async function getRun(id: string): Promise<Run | null> {const { rows } = await q("SELECT * FROM runs WHERE id = $1", [id]);return rows[0] ? rowToRun(rows[0]) : null;@@ -168,7 +204,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaconst { rowCount } = await q(`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,error_attempts=error_attempts+$4, retry_after=$5- WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,+ WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,[run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],);return { requeued: rowCount > 0, applied: rowCount > 0 };@@ -181,7 +217,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaconst { rowCount } = await q(`UPDATE runs SET status='failed', result=$4, lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, finished_at=$5,error_attempts=error_attempts+$6- WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,+ WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,[run.id, run.leaseToken, ifExpiredAt, JSON.stringify(result), Date.now(), countsAsError ? 1 : 0],);if (rowCount > 0) settle(await getRun(run.id));@@ -189,6 +225,21 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla}const runs: RunStore = {+ subscribeAvailable(listener, options) {+ availabilityListeners.set(listener, options?.pollMs ?? 50);+ clearTimeout(availabilityTimer);+ availabilityTimer = undefined;+ const off = available.subscribe(listener, options);+ watchAvailability();+ return () => {+ off();+ availabilityListeners.delete(listener);+ if (availabilityListeners.size === 0) {+ clearTimeout(availabilityTimer);+ availabilityTimer = undefined;+ }+ };+ },...(Number.isFinite(maxClaims) ? { maxClaims } : {}),async enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {@@ -197,7 +248,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at)VALUES ($1,$2,'pending',$3,$4,0,$5,$6)ON CONFLICT (idempotency_key) DO UPDATE SET id = runs.id- RETURNING *`,+ RETURNING *, pg_notify('qm_run_available', 'null')`,[id, sessionId, JSON.stringify(request), dedupKey ?? null, maxAttempts, Date.now()],);const run = rowToRun(rows[0]!);@@ -263,7 +314,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaasync releaseLease(runId, leaseToken): Promise<boolean> {const { rowCount } = await q(- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running'",+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",[runId, leaseToken],);return rowCount > 0;@@ -273,7 +324,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaconst { rowCount } = await q(`UPDATE runs SET status='done', result=$1, lease_token=NULL, lease_expires_at=NULL, finished_at=$2,idempotency_key = CASE WHEN $5 THEN NULL ELSE idempotency_key END- WHERE id=$3 AND lease_token=$4`,+ WHERE id=$3 AND lease_token=$4 RETURNING pg_notify('qm_run_available', 'null')`,[JSON.stringify(result), Date.now(), runId, leaseToken, releasesDedupKey(result)],);if (rowCount > 0) {@@ -429,6 +480,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla},async close(): Promise<void> {+ availabilityClosed = true;+ clearTimeout(availabilityTimer);+ availabilityListeners.clear();+ await availabilityProbe;+ await available.close?.();await closePool();},};diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex c3702fb1c..29e941269 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -1,3 +1,4 @@+import type { SubscribeOptions } from "../util/event-bus.ts";import type { TurnResult } from "../types.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";@@ -51,6 +52,8 @@ export interface EnqueueResult {export interface RunStore {readonly maxClaims?: number;+ subscribeAvailable?(listener: () => void, options?: SubscribeOptions & { pollMs?: number }): () => void;+enqueue(input: EnqueueInput): Promise<EnqueueResult>;getByDedupKey(dedupKey: string): Promise<Run | null>;diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex ce6ce659a..b89d55cbb 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -108,6 +108,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgrounexport interface WorkerDeps extends ProcessDeps {pollMs?: number;+ recoveryPollMs?: number;workerId?: string;sessions: SessionStore;canClaim?: () => boolean;@@ -126,6 +127,27 @@ const STOP_DRAIN_MS = 2_000;export function createWorker(deps: WorkerDeps): Worker {const workerId = deps.workerId ?? `w-${randomUUID().slice(0, 8)}`;const pollMs = deps.pollMs ?? 50;+ const recoveryPollMs = deps.recoveryPollMs ?? 5_000;+ const notifications = Boolean(deps.runs.subscribeAvailable);+ let generation = 0;+ let wake: (() => void) | null = null;+ let unsubscribe: (() => void) | undefined;+ const notify = (): void => {+ generation++;+ wake?.();+ };+ async function waitForWork(observed: number): Promise<void> {+ if (stopped || observed !== generation) return;+ await new Promise<void>((resolve) => {+ const done = (): void => {+ clearTimeout(timer);+ wake = null;+ resolve();+ };+ const timer = setTimeout(done, notifications ? recoveryPollMs : pollMs);+ wake = done;+ });+ }let stopped = false;let loopDone: Promise<void> | null = null;let inFlight: { runId: string; leaseToken: string; threadRef: string } | null = null;@@ -139,6 +161,7 @@ export function createWorker(deps: WorkerDeps): Worker {await sleep(pollMs);continue;}+ const observed = generation;let run: Run | null;try {run = await deps.runs.claim(workerId, deps.leaseTtlMs);@@ -151,7 +174,7 @@ export function createWorker(deps: WorkerDeps): Worker {continue;}if (!run) {- await sleep(pollMs);+ await waitForWork(observed);continue;}if (stopped) {@@ -178,6 +201,10 @@ export function createWorker(deps: WorkerDeps): Worker {start() {if (loopDone) return;stopped = false;+ unsubscribe = deps.runs.subscribeAvailable?.(notify, {+ pollMs,+ onResync: notify,+ });loopDone = loop();},busy() {@@ -205,6 +232,9 @@ export function createWorker(deps: WorkerDeps): Worker {},async stop(drainMs = STOP_DRAIN_MS) {stopped = true;+ unsubscribe?.();+ unsubscribe = undefined;+ notify();await Promise.race([loopDone, sleep(drainMs, { unref: true })]);loopDone = null;},diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex 689b45cf5..fe064b799 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -1,3 +1,4 @@+import "./run-availability.ts";import { test, before } from "node:test";import { randomUUID } from "node:crypto";import assert from "node:assert/strict";diff --git a/test/run-availability.ts b/test/run-availability.tsnew file mode 100644index 000000000..6a9093d41--- /dev/null+++ b/test/run-availability.ts@@ -0,0 +1,323 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { randomUUID } from "node:crypto";+import { setTimeout as sleep } from "node:timers/promises";+import pg from "pg";+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";+import { createMemoryEventBus } from "../src/util/event-bus.ts";+import { createWorker } from "../src/runs/worker.ts";+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";++const actor = { id: "internal:U1", type: "internal" as const };+const request: OrchestratorInput = {+ actor,+ conversation: { kind: "dm", threadRef: "availability", audience: [actor] },+ origin: { kind: "direct" },+ text: "availability",+};+async function until(predicate: () => boolean): Promise<void> {+ const deadline = Date.now() + 3_000;+ while (!predicate()) {+ assert.ok(Date.now() < deadline, "condition reached before deadline");+ await sleep(5);+ }+}++async function isolatedPostgres(): Promise<{ url: string; admin: pg.Pool; cleanup(): Promise<void> }> {+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL! });+ const schema = `run_availability_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new URL(process.env.DATABASE_URL!);+ url.searchParams.set("options", `-c search_path=${schema}`);+ url.searchParams.set("application_name", schema);+ return {+ url: url.toString(),+ admin,+ async cleanup() {+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ await admin.end();+ },+ };+}++for (const duringClaim of [false, true]) {+ test(`notification wakes idle worker without a lost wakeup (during claim: ${duringClaim})`, async () => {+ const { runs } = createMemoryRunStore();+ const bus = createMemoryEventBus<null>("availability test");+ runs.subscribeAvailable = (listener, options) => {+ const off = bus.subscribe(listener, options);+ options?.onResync?.();+ return off;+ };+ const claim = runs.claim.bind(runs);+ let claims = 0;+ let finished = false;+ runs.claim = async (...args) => {+ claims++;+ const result = await claim(...args);+ if (duringClaim && claims === 1) {+ await runs.enqueue({ sessionId: "availability", request });+ bus.emit(null);+ }+ return result;+ };+ const worker = createWorker({+ runs,+ sessions: createMemorySessionStore(),+ orchestrator: {+ handleTurn: async () => {+ finished = true;+ return { status: "ok", sessionId: "availability" };+ },+ } as unknown as Orchestrator,+ leaseTtlMs: 5_000,+ pollMs: 10,+ recoveryPollMs: 10_000,+ });+ worker.start();+ try {+ if (!duringClaim) {+ await until(() => claims === 1);+ await sleep(100);+ assert.equal(claims, 1, "healthy idle listener does not repeatedly claim");+ await runs.enqueue({ sessionId: "availability", request });+ bus.emit(null);+ }+ await until(() => finished);+ } finally {+ await worker.stop();+ }+ assert.equal(bus.size(), 0);+ });+}++test("missed notifications recover and reconnect checks immediately", async () => {+ const { runs } = createMemoryRunStore();+ const bus = createMemoryEventBus<null>("availability test");+ runs.subscribeAvailable = (listener, options) => {+ const off = bus.subscribe(listener, options);+ options?.onResync?.();+ return off;+ };+ let claims = 0;+ runs.claim = async () => {+ claims++;+ return null;+ };+ const worker = createWorker({+ runs,+ sessions: createMemorySessionStore(),+ orchestrator: {} as Orchestrator,+ leaseTtlMs: 5_000,+ pollMs: 10,+ recoveryPollMs: 200,+ });+ worker.start();+ try {+ await until(() => claims >= 2);+ bus.resync();+ const reconnectClaims = claims;+ await until(() => claims > reconnectClaims);+ const healthyClaims = claims;+ await sleep(50);+ assert.equal(claims, healthyClaims);+ } finally {+ await worker.stop();+ }+});++test(+ "Postgres notifies across stores for enqueue, unblock, retry, release and reconnect",+ { skip: !process.env.DATABASE_URL },+ async () => {+ const { url, admin, cleanup } = await isolatedPostgres();+ const writer = createPostgresRunStore(url);+ const reader = createPostgresRunStore(url);+ let notifications = 0;+ let resyncs = 0;+ const off = reader.runs.subscribeAvailable!(() => notifications++, {+ pollMs: 60_000,+ onResync: () => resyncs++,+ });+ const ids: string[] = [];+ try {+ await writer.runs.list();+ await until(() => resyncs === 1);+ const sessionId = randomUUID();+ const enqueue = async (): Promise<void> => {+ const before = notifications;+ ids.push((await writer.runs.enqueue({ sessionId, request })).run.id);+ await until(() => notifications > before);+ };+ await enqueue();+ let claimed = await writer.runs.claimById(ids[0]!, "test", 5_000);+ assert.ok(claimed);+ await enqueue();+ assert.equal(await reader.runs.claimById(ids[1]!, "test2", 5_000), null);+ let before = notifications;+ await writer.runs.complete(claimed.id, claimed.leaseToken!, { status: "ok", sessionId });+ await until(() => notifications > before);+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);+ assert.ok(claimed);+ before = notifications;+ await writer.runs.fail(claimed.id, claimed.leaseToken!, "retry");+ await until(() => notifications > before);+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);+ assert.ok(claimed);+ before = notifications;+ await writer.runs.releaseLease(claimed.id, claimed.leaseToken!);+ await until(() => notifications > before);+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);+ assert.ok(claimed);+ before = notifications;+ await writer.runs.fail(claimed.id, claimed.leaseToken!, "terminal", { retry: false });+ await until(() => notifications > before);+ await enqueue();+ claimed = await reader.runs.claimById(ids[2]!, "expired", 1);+ assert.ok(claimed);+ await sleep(5);+ before = notifications;+ assert.equal((await writer.runs.reapExpired()).requeued, 1);+ await until(() => notifications > before);+ await admin.query(+ "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname=current_database() AND query='LISTEN qm_run_available' AND application_name=$1",+ [new URL(url).searchParams.get("application_name")],+ );+ ids.push((await writer.runs.enqueue({ sessionId, request })).run.id);+ await until(() => resyncs === 2);+ await enqueue();+ } finally {+ off();+ await reader.close();+ await writer.close();+ await cleanup();+ }+ },+);++test(+ "Postgres workers pick up new runs promptly and preserve single claims under contention",+ { skip: !process.env.DATABASE_URL },+ async (t) => {+ const { url, cleanup } = await isolatedPostgres();+ const writer = createPostgresRunStore(url);+ const reader = createPostgresRunStore(url);+ const prefix = randomUUID();+ const observed = new Map<string, number>();+ const enqueueTimes = new Map<string, number>();+ const latencies: number[] = [];+ const claim = reader.runs.claim.bind(reader.runs);+ let claims = 0;+ reader.runs.claim = async (...args) => {+ claims++;+ return claim(...args);+ };+ const workers = Array.from({ length: 2 }, () =>+ createWorker({+ runs: reader.runs,+ sessions: createMemorySessionStore(),+ orchestrator: {+ handleTurn: async (input: OrchestratorInput) => {+ const id = input.conversation.threadRef;+ observed.set(id, (observed.get(id) ?? 0) + 1);+ latencies.push(performance.now() - enqueueTimes.get(id)!);+ return { status: "ok", sessionId: id };+ },+ } as unknown as Orchestrator,+ leaseTtlMs: 5_000,+ pollMs: 60_000,+ }),+ );+ try {+ await writer.runs.list();+ for (const worker of workers) worker.start();+ await until(() => claims >= 2);+ await sleep(100);+ const before = claims;+ await sleep(1_000);+ assert.equal(claims, before, "idle workers issue no claim queries between recovery polls");+ for (let i = 0; i < 20; i++) {+ const sessionId = `${prefix}-${i}`;+ enqueueTimes.set(sessionId, performance.now());+ await writer.runs.enqueue({+ sessionId,+ request: { ...request, conversation: { ...request.conversation, threadRef: sessionId } },+ });+ await until(() => observed.has(sessionId));+ }+ assert.equal(observed.size, 20);+ assert.ok([...observed.values()].every((count) => count === 1));+ latencies.sort((a, b) => a - b);+ t.diagnostic(+ `enqueue-to-handler latency, local Postgres: p50=${latencies[9]!.toFixed(1)}ms, p95=${latencies[18]!.toFixed(1)}ms, max=${latencies[19]!.toFixed(1)}ms`,+ );+ assert.ok(latencies[19]! < 1_000, "pickup does not wait for the five-second recovery poll");+ } finally {+ await Promise.all(workers.map((worker) => worker.stop()));+ await reader.close();+ await writer.close();+ await cleanup();+ }+ },+);++test(+ "shared watchdog preserves retry deadlines and catches missing notifications",+ { skip: !process.env.DATABASE_URL },+ async () => {+ const { url, cleanup } = await isolatedPostgres();+ const writer = createPostgresRunStore(url);+ const reader = createPostgresRunStore(url);+ const direct = new pg.Pool({ connectionString: url });+ const observed: Array<{ id: string; at: number }> = [];+ const workers = Array.from({ length: 16 }, () =>+ createWorker({+ runs: reader.runs,+ sessions: createMemorySessionStore(),+ orchestrator: {+ handleTurn: async (input: OrchestratorInput) => {+ observed.push({ id: input.text, at: Date.now() });+ return { status: "ok", sessionId: "scheduled" };+ },+ } as unknown as Orchestrator,+ leaseTtlMs: 5_000,+ pollMs: 250,+ }),+ );+ try {+ const first = (await writer.runs.enqueue({ sessionId: "scheduled", request: { ...request, text: "first" } })).run;+ const claimed = await writer.runs.claimById(first.id, "setup", 5_000);+ await writer.runs.fail(first.id, claimed!.leaseToken!, "transient", { retryAfterMs: 500 });+ const { rows } = await direct.query("SELECT retry_after FROM runs WHERE id=$1", [first.id]);+ const deadline = Number(rows[0].retry_after);+ await writer.runs.enqueue({ sessionId: "scheduled", request: { ...request, text: "second" } });+ for (const worker of workers) worker.start();+ await until(() => observed.length === 2);+ assert.deepEqual(+ observed.map(({ id }) => id),+ ["first", "second"],+ );+ assert.ok(observed[0]!.at >= deadline, "retry never claims before its deadline");+ assert.ok(+ observed[0]!.at - deadline < 750,+ "deadline pickup uses the watchdog, not five-second recovery polling",+ );+ const insertedAt = Date.now();+ await direct.query(+ "INSERT INTO runs(id, session_id, status, request, created_at) VALUES($1,$2,'pending',$3,$4)",+ [randomUUID(), "missed", JSON.stringify({ ...request, text: "missed" }), insertedAt],+ );+ await until(() => observed.length === 3);+ assert.equal(observed[2]!.id, "missed");+ assert.ok(observed[2]!.at - insertedAt < 750, "missing NOTIFY retains the original polling cadence");+ } finally {+ await Promise.all(workers.map((worker) => worker.stop()));+ await reader.close();+ await writer.close();+ await direct.end();+ await cleanup();+ }+ },+);