Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Wake run workers through PostgreSQL notifications (#1255)

Regan Bellbf8f3264ea7a
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 252ffd178..bfc85e2a6 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -1,3 +1,4 @@
+import { createPostgresNotifyBus } from "../persistence/postgres-notify-bus.ts";
import { randomUUID } from "node:crypto";
import { EventEmitter } from "node:events";
import { createPgPool } from "../persistence/pg-pool.ts";
@@ -7,7 +8,7 @@ import type { OrchestratorInput } from "../core/orchestrator.ts";
import { resolveTurnOrigin } from "../core/turn-origin.ts";
import type { EnqueueInput, EnqueueResult, ReapEvent, Run, RunDeliveryState, RunStore } from "./run-store.ts";
import { isTerminal, releasesDedupKey } from "./run-store.ts";
-import { errMessage } from "../util/errors.ts";
+import { errMessage, swallow } from "../util/errors.ts";
import type { LedgerBegin, ToolLedger } from "./tool-ledger.ts";
export interface PostgresRuntime {
@@ -46,6 +47,7 @@ function rowToRun(r: Record<string, unknown>): Run {
const FENCE_HOLD_MS = 600_000;
export function createPostgresRunStore(connectionString: string, opts?: { maxClaims?: number }): PostgresRuntime {
+ const available = createPostgresNotifyBus<null>(connectionString, "qm_run_available", "run availability");
const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;
const events = new EventEmitter();
events.setMaxListeners(0);
@@ -144,6 +146,40 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
],
);
+ const availabilityListeners = new Map<() => void, number>();
+ let availabilityTimer: ReturnType<typeof setTimeout> | undefined;
+ let availabilityProbe: Promise<void> | null = null;
+ let availabilityClosed = false;
+
+ function watchAvailability(): void {
+ if (availabilityClosed || availabilityListeners.size === 0 || availabilityTimer || availabilityProbe) return;
+ availabilityTimer = setTimeout(
+ () => {
+ availabilityTimer = undefined;
+ availabilityProbe = q(
+ `SELECT EXISTS (
+ SELECT 1 FROM runs r WHERE r.status='pending' AND r.retry_after <= $1
+ AND NOT EXISTS (
+ SELECT 1 FROM runs blocked WHERE blocked.session_id=r.session_id
+ AND (blocked.status='running' OR (blocked.status='pending' AND blocked.retry_after > $1))
+ )
+ ) AS available`,
+ [Date.now()],
+ )
+ .then(({ rows }) => {
+ if (rows[0]?.available) for (const listener of availabilityListeners.keys()) listener();
+ })
+ .catch((error: unknown) => swallow("run availability probe", error))
+ .finally(() => {
+ availabilityProbe = null;
+ watchAvailability();
+ });
+ },
+ Math.min(...availabilityListeners.values()),
+ );
+ availabilityTimer.unref();
+ }
+
async function getRun(id: string): Promise<Run | null> {
const { rows } = await q("SELECT * FROM runs WHERE id = $1", [id]);
return rows[0] ? rowToRun(rows[0]) : null;
@@ -168,7 +204,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
const { rowCount } = await q(
`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,
error_attempts=error_attempts+$4, retry_after=$5
- WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,
+ WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,
[run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],
);
return { requeued: rowCount > 0, applied: rowCount > 0 };
@@ -181,7 +217,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
const { rowCount } = await q(
`UPDATE runs SET status='failed', result=$4, lease_token=NULL, lease_expires_at=NULL, worker_id=NULL, finished_at=$5,
error_attempts=error_attempts+$6
- WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,
+ WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3) RETURNING pg_notify('qm_run_available', 'null')`,
[run.id, run.leaseToken, ifExpiredAt, JSON.stringify(result), Date.now(), countsAsError ? 1 : 0],
);
if (rowCount > 0) settle(await getRun(run.id));
@@ -189,6 +225,21 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
}
const runs: RunStore = {
+ subscribeAvailable(listener, options) {
+ availabilityListeners.set(listener, options?.pollMs ?? 50);
+ clearTimeout(availabilityTimer);
+ availabilityTimer = undefined;
+ const off = available.subscribe(listener, options);
+ watchAvailability();
+ return () => {
+ off();
+ availabilityListeners.delete(listener);
+ if (availabilityListeners.size === 0) {
+ clearTimeout(availabilityTimer);
+ availabilityTimer = undefined;
+ }
+ };
+ },
...(Number.isFinite(maxClaims) ? { maxClaims } : {}),
async enqueue({ sessionId, request, dedupKey, maxAttempts = 3 }: EnqueueInput): Promise<EnqueueResult> {
@@ -197,7 +248,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
`INSERT INTO runs(id, session_id, status, request, idempotency_key, attempts, max_attempts, created_at)
VALUES ($1,$2,'pending',$3,$4,0,$5,$6)
ON CONFLICT (idempotency_key) DO UPDATE SET id = runs.id
- RETURNING *`,
+ RETURNING *, pg_notify('qm_run_available', 'null')`,
[id, sessionId, JSON.stringify(request), dedupKey ?? null, maxAttempts, Date.now()],
);
const run = rowToRun(rows[0]!);
@@ -263,7 +314,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
async releaseLease(runId, leaseToken): Promise<boolean> {
const { rowCount } = await q(
- "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running'",
+ "UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL WHERE id=$1 AND lease_token=$2 AND status='running' RETURNING pg_notify('qm_run_available', 'null')",
[runId, leaseToken],
);
return rowCount > 0;
@@ -273,7 +324,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
const { rowCount } = await q(
`UPDATE runs SET status='done', result=$1, lease_token=NULL, lease_expires_at=NULL, finished_at=$2,
idempotency_key = CASE WHEN $5 THEN NULL ELSE idempotency_key END
- WHERE id=$3 AND lease_token=$4`,
+ WHERE id=$3 AND lease_token=$4 RETURNING pg_notify('qm_run_available', 'null')`,
[JSON.stringify(result), Date.now(), runId, leaseToken, releasesDedupKey(result)],
);
if (rowCount > 0) {
@@ -429,6 +480,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
},
async close(): Promise<void> {
+ availabilityClosed = true;
+ clearTimeout(availabilityTimer);
+ availabilityListeners.clear();
+ await availabilityProbe;
+ await available.close?.();
await closePool();
},
};
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index c3702fb1c..29e941269 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -1,3 +1,4 @@
+import type { SubscribeOptions } from "../util/event-bus.ts";
import type { TurnResult } from "../types.ts";
import type { OrchestratorInput } from "../core/orchestrator.ts";
@@ -51,6 +52,8 @@ export interface EnqueueResult {
export interface RunStore {
readonly maxClaims?: number;
+ subscribeAvailable?(listener: () => void, options?: SubscribeOptions & { pollMs?: number }): () => void;
+
enqueue(input: EnqueueInput): Promise<EnqueueResult>;
getByDedupKey(dedupKey: string): Promise<Run | null>;
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index ce6ce659a..b89d55cbb 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -108,6 +108,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun
export interface WorkerDeps extends ProcessDeps {
pollMs?: number;
+ recoveryPollMs?: number;
workerId?: string;
sessions: SessionStore;
canClaim?: () => boolean;
@@ -126,6 +127,27 @@ const STOP_DRAIN_MS = 2_000;
export function createWorker(deps: WorkerDeps): Worker {
const workerId = deps.workerId ?? `w-${randomUUID().slice(0, 8)}`;
const pollMs = deps.pollMs ?? 50;
+ const recoveryPollMs = deps.recoveryPollMs ?? 5_000;
+ const notifications = Boolean(deps.runs.subscribeAvailable);
+ let generation = 0;
+ let wake: (() => void) | null = null;
+ let unsubscribe: (() => void) | undefined;
+ const notify = (): void => {
+ generation++;
+ wake?.();
+ };
+ async function waitForWork(observed: number): Promise<void> {
+ if (stopped || observed !== generation) return;
+ await new Promise<void>((resolve) => {
+ const done = (): void => {
+ clearTimeout(timer);
+ wake = null;
+ resolve();
+ };
+ const timer = setTimeout(done, notifications ? recoveryPollMs : pollMs);
+ wake = done;
+ });
+ }
let stopped = false;
let loopDone: Promise<void> | null = null;
let inFlight: { runId: string; leaseToken: string; threadRef: string } | null = null;
@@ -139,6 +161,7 @@ export function createWorker(deps: WorkerDeps): Worker {
await sleep(pollMs);
continue;
}
+ const observed = generation;
let run: Run | null;
try {
run = await deps.runs.claim(workerId, deps.leaseTtlMs);
@@ -151,7 +174,7 @@ export function createWorker(deps: WorkerDeps): Worker {
continue;
}
if (!run) {
- await sleep(pollMs);
+ await waitForWork(observed);
continue;
}
if (stopped) {
@@ -178,6 +201,10 @@ export function createWorker(deps: WorkerDeps): Worker {
start() {
if (loopDone) return;
stopped = false;
+ unsubscribe = deps.runs.subscribeAvailable?.(notify, {
+ pollMs,
+ onResync: notify,
+ });
loopDone = loop();
},
busy() {
@@ -205,6 +232,9 @@ export function createWorker(deps: WorkerDeps): Worker {
},
async stop(drainMs = STOP_DRAIN_MS) {
stopped = true;
+ unsubscribe?.();
+ unsubscribe = undefined;
+ notify();
await Promise.race([loopDone, sleep(drainMs, { unref: true })]);
loopDone = null;
},
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index 689b45cf5..fe064b799 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -1,3 +1,4 @@
+import "./run-availability.ts";
import { test, before } from "node:test";
import { randomUUID } from "node:crypto";
import assert from "node:assert/strict";
diff --git a/test/run-availability.ts b/test/run-availability.ts
new file mode 100644
index 000000000..6a9093d41
--- /dev/null
+++ b/test/run-availability.ts
@@ -0,0 +1,323 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { setTimeout as sleep } from "node:timers/promises";
+import pg from "pg";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";
+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
+import { createMemoryEventBus } from "../src/util/event-bus.ts";
+import { createWorker } from "../src/runs/worker.ts";
+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";
+
+const actor = { id: "internal:U1", type: "internal" as const };
+const request: OrchestratorInput = {
+ actor,
+ conversation: { kind: "dm", threadRef: "availability", audience: [actor] },
+ origin: { kind: "direct" },
+ text: "availability",
+};
+async function until(predicate: () => boolean): Promise<void> {
+ const deadline = Date.now() + 3_000;
+ while (!predicate()) {
+ assert.ok(Date.now() < deadline, "condition reached before deadline");
+ await sleep(5);
+ }
+}
+
+async function isolatedPostgres(): Promise<{ url: string; admin: pg.Pool; cleanup(): Promise<void> }> {
+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL! });
+ const schema = `run_availability_${randomUUID().replaceAll("-", "")}`;
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new URL(process.env.DATABASE_URL!);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ url.searchParams.set("application_name", schema);
+ return {
+ url: url.toString(),
+ admin,
+ async cleanup() {
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ await admin.end();
+ },
+ };
+}
+
+for (const duringClaim of [false, true]) {
+ test(`notification wakes idle worker without a lost wakeup (during claim: ${duringClaim})`, async () => {
+ const { runs } = createMemoryRunStore();
+ const bus = createMemoryEventBus<null>("availability test");
+ runs.subscribeAvailable = (listener, options) => {
+ const off = bus.subscribe(listener, options);
+ options?.onResync?.();
+ return off;
+ };
+ const claim = runs.claim.bind(runs);
+ let claims = 0;
+ let finished = false;
+ runs.claim = async (...args) => {
+ claims++;
+ const result = await claim(...args);
+ if (duringClaim && claims === 1) {
+ await runs.enqueue({ sessionId: "availability", request });
+ bus.emit(null);
+ }
+ return result;
+ };
+ const worker = createWorker({
+ runs,
+ sessions: createMemorySessionStore(),
+ orchestrator: {
+ handleTurn: async () => {
+ finished = true;
+ return { status: "ok", sessionId: "availability" };
+ },
+ } as unknown as Orchestrator,
+ leaseTtlMs: 5_000,
+ pollMs: 10,
+ recoveryPollMs: 10_000,
+ });
+ worker.start();
+ try {
+ if (!duringClaim) {
+ await until(() => claims === 1);
+ await sleep(100);
+ assert.equal(claims, 1, "healthy idle listener does not repeatedly claim");
+ await runs.enqueue({ sessionId: "availability", request });
+ bus.emit(null);
+ }
+ await until(() => finished);
+ } finally {
+ await worker.stop();
+ }
+ assert.equal(bus.size(), 0);
+ });
+}
+
+test("missed notifications recover and reconnect checks immediately", async () => {
+ const { runs } = createMemoryRunStore();
+ const bus = createMemoryEventBus<null>("availability test");
+ runs.subscribeAvailable = (listener, options) => {
+ const off = bus.subscribe(listener, options);
+ options?.onResync?.();
+ return off;
+ };
+ let claims = 0;
+ runs.claim = async () => {
+ claims++;
+ return null;
+ };
+ const worker = createWorker({
+ runs,
+ sessions: createMemorySessionStore(),
+ orchestrator: {} as Orchestrator,
+ leaseTtlMs: 5_000,
+ pollMs: 10,
+ recoveryPollMs: 200,
+ });
+ worker.start();
+ try {
+ await until(() => claims >= 2);
+ bus.resync();
+ const reconnectClaims = claims;
+ await until(() => claims > reconnectClaims);
+ const healthyClaims = claims;
+ await sleep(50);
+ assert.equal(claims, healthyClaims);
+ } finally {
+ await worker.stop();
+ }
+});
+
+test(
+ "Postgres notifies across stores for enqueue, unblock, retry, release and reconnect",
+ { skip: !process.env.DATABASE_URL },
+ async () => {
+ const { url, admin, cleanup } = await isolatedPostgres();
+ const writer = createPostgresRunStore(url);
+ const reader = createPostgresRunStore(url);
+ let notifications = 0;
+ let resyncs = 0;
+ const off = reader.runs.subscribeAvailable!(() => notifications++, {
+ pollMs: 60_000,
+ onResync: () => resyncs++,
+ });
+ const ids: string[] = [];
+ try {
+ await writer.runs.list();
+ await until(() => resyncs === 1);
+ const sessionId = randomUUID();
+ const enqueue = async (): Promise<void> => {
+ const before = notifications;
+ ids.push((await writer.runs.enqueue({ sessionId, request })).run.id);
+ await until(() => notifications > before);
+ };
+ await enqueue();
+ let claimed = await writer.runs.claimById(ids[0]!, "test", 5_000);
+ assert.ok(claimed);
+ await enqueue();
+ assert.equal(await reader.runs.claimById(ids[1]!, "test2", 5_000), null);
+ let before = notifications;
+ await writer.runs.complete(claimed.id, claimed.leaseToken!, { status: "ok", sessionId });
+ await until(() => notifications > before);
+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);
+ assert.ok(claimed);
+ before = notifications;
+ await writer.runs.fail(claimed.id, claimed.leaseToken!, "retry");
+ await until(() => notifications > before);
+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);
+ assert.ok(claimed);
+ before = notifications;
+ await writer.runs.releaseLease(claimed.id, claimed.leaseToken!);
+ await until(() => notifications > before);
+ claimed = await reader.runs.claimById(ids[1]!, "test2", 5_000);
+ assert.ok(claimed);
+ before = notifications;
+ await writer.runs.fail(claimed.id, claimed.leaseToken!, "terminal", { retry: false });
+ await until(() => notifications > before);
+ await enqueue();
+ claimed = await reader.runs.claimById(ids[2]!, "expired", 1);
+ assert.ok(claimed);
+ await sleep(5);
+ before = notifications;
+ assert.equal((await writer.runs.reapExpired()).requeued, 1);
+ await until(() => notifications > before);
+ await admin.query(
+ "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname=current_database() AND query='LISTEN qm_run_available' AND application_name=$1",
+ [new URL(url).searchParams.get("application_name")],
+ );
+ ids.push((await writer.runs.enqueue({ sessionId, request })).run.id);
+ await until(() => resyncs === 2);
+ await enqueue();
+ } finally {
+ off();
+ await reader.close();
+ await writer.close();
+ await cleanup();
+ }
+ },
+);
+
+test(
+ "Postgres workers pick up new runs promptly and preserve single claims under contention",
+ { skip: !process.env.DATABASE_URL },
+ async (t) => {
+ const { url, cleanup } = await isolatedPostgres();
+ const writer = createPostgresRunStore(url);
+ const reader = createPostgresRunStore(url);
+ const prefix = randomUUID();
+ const observed = new Map<string, number>();
+ const enqueueTimes = new Map<string, number>();
+ const latencies: number[] = [];
+ const claim = reader.runs.claim.bind(reader.runs);
+ let claims = 0;
+ reader.runs.claim = async (...args) => {
+ claims++;
+ return claim(...args);
+ };
+ const workers = Array.from({ length: 2 }, () =>
+ createWorker({
+ runs: reader.runs,
+ sessions: createMemorySessionStore(),
+ orchestrator: {
+ handleTurn: async (input: OrchestratorInput) => {
+ const id = input.conversation.threadRef;
+ observed.set(id, (observed.get(id) ?? 0) + 1);
+ latencies.push(performance.now() - enqueueTimes.get(id)!);
+ return { status: "ok", sessionId: id };
+ },
+ } as unknown as Orchestrator,
+ leaseTtlMs: 5_000,
+ pollMs: 60_000,
+ }),
+ );
+ try {
+ await writer.runs.list();
+ for (const worker of workers) worker.start();
+ await until(() => claims >= 2);
+ await sleep(100);
+ const before = claims;
+ await sleep(1_000);
+ assert.equal(claims, before, "idle workers issue no claim queries between recovery polls");
+ for (let i = 0; i < 20; i++) {
+ const sessionId = `${prefix}-${i}`;
+ enqueueTimes.set(sessionId, performance.now());
+ await writer.runs.enqueue({
+ sessionId,
+ request: { ...request, conversation: { ...request.conversation, threadRef: sessionId } },
+ });
+ await until(() => observed.has(sessionId));
+ }
+ assert.equal(observed.size, 20);
+ assert.ok([...observed.values()].every((count) => count === 1));
+ latencies.sort((a, b) => a - b);
+ t.diagnostic(
+ `enqueue-to-handler latency, local Postgres: p50=${latencies[9]!.toFixed(1)}ms, p95=${latencies[18]!.toFixed(1)}ms, max=${latencies[19]!.toFixed(1)}ms`,
+ );
+ assert.ok(latencies[19]! < 1_000, "pickup does not wait for the five-second recovery poll");
+ } finally {
+ await Promise.all(workers.map((worker) => worker.stop()));
+ await reader.close();
+ await writer.close();
+ await cleanup();
+ }
+ },
+);
+
+test(
+ "shared watchdog preserves retry deadlines and catches missing notifications",
+ { skip: !process.env.DATABASE_URL },
+ async () => {
+ const { url, cleanup } = await isolatedPostgres();
+ const writer = createPostgresRunStore(url);
+ const reader = createPostgresRunStore(url);
+ const direct = new pg.Pool({ connectionString: url });
+ const observed: Array<{ id: string; at: number }> = [];
+ const workers = Array.from({ length: 16 }, () =>
+ createWorker({
+ runs: reader.runs,
+ sessions: createMemorySessionStore(),
+ orchestrator: {
+ handleTurn: async (input: OrchestratorInput) => {
+ observed.push({ id: input.text, at: Date.now() });
+ return { status: "ok", sessionId: "scheduled" };
+ },
+ } as unknown as Orchestrator,
+ leaseTtlMs: 5_000,
+ pollMs: 250,
+ }),
+ );
+ try {
+ const first = (await writer.runs.enqueue({ sessionId: "scheduled", request: { ...request, text: "first" } })).run;
+ const claimed = await writer.runs.claimById(first.id, "setup", 5_000);
+ await writer.runs.fail(first.id, claimed!.leaseToken!, "transient", { retryAfterMs: 500 });
+ const { rows } = await direct.query("SELECT retry_after FROM runs WHERE id=$1", [first.id]);
+ const deadline = Number(rows[0].retry_after);
+ await writer.runs.enqueue({ sessionId: "scheduled", request: { ...request, text: "second" } });
+ for (const worker of workers) worker.start();
+ await until(() => observed.length === 2);
+ assert.deepEqual(
+ observed.map(({ id }) => id),
+ ["first", "second"],
+ );
+ assert.ok(observed[0]!.at >= deadline, "retry never claims before its deadline");
+ assert.ok(
+ observed[0]!.at - deadline < 750,
+ "deadline pickup uses the watchdog, not five-second recovery polling",
+ );
+ const insertedAt = Date.now();
+ await direct.query(
+ "INSERT INTO runs(id, session_id, status, request, created_at) VALUES($1,$2,'pending',$3,$4)",
+ [randomUUID(), "missed", JSON.stringify({ ...request, text: "missed" }), insertedAt],
+ );
+ await until(() => observed.length === 3);
+ assert.equal(observed[2]!.id, "missed");
+ assert.ok(observed[2]!.at - insertedAt < 750, "missing NOTIFY retains the original polling cadence");
+ } finally {
+ await Promise.all(workers.map((worker) => worker.stop()));
+ await reader.close();
+ await writer.close();
+ await direct.end();
+ await cleanup();
+ }
+ },
+);