Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Revert the wake-one-worker half of the run-claim change, keep OFFSET 0 (#2526)

Regan Bellf5d40e73f397
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index a931a69fe..1f3dcaaea 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -197,19 +197,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
],
);
- const availabilityListeners = new Map<() => boolean | void, number>();
- let availabilityOff: (() => void) | undefined;
- let wakeCursor = 0;
- function wakeOne(): void {
- const listeners = [...availabilityListeners.keys()];
- for (let i = 0; i < listeners.length; i++) {
- const at = (wakeCursor + i) % listeners.length;
- if (listeners[at]!() === true) {
- wakeCursor = at + 1;
- return;
- }
- }
- }
+ const availabilityListeners = new Map<() => void, number>();
let availabilityTimer: ReturnType<typeof setTimeout> | undefined;
let availabilityProbe: Promise<void> | null = null;
let availabilityClosed = false;
@@ -230,7 +218,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
[Date.now()],
)
.then(({ rows }) => {
- if (rows[0]?.available) wakeOne();
+ if (rows[0]?.available) for (const listener of availabilityListeners.keys()) listener();
})
.catch((error: unknown) => swallow("run availability probe", error))
.finally(() => {
@@ -317,9 +305,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
) RETURNING *`,
[token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],
);
- if (!rows[0]) return null;
- if (runId === undefined && sessionId === undefined) wakeOne();
- return rowToRun(rows[0]);
+ return rows[0] ? rowToRun(rows[0]) : null;
} catch (err) {
if (isUniqueViolation(err)) return null;
throw err;
@@ -331,15 +317,12 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
availabilityListeners.set(listener, options?.pollMs ?? 50);
clearTimeout(availabilityTimer);
availabilityTimer = undefined;
- availabilityOff ??= available.subscribe(wakeOne);
- const off = available.subscribe(() => {}, options);
+ const off = available.subscribe(listener, options);
watchAvailability();
return () => {
off();
availabilityListeners.delete(listener);
if (availabilityListeners.size === 0) {
- availabilityOff?.();
- availabilityOff = undefined;
clearTimeout(availabilityTimer);
availabilityTimer = undefined;
}
@@ -682,8 +665,6 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
availabilityClosed = true;
clearTimeout(availabilityTimer);
availabilityListeners.clear();
- availabilityOff?.();
- availabilityOff = undefined;
await availabilityProbe;
await available.close?.();
await closePool();
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 39f5b2887..4a8b82748 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -91,7 +91,7 @@ export interface EnqueueResult {
export interface RunStore {
readonly maxClaims?: number;
- subscribeAvailable?(listener: () => boolean | void, options?: SubscribeOptions & { pollMs?: number }): () => void;
+ subscribeAvailable?(listener: () => void, options?: SubscribeOptions & { pollMs?: number }): () => void;
enqueue(input: EnqueueInput): Promise<EnqueueResult>;
getByDedupKey(dedupKey: string): Promise<Run | null>;
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index d612fd849..677877356 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -189,11 +189,9 @@ export function createWorker(deps: WorkerDeps): Worker {
let generation = 0;
let wake: (() => void) | null = null;
let unsubscribe: (() => void) | undefined;
- const notify = (): boolean => {
+ const notify = (): void => {
generation++;
- if (!wake) return false;
- wake();
- return true;
+ wake?.();
};
async function waitForWork(observed: number): Promise<void> {
if (stopped || observed !== generation) return;
diff --git a/test/run-availability.ts b/test/run-availability.ts
index d479b2ca6..d3c68421b 100644
--- a/test/run-availability.ts
+++ b/test/run-availability.ts
@@ -134,15 +134,10 @@ test(
const reader = createPostgresRunStore(url);
let notifications = 0;
let resyncs = 0;
- const off = reader.runs.subscribeAvailable!(
- () => {
- notifications++;
- },
- {
- pollMs: 60_000,
- onResync: () => resyncs++,
- },
- );
+ const off = reader.runs.subscribeAvailable!(() => notifications++, {
+ pollMs: 60_000,
+ onResync: () => resyncs++,
+ });
const ids: string[] = [];
try {
await writer.runs.list();
@@ -358,110 +353,3 @@ test(
}
},
);
-
-test(
- "Postgres wakes one idle worker per notification and a backlog still fans out",
- { skip: !process.env.DATABASE_URL },
- async () => {
- const { url, cleanup } = await isolatedPostgres();
- const writer = createPostgresRunStore(url);
- const reader = createPostgresRunStore(url);
- const direct = new pg.Pool({ connectionString: url });
- const claim = reader.runs.claim.bind(reader.runs);
- let claims = 0;
- let claiming = 0;
- reader.runs.claim = async (...args) => {
- claims++;
- claiming++;
- try {
- return await claim(...args);
- } finally {
- claiming--;
- }
- };
- let resyncs = 0;
- const subscribe = reader.runs.subscribeAvailable!.bind(reader.runs);
- reader.runs.subscribeAvailable = (listener, options) =>
- subscribe(listener, {
- ...options,
- onResync: () => {
- resyncs++;
- options?.onResync?.();
- },
- });
- const handled: string[] = [];
- let releaseHeld!: () => void;
- const held = new Promise<void>((resolve) => {
- releaseHeld = resolve;
- });
- const workers = Array.from({ length: 12 }, () =>
- createWorker({
- runs: reader.runs,
- orchestrator: {
- handleTurn: async (input: OrchestratorInput) => {
- handled.push(input.conversation.threadRef);
- if (input.text === "hold") await held;
- return { status: "ok", sessionId: input.conversation.threadRef };
- },
- } as unknown as Orchestrator,
- leaseTtlMs: 5_000,
- pollMs: 60_000,
- }),
- );
- const enqueue = (sessionId: string, text = "go") =>
- writer.runs.enqueue({
- sessionId,
- request: { ...request, text, conversation: { ...request.conversation, threadRef: sessionId } },
- });
- try {
- await writer.runs.list();
- for (const worker of workers) worker.start();
- await until(() => resyncs >= workers.length);
- await until(() => claiming === 0);
- await sleep(100);
- let before = claims;
- await enqueue("single");
- await until(() => handled.includes("single"));
- await sleep(200);
- assert.ok(
- claims - before <= 4,
- `one run costs a handful of claims, not one per idle worker (${claims - before})`,
- );
-
- before = claims;
- const sessions = Array.from({ length: 6 }, (_, i) => `backlog-${i}`);
- const startedAt = Date.now();
- for (const sessionId of sessions)
- await direct.query(
- "INSERT INTO runs(id, session_id, status, request, created_at) VALUES($1,$2,'pending',$3,$4)",
- [
- randomUUID(),
- sessionId,
- JSON.stringify({
- ...request,
- text: "hold",
- conversation: { ...request.conversation, threadRef: sessionId },
- }),
- Date.now(),
- ],
- );
- await direct.query("SELECT pg_notify('qm_run_available', 'null')");
- await until(() => sessions.every((sessionId) => handled.includes(sessionId)));
- assert.ok(
- Date.now() - startedAt < 2_000,
- "one signal drains a backlog across idle workers without recovery polls",
- );
- const { rows } = await direct.query(
- "SELECT count(DISTINCT worker_id)::int AS n FROM runs WHERE status='running'",
- );
- assert.equal(rows[0].n, sessions.length, "each backlog run is held by its own worker");
- } finally {
- releaseHeld();
- await Promise.all(workers.map((worker) => worker.stop()));
- await reader.close();
- await writer.close();
- await direct.end();
- await cleanup();
- }
- },
-);