Multiplayer agent harness for work.
Preserve session queue order across concurrent and inline claims (#1289)
Regan Bell84e649c68256
diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.tsindex 559b46383..27dc1e7a0 100644--- a/src/api/app-helpers.ts+++ b/src/api/app-helpers.ts@@ -211,11 +211,15 @@ export function createAppHelpers(deps: AppDeps, app: App) {run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },);}- const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);+ const claimed = await deps.runs.claimForSession(run.sessionId, "inline", deps.leaseTtlMs);if (claimed) {- return withAdminLink(- await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),+ const result = processRun(+ { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },+ claimed,);+ if (claimed.id === runId) return withAdminLink(await result);+ await result.catch((error: unknown) => swallow("inline predecessor run failed", error));+ continue;}const remaining = deadline - performance.now();if (remaining <= 0) throw new Error(`run ${runId} did not finish within ${timeoutMs}ms`);diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex c0bb961af..29541dab3 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -33,6 +33,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntireturn false;}+ function pendingRuns(now: number): Run[] {+ return [...runs.values()]+ .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))+ .sort((a, b) => a.createdAt - b.createdAt);+ }+function settle(run: Run): void {if (!isTerminal(run.status)) return;events.emit(run.id, run);@@ -76,20 +82,23 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiasync claim(workerId, ttlMs) {const now = Date.now();- const pending = [...runs.values()]- .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))- .sort((a, b) => a.createdAt - b.createdAt);- const run = pending[0];+ const run = pendingRuns(now)[0];if (!run) return null;return lease(run, workerId, ttlMs);},async claimById(runId, workerId, ttlMs) {const run = runs.get(runId);- if (!run || run.status !== "pending" || sessionUnavailable(run.sessionId, Date.now())) return null;+ if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)+ return null;return lease(run, workerId, ttlMs);},+ async claimForSession(sessionId, workerId, ttlMs) {+ const run = pendingRuns(Date.now()).find((pending) => pending.sessionId === sessionId);+ return run ? lease(run, workerId, ttlMs) : null;+ },+async heartbeat(runId, leaseToken, ttlMs) {const run = runs.get(runId);if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 468d95a96..ee9d05701 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -231,6 +231,34 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClareturn { requeued: false, applied: rowCount > 0 };}+ async function claim(workerId: string, ttlMs: number, runId?: string, sessionId?: string): Promise<Run | null> {+ const token = randomUUID();+ const now = Date.now();+ try {+ const { rows } = await q(+ `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,+ attempts=attempts+1, started_at=COALESCE(started_at,$4)+ WHERE id = (+ SELECT candidate.id FROM runs candidate WHERE candidate.status='pending'+ AND candidate.retry_after <= $4+ AND ($5::text IS NULL OR candidate.id=$5)+ AND ($6::text IS NULL OR candidate.session_id=$6)+ AND NOT EXISTS (+ SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id+ AND (sibling.status='running' OR (sibling.status='pending'+ AND (sibling.retry_after > $4 OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))+ )+ ORDER BY candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1+ ) RETURNING *`,+ [token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],+ );+ return rows[0] ? rowToRun(rows[0]) : null;+ } catch (err) {+ if (isUniqueViolation(err)) return null;+ throw err;+ }+ }+const runs: RunStore = {subscribeAvailable(listener, options) {availabilityListeners.set(listener, options?.pollMs ?? 50);@@ -270,49 +298,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClareturn rows[0] ? rowToRun(rows[0]) : null;},- async claim(workerId, ttlMs): Promise<Run | null> {- const token = randomUUID();- const now = Date.now();- try {- const { rows } = await q(- `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,- attempts=attempts+1, started_at=COALESCE(started_at,$4)- WHERE id = (- SELECT id FROM runs WHERE status='pending'- AND retry_after <= $4- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))- ORDER BY created_at ASC, seq ASC FOR UPDATE SKIP LOCKED LIMIT 1- ) RETURNING *`,- [token, now + ttlMs, workerId, now],- );- return rows[0] ? rowToRun(rows[0]) : null;- } catch (err) {- if (isUniqueViolation(err)) return null;- throw err;- }- },+ claim,- async claimById(runId, workerId, ttlMs): Promise<Run | null> {- const token = randomUUID();- const now = Date.now();- try {- const { rows } = await q(- `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,- attempts=attempts+1, started_at=COALESCE(started_at,$4)- WHERE id = (- SELECT id FROM runs WHERE id=$5 AND status='pending'- AND retry_after <= $4- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))- FOR UPDATE SKIP LOCKED LIMIT 1- ) RETURNING *`,- [token, now + ttlMs, workerId, now, runId],- );- return rows[0] ? rowToRun(rows[0]) : null;- } catch (err) {- if (isUniqueViolation(err)) return null;- throw err;- }- },+ claimById: (runId, workerId, ttlMs) => claim(workerId, ttlMs, runId),+ claimForSession: (sessionId, workerId, ttlMs) => claim(workerId, ttlMs, undefined, sessionId),async heartbeat(runId, leaseToken, ttlMs): Promise<boolean> {const { rowCount } = await q(diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 7f7a8d750..00f6f0f18 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -61,6 +61,8 @@ export interface RunStore {claimById(runId: string, workerId: string, ttlMs: number): Promise<Run | null>;+ claimForSession(sessionId: string, workerId: string, ttlMs: number): Promise<Run | null>;+heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;releaseLease(runId: string, leaseToken: string): Promise<boolean>;diff --git a/test/orchestrator.test.ts b/test/orchestrator.test.tsindex a96edea64..043dec017 100644--- a/test/orchestrator.test.ts+++ b/test/orchestrator.test.ts@@ -297,26 +297,40 @@ test("a cron-delivered digest lands as a delivery event with origin, not recipieassert.doesNotMatch(footer, /deploy digest ready/);});-test("a setup-phase failure after the lease is acquired does NOT wedge the thread (lease released)", async () => {- const built = freshApp();- const { app, connectorTokens } = built;-- const realConnectorAccessToken = connectorTokens.connectorAccessToken.bind(connectorTokens);- let injectFailure = true;- connectorTokens.connectorAccessToken = async (...args: Parameters<typeof realConnectorAccessToken>) => {- if (injectFailure) {- injectFailure = false;- throw new Error("injected setup failure");- }- return realConnectorAccessToken(...args);- };-- await assert.rejects(app.turn(dm("hi")), /injected setup failure/);-- const res = await app.turn(dm("hi again"));- assert.equal(res.status, "ok", res.reason);- assert.doesNotMatch(res.reason ?? "", /session busy/);-});+test(+ "an inline turn retries its failed predecessor before executing without a background worker",+ { timeout: 10_000 },+ async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });+ const built = freshApp();+ const { app, connectorTokens } = built;++ const realConnectorAccessToken = connectorTokens.connectorAccessToken.bind(connectorTokens);+ const attempted: string[] = [];+ const claimForSession = built.runs.claimForSession.bind(built.runs);+ built.runs.claimForSession = async (...args) => {+ const run = await claimForSession(...args);+ if (run) attempted.push(run.request.text);+ return run;+ };+ let injectFailure = true;+ connectorTokens.connectorAccessToken = async (...args: Parameters<typeof realConnectorAccessToken>) => {+ if (injectFailure) {+ injectFailure = false;+ throw new Error("injected setup failure");+ }+ return realConnectorAccessToken(...args);+ };++ await assert.rejects(app.turn(dm("hi")), /injected setup failure/);+ t.mock.timers.tick(18_000);++ const res = await app.turn(dm("hi again"));+ assert.equal(res.status, "ok", res.reason);+ assert.doesNotMatch(res.reason ?? "", /session busy/);+ assert.deepEqual(attempted, ["hi", "hi", "hi again"]);+ },+);test("a retried run RESUMES the interrupted turn from the durable ledger instead of restarting it", async () => {const { app } = freshApp();diff --git a/test/run-availability.ts b/test/run-availability.tsindex 6a9093d41..5531bd88c 100644--- a/test/run-availability.ts+++ b/test/run-availability.ts@@ -321,3 +321,40 @@ test(}},);++test(+ "Postgres claims preserve a locked session head while other sessions progress",+ { skip: !process.env.DATABASE_URL },+ async () => {+ const { url, cleanup } = await isolatedPostgres();+ const runtime = createPostgresRunStore(url);+ const pool = new pg.Pool({ connectionString: url });+ const holder = await pool.connect();+ try {+ const first = (await runtime.runs.enqueue({ sessionId: "ordered", request })).run;+ const second = (await runtime.runs.enqueue({ sessionId: "ordered", request })).run;+ const other = (await runtime.runs.enqueue({ sessionId: "independent", request })).run;+ await pool.query("UPDATE runs SET created_at = 1 WHERE session_id = 'ordered'");+ await holder.query("BEGIN");+ await holder.query("SELECT id FROM runs WHERE id = $1 FOR UPDATE", [first.id]);+ const claimedOther = await runtime.runs.claim("other-worker", 5_000);+ assert.equal(claimedOther?.id, other.id, "a locked head blocks its siblings without blocking another session");+ assert.equal(await runtime.runs.claim("waiting-worker", 5_000), null);+ assert.equal(await runtime.runs.claimById(second.id, "inline-worker", 5_000), null);+ assert.equal(await runtime.runs.claimForSession("ordered", "inline-worker", 5_000), null);+ await holder.query("ROLLBACK");+ const claimedFirst = await runtime.runs.claimForSession("ordered", "head-worker", 5_000);+ assert.equal(claimedFirst?.id, first.id);+ assert.equal(await runtime.runs.claimById(second.id, "inline-worker", 5_000), null);+ await runtime.runs.complete(first.id, claimedFirst!.leaseToken!, { status: "ok" });+ const claimedSecond = await runtime.runs.claimById(second.id, "inline-worker", 5_000);+ assert.equal(claimedSecond?.id, second.id);+ } finally {+ await holder.query("ROLLBACK");+ holder.release();+ await runtime.close();+ await pool.end();+ await cleanup();+ }+ },+);diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.tsindex 3242fd41b..87ddc41f9 100644--- a/test/run-retry-delay.test.ts+++ b/test/run-retry-delay.test.ts@@ -49,13 +49,15 @@ for (const backend of ["memory", "postgres"] as const) {const other = (await runs.enqueue({ sessionId: randomUUID(), request })).run;assert.equal(await runs.claimById(first.id, "worker-2", 60_000), null);assert.equal(await runs.claimById(later.id, "worker-2", 60_000), null);+ assert.equal(await runs.claimForSession(sessionId, "inline-worker", 60_000), null);const unrelated = await runs.claim("worker-2", 60_000);assert.equal(unrelated?.id, other.id);await runs.complete(other.id, unrelated!.leaseToken!, { status: "ok", sessionId: other.sessionId });t.mock.timers.tick(29_999);assert.equal(await runs.claim("worker-3", 60_000), null);t.mock.timers.tick(1);- const retried = await runs.claim("worker-3", 60_000);+ assert.equal(await runs.claimById(later.id, "inline-worker", 60_000), null);+ const retried = await runs.claimForSession(sessionId, "worker-3", 60_000);assert.equal(retried?.id, first.id);assert.equal(retried?.attempts, 2);assert.equal(retried?.errorAttempts, 1);diff --git a/test/run-store.test.ts b/test/run-store.test.tsindex 0b9a3e6d1..3eaa57f5a 100644--- a/test/run-store.test.ts+++ b/test/run-store.test.ts@@ -166,6 +166,19 @@ for (const backend of backends) {assert.equal(await runs.claim("w3", 5_000), null);});+ test(`[${backend.name}] inline claims cannot overtake an earlier pending run`, async () => {+ const { runs } = backend.make();+ const unrelated = (await runs.enqueue({ sessionId: "unrelated", request: turn("other") })).run;+ const first = (await runs.enqueue({ sessionId: "inline-order", request: turn("first") })).run;+ const second = (await runs.enqueue({ sessionId: "inline-order", request: turn("second") })).run;+ assert.equal(await runs.claimById(second.id, "second-worker", 5_000), null);+ const claimedFirst = await runs.claimForSession("inline-order", "first-worker", 5_000);+ assert.equal((await runs.get(unrelated.id))?.status, "pending");+ assert.equal(claimedFirst?.id, first.id);+ await runs.complete(first.id, claimedFirst!.leaseToken!, { status: "ok" });+ assert.equal((await runs.claimById(second.id, "second-worker", 5_000))?.id, second.id);+ });+test(`[${backend.name}] lease fencing on heartbeat/complete`, async () => {const { runs } = backend.make();const r = (await runs.enqueue({ sessionId: "s1", request: turn("hi") })).run;