Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Preserve session queue order across concurrent and inline claims (#1289)

Regan Bell84e649c68256
diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.ts
index 559b46383..27dc1e7a0 100644
--- a/src/api/app-helpers.ts
+++ b/src/api/app-helpers.ts
@@ -211,11 +211,15 @@ export function createAppHelpers(deps: AppDeps, app: App) {
run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },
);
}
- const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);
+ const claimed = await deps.runs.claimForSession(run.sessionId, "inline", deps.leaseTtlMs);
if (claimed) {
- return withAdminLink(
- await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),
+ const result = processRun(
+ { runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs },
+ claimed,
);
+ if (claimed.id === runId) return withAdminLink(await result);
+ await result.catch((error: unknown) => swallow("inline predecessor run failed", error));
+ continue;
}
const remaining = deadline - performance.now();
if (remaining <= 0) throw new Error(`run ${runId} did not finish within ${timeoutMs}ms`);
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index c0bb961af..29541dab3 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -33,6 +33,12 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
return false;
}
+ function pendingRuns(now: number): Run[] {
+ return [...runs.values()]
+ .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))
+ .sort((a, b) => a.createdAt - b.createdAt);
+ }
+
function settle(run: Run): void {
if (!isTerminal(run.status)) return;
events.emit(run.id, run);
@@ -76,20 +82,23 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
async claim(workerId, ttlMs) {
const now = Date.now();
- const pending = [...runs.values()]
- .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))
- .sort((a, b) => a.createdAt - b.createdAt);
- const run = pending[0];
+ const run = pendingRuns(now)[0];
if (!run) return null;
return lease(run, workerId, ttlMs);
},
async claimById(runId, workerId, ttlMs) {
const run = runs.get(runId);
- if (!run || run.status !== "pending" || sessionUnavailable(run.sessionId, Date.now())) return null;
+ if (!run || pendingRuns(Date.now()).find((pending) => pending.sessionId === run.sessionId)?.id !== runId)
+ return null;
return lease(run, workerId, ttlMs);
},
+ async claimForSession(sessionId, workerId, ttlMs) {
+ const run = pendingRuns(Date.now()).find((pending) => pending.sessionId === sessionId);
+ return run ? lease(run, workerId, ttlMs) : null;
+ },
+
async heartbeat(runId, leaseToken, ttlMs) {
const run = runs.get(runId);
if (!run || run.status !== "running" || run.leaseToken !== leaseToken) return false;
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 468d95a96..ee9d05701 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -231,6 +231,34 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
return { requeued: false, applied: rowCount > 0 };
}
+ async function claim(workerId: string, ttlMs: number, runId?: string, sessionId?: string): Promise<Run | null> {
+ const token = randomUUID();
+ const now = Date.now();
+ try {
+ const { rows } = await q(
+ `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,
+ attempts=attempts+1, started_at=COALESCE(started_at,$4)
+ WHERE id = (
+ SELECT candidate.id FROM runs candidate WHERE candidate.status='pending'
+ AND candidate.retry_after <= $4
+ AND ($5::text IS NULL OR candidate.id=$5)
+ AND ($6::text IS NULL OR candidate.session_id=$6)
+ AND NOT EXISTS (
+ SELECT 1 FROM runs sibling WHERE sibling.session_id=candidate.session_id
+ AND (sibling.status='running' OR (sibling.status='pending'
+ AND (sibling.retry_after > $4 OR (sibling.created_at, sibling.seq) < (candidate.created_at, candidate.seq))))
+ )
+ ORDER BY candidate.created_at ASC, candidate.seq ASC FOR UPDATE SKIP LOCKED LIMIT 1
+ ) RETURNING *`,
+ [token, now + ttlMs, workerId, now, runId ?? null, sessionId ?? null],
+ );
+ return rows[0] ? rowToRun(rows[0]) : null;
+ } catch (err) {
+ if (isUniqueViolation(err)) return null;
+ throw err;
+ }
+ }
+
const runs: RunStore = {
subscribeAvailable(listener, options) {
availabilityListeners.set(listener, options?.pollMs ?? 50);
@@ -270,49 +298,10 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
return rows[0] ? rowToRun(rows[0]) : null;
},
- async claim(workerId, ttlMs): Promise<Run | null> {
- const token = randomUUID();
- const now = Date.now();
- try {
- const { rows } = await q(
- `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,
- attempts=attempts+1, started_at=COALESCE(started_at,$4)
- WHERE id = (
- SELECT id FROM runs WHERE status='pending'
- AND retry_after <= $4
- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))
- ORDER BY created_at ASC, seq ASC FOR UPDATE SKIP LOCKED LIMIT 1
- ) RETURNING *`,
- [token, now + ttlMs, workerId, now],
- );
- return rows[0] ? rowToRun(rows[0]) : null;
- } catch (err) {
- if (isUniqueViolation(err)) return null;
- throw err;
- }
- },
+ claim,
- async claimById(runId, workerId, ttlMs): Promise<Run | null> {
- const token = randomUUID();
- const now = Date.now();
- try {
- const { rows } = await q(
- `UPDATE runs SET status='running', lease_token=$1, lease_expires_at=$2, worker_id=$3,
- attempts=attempts+1, started_at=COALESCE(started_at,$4)
- WHERE id = (
- SELECT id FROM runs WHERE id=$5 AND status='pending'
- AND retry_after <= $4
- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))
- FOR UPDATE SKIP LOCKED LIMIT 1
- ) RETURNING *`,
- [token, now + ttlMs, workerId, now, runId],
- );
- return rows[0] ? rowToRun(rows[0]) : null;
- } catch (err) {
- if (isUniqueViolation(err)) return null;
- throw err;
- }
- },
+ claimById: (runId, workerId, ttlMs) => claim(workerId, ttlMs, runId),
+ claimForSession: (sessionId, workerId, ttlMs) => claim(workerId, ttlMs, undefined, sessionId),
async heartbeat(runId, leaseToken, ttlMs): Promise<boolean> {
const { rowCount } = await q(
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 7f7a8d750..00f6f0f18 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -61,6 +61,8 @@ export interface RunStore {
claimById(runId: string, workerId: string, ttlMs: number): Promise<Run | null>;
+ claimForSession(sessionId: string, workerId: string, ttlMs: number): Promise<Run | null>;
+
heartbeat(runId: string, leaseToken: string, ttlMs: number): Promise<boolean>;
releaseLease(runId: string, leaseToken: string): Promise<boolean>;
diff --git a/test/orchestrator.test.ts b/test/orchestrator.test.ts
index a96edea64..043dec017 100644
--- a/test/orchestrator.test.ts
+++ b/test/orchestrator.test.ts
@@ -297,26 +297,40 @@ test("a cron-delivered digest lands as a delivery event with origin, not recipie
assert.doesNotMatch(footer, /deploy digest ready/);
});
-test("a setup-phase failure after the lease is acquired does NOT wedge the thread (lease released)", async () => {
- const built = freshApp();
- const { app, connectorTokens } = built;
-
- const realConnectorAccessToken = connectorTokens.connectorAccessToken.bind(connectorTokens);
- let injectFailure = true;
- connectorTokens.connectorAccessToken = async (...args: Parameters<typeof realConnectorAccessToken>) => {
- if (injectFailure) {
- injectFailure = false;
- throw new Error("injected setup failure");
- }
- return realConnectorAccessToken(...args);
- };
-
- await assert.rejects(app.turn(dm("hi")), /injected setup failure/);
-
- const res = await app.turn(dm("hi again"));
- assert.equal(res.status, "ok", res.reason);
- assert.doesNotMatch(res.reason ?? "", /session busy/);
-});
+test(
+ "an inline turn retries its failed predecessor before executing without a background worker",
+ { timeout: 10_000 },
+ async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
+ const built = freshApp();
+ const { app, connectorTokens } = built;
+
+ const realConnectorAccessToken = connectorTokens.connectorAccessToken.bind(connectorTokens);
+ const attempted: string[] = [];
+ const claimForSession = built.runs.claimForSession.bind(built.runs);
+ built.runs.claimForSession = async (...args) => {
+ const run = await claimForSession(...args);
+ if (run) attempted.push(run.request.text);
+ return run;
+ };
+ let injectFailure = true;
+ connectorTokens.connectorAccessToken = async (...args: Parameters<typeof realConnectorAccessToken>) => {
+ if (injectFailure) {
+ injectFailure = false;
+ throw new Error("injected setup failure");
+ }
+ return realConnectorAccessToken(...args);
+ };
+
+ await assert.rejects(app.turn(dm("hi")), /injected setup failure/);
+ t.mock.timers.tick(18_000);
+
+ const res = await app.turn(dm("hi again"));
+ assert.equal(res.status, "ok", res.reason);
+ assert.doesNotMatch(res.reason ?? "", /session busy/);
+ assert.deepEqual(attempted, ["hi", "hi", "hi again"]);
+ },
+);
test("a retried run RESUMES the interrupted turn from the durable ledger instead of restarting it", async () => {
const { app } = freshApp();
diff --git a/test/run-availability.ts b/test/run-availability.ts
index 6a9093d41..5531bd88c 100644
--- a/test/run-availability.ts
+++ b/test/run-availability.ts
@@ -321,3 +321,40 @@ test(
}
},
);
+
+test(
+ "Postgres claims preserve a locked session head while other sessions progress",
+ { skip: !process.env.DATABASE_URL },
+ async () => {
+ const { url, cleanup } = await isolatedPostgres();
+ const runtime = createPostgresRunStore(url);
+ const pool = new pg.Pool({ connectionString: url });
+ const holder = await pool.connect();
+ try {
+ const first = (await runtime.runs.enqueue({ sessionId: "ordered", request })).run;
+ const second = (await runtime.runs.enqueue({ sessionId: "ordered", request })).run;
+ const other = (await runtime.runs.enqueue({ sessionId: "independent", request })).run;
+ await pool.query("UPDATE runs SET created_at = 1 WHERE session_id = 'ordered'");
+ await holder.query("BEGIN");
+ await holder.query("SELECT id FROM runs WHERE id = $1 FOR UPDATE", [first.id]);
+ const claimedOther = await runtime.runs.claim("other-worker", 5_000);
+ assert.equal(claimedOther?.id, other.id, "a locked head blocks its siblings without blocking another session");
+ assert.equal(await runtime.runs.claim("waiting-worker", 5_000), null);
+ assert.equal(await runtime.runs.claimById(second.id, "inline-worker", 5_000), null);
+ assert.equal(await runtime.runs.claimForSession("ordered", "inline-worker", 5_000), null);
+ await holder.query("ROLLBACK");
+ const claimedFirst = await runtime.runs.claimForSession("ordered", "head-worker", 5_000);
+ assert.equal(claimedFirst?.id, first.id);
+ assert.equal(await runtime.runs.claimById(second.id, "inline-worker", 5_000), null);
+ await runtime.runs.complete(first.id, claimedFirst!.leaseToken!, { status: "ok" });
+ const claimedSecond = await runtime.runs.claimById(second.id, "inline-worker", 5_000);
+ assert.equal(claimedSecond?.id, second.id);
+ } finally {
+ await holder.query("ROLLBACK");
+ holder.release();
+ await runtime.close();
+ await pool.end();
+ await cleanup();
+ }
+ },
+);
diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.ts
index 3242fd41b..87ddc41f9 100644
--- a/test/run-retry-delay.test.ts
+++ b/test/run-retry-delay.test.ts
@@ -49,13 +49,15 @@ for (const backend of ["memory", "postgres"] as const) {
const other = (await runs.enqueue({ sessionId: randomUUID(), request })).run;
assert.equal(await runs.claimById(first.id, "worker-2", 60_000), null);
assert.equal(await runs.claimById(later.id, "worker-2", 60_000), null);
+ assert.equal(await runs.claimForSession(sessionId, "inline-worker", 60_000), null);
const unrelated = await runs.claim("worker-2", 60_000);
assert.equal(unrelated?.id, other.id);
await runs.complete(other.id, unrelated!.leaseToken!, { status: "ok", sessionId: other.sessionId });
t.mock.timers.tick(29_999);
assert.equal(await runs.claim("worker-3", 60_000), null);
t.mock.timers.tick(1);
- const retried = await runs.claim("worker-3", 60_000);
+ assert.equal(await runs.claimById(later.id, "inline-worker", 60_000), null);
+ const retried = await runs.claimForSession(sessionId, "worker-3", 60_000);
assert.equal(retried?.id, first.id);
assert.equal(retried?.attempts, 2);
assert.equal(retried?.errorAttempts, 1);
diff --git a/test/run-store.test.ts b/test/run-store.test.ts
index 0b9a3e6d1..3eaa57f5a 100644
--- a/test/run-store.test.ts
+++ b/test/run-store.test.ts
@@ -166,6 +166,19 @@ for (const backend of backends) {
assert.equal(await runs.claim("w3", 5_000), null);
});
+ test(`[${backend.name}] inline claims cannot overtake an earlier pending run`, async () => {
+ const { runs } = backend.make();
+ const unrelated = (await runs.enqueue({ sessionId: "unrelated", request: turn("other") })).run;
+ const first = (await runs.enqueue({ sessionId: "inline-order", request: turn("first") })).run;
+ const second = (await runs.enqueue({ sessionId: "inline-order", request: turn("second") })).run;
+ assert.equal(await runs.claimById(second.id, "second-worker", 5_000), null);
+ const claimedFirst = await runs.claimForSession("inline-order", "first-worker", 5_000);
+ assert.equal((await runs.get(unrelated.id))?.status, "pending");
+ assert.equal(claimedFirst?.id, first.id);
+ await runs.complete(first.id, claimedFirst!.leaseToken!, { status: "ok" });
+ assert.equal((await runs.claimById(second.id, "second-worker", 5_000))?.id, second.id);
+ });
+
test(`[${backend.name}] lease fencing on heartbeat/complete`, async () => {
const { runs } = backend.make();
const r = (await runs.enqueue({ sessionId: "s1", request: turn("hi") })).run;