Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: back off failed-turn retries (#1005)

Joshua Francecbfef67913c1
diff --git a/package.json b/package.json
index a5ec15fab..e39f52504 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node --test --test-concurrency=1 test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts",
+ "test:pg": "node --test --test-concurrency=1 test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.ts
index e12df8326..a3fc597dd 100644
--- a/src/api/app-helpers.ts
+++ b/src/api/app-helpers.ts
@@ -10,7 +10,8 @@ import type {
import { orgId as orgIdOf } from "../config.ts";
import { isManageableCreationScope, parseScopeId, scopeId } from "../types.ts";
import { type ListOwnedOptions } from "../files/file-artifact-store.ts";
-import type { Run } from "../runs/run-store.ts";
+import { isTerminal, type Run } from "../runs/run-store.ts";
+import { sleep } from "../util/async.ts";
import type { RunSignal } from "../runs/run-signal-store.ts";
import { processRun } from "../runs/worker.ts";
import { deployRef, encodeRef, parseRef } from "../acl/resource-ref.ts";
@@ -199,16 +200,26 @@ export function createAppHelpers(deps: AppDeps, app: App) {
}
async function drive(runId: string): Promise<TurnResult> {
- const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);
- if (claimed) {
- return withAdminLink(
- await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),
- );
+ const timeoutMs = deps.runWaitMs ?? 60_000;
+ const deadline = performance.now() + timeoutMs;
+ for (;;) {
+ const run = await deps.runs.get(runId);
+ if (!run) throw new Error(`run ${runId} not found`);
+ if (isTerminal(run.status)) {
+ return withAdminLink(
+ run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },
+ );
+ }
+ const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);
+ if (claimed) {
+ return withAdminLink(
+ await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),
+ );
+ }
+ const remaining = deadline - performance.now();
+ if (remaining <= 0) throw new Error(`run ${runId} did not finish within ${timeoutMs}ms`);
+ await sleep(Math.min(100, remaining));
}
- const finished = await deps.runs.waitFor(runId, deps.runWaitMs);
- return withAdminLink(
- finished.result ?? { status: "failed", sessionId: finished.sessionId, reason: "run produced no result" },
- );
}
async function mayUseSharedScope(kind: "channel" | "group", ref: string, actor: Principal): Promise<boolean> {
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index 56b9efd83..7ad6a2ebe 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -14,15 +14,20 @@ const FENCE_HOLD_MS = 600_000;
export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntime {
const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;
const runs = new Map<string, Run>();
+ const retryAfter = new Map<string, number>();
const byKey = new Map<string, string>();
const ledger = new Map<string, string>();
const events = new EventEmitter();
events.setMaxListeners(0);
const terminalListeners: Array<(run: Run) => void> = [];
- function sessionHasRunning(sessionId: string, exceptId?: string): boolean {
+ function sessionUnavailable(sessionId: string, now: number): boolean {
for (const r of runs.values()) {
- if (r.sessionId === sessionId && r.status === "running" && r.id !== exceptId) return true;
+ if (
+ r.sessionId === sessionId &&
+ (r.status === "running" || (r.status === "pending" && (retryAfter.get(r.id) ?? 0) > now))
+ )
+ return true;
}
return false;
}
@@ -69,8 +74,9 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
},
async claim(workerId, ttlMs) {
+ const now = Date.now();
const pending = [...runs.values()]
- .filter((r) => r.status === "pending" && !sessionHasRunning(r.sessionId))
+ .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now))
.sort((a, b) => a.createdAt - b.createdAt);
const run = pending[0];
if (!run) return null;
@@ -79,7 +85,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
async claimById(runId, workerId, ttlMs) {
const run = runs.get(runId);
- if (!run || run.status !== "pending" || sessionHasRunning(run.sessionId)) return null;
+ if (!run || run.status !== "pending" || sessionUnavailable(run.sessionId, Date.now())) return null;
return lease(run, workerId, ttlMs);
},
@@ -119,7 +125,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
async fail(runId, leaseToken, error, opts) {
const run = runs.get(runId);
if (!run || run.leaseToken !== leaseToken) return { requeued: false };
- return { requeued: retire(run, error, opts?.retry !== false, { countsAsError: true }).requeued };
+ return {
+ requeued: retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })
+ .requeued,
+ };
},
async noteTurnUserSeq(runId: string, seq: number) {
@@ -167,6 +176,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
const run = runs.get(runId);
if (!run || run.status !== "pending") return false;
runs.delete(runId);
+ retryAfter.delete(runId);
if (run.dedupKey) byKey.delete(run.dedupKey);
return true;
},
@@ -232,6 +242,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
};
function lease(run: Run, workerId: string, ttlMs: number): Run {
+ retryAfter.delete(run.id);
run.status = "running";
run.leaseToken = randomUUID();
run.leaseExpiresAt = Date.now() + ttlMs;
@@ -245,7 +256,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
run: Run,
error: string,
retry: boolean,
- opts?: { ifExpiredAt?: number; countsAsError?: boolean },
+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },
): { requeued: boolean; applied: boolean } {
if (run.status !== "running") return { requeued: false, applied: false };
if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {
@@ -258,6 +269,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
const overClaimed = run.attempts >= maxClaims;
if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {
run.status = "pending";
+ retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));
return { requeued: true, applied: true };
}
run.status = "failed";
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 7590a8506..252ffd178 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -101,6 +101,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
expectedChecksum: "8594c46c02ee90d43292a4c083fedea6c72d93b5f414f75e9e20d2db51b1b595",
statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS turn_user_seq BIGINT`],
},
+ {
+ id: "runs/store/0004",
+ statements: [
+ `SET LOCAL lock_timeout = '3s'`,
+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS retry_after BIGINT NOT NULL DEFAULT 0`,
+ ],
+ },
],
[
{
@@ -151,7 +158,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
run: Run,
error: string,
retry: boolean,
- opts?: { ifExpiredAt?: number; countsAsError?: boolean },
+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },
): Promise<{ requeued: boolean; applied: boolean }> {
const ifExpiredAt = opts?.ifExpiredAt ?? null;
const countsAsError = opts?.countsAsError ?? false;
@@ -160,9 +167,9 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
if (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {
const { rowCount } = await q(
`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,
- error_attempts=error_attempts+$4
+ error_attempts=error_attempts+$4, retry_after=$5
WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,
- [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0],
+ [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],
);
return { requeued: rowCount > 0, applied: rowCount > 0 };
}
@@ -211,7 +218,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
attempts=attempts+1, started_at=COALESCE(started_at,$4)
WHERE id = (
SELECT id FROM runs WHERE status='pending'
- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running')
+ AND retry_after <= $4
+ AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))
ORDER BY created_at ASC, seq ASC FOR UPDATE SKIP LOCKED LIMIT 1
) RETURNING *`,
[token, now + ttlMs, workerId, now],
@@ -232,7 +240,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
attempts=attempts+1, started_at=COALESCE(started_at,$4)
WHERE id = (
SELECT id FROM runs WHERE id=$5 AND status='pending'
- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running')
+ AND retry_after <= $4
+ AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))
FOR UPDATE SKIP LOCKED LIMIT 1
) RETURNING *`,
[token, now + ttlMs, workerId, now, runId],
@@ -277,7 +286,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
async fail(runId, leaseToken, error, opts): Promise<{ requeued: boolean }> {
const run = await getRun(runId);
if (!run || run.leaseToken !== leaseToken) return { requeued: false };
- return { requeued: (await retire(run, error, opts?.retry !== false, { countsAsError: true })).requeued };
+ return {
+ requeued: (
+ await retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })
+ ).requeued,
+ };
},
async noteTurnUserSeq(runId: string, seq: number): Promise<boolean> {
diff --git a/src/runs/retry-delay.ts b/src/runs/retry-delay.ts
new file mode 100644
index 000000000..a325798eb
--- /dev/null
+++ b/src/runs/retry-delay.ts
@@ -0,0 +1,4 @@
+export function retryDelay(priorErrors: number): number {
+ const base = Math.min(60_000, 15_000 * 2 ** Math.min(Math.max(0, priorErrors), 2));
+ return Math.min(60_000, Math.round(base * (1 + Math.random() * 0.2)));
+}
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 376827845..c3702fb1c 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -64,7 +64,12 @@ export interface RunStore {
complete(runId: string, leaseToken: string, result: TurnResult): Promise<boolean>;
- fail(runId: string, leaseToken: string, error: string, opts?: { retry?: boolean }): Promise<{ requeued: boolean }>;
+ fail(
+ runId: string,
+ leaseToken: string,
+ error: string,
+ opts?: { retry?: boolean; retryAfterMs?: number },
+ ): Promise<{ requeued: boolean }>;
setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState): Promise<boolean>;
diff --git a/src/runs/worker.ts b/src/runs/worker.ts
index 6526747f2..56f7b2f04 100644
--- a/src/runs/worker.ts
+++ b/src/runs/worker.ts
@@ -9,6 +9,7 @@ import { errorParks, type Run, type RunStore } from "./run-store.ts";
import type { SessionStore } from "../sessions/session-store.ts";
import { errMessage, swallow } from "../util/errors.ts";
import { sleep } from "../util/async.ts";
+import { retryDelay } from "./retry-delay.ts";
export interface ProcessDeps {
runs: RunStore;
@@ -88,6 +89,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun
});
await deps.runs.fail(run.id, token, turnFailureMessage(err), {
retry: !(err instanceof NonRetryableTurnError),
+ retryAfterMs: retryDelay(run.errorAttempts),
});
throw err;
} finally {
diff --git a/test/inline-retry.test.ts b/test/inline-retry.test.ts
new file mode 100644
index 000000000..fe5726fa2
--- /dev/null
+++ b/test/inline-retry.test.ts
@@ -0,0 +1,67 @@
+import "./support/auto-fake-sprites.ts";
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { setTimeout as sleep } from "node:timers/promises";
+import { buildApp } from "../src/wiring.ts";
+import { testConfig } from "./support/test-config.ts";
+import type { TurnRequest } from "../src/types.ts";
+
+const request: TurnRequest = {
+ surface: "test",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "inline-retry" },
+ text: "hello",
+ idempotencyKey: "inline-retry",
+};
+
+for (const laterMessage of [false, true]) {
+ test(
+ `inline execution waits out backoff without a worker: later message=${laterMessage}`,
+ { timeout: 5_000 },
+ async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
+ const built = buildApp(testConfig({ runWaitMs: 1_000 }));
+ const queued = await built.app.turn({ ...request, async: true });
+ const claimed = await built.runs.claimById(queued.runId!, "setup", 60_000);
+ await built.runs.fail(claimed!.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 30_000 });
+ let settled = false;
+ const pending = built.app.turn(laterMessage ? { ...request, text: "next", idempotencyKey: "next" } : request);
+ const outcome = pending.then((result) => {
+ settled = true;
+ return result;
+ });
+ await sleep(50);
+ assert.equal(settled, false);
+ assert.equal((await built.runs.get(queued.runId!))?.attempts, 1);
+ t.mock.timers.tick(30_000);
+ const result = await outcome;
+ assert.equal(result.status, "ok");
+ assert.match(result.reply!, laterMessage ? /next/ : /hello/);
+ },
+ );
+}
+
+test("inline waiting respects its timeout without clearing a retry deadline", async () => {
+ const built = buildApp(testConfig({ runWaitMs: 100 }));
+ const queued = await built.app.turn({ ...request, async: true });
+ const claimed = await built.runs.claimById(queued.runId!, "setup", 60_000);
+ await built.runs.fail(claimed!.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 30_000 });
+ const keepAlive = setInterval(() => {}, 100);
+ try {
+ await assert.rejects(built.app.turn(request), /did not finish within 100ms/);
+ assert.equal(await built.runs.claimById(claimed!.id, "other", 60_000), null);
+ } finally {
+ clearInterval(keepAlive);
+ }
+});
+
+test("inline waiting returns another worker's result without executing twice", async () => {
+ const built = buildApp(testConfig({ runWaitMs: 1_000 }));
+ const queued = await built.app.turn({ ...request, async: true });
+ const claimed = await built.runs.claimById(queued.runId!, "other", 60_000);
+ const pending = built.app.turn(request);
+ await sleep(50);
+ await built.runs.complete(claimed!.id, claimed!.leaseToken!, { status: "ok", reply: "other worker" });
+ assert.equal((await pending).reply, "other worker");
+ assert.equal((await built.runs.get(claimed!.id))?.attempts, 1);
+});
diff --git a/test/process-run.test.ts b/test/process-run.test.ts
index 69b577763..97690d3ef 100644
--- a/test/process-run.test.ts
+++ b/test/process-run.test.ts
@@ -170,7 +170,7 @@ test("processRun heartbeats the lease while the turn runs, and the beat stops wi
});
test("a retryable turn failure requeues the run, rethrows, and stops the heartbeat", async (t) => {
- t.mock.timers.enable({ apis: ["setInterval"] });
+ t.mock.timers.enable({ apis: ["setInterval", "Date"], now: Date.now() });
const store = createMemoryRunStore();
const { runs, beats } = spyHeartbeats(store.runs);
@@ -195,6 +195,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe
assert.equal(requeued?.status, "pending", "an ordinary failure goes back on the queue");
assert.equal(requeued?.attempts, 1);
+ assert.equal(await runs.claim("w2", 9_000), null);
t.mock.timers.tick(30_000);
await microtasks();
assert.equal(beats.length, 1, "no heartbeat leaks past the failure");
@@ -221,7 +222,8 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe
);
});
-test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async () => {
+test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
const { runs } = createMemoryRunStore();
const seen: OrchestratorInput[] = [];
const orchestrator = fakeOrchestrator(async (input) => {
@@ -236,6 +238,8 @@ test("finalAttempt marks the attempt whose error would park the run, from the cl
assert.equal(seen[0]?.finalAttempt, false, "budget remains — the orchestrator must not record a terminal failure");
assert.equal((await runs.get(first!.id))?.status, "pending");
+ assert.equal(await runs.claim("w1", 5_000), null);
+ t.mock.timers.tick(60_000);
const second = await runs.claim("w1", 5_000);
await assert.rejects(processRun(deps, second!), /hiccup/);
assert.equal(seen[1]?.finalAttempt, true, "the last budgeted attempt is marked — an error now is terminal");
diff --git a/test/retry-delay.test.ts b/test/retry-delay.test.ts
new file mode 100644
index 000000000..f3e6c30d8
--- /dev/null
+++ b/test/retry-delay.test.ts
@@ -0,0 +1,71 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { retryDelay } from "../src/runs/retry-delay.ts";
+import { processRun } from "../src/runs/worker.ts";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
+import { NonRetryableTurnError } from "../src/core/turn-error.ts";
+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";
+
+const request: OrchestratorInput = {
+ actor: { id: "retry-test", type: "internal" },
+ conversation: { kind: "dm", threadRef: "retry-test", audience: [] },
+ origin: { kind: "direct" },
+ text: "test",
+};
+
+test("retry delays grow with bounded jitter and stay capped", (t) => {
+ for (const random of [0, 0.5, 0.999999]) {
+ t.mock.method(Math, "random", () => random);
+ for (const [attempt, base] of [
+ [0, 15000],
+ [1, 30000],
+ [2, 60000],
+ [100, 60000],
+ ]) {
+ assert.equal(retryDelay(attempt!), Math.min(60000, Math.round(base! * (1 + random * 0.2))));
+ }
+ t.mock.restoreAll();
+ }
+});
+
+test("workers delay all failed-turn retries and respect terminal budgets", async (t) => {
+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });
+ const { runs } = createMemoryRunStore();
+ const run = (await runs.enqueue({ sessionId: "retry-test", request })).run;
+ const error = new Error("dependency unavailable");
+ const orchestrator = {
+ handleTurn: async () => {
+ throw error;
+ },
+ } as unknown as Orchestrator;
+ for (let i = 0; i < 3; i++) {
+ const claimed = await runs.claim("worker", 60_000);
+ assert.ok(claimed);
+ await assert.rejects(processRun({ runs, orchestrator, leaseTtlMs: 60_000 }, claimed), error);
+ if (i < 2) {
+ assert.equal((await runs.get(run.id))?.status, "pending");
+ assert.equal(await runs.claim("other-worker", 60_000), null);
+ t.mock.timers.tick(60_000);
+ }
+ }
+ assert.equal((await runs.get(run.id))?.status, "failed");
+ assert.equal((await runs.get(run.id))?.errorAttempts, 3);
+ for (const error of [new Error("ordinary"), new NonRetryableTurnError("permanent")]) {
+ const { run } = await runs.enqueue({ sessionId: "other", request });
+ const lease = await runs.claim("worker", 60_000);
+ assert.ok(lease);
+ const orchestrator = {
+ handleTurn: async () => {
+ throw error;
+ },
+ } as unknown as Orchestrator;
+ await assert.rejects(processRun({ runs, orchestrator, leaseTtlMs: 60_000 }, lease), error);
+ if (error instanceof NonRetryableTurnError) assert.equal((await runs.get(run.id))?.status, "failed");
+ else {
+ assert.equal(await runs.claimById(run.id, "other", 60_000), null);
+ t.mock.timers.tick(60_000);
+ assert.ok(await runs.claimById(run.id, "other", 60_000));
+ await runs.fail(run.id, (await runs.get(run.id))!.leaseToken!, "done", { retry: false });
+ }
+ }
+});
diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.ts
new file mode 100644
index 000000000..3242fd41b
--- /dev/null
+++ b/test/run-retry-delay.test.ts
@@ -0,0 +1,132 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import type { Pool } from "pg";
+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";
+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";
+import type { OrchestratorInput } from "../src/core/orchestrator.ts";
+
+const request: OrchestratorInput = {
+ actor: { id: "retry-test", type: "internal" },
+ conversation: { kind: "dm", threadRef: "retry-test", audience: [] },
+ origin: { kind: "direct" },
+ text: "test",
+};
+
+for (const backend of ["memory", "postgres"] as const) {
+ test(
+ `${backend}: retry deadline is respected across claim paths and preserves session order`,
+ {
+ skip: backend === "postgres" && !process.env.DATABASE_URL,
+ },
+ async (t) => {
+ let connectionString = process.env.DATABASE_URL!;
+ let admin: Pool | undefined;
+ const schema = `retry_${randomUUID().replaceAll("-", "")}`;
+ if (backend === "postgres") {
+ const pg = (await import("pg")).default;
+ admin = new pg.Pool({ connectionString });
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new URL(connectionString);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ connectionString = url.toString();
+ }
+ const runtime = backend === "memory" ? createMemoryRunStore() : createPostgresRunStore(connectionString);
+ let { runs } = runtime;
+ const sessionId = randomUUID();
+ try {
+ const first = (await runs.enqueue({ sessionId, request })).run;
+ const lease = await runs.claimById(first.id, "worker-1", 60_000);
+ assert.ok(lease?.leaseToken);
+ const now = Date.now();
+ t.mock.timers.enable({ apis: ["Date"], now });
+ await runs.fail(first.id, lease.leaseToken, "temporary outage", { retryAfterMs: 30_000 });
+ if (backend === "postgres") {
+ await runs.close?.();
+ runs = createPostgresRunStore(connectionString).runs;
+ }
+ const later = (await runs.enqueue({ sessionId, request })).run;
+ const other = (await runs.enqueue({ sessionId: randomUUID(), request })).run;
+ assert.equal(await runs.claimById(first.id, "worker-2", 60_000), null);
+ assert.equal(await runs.claimById(later.id, "worker-2", 60_000), null);
+ const unrelated = await runs.claim("worker-2", 60_000);
+ assert.equal(unrelated?.id, other.id);
+ await runs.complete(other.id, unrelated!.leaseToken!, { status: "ok", sessionId: other.sessionId });
+ t.mock.timers.tick(29_999);
+ assert.equal(await runs.claim("worker-3", 60_000), null);
+ t.mock.timers.tick(1);
+ const retried = await runs.claim("worker-3", 60_000);
+ assert.equal(retried?.id, first.id);
+ assert.equal(retried?.attempts, 2);
+ assert.equal(retried?.errorAttempts, 1);
+ await runs.complete(first.id, retried!.leaseToken!, { status: "ok", sessionId });
+ const next = await runs.claim("worker-3", 60_000);
+ assert.equal(next?.id, later.id);
+ await runs.complete(later.id, next!.leaseToken!, { status: "ok", sessionId });
+ } finally {
+ t.mock.timers.reset();
+ await runs.close?.();
+ if (admin) {
+ try {
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ } finally {
+ await admin.end();
+ }
+ }
+ }
+ },
+ );
+}
+
+test("memory: one claim uses one clock snapshot across the retry boundary", async (t) => {
+ const { runs } = createMemoryRunStore();
+ const first = (await runs.enqueue({ sessionId: "same", request })).run;
+ const claimed = await runs.claim("w1", 60_000);
+ const now = Date.now();
+ t.mock.method(Date, "now", () => now);
+ await runs.fail(first.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 1_000 });
+ await runs.enqueue({ sessionId: "same", request });
+ let reads = 0;
+ t.mock.method(Date, "now", () => now + (++reads === 1 ? 999 : 1_000));
+ assert.equal(await runs.claim("w2", 60_000), null);
+ assert.equal((await runs.claim("w2", 60_000))?.id, first.id);
+});
+
+test(
+ "postgres: retry migration bounds lock waiting and can be retried",
+ { skip: !process.env.DATABASE_URL },
+ async () => {
+ const pg = (await import("pg")).default;
+ const { applyPgMigrations, registeredPgMigrations } = await import("../src/persistence/pg-pool.ts");
+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL });
+ const schema = `retry_lock_${randomUUID().replaceAll("-", "")}`;
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new URL(process.env.DATABASE_URL!);
+ url.searchParams.set("options", `-c search_path=${schema} -c statement_timeout=5000`);
+ const runtime = createPostgresRunStore(url.toString());
+ const pool = new pg.Pool({ connectionString: url.toString() });
+ const holder = await pool.connect();
+ try {
+ const migrations = registeredPgMigrations(url.toString());
+ await applyPgMigrations(
+ pool,
+ migrations.filter((m) => m.id !== "runs/store/0004"),
+ );
+ await holder.query("BEGIN");
+ await holder.query("SELECT * FROM runs");
+ await assert.rejects(applyPgMigrations(pool, migrations), { code: "55P03" });
+ assert.equal((await pool.query("SELECT count(*) FROM runs")).rows[0].count, "0");
+ await holder.query("ROLLBACK");
+ await applyPgMigrations(pool, migrations);
+ await applyPgMigrations(pool, migrations);
+ await pool.query("SELECT retry_after FROM runs");
+ } finally {
+ await holder.query("ROLLBACK");
+ holder.release();
+ await runtime.close();
+ await pool.end();
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ await admin.end();
+ }
+ },
+);