Multiplayer agent harness for work.
fix: back off failed-turn retries (#1005)
Joshua Francecbfef67913c1
diff --git a/package.json b/package.jsonindex a5ec15fab..e39f52504 100644--- a/package.json+++ b/package.json@@ -36,7 +36,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node --test --test-concurrency=1 test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts",+ "test:pg": "node --test --test-concurrency=1 test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/src/api/app-helpers.ts b/src/api/app-helpers.tsindex e12df8326..a3fc597dd 100644--- a/src/api/app-helpers.ts+++ b/src/api/app-helpers.ts@@ -10,7 +10,8 @@ import type {import { orgId as orgIdOf } from "../config.ts";import { isManageableCreationScope, parseScopeId, scopeId } from "../types.ts";import { type ListOwnedOptions } from "../files/file-artifact-store.ts";-import type { Run } from "../runs/run-store.ts";+import { isTerminal, type Run } from "../runs/run-store.ts";+import { sleep } from "../util/async.ts";import type { RunSignal } from "../runs/run-signal-store.ts";import { processRun } from "../runs/worker.ts";import { deployRef, encodeRef, parseRef } from "../acl/resource-ref.ts";@@ -199,16 +200,26 @@ export function createAppHelpers(deps: AppDeps, app: App) {}async function drive(runId: string): Promise<TurnResult> {- const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);- if (claimed) {- return withAdminLink(- await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),- );+ const timeoutMs = deps.runWaitMs ?? 60_000;+ const deadline = performance.now() + timeoutMs;+ for (;;) {+ const run = await deps.runs.get(runId);+ if (!run) throw new Error(`run ${runId} not found`);+ if (isTerminal(run.status)) {+ return withAdminLink(+ run.result ?? { status: "failed", sessionId: run.sessionId, reason: "run produced no result" },+ );+ }+ const claimed = await deps.runs.claimById(runId, "inline", deps.leaseTtlMs);+ if (claimed) {+ return withAdminLink(+ await processRun({ runs: deps.runs, orchestrator: deps.orchestrator, leaseTtlMs: deps.leaseTtlMs }, claimed),+ );+ }+ const remaining = deadline - performance.now();+ if (remaining <= 0) throw new Error(`run ${runId} did not finish within ${timeoutMs}ms`);+ await sleep(Math.min(100, remaining));}- const finished = await deps.runs.waitFor(runId, deps.runWaitMs);- return withAdminLink(- finished.result ?? { status: "failed", sessionId: finished.sessionId, reason: "run produced no result" },- );}async function mayUseSharedScope(kind: "channel" | "group", ref: string, actor: Principal): Promise<boolean> {diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex 56b9efd83..7ad6a2ebe 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -14,15 +14,20 @@ const FENCE_HOLD_MS = 600_000;export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntime {const maxClaims = opts?.maxClaims ?? Number.POSITIVE_INFINITY;const runs = new Map<string, Run>();+ const retryAfter = new Map<string, number>();const byKey = new Map<string, string>();const ledger = new Map<string, string>();const events = new EventEmitter();events.setMaxListeners(0);const terminalListeners: Array<(run: Run) => void> = [];- function sessionHasRunning(sessionId: string, exceptId?: string): boolean {+ function sessionUnavailable(sessionId: string, now: number): boolean {for (const r of runs.values()) {- if (r.sessionId === sessionId && r.status === "running" && r.id !== exceptId) return true;+ if (+ r.sessionId === sessionId &&+ (r.status === "running" || (r.status === "pending" && (retryAfter.get(r.id) ?? 0) > now))+ )+ return true;}return false;}@@ -69,8 +74,9 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti},async claim(workerId, ttlMs) {+ const now = Date.now();const pending = [...runs.values()]- .filter((r) => r.status === "pending" && !sessionHasRunning(r.sessionId))+ .filter((r) => r.status === "pending" && !sessionUnavailable(r.sessionId, now)).sort((a, b) => a.createdAt - b.createdAt);const run = pending[0];if (!run) return null;@@ -79,7 +85,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiasync claimById(runId, workerId, ttlMs) {const run = runs.get(runId);- if (!run || run.status !== "pending" || sessionHasRunning(run.sessionId)) return null;+ if (!run || run.status !== "pending" || sessionUnavailable(run.sessionId, Date.now())) return null;return lease(run, workerId, ttlMs);},@@ -119,7 +125,10 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntiasync fail(runId, leaseToken, error, opts) {const run = runs.get(runId);if (!run || run.leaseToken !== leaseToken) return { requeued: false };- return { requeued: retire(run, error, opts?.retry !== false, { countsAsError: true }).requeued };+ return {+ requeued: retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })+ .requeued,+ };},async noteTurnUserSeq(runId: string, seq: number) {@@ -167,6 +176,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunticonst run = runs.get(runId);if (!run || run.status !== "pending") return false;runs.delete(runId);+ retryAfter.delete(runId);if (run.dedupKey) byKey.delete(run.dedupKey);return true;},@@ -232,6 +242,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti};function lease(run: Run, workerId: string, ttlMs: number): Run {+ retryAfter.delete(run.id);run.status = "running";run.leaseToken = randomUUID();run.leaseExpiresAt = Date.now() + ttlMs;@@ -245,7 +256,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRuntirun: Run,error: string,retry: boolean,- opts?: { ifExpiredAt?: number; countsAsError?: boolean },+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },): { requeued: boolean; applied: boolean } {if (run.status !== "running") return { requeued: false, applied: false };if (opts?.ifExpiredAt !== undefined && (run.leaseExpiresAt === null || run.leaseExpiresAt > opts.ifExpiredAt)) {@@ -258,6 +269,7 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunticonst overClaimed = run.attempts >= maxClaims;if (retry && run.errorAttempts < run.maxAttempts && !overClaimed) {run.status = "pending";+ retryAfter.set(run.id, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0));return { requeued: true, applied: true };}run.status = "failed";diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 7590a8506..252ffd178 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -101,6 +101,13 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaexpectedChecksum: "8594c46c02ee90d43292a4c083fedea6c72d93b5f414f75e9e20d2db51b1b595",statements: [`SET LOCAL lock_timeout = '3s'`, `ALTER TABLE runs ADD COLUMN IF NOT EXISTS turn_user_seq BIGINT`],},+ {+ id: "runs/store/0004",+ statements: [+ `SET LOCAL lock_timeout = '3s'`,+ `ALTER TABLE runs ADD COLUMN IF NOT EXISTS retry_after BIGINT NOT NULL DEFAULT 0`,+ ],+ },],[{@@ -151,7 +158,7 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClarun: Run,error: string,retry: boolean,- opts?: { ifExpiredAt?: number; countsAsError?: boolean },+ opts?: { ifExpiredAt?: number; countsAsError?: boolean; retryAfterMs?: number },): Promise<{ requeued: boolean; applied: boolean }> {const ifExpiredAt = opts?.ifExpiredAt ?? null;const countsAsError = opts?.countsAsError ?? false;@@ -160,9 +167,9 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaif (retry && errorAttemptsAfter < run.maxAttempts && !overClaimed) {const { rowCount } = await q(`UPDATE runs SET status='pending', lease_token=NULL, lease_expires_at=NULL, worker_id=NULL,- error_attempts=error_attempts+$4+ error_attempts=error_attempts+$4, retry_after=$5WHERE id=$1 AND lease_token=$2 AND status='running' AND ($3::bigint IS NULL OR lease_expires_at <= $3)`,- [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0],+ [run.id, run.leaseToken, ifExpiredAt, countsAsError ? 1 : 0, Date.now() + Math.max(0, opts?.retryAfterMs ?? 0)],);return { requeued: rowCount > 0, applied: rowCount > 0 };}@@ -211,7 +218,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaattempts=attempts+1, started_at=COALESCE(started_at,$4)WHERE id = (SELECT id FROM runs WHERE status='pending'- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running')+ AND retry_after <= $4+ AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))ORDER BY created_at ASC, seq ASC FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`,[token, now + ttlMs, workerId, now],@@ -232,7 +240,8 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaattempts=attempts+1, started_at=COALESCE(started_at,$4)WHERE id = (SELECT id FROM runs WHERE id=$5 AND status='pending'- AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running')+ AND retry_after <= $4+ AND session_id NOT IN (SELECT session_id FROM runs WHERE status='running' OR (status='pending' AND retry_after > $4))FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`,[token, now + ttlMs, workerId, now, runId],@@ -277,7 +286,11 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxClaasync fail(runId, leaseToken, error, opts): Promise<{ requeued: boolean }> {const run = await getRun(runId);if (!run || run.leaseToken !== leaseToken) return { requeued: false };- return { requeued: (await retire(run, error, opts?.retry !== false, { countsAsError: true })).requeued };+ return {+ requeued: (+ await retire(run, error, opts?.retry !== false, { countsAsError: true, retryAfterMs: opts?.retryAfterMs })+ ).requeued,+ };},async noteTurnUserSeq(runId: string, seq: number): Promise<boolean> {diff --git a/src/runs/retry-delay.ts b/src/runs/retry-delay.tsnew file mode 100644index 000000000..a325798eb--- /dev/null+++ b/src/runs/retry-delay.ts@@ -0,0 +1,4 @@+export function retryDelay(priorErrors: number): number {+ const base = Math.min(60_000, 15_000 * 2 ** Math.min(Math.max(0, priorErrors), 2));+ return Math.min(60_000, Math.round(base * (1 + Math.random() * 0.2)));+}diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 376827845..c3702fb1c 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -64,7 +64,12 @@ export interface RunStore {complete(runId: string, leaseToken: string, result: TurnResult): Promise<boolean>;- fail(runId: string, leaseToken: string, error: string, opts?: { retry?: boolean }): Promise<{ requeued: boolean }>;+ fail(+ runId: string,+ leaseToken: string,+ error: string,+ opts?: { retry?: boolean; retryAfterMs?: number },+ ): Promise<{ requeued: boolean }>;setDeliveryState(runId: string, leaseToken: string | null, state: RunDeliveryState): Promise<boolean>;diff --git a/src/runs/worker.ts b/src/runs/worker.tsindex 6526747f2..56f7b2f04 100644--- a/src/runs/worker.ts+++ b/src/runs/worker.ts@@ -9,6 +9,7 @@ import { errorParks, type Run, type RunStore } from "./run-store.ts";import type { SessionStore } from "../sessions/session-store.ts";import { errMessage, swallow } from "../util/errors.ts";import { sleep } from "../util/async.ts";+import { retryDelay } from "./retry-delay.ts";export interface ProcessDeps {runs: RunStore;@@ -88,6 +89,7 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun});await deps.runs.fail(run.id, token, turnFailureMessage(err), {retry: !(err instanceof NonRetryableTurnError),+ retryAfterMs: retryDelay(run.errorAttempts),});throw err;} finally {diff --git a/test/inline-retry.test.ts b/test/inline-retry.test.tsnew file mode 100644index 000000000..fe5726fa2--- /dev/null+++ b/test/inline-retry.test.ts@@ -0,0 +1,67 @@+import "./support/auto-fake-sprites.ts";+import { test } from "node:test";+import assert from "node:assert/strict";+import { setTimeout as sleep } from "node:timers/promises";+import { buildApp } from "../src/wiring.ts";+import { testConfig } from "./support/test-config.ts";+import type { TurnRequest } from "../src/types.ts";++const request: TurnRequest = {+ surface: "test",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "inline-retry" },+ text: "hello",+ idempotencyKey: "inline-retry",+};++for (const laterMessage of [false, true]) {+ test(+ `inline execution waits out backoff without a worker: later message=${laterMessage}`,+ { timeout: 5_000 },+ async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });+ const built = buildApp(testConfig({ runWaitMs: 1_000 }));+ const queued = await built.app.turn({ ...request, async: true });+ const claimed = await built.runs.claimById(queued.runId!, "setup", 60_000);+ await built.runs.fail(claimed!.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 30_000 });+ let settled = false;+ const pending = built.app.turn(laterMessage ? { ...request, text: "next", idempotencyKey: "next" } : request);+ const outcome = pending.then((result) => {+ settled = true;+ return result;+ });+ await sleep(50);+ assert.equal(settled, false);+ assert.equal((await built.runs.get(queued.runId!))?.attempts, 1);+ t.mock.timers.tick(30_000);+ const result = await outcome;+ assert.equal(result.status, "ok");+ assert.match(result.reply!, laterMessage ? /next/ : /hello/);+ },+ );+}++test("inline waiting respects its timeout without clearing a retry deadline", async () => {+ const built = buildApp(testConfig({ runWaitMs: 100 }));+ const queued = await built.app.turn({ ...request, async: true });+ const claimed = await built.runs.claimById(queued.runId!, "setup", 60_000);+ await built.runs.fail(claimed!.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 30_000 });+ const keepAlive = setInterval(() => {}, 100);+ try {+ await assert.rejects(built.app.turn(request), /did not finish within 100ms/);+ assert.equal(await built.runs.claimById(claimed!.id, "other", 60_000), null);+ } finally {+ clearInterval(keepAlive);+ }+});++test("inline waiting returns another worker's result without executing twice", async () => {+ const built = buildApp(testConfig({ runWaitMs: 1_000 }));+ const queued = await built.app.turn({ ...request, async: true });+ const claimed = await built.runs.claimById(queued.runId!, "other", 60_000);+ const pending = built.app.turn(request);+ await sleep(50);+ await built.runs.complete(claimed!.id, claimed!.leaseToken!, { status: "ok", reply: "other worker" });+ assert.equal((await pending).reply, "other worker");+ assert.equal((await built.runs.get(claimed!.id))?.attempts, 1);+});diff --git a/test/process-run.test.ts b/test/process-run.test.tsindex 69b577763..97690d3ef 100644--- a/test/process-run.test.ts+++ b/test/process-run.test.ts@@ -170,7 +170,7 @@ test("processRun heartbeats the lease while the turn runs, and the beat stops wi});test("a retryable turn failure requeues the run, rethrows, and stops the heartbeat", async (t) => {- t.mock.timers.enable({ apis: ["setInterval"] });+ t.mock.timers.enable({ apis: ["setInterval", "Date"], now: Date.now() });const store = createMemoryRunStore();const { runs, beats } = spyHeartbeats(store.runs);@@ -195,6 +195,7 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbeassert.equal(requeued?.status, "pending", "an ordinary failure goes back on the queue");assert.equal(requeued?.attempts, 1);+ assert.equal(await runs.claim("w2", 9_000), null);t.mock.timers.tick(30_000);await microtasks();assert.equal(beats.length, 1, "no heartbeat leaks past the failure");@@ -221,7 +222,8 @@ test("a retryable turn failure requeues the run, rethrows, and stops the heartbe);});-test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async () => {+test("finalAttempt marks the attempt whose error would park the run, from the claim-time budget", async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });const { runs } = createMemoryRunStore();const seen: OrchestratorInput[] = [];const orchestrator = fakeOrchestrator(async (input) => {@@ -236,6 +238,8 @@ test("finalAttempt marks the attempt whose error would park the run, from the classert.equal(seen[0]?.finalAttempt, false, "budget remains — the orchestrator must not record a terminal failure");assert.equal((await runs.get(first!.id))?.status, "pending");+ assert.equal(await runs.claim("w1", 5_000), null);+ t.mock.timers.tick(60_000);const second = await runs.claim("w1", 5_000);await assert.rejects(processRun(deps, second!), /hiccup/);assert.equal(seen[1]?.finalAttempt, true, "the last budgeted attempt is marked — an error now is terminal");diff --git a/test/retry-delay.test.ts b/test/retry-delay.test.tsnew file mode 100644index 000000000..f3e6c30d8--- /dev/null+++ b/test/retry-delay.test.ts@@ -0,0 +1,71 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { retryDelay } from "../src/runs/retry-delay.ts";+import { processRun } from "../src/runs/worker.ts";+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";+import { NonRetryableTurnError } from "../src/core/turn-error.ts";+import type { Orchestrator, OrchestratorInput } from "../src/core/orchestrator.ts";++const request: OrchestratorInput = {+ actor: { id: "retry-test", type: "internal" },+ conversation: { kind: "dm", threadRef: "retry-test", audience: [] },+ origin: { kind: "direct" },+ text: "test",+};++test("retry delays grow with bounded jitter and stay capped", (t) => {+ for (const random of [0, 0.5, 0.999999]) {+ t.mock.method(Math, "random", () => random);+ for (const [attempt, base] of [+ [0, 15000],+ [1, 30000],+ [2, 60000],+ [100, 60000],+ ]) {+ assert.equal(retryDelay(attempt!), Math.min(60000, Math.round(base! * (1 + random * 0.2))));+ }+ t.mock.restoreAll();+ }+});++test("workers delay all failed-turn retries and respect terminal budgets", async (t) => {+ t.mock.timers.enable({ apis: ["Date"], now: Date.now() });+ const { runs } = createMemoryRunStore();+ const run = (await runs.enqueue({ sessionId: "retry-test", request })).run;+ const error = new Error("dependency unavailable");+ const orchestrator = {+ handleTurn: async () => {+ throw error;+ },+ } as unknown as Orchestrator;+ for (let i = 0; i < 3; i++) {+ const claimed = await runs.claim("worker", 60_000);+ assert.ok(claimed);+ await assert.rejects(processRun({ runs, orchestrator, leaseTtlMs: 60_000 }, claimed), error);+ if (i < 2) {+ assert.equal((await runs.get(run.id))?.status, "pending");+ assert.equal(await runs.claim("other-worker", 60_000), null);+ t.mock.timers.tick(60_000);+ }+ }+ assert.equal((await runs.get(run.id))?.status, "failed");+ assert.equal((await runs.get(run.id))?.errorAttempts, 3);+ for (const error of [new Error("ordinary"), new NonRetryableTurnError("permanent")]) {+ const { run } = await runs.enqueue({ sessionId: "other", request });+ const lease = await runs.claim("worker", 60_000);+ assert.ok(lease);+ const orchestrator = {+ handleTurn: async () => {+ throw error;+ },+ } as unknown as Orchestrator;+ await assert.rejects(processRun({ runs, orchestrator, leaseTtlMs: 60_000 }, lease), error);+ if (error instanceof NonRetryableTurnError) assert.equal((await runs.get(run.id))?.status, "failed");+ else {+ assert.equal(await runs.claimById(run.id, "other", 60_000), null);+ t.mock.timers.tick(60_000);+ assert.ok(await runs.claimById(run.id, "other", 60_000));+ await runs.fail(run.id, (await runs.get(run.id))!.leaseToken!, "done", { retry: false });+ }+ }+});diff --git a/test/run-retry-delay.test.ts b/test/run-retry-delay.test.tsnew file mode 100644index 000000000..3242fd41b--- /dev/null+++ b/test/run-retry-delay.test.ts@@ -0,0 +1,132 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { randomUUID } from "node:crypto";+import type { Pool } from "pg";+import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";+import { createPostgresRunStore } from "../src/runs/postgres-run-store.ts";+import type { OrchestratorInput } from "../src/core/orchestrator.ts";++const request: OrchestratorInput = {+ actor: { id: "retry-test", type: "internal" },+ conversation: { kind: "dm", threadRef: "retry-test", audience: [] },+ origin: { kind: "direct" },+ text: "test",+};++for (const backend of ["memory", "postgres"] as const) {+ test(+ `${backend}: retry deadline is respected across claim paths and preserves session order`,+ {+ skip: backend === "postgres" && !process.env.DATABASE_URL,+ },+ async (t) => {+ let connectionString = process.env.DATABASE_URL!;+ let admin: Pool | undefined;+ const schema = `retry_${randomUUID().replaceAll("-", "")}`;+ if (backend === "postgres") {+ const pg = (await import("pg")).default;+ admin = new pg.Pool({ connectionString });+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new URL(connectionString);+ url.searchParams.set("options", `-c search_path=${schema}`);+ connectionString = url.toString();+ }+ const runtime = backend === "memory" ? createMemoryRunStore() : createPostgresRunStore(connectionString);+ let { runs } = runtime;+ const sessionId = randomUUID();+ try {+ const first = (await runs.enqueue({ sessionId, request })).run;+ const lease = await runs.claimById(first.id, "worker-1", 60_000);+ assert.ok(lease?.leaseToken);+ const now = Date.now();+ t.mock.timers.enable({ apis: ["Date"], now });+ await runs.fail(first.id, lease.leaseToken, "temporary outage", { retryAfterMs: 30_000 });+ if (backend === "postgres") {+ await runs.close?.();+ runs = createPostgresRunStore(connectionString).runs;+ }+ const later = (await runs.enqueue({ sessionId, request })).run;+ const other = (await runs.enqueue({ sessionId: randomUUID(), request })).run;+ assert.equal(await runs.claimById(first.id, "worker-2", 60_000), null);+ assert.equal(await runs.claimById(later.id, "worker-2", 60_000), null);+ const unrelated = await runs.claim("worker-2", 60_000);+ assert.equal(unrelated?.id, other.id);+ await runs.complete(other.id, unrelated!.leaseToken!, { status: "ok", sessionId: other.sessionId });+ t.mock.timers.tick(29_999);+ assert.equal(await runs.claim("worker-3", 60_000), null);+ t.mock.timers.tick(1);+ const retried = await runs.claim("worker-3", 60_000);+ assert.equal(retried?.id, first.id);+ assert.equal(retried?.attempts, 2);+ assert.equal(retried?.errorAttempts, 1);+ await runs.complete(first.id, retried!.leaseToken!, { status: "ok", sessionId });+ const next = await runs.claim("worker-3", 60_000);+ assert.equal(next?.id, later.id);+ await runs.complete(later.id, next!.leaseToken!, { status: "ok", sessionId });+ } finally {+ t.mock.timers.reset();+ await runs.close?.();+ if (admin) {+ try {+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ } finally {+ await admin.end();+ }+ }+ }+ },+ );+}++test("memory: one claim uses one clock snapshot across the retry boundary", async (t) => {+ const { runs } = createMemoryRunStore();+ const first = (await runs.enqueue({ sessionId: "same", request })).run;+ const claimed = await runs.claim("w1", 60_000);+ const now = Date.now();+ t.mock.method(Date, "now", () => now);+ await runs.fail(first.id, claimed!.leaseToken!, "temporary", { retryAfterMs: 1_000 });+ await runs.enqueue({ sessionId: "same", request });+ let reads = 0;+ t.mock.method(Date, "now", () => now + (++reads === 1 ? 999 : 1_000));+ assert.equal(await runs.claim("w2", 60_000), null);+ assert.equal((await runs.claim("w2", 60_000))?.id, first.id);+});++test(+ "postgres: retry migration bounds lock waiting and can be retried",+ { skip: !process.env.DATABASE_URL },+ async () => {+ const pg = (await import("pg")).default;+ const { applyPgMigrations, registeredPgMigrations } = await import("../src/persistence/pg-pool.ts");+ const admin = new pg.Pool({ connectionString: process.env.DATABASE_URL });+ const schema = `retry_lock_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new URL(process.env.DATABASE_URL!);+ url.searchParams.set("options", `-c search_path=${schema} -c statement_timeout=5000`);+ const runtime = createPostgresRunStore(url.toString());+ const pool = new pg.Pool({ connectionString: url.toString() });+ const holder = await pool.connect();+ try {+ const migrations = registeredPgMigrations(url.toString());+ await applyPgMigrations(+ pool,+ migrations.filter((m) => m.id !== "runs/store/0004"),+ );+ await holder.query("BEGIN");+ await holder.query("SELECT * FROM runs");+ await assert.rejects(applyPgMigrations(pool, migrations), { code: "55P03" });+ assert.equal((await pool.query("SELECT count(*) FROM runs")).rows[0].count, "0");+ await holder.query("ROLLBACK");+ await applyPgMigrations(pool, migrations);+ await applyPgMigrations(pool, migrations);+ await pool.query("SELECT retry_after FROM runs");+ } finally {+ await holder.query("ROLLBACK");+ holder.release();+ await runtime.close();+ await pool.end();+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ await admin.end();+ }+ },+);