Multiplayer agent harness for work.
Retry transient sandbox provider errors and report request ids (#1412)
Regan Bell7832ccf3b179
diff --git a/src/sandbox/agent37-sandbox.ts b/src/sandbox/agent37-sandbox.tsindex 4a9cefcc7..0db15f3fe 100644--- a/src/sandbox/agent37-sandbox.ts+++ b/src/sandbox/agent37-sandbox.ts@@ -1,8 +1,8 @@import { randomUUID } from "node:crypto";import type { WorkspaceLayer } from "../types.ts";import type { WorkspaceStore } from "../workspace/workspace-store.ts";-import { createKeyedQueue, sleep } from "../util/async.ts";-import { swallowAs, errMessage } from "../util/errors.ts";+import { createKeyedQueue, fetchWithRetry, sleep } from "../util/async.ts";+import { httpFailure, swallowAs, errMessage } from "../util/errors.ts";import { shq } from "../util/shell.ts";import { nonInteractiveShellPrefix } from "./sandbox-env.ts";import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";@@ -109,7 +109,13 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37Sanconst scratchKeyByName = new Map<string, string>();const activeScratch = new Map<string, number>();- async function api(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<Response> {+ function send(+ method: string,+ path: string,+ body?: unknown,+ timeoutMs = 60_000,+ signal?: AbortSignal,+ ): Promise<Response> {return fetchImpl(`${baseUrl}${path}`, {method,headers: {@@ -117,14 +123,21 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San...(body !== undefined ? { "content-type": "application/json" } : {}),},...(body !== undefined ? { body: JSON.stringify(body) } : {}),- signal: AbortSignal.timeout(timeoutMs),+ signal: signal ?? AbortSignal.timeout(timeoutMs),});}+ function api(method: string, path: string, body?: unknown, timeoutMs?: number): Promise<Response> {+ const operation = (signal?: AbortSignal) => send(method, path, body, timeoutMs, signal);+ return method === "GET" || method === "DELETE"+ ? fetchWithRetry(operation, "idempotent", { timeoutMs })+ : operation();+ }+async function apiJson<T>(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<T> {const res = await api(method, path, body, timeoutMs);if (!res.ok) {- throw new Error(`agent37 ${method} ${path}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`agent37 ${method} ${path}: ${await httpFailure(res)}`);}return (await res.json()) as T;}@@ -146,7 +159,7 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37Sanconst res = await api("POST", `/v1/instances/${encodeURIComponent(id)}/start`, undefined, START_TIMEOUT_MS);if (res.ok) continue;if (res.status !== 400 && res.status !== 409) {- throw new Error(`agent37 start ${id}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`agent37 start ${id}: ${await httpFailure(res)}`);}}await sleep(READY_POLL_MS);@@ -154,12 +167,14 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San}async function createInstance(name: string): Promise<InstanceInfo> {- const info = await apiJson<InstanceInfo>(- "POST",- "/v1/instances",- { template, name, resources, auto_sleep: true },- CREATE_TIMEOUT_MS,+ const res = await fetchWithRetry(+ (signal) =>+ send("POST", "/v1/instances", { template, name, resources, auto_sleep: true }, CREATE_TIMEOUT_MS, signal),+ "refused",+ { timeoutMs: CREATE_TIMEOUT_MS },);+ if (!res.ok) throw new Error(`agent37 create ${name}: ${await httpFailure(res)}`);+ const info = (await res.json()) as InstanceInfo;await ensureRunning(info.id);return info;}@@ -181,7 +196,7 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San}const res = await api("DELETE", `/v1/instances/${encodeURIComponent(found)}`, undefined, 120_000);if (!res.ok && res.status !== 404) {- throw new Error(`agent37 delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`agent37 delete ${name}: ${await httpFailure(res)}`);}idByName.delete(name);}@@ -190,21 +205,21 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37Sanconst id = await instanceIdFor(name);const body = { command: script };const timeoutMs = timeoutSec * 1000 + 2 * EXIT_GRACE_MS;- const first = await api("POST", `/v1/instances/${encodeURIComponent(id)}/exec`, body, timeoutMs);+ const first = await send("POST", `/v1/instances/${encodeURIComponent(id)}/exec`, body, timeoutMs);let res = first;if (!first.ok) {- const detail = (await first.text()).slice(0, 200);+ const detail = await httpFailure(first);if (first.status === 404) idByName.delete(name);const notRunning = first.status === 400 && /running instances/i.test(detail);const freezing = first.status === 409;if (first.status !== 404 && !notRunning && !freezing) {- throw new Error(`agent37 exec ${name}: http ${first.status} ${detail}`);+ throw new Error(`agent37 exec ${name}: ${detail}`);}const retryId = await instanceIdFor(name);await ensureRunning(retryId);- res = await api("POST", `/v1/instances/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);+ res = await send("POST", `/v1/instances/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);if (!res.ok) {- throw new Error(`agent37 exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`agent37 exec ${name}: ${await httpFailure(res)}`);}}const parsed = (await res.json()) as InstanceExecResponse;diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.tsindex 48d16e4d1..a6b61b931 100644--- a/src/sandbox/aws-microvm-api.ts+++ b/src/sandbox/aws-microvm-api.ts@@ -1,8 +1,8 @@import { SignatureV4 } from "@smithy/signature-v4";import { Sha256 } from "@aws-crypto/sha256-js";import { defaultProvider } from "@aws-sdk/credential-provider-node";-import { sleep } from "../util/async.ts";-import { errMessage, swallow } from "../util/errors.ts";+import { fetchWithRetry, sleep } from "../util/async.ts";+import { errMessage, swallow, withRequestId } from "../util/errors.ts";type CredentialProvider = () => Promise<{accessKeyId: string;@@ -111,11 +111,16 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {body: payload,query: {},});- const res = await doFetch(`https://${host}${path}`, {- method,- headers: signed.headers as Record<string, string>,- ...(payload ? { body: payload } : {}),- });+ const res = await fetchWithRetry(+ (signal) =>+ doFetch(`https://${host}${path}`, {+ method,+ headers: signed.headers as Record<string, string>,+ signal,+ ...(payload ? { body: payload } : {}),+ }),+ method === "POST" ? "refused" : "idempotent",+ );const text = await res.text();let json: unknown = undefined;if (text.trim()) {@@ -128,7 +133,11 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {if (res.status < 200 || res.status >= 300) {const msg = (json as { message?: string } | undefined)?.message ?? text ?? `HTTP ${res.status}`;const awsCode = res.headers.get("x-amzn-errortype") ?? undefined;- throw new AwsApiError(`lambda-microvms ${method} ${path} -> ${res.status}: ${msg}`, res.status, awsCode);+ throw new AwsApiError(+ withRequestId(`lambda-microvms ${method} ${path} -> ${res.status}: ${msg}`, res.headers),+ res.status,+ awsCode,+ );}return { status: res.status, json: (json ?? {}) as T };}diff --git a/src/sandbox/smolmachines-sandbox.ts b/src/sandbox/smolmachines-sandbox.tsindex d6893f35a..0b095b98e 100644--- a/src/sandbox/smolmachines-sandbox.ts+++ b/src/sandbox/smolmachines-sandbox.ts@@ -1,7 +1,7 @@import { randomUUID } from "node:crypto";import type { WorkspaceStore } from "../workspace/workspace-store.ts";-import { sleep } from "../util/async.ts";-import { swallowAs } from "../util/errors.ts";+import { fetchWithRetry, sleep } from "../util/async.ts";+import { httpFailure, swallowAs } from "../util/errors.ts";import { shq } from "../util/shell.ts";import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";import {@@ -75,23 +75,35 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolmconst idByName = new Map<string, string>();- async function api(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<Response> {- const res = await fetchImpl(`${baseUrl}${path}`, {+ function send(+ method: string,+ path: string,+ body?: unknown,+ timeoutMs = 60_000,+ signal?: AbortSignal,+ ): Promise<Response> {+ return fetchImpl(`${baseUrl}${path}`, {method,headers: {authorization: `Bearer ${opts.token ?? ""}`,...(body !== undefined ? { "content-type": "application/json" } : {}),},...(body !== undefined ? { body: JSON.stringify(body) } : {}),- signal: AbortSignal.timeout(timeoutMs),+ signal: signal ?? AbortSignal.timeout(timeoutMs),});- return res;+ }++ function api(method: string, path: string, body?: unknown, timeoutMs?: number): Promise<Response> {+ const operation = (signal?: AbortSignal) => send(method, path, body, timeoutMs, signal);+ return method === "GET" || method === "DELETE"+ ? fetchWithRetry(operation, "idempotent", { timeoutMs })+ : operation();}async function apiJson<T>(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<T> {const res = await api(method, path, body, timeoutMs);if (!res.ok) {- throw new Error(`smolmachines ${method} ${path}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines ${method} ${path}: ${await httpFailure(res)}`);}return (await res.json()) as T;}@@ -128,7 +140,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolmreturn { info: existing, created: false };}if (!res.ok) {- throw new Error(`smolmachines create ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines create ${name}: ${await httpFailure(res)}`);}const info = (await res.json()) as MachineInfo;await startMachine(info.id);@@ -138,7 +150,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolmasync function startMachine(id: string): Promise<void> {const res = await api("POST", `/v1/machines/${encodeURIComponent(id)}/start`);if (!res.ok && res.status !== 409) {- throw new Error(`smolmachines start ${id}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines start ${id}: ${await httpFailure(res)}`);}await awaitRunning(id);}@@ -160,7 +172,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm}const res = await api("DELETE", `/v1/machines/${encodeURIComponent(found)}`);if (!res.ok && res.status !== 404) {- throw new Error(`smolmachines delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines delete ${name}: ${await httpFailure(res)}`);}idByName.delete(name);}@@ -178,19 +190,19 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm...(stdinB64 !== undefined ? { stdin: stdinB64 } : {}),};const timeoutMs = timeoutSec * 1000 + 2 * EXIT_GRACE_MS;- const first = await api("POST", `/v1/machines/${encodeURIComponent(id)}/exec`, body, timeoutMs);+ const first = await send("POST", `/v1/machines/${encodeURIComponent(id)}/exec`, body, timeoutMs);let res = first;if (!first.ok) {- const detail = (await first.text()).slice(0, 200);+ const detail = await httpFailure(first);if (first.status === 404) idByName.delete(name);if (first.status !== 409 && first.status !== 404 && !/stopped|suspended/i.test(detail)) {- throw new Error(`smolmachines exec ${name}: http ${first.status} ${detail}`);+ throw new Error(`smolmachines exec ${name}: ${detail}`);}const retryId = await machineIdFor(name);await startMachine(retryId);- res = await api("POST", `/v1/machines/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);+ res = await send("POST", `/v1/machines/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);if (!res.ok) {- throw new Error(`smolmachines exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines exec ${name}: ${await httpFailure(res)}`);}}const parsed = (await res.json()) as MachineExecResponse;@@ -279,26 +291,36 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolmasync function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {const id = await machineIdFor(name);- const res = await fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {- method: "PUT",- headers: { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/octet-stream" },- body: Buffer.from(data),- signal: AbortSignal.timeout(FILE_TRANSFER_TIMEOUT_MS),- });+ const res = await fetchWithRetry(+ (signal) =>+ fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {+ method: "PUT",+ headers: { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/octet-stream" },+ body: Buffer.from(data),+ signal,+ }),+ "idempotent",+ { timeoutMs: FILE_TRANSFER_TIMEOUT_MS },+ );if (!res.ok) {- throw new Error(`smolmachines write ${absPath}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines write ${absPath}: ${await httpFailure(res)}`);}}async function readAbsBytes(name: string, absPath: string): Promise<Uint8Array | null> {const id = await machineIdFor(name);- const res = await fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal: AbortSignal.timeout(FILE_TRANSFER_TIMEOUT_MS),- });+ const res = await fetchWithRetry(+ (signal) =>+ fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {+ headers: { authorization: `Bearer ${opts.token ?? ""}` },+ signal,+ }),+ "idempotent",+ { timeoutMs: FILE_TRANSFER_TIMEOUT_MS },+ );if (res.status === 404) return null;if (!res.ok) {- throw new Error(`smolmachines read ${absPath}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ throw new Error(`smolmachines read ${absPath}: ${await httpFailure(res)}`);}return new Uint8Array(await res.arrayBuffer());}diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.tsindex 9c64e5f74..15257e47f 100644--- a/src/sandbox/sprites-sandbox.ts+++ b/src/sandbox/sprites-sandbox.ts@@ -2,8 +2,8 @@ import { randomUUID } from "node:crypto";import { Agent, fetch as undiciFetch } from "undici";import { SpritesClient } from "@fly/sprites";import type { WorkspaceStore } from "../workspace/workspace-store.ts";-import { sleep } from "../util/async.ts";-import { swallow, swallowAs, errMessage } from "../util/errors.ts";+import { fetchWithRetry, sleep } from "../util/async.ts";+import { httpFailure, swallow, swallowAs, errMessage } from "../util/errors.ts";import { shq } from "../util/shell.ts";import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";import {@@ -104,7 +104,7 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan...(body ? { body: Buffer.from(body) } : {}),signal: AbortSignal.timeout(timeoutSec * 1000 + EXIT_GRACE_MS),} as RequestInit);- if (!res.ok) throw new Error(`sprites exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ if (!res.ok) throw new Error(`sprites exec ${name}: ${await httpFailure(res)}`);const raw = Buffer.from(await res.arrayBuffer());let rc = 0;const out: Buffer[] = [];@@ -214,15 +214,21 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanif (egressPolicyByName.get(name) === want) return;const url = `${baseUrl}/v1/sprites/${encodeURIComponent(name)}/policy/network`;const headers = { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/json" };- const res = await fetchImpl(url, {- method: "POST",- headers,- body: JSON.stringify({ rules }),- signal: AbortSignal.timeout(30_000),+ const res = await fetchWithRetry(+ (signal) =>+ fetchImpl(url, {+ method: "POST",+ headers,+ body: JSON.stringify({ rules }),+ signal,+ }),+ "idempotent",+ { timeoutMs: 30_000 },+ );+ if (!res.ok) throw new Error(`sprites egress policy ${name}: ${await httpFailure(res)}`);+ const check = await fetchWithRetry((signal) => fetchImpl(url, { headers, signal }), "idempotent", {+ timeoutMs: 30_000,});- if (!res.ok)- throw new Error(`sprites egress policy ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);- const check = await fetchImpl(url, { headers, signal: AbortSignal.timeout(30_000) });const got = (await check.json().catch(() => null)) as {rules?: Array<{ domain?: string; action?: string }>;} | null;@@ -398,13 +404,17 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanasync destroyScope(scopeId: string): Promise<void> {return base.provisionQueue(scopeId, async () => {const name = sandboxScopeName(prefix, scopeId);- const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}`, {- method: "DELETE",- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal: AbortSignal.timeout(RESTART_TIMEOUT_MS),- });- if (!res.ok && res.status !== 404)- throw new Error(`sprites delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);+ const res = await fetchWithRetry(+ (signal) =>+ fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}`, {+ method: "DELETE",+ headers: { authorization: `Bearer ${opts.token ?? ""}` },+ signal,+ }),+ "idempotent",+ { timeoutMs: RESTART_TIMEOUT_MS },+ );+ if (!res.ok && res.status !== 404) throw new Error(`sprites delete ${name}: ${await httpFailure(res)}`);ensured.delete(name);pressureEpisodes.delete(name);egressPolicyByName.delete(name);@@ -414,11 +424,16 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSanasync computerStatus(scopeId: string) {const name = sandboxScopeName(prefix, scopeId);const spriteJson = async (path: string): Promise<{ status?: string } | null> => {- const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}${path}`, {- headers: { authorization: `Bearer ${opts.token ?? ""}` },- signal: AbortSignal.timeout(CHECK_TIMEOUT_MS),- });- if (!res.ok) throw new Error(`http ${res.status}`);+ const res = await fetchWithRetry(+ (signal) =>+ fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}${path}`, {+ headers: { authorization: `Bearer ${opts.token ?? ""}` },+ signal,+ }),+ "idempotent",+ { timeoutMs: CHECK_TIMEOUT_MS },+ );+ if (!res.ok) throw new Error(await httpFailure(res));return (await res.json().catch(() => null)) as { status?: string } | null;};const [machineOut, listedOut] = await Promise.allSettled([spriteJson("/check"), spriteJson("")]);@@ -458,14 +473,14 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan});const plain = await restart(false);if (plain.ok) return;- const plainDetail = `http ${plain.status} ${(await plain.text()).slice(0, 200)}`;+ const plainDetail = await httpFailure(plain);if (plain.status !== 409 && plain.status !== 502) throw new Error(`sprites restart ${name}: ${plainDetail}`);let forcedDetail = "";for (let attempt = 0; attempt < FORCED_RESTART_ATTEMPTS; attempt++) {if (attempt > 0) await sleep(FORCED_RESTART_RETRY_MS);const forced = await restart(true);if (forced.ok) return;- forcedDetail = `http ${forced.status} ${(await forced.text()).slice(0, 200)}`;+ forcedDetail = await httpFailure(forced);if (forced.status !== 409 && forced.status !== 502) break;}throw new Error(`sprites restart ${name}: ${plainDetail}; forced retry: ${forcedDetail}`);diff --git a/src/security/security-screener.ts b/src/security/security-screener.tsindex 7149a543f..79a1cf171 100644--- a/src/security/security-screener.ts+++ b/src/security/security-screener.ts@@ -4,6 +4,7 @@ import { setTimeout as delay } from "node:timers/promises";import type { ScopeId } from "../types.ts";import type { SecurityScreenVerdict } from "./security-posture.ts";import { swallow } from "../util/errors.ts";+import { retryAfterMs } from "../util/async.ts";export type SecurityScreenHook = "user_input" | "tool_response";@@ -193,14 +194,6 @@ function classification(body: string, provider: string): SecurityScreenClassific};}-function retryDelayMs(response: Response, fallback: number): number {- const retryAfterHeader = response.headers.get("retry-after");- const retryAfter =- retryAfterHeader === null || retryAfterHeader.trim() === "" ? Number.NaN : Number(retryAfterHeader);- if (Number.isFinite(retryAfter) && retryAfter >= 0) return Math.min(retryAfter * 1_000, 30_000);- return fallback;-}-export function createSecurityScreenProxy(opts: {provider: string;endpoint: string;@@ -255,7 +248,7 @@ export function createSecurityScreenProxy(opts: {signal: chunkSignal,});if (response.status === 429 && attempt < SECURITY_SCREEN_RETRY_MS.length) {- const waitMs = retryDelayMs(response, SECURITY_SCREEN_RETRY_MS[attempt]!);+ const waitMs = retryAfterMs(response.headers) ?? SECURITY_SCREEN_RETRY_MS[attempt]!;await cancelResponseBody(response);await delay(waitMs, undefined, { signal: chunkSignal });continue;diff --git a/src/util/async.ts b/src/util/async.tsindex 6479fec0b..a7b8f3bfe 100644--- a/src/util/async.ts+++ b/src/util/async.ts@@ -1,3 +1,5 @@+import { setTimeout as delay } from "node:timers/promises";+export const sleep = (ms: number, opts?: { unref?: boolean }): Promise<void> =>new Promise((r) => {const t = setTimeout(r, ms);@@ -58,3 +60,78 @@ export async function withAbort<T>(start: () => Promise<T>, signal?: AbortSignalsignal.removeEventListener("abort", onAbort);}}++export interface BackoffOptions {+ attempts?: number;+ baseDelayMs?: number;+ maxDelayMs?: number;+}++const DEFAULT_ATTEMPTS = 4;+const DEFAULT_BASE_DELAY_MS = 500;+const DEFAULT_MAX_DELAY_MS = 8_000;+const RETRY_AFTER_CAP_MS = 30_000;+const REFUSED_STATUSES: ReadonlySet<number> = new Set([429]);+const TRANSIENT_STATUSES: ReadonlySet<number> = new Set([429, 500, 502, 503, 504]);++export function jitteredBackoffMs(attempt: number, opts: BackoffOptions = {}): number {+ const base = opts.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;+ const max = opts.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;+ const ceiling = Math.min(max, base * 2 ** Math.max(0, attempt - 1));+ return Math.round(ceiling * (0.5 + Math.random() / 2));+}++export function retryAfterMs(headers: Headers, now = Date.now()): number | undefined {+ const raw = headers.get("retry-after")?.trim();+ if (!raw) return undefined;+ const seconds = Number(raw);+ if (Number.isFinite(seconds)) return seconds < 0 ? undefined : Math.min(seconds * 1000, RETRY_AFTER_CAP_MS);+ const at = Date.parse(raw);+ if (Number.isNaN(at)) return undefined;+ return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);+}++export type RetryClass = "idempotent" | "refused";++function isAbortError(e: unknown): boolean {+ const name = (e as { name?: unknown } | null)?.name;+ return name === "AbortError" || name === "TimeoutError";+}++export interface RetryOptions extends BackoffOptions {+ signal?: AbortSignal;+ timeoutMs?: number;+}++export async function fetchWithRetry(+ send: (signal: AbortSignal) => Promise<Response>,+ retry: RetryClass,+ opts: RetryOptions = {},+): Promise<Response> {+ const attempts = opts.attempts ?? DEFAULT_ATTEMPTS;+ const deadline = AbortSignal.timeout(opts.timeoutMs ?? 60_000);+ const signal = opts.signal ? AbortSignal.any([opts.signal, deadline]) : deadline;+ const wait = async (ms: number): Promise<void> => {+ try {+ await delay(ms, undefined, { signal });+ } catch (error) {+ signal.throwIfAborted();+ throw error;+ }+ };+ const statuses = retry === "idempotent" ? TRANSIENT_STATUSES : REFUSED_STATUSES;+ for (let attempt = 1; ; attempt++) {+ let res: Response;+ try {+ res = await withAbort(() => send(signal), signal);+ } catch (e) {+ signal.throwIfAborted();+ if (retry !== "idempotent" || attempt >= attempts || isAbortError(e)) throw e;+ await wait(jitteredBackoffMs(attempt, opts));+ continue;+ }+ if (!statuses.has(res.status) || attempt >= attempts) return res;+ await withAbort(() => res.body?.cancel().catch(() => undefined) ?? Promise.resolve(), signal);+ await wait(retryAfterMs(res.headers) ?? jitteredBackoffMs(attempt, opts));+ }+}diff --git a/src/util/errors.ts b/src/util/errors.tsindex 1eaec7bf2..4e48f046e 100644--- a/src/util/errors.ts+++ b/src/util/errors.ts@@ -84,3 +84,18 @@ export function reportFailureAs<T>(context: string, fallback: T, detail?: stringreturn fallback;};}++const REQUEST_ID_HEADERS = ["x-request-id", "x-amzn-requestid", "fly-request-id"];++export function withRequestId(message: string, headers: Headers): string {+ for (const name of REQUEST_ID_HEADERS) {+ const value = headers.get(name);+ if (value) return `${message} [request id ${value}]`;+ }+ return message;+}++export async function httpFailure(res: Response, bodyChars = 200): Promise<string> {+ const body = (await res.text().catch(() => "")).slice(0, bodyChars);+ return withRequestId(`http ${res.status} ${body}`, res.headers);+}diff --git a/test/agent37-sandbox.test.ts b/test/agent37-sandbox.test.tsindex b7707a845..74eeec70a 100644--- a/test/agent37-sandbox.test.ts+++ b/test/agent37-sandbox.test.ts@@ -265,3 +265,62 @@ test("profile advertises resident disk and process sessions", () => {assert.equal(sandbox.profile.processSessions, true);assert.equal(sandbox.profile.egressEnforcement, "none");});++test("instance create retries 429 refusals but not an ambiguous 5xx", async () => {+ fake.failNext(429, {+ headers: { "retry-after": "0" },+ match: (c) => c.method === "POST" && c.path === "/v1/instances",+ });+ const h = await sandbox.provision(layers);+ assert.equal(h.coldStart, true);+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, 2);++ const otherScope = scopeId("personal", "other");+ fake.failNext(500, {+ headers: { "x-request-id": "req-create" },+ match: (c) => c.method === "POST" && c.path === "/v1/instances",+ });+ const posts = fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length;+ await assert.rejects(+ sandbox.provision([{ scopeId: otherScope, mountPath: "/", mode: "rw" }]),+ /agent37 create .*: http 500 .*\[request id req-create\]/,+ );+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, posts + 1);+});++for (const failure of ["disconnect", "503"]) {+ test(`accepted create followed by ${failure} never duplicates the instance`, async () => {+ let injected = false;+ const adapter = make({+ fetchImpl: async (input: string | URL | Request, init?: RequestInit) => {+ const response = await fake.fetchImpl(input, init);+ if (!injected && init?.method === "POST" && new URL(String(input)).pathname === "/v1/instances") {+ injected = true;+ if (failure === "disconnect") throw new TypeError("connection lost after create");+ return new Response("upstream unavailable", { status: 503, headers: { "retry-after": "0" } });+ }+ return response;+ },+ });+ await assert.rejects(adapter.provision(layers), /connection lost after create|http 503/);+ assert.equal(fake.names().length, 1);+ const recovered = await adapter.provision(layers);+ assert.equal(fake.names().length, 1);+ assert.equal((await adapter.run(recovered, "echo recovered")).stdout.trim(), "recovered");+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, 1);+ });+}++test("instance listing retries 429 with Retry-After; exec is never retried and names the request id", async () => {+ fake.failNext(429, {+ headers: { "retry-after": "0" },+ match: (c) => c.method === "GET" && c.path === "/v1/instances",+ });+ const h = await sandbox.provision(layers);+ assert.equal(fake.calls.filter((c) => c.method === "GET" && c.path === "/v1/instances").length, 2);++ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;+ fake.failNext(502, { headers: { "x-request-id": "req-exec" }, match: (c) => c.path.endsWith("/exec") });+ await assert.rejects(sandbox.run(h, "echo hi"), /agent37 exec .*: http 502 .*\[request id req-exec\]/);+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);+});diff --git a/test/aws-microvm-api.test.ts b/test/aws-microvm-api.test.tsindex 6cffe702f..30e980733 100644--- a/test/aws-microvm-api.test.ts+++ b/test/aws-microvm-api.test.ts@@ -15,7 +15,9 @@ interface Recorded {headers: Record<string, string>;}-function fakeFetch(reply: (rec: Recorded, n: number) => { status: number; body?: unknown }): {+function fakeFetch(+ reply: (rec: Recorded, n: number) => { status: number; body?: unknown; headers?: Record<string, string> },+): {fetchImpl: typeof fetch;calls: Recorded[];} {@@ -32,7 +34,7 @@ function fakeFetch(reply: (rec: Recorded, n: number) => { status: number; body?:calls.push(rec);const r = reply(rec, calls.length - 1);const text = r.body === undefined ? "" : JSON.stringify(r.body);- return new Response(text, { status: r.status, headers: { "content-type": "application/json" } });+ return new Response(text, { status: r.status, headers: { "content-type": "application/json", ...r.headers } });}) as unknown as typeof fetch;return { fetchImpl, calls };}@@ -183,3 +185,46 @@ test("readiness reuses only the supplied observation and checks again on the nexassert.equal(reads, 1);await assert.rejects(client.ensureRunning("one", "endpoint", { microvmId: "two", state: "RUNNING" }), /another body/);});++test("control-plane reads retry throttling and outages, and failures carry the AWS request id", async () => {+ const replies = [+ { status: 429, body: { message: "Rate exceeded" }, headers: { "retry-after": "0" } },+ { status: 503, body: { message: "unavailable" }, headers: { "retry-after": "0" } },+ { status: 200, body: { microvmId: "mvm-1", endpoint: "e", state: "RUNNING" } },+ ];+ const { fetchImpl, calls } = fakeFetch((_rec, n) => replies[n]!);+ const api = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl });+ assert.equal((await api.getMicrovm("mvm-1")).state, "RUNNING");+ assert.equal(calls.length, 3);++ const failing = fakeFetch(() => ({+ status: 500,+ body: { message: "boom" },+ headers: { "x-amzn-requestid": "aws-req-1" },+ }));+ const api2 = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: failing.fetchImpl });+ await assert.rejects(api2.getMicrovm("mvm-1"), (e: unknown) => {+ assert.ok(e instanceof AwsApiError);+ assert.equal(e.status, 500);+ assert.match(e.message, /-> 500: boom \[request id aws-req-1\]$/);+ return true;+ });+ assert.equal(failing.calls.length, 4, "an idempotent read is retried until attempts run out");+});++test("runMicrovm retries a throttled create but never an ambiguous server error", async () => {+ const throttled = fakeFetch((_rec, n) =>+ n === 0+ ? { status: 429, body: { message: "Rate exceeded" }, headers: { "retry-after": "0" } }+ : { status: 200, body: { microvmId: "mvm-2", endpoint: "e", state: "PENDING" } },+ );+ const api = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: throttled.fetchImpl });+ const run = { imageIdentifier: "img", ingressNetworkConnectors: [], egressNetworkConnectors: [] };+ assert.equal((await api.runMicrovm(run)).microvmId, "mvm-2");+ assert.equal(throttled.calls.length, 2);++ const broken = fakeFetch(() => ({ status: 502, body: { message: "bad gateway" } }));+ const api2 = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: broken.fetchImpl });+ await assert.rejects(api2.runMicrovm(run), /-> 502: bad gateway/);+ assert.equal(broken.calls.length, 1);+});diff --git a/test/errors.test.ts b/test/errors.test.tsindex dec636fc6..0e308321b 100644--- a/test/errors.test.ts+++ b/test/errors.test.ts@@ -1,6 +1,14 @@import { test } from "node:test";import assert from "node:assert/strict";-import { asError, errMessage, errorAlreadyReported, failureCode, reportFailure } from "../src/util/errors.ts";+import {+ asError,+ errMessage,+ errorAlreadyReported,+ failureCode,+ reportFailure,+ httpFailure,+ withRequestId,+} from "../src/util/errors.ts";import { WorkAdmissionClosed } from "../src/util/admitted-work.ts";import { errMessage as pluginErrMessage } from "../plugins/chassis/src/errors.ts";import { runInNewContext } from "node:vm";@@ -113,3 +121,15 @@ test("reportFailure logs every failure but marks only reportable ones as reporte"[failed] worker: retry (run=r1): a string throw",]);});++test("httpFailure names the status, a clipped body, and the provider request id", async () => {+ const res = new Response("x".repeat(300), { status: 502, headers: { "x-request-id": "req-7" } });+ const text = await httpFailure(res);+ assert.match(text, /^http 502 x{200} \[request id req-7\]$/);+ assert.equal(await httpFailure(new Response("plain", { status: 429 })), "http 429 plain");+ assert.equal(+ withRequestId("lambda -> 429: throttled", new Headers({ "x-amzn-requestid": "aws-1" })),+ "lambda -> 429: throttled [request id aws-1]",+ );+ assert.equal(withRequestId("fine", new Headers()), "fine");+});diff --git a/test/sandbox-destroy-scope.test.ts b/test/sandbox-destroy-scope.test.tsindex 4751b7929..f4d5d696c 100644--- a/test/sandbox-destroy-scope.test.ts+++ b/test/sandbox-destroy-scope.test.ts@@ -47,7 +47,7 @@ test("Sprites scope deletion sends only DELETE, retries provider errors, and acctoken: "test",fetchImpl: async (input, init) => {calls.push(`${init?.method} ${new URL(String(input)).pathname}`);- return new Response(null, { status });+ return new Response(null, { status, headers: { "retry-after": "0" } });},});await assert.rejects(sandbox.destroyScope!("scope"), /503/);@@ -55,7 +55,7 @@ test("Sprites scope deletion sends only DELETE, retries provider errors, and accawait sandbox.destroyScope!("scope");status = 404;await sandbox.destroyScope!("scope");- assert.deepEqual(calls, Array(3).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`));+ assert.deepEqual(calls, Array(6).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`));});for (const [name, create, resource] of [@@ -75,7 +75,7 @@ for (const [name, create, resource] of [const rows = exists ? [{ id: "existing", name: sandboxScopeName("qm", "scope"), status: "stopped" }] : [];return Response.json(resource === "instances" ? { data: rows } : rows);}- return new Response(null, { status });+ return new Response(null, { status, headers: { "retry-after": "0" } });},});await assert.rejects(sandbox.destroyScope!("scope"), /503/);@@ -85,7 +85,7 @@ for (const [name, create, resource] of [await sandbox.destroyScope!("scope");assert.deepEqual(calls, [`GET /v1/${resource}`,- `DELETE /v1/${resource}/existing`,+ ...Array(4).fill(`DELETE /v1/${resource}/existing`),`GET /v1/${resource}`,`DELETE /v1/${resource}/existing`,`GET /v1/${resource}`,diff --git a/test/smolmachines-sandbox.test.ts b/test/smolmachines-sandbox.test.tsindex 7ca38742b..799067656 100644--- a/test/smolmachines-sandbox.test.ts+++ b/test/smolmachines-sandbox.test.ts@@ -225,3 +225,24 @@ test("read and write refuse parent path segments before any provider request", a[],);});++test("control-plane 429 and 5xx are retried with Retry-After honored; exhausted retries name the request id", async () => {+ fake.failNext(429, { headers: { "retry-after": "0" } });+ fake.failNext(503);+ const h = await sandbox.provision(layers);+ assert.equal(h.coldStart, true);+ const listing = fake.calls.filter((c) => c.method === "GET" && c.path === "/v1/machines");+ assert.equal(listing.length, 3, "the machine listing was retried after the 429 and the 503");++ const other = make();+ for (let i = 1; i <= 4; i++) fake.failNext(429, { headers: { "retry-after": "0", "x-request-id": `req-${i}` } });+ await assert.rejects(other.provision(layers), /smolmachines GET \/v1\/machines: http 429 .*\[request id req-4\]/);+});++test("exec is never retried and its failure names the request id", async () => {+ const h = await sandbox.provision(layers);+ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;+ fake.failNext(502, { headers: { "x-request-id": "req-exec" }, match: (c) => c.path.endsWith("/exec") });+ await assert.rejects(sandbox.run(h, "echo hi"), /smolmachines exec .*: http 502 .*\[request id req-exec\]/);+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);+});diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.tsindex 613e73c72..4a77d32ff 100644--- a/test/sprites-sandbox.test.ts+++ b/test/sprites-sandbox.test.ts@@ -518,3 +518,27 @@ test("every sprites fetch rides one HTTP/1.1 dispatcher, so a bad request failsassert.ok(optionsKey, "undici Agent must expose its options");assert.equal(dispatcher[optionsKey]?.allowH2, false);});++test("control-plane reads retry 429 with Retry-After while exec is never retried", async () => {+ const h = await sandbox.provision(layers);+ fake.failNext(429, { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });+ assert.deepEqual(await sandbox.computerStatus!(scope), {+ machine: "healthy",+ listed: "warm",+ provisioned: true,+ guestResponsive: true,+ });+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/check")).length, 2);++ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;+ fake.failNext(502, { headers: { "fly-request-id": "fly-9" }, match: (c) => c.path.endsWith("/exec") });+ await assert.rejects(sandbox.run(h, "echo hi"), /sprites exec .*: http 502 .*\[request id fly-9\]/);+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);++ for (let i = 1; i <= 4; i++)+ fake.failNext(503, {+ headers: { "retry-after": "0", "x-request-id": `req-${i}` },+ match: (c) => c.method === "DELETE",+ });+ await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*: http 503 .*\[request id req-4\]/);+});diff --git a/test/support/fake-agent37.ts b/test/support/fake-agent37.tsindex 0ff06e1cf..ddb580c3f 100644--- a/test/support/fake-agent37.ts+++ b/test/support/fake-agent37.ts@@ -20,6 +20,11 @@ interface FakeInstance {home: string;}+export interface InjectedFailure {+ headers?: Record<string, string>;+ match?: (call: { method: string; path: string }) => boolean;+}+export interface FakeInstanceView {status: string;template?: string;@@ -35,6 +40,7 @@ export interface FakeAgent37 {sleep(name: string, opts?: { freezing?: number }): void;stop(name: string): void;fail(name: string): void;+ failNext(status: number, opts?: InjectedFailure): void;execScripts(): string[];cleanup(): void;}@@ -48,6 +54,7 @@ export function installFakeAgent37(): FakeAgent37 {const execScripts: string[] = [];const calls: Agent37Call[] = [];let nextId = 1;+ const injected: Array<InjectedFailure & { status: number }> = [];const live = (): FakeInstance[] => [...instances.values()].filter((i) => i.status !== "deleted").reverse();const byName = (name: string): FakeInstance | undefined => live().find((i) => i.name === name);@@ -131,6 +138,11 @@ export function installFakeAgent37(): FakeAgent37 {const url = new URL(typeof input === "string" ? input : input.toString());const method = init?.method ?? "GET";calls.push({ method, path: url.pathname });+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));+ if (at >= 0) {+ const [next] = injected.splice(at, 1);+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });+ }if (url.pathname === "/v1/instances" && method === "GET") {return Response.json({ data: live().map(info) });}@@ -222,6 +234,9 @@ export function installFakeAgent37(): FakeAgent37 {const m = byName(name);if (m) m.status = "failed";},+ failNext: (status, opts = {}) => {+ injected.push({ status, ...opts });+ },execScripts: () => [...execScripts],cleanup: () => rmSync(root, { recursive: true, force: true }),};diff --git a/test/support/fake-smolmachines.ts b/test/support/fake-smolmachines.tsindex 652166db4..8b7b88776 100644--- a/test/support/fake-smolmachines.ts+++ b/test/support/fake-smolmachines.ts@@ -18,6 +18,11 @@ interface FakeMachine {home: string;}+export interface InjectedFailure {+ headers?: Record<string, string>;+ match?: (call: { method: string; path: string }) => boolean;+}+export interface FakeSmolmachines {fetchImpl: typeof fetch;calls: SmolCall[];@@ -25,6 +30,7 @@ export interface FakeSmolmachines {names(): string[];machine(name: string): { state: string; ephemeral: boolean; resources?: Record<string, number> } | null;stop(name: string): void;+ failNext(status: number, opts?: InjectedFailure): void;execScripts(): string[];reset(): void;cleanup(): void;@@ -38,6 +44,7 @@ export function installFakeSmolmachines(): FakeSmolmachines {const execScripts: string[] = [];const calls: SmolCall[] = [];let nextId = 1;+ const injected: Array<InjectedFailure & { status: number }> = [];const byName = (name: string): FakeMachine | undefined => [...machines.values()].find((m) => m.name === name);@@ -115,6 +122,11 @@ export function installFakeSmolmachines(): FakeSmolmachines {const url = new URL(typeof input === "string" ? input : input.toString());const method = init?.method ?? "GET";calls.push({ method, path: url.pathname });+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));+ if (at >= 0) {+ const [next] = injected.splice(at, 1);+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });+ }if (url.pathname === "/v1/machines" && method === "GET") {return Response.json([...machines.values()].map(info));}@@ -191,12 +203,16 @@ export function installFakeSmolmachines(): FakeSmolmachines {const m = byName(name);if (m) m.state = "stopped";},+ failNext: (status, opts = {}) => {+ injected.push({ status, ...opts });+ },execScripts: () => [...execScripts],reset: () => {for (const m of machines.values()) rmSync(m.home, { recursive: true, force: true });machines.clear();execScripts.length = 0;calls.length = 0;+ injected.length = 0;},cleanup: () => rmSync(root, { recursive: true, force: true }),};diff --git a/test/support/fake-sprites.ts b/test/support/fake-sprites.tsindex 81bca945f..0f01a25c1 100644--- a/test/support/fake-sprites.ts+++ b/test/support/fake-sprites.ts@@ -17,6 +17,11 @@ export interface SpritesCall {script?: string;}+export interface InjectedFailure {+ headers?: Record<string, string>;+ match?: (call: { method: string; path: string }) => boolean;+}+export interface FakeSprites {client: SpritesClientLike;fetchImpl: typeof fetch;@@ -28,6 +33,7 @@ export interface FakeSprites {execScripts(): string[];stallAfterRun(name: string): void;fail502(name: string): void;+ failNext(status: number, opts?: InjectedFailure): void;refuseRestart(name: string): void;refuseForcedRestart(name: string): void;setPressure(name: string, p: { full10: number; full60: number; load1: number }): void;@@ -50,6 +56,7 @@ export function installFakeSprites(): FakeSprites {const refusedRestart = new Set<string>();const refusedForcedRestart = new Set<string>();const restarts: string[] = [];+ const injected: Array<InjectedFailure & { status: number }> = [];const ensureDir = (name: string): string => {let s = sprites.get(name);@@ -114,6 +121,11 @@ export function installFakeSprites(): FakeSprites {const url = new URL(typeof input === "string" ? input : input.toString());const method = init?.method ?? "GET";calls.push({ method, path: url.pathname });+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));+ if (at >= 0) {+ const [next] = injected.splice(at, 1);+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });+ }const health = /^\/v1\/sprites\/([^/]+)\/check$/.exec(url.pathname);if (health) {const name = decodeURIComponent(health[1]!);@@ -211,6 +223,9 @@ export function installFakeSprites(): FakeSprites {fail502: (name) => {gateway502.add(name);},+ failNext: (status, opts = {}) => {+ injected.push({ status, ...opts });+ },refuseRestart: (name) => {refusedRestart.add(name);},@@ -235,6 +250,7 @@ export function installFakeSprites(): FakeSprites {refusedRestart.clear();refusedForcedRestart.clear();restarts.length = 0;+ injected.length = 0;},cleanup: () => rmSync(root, { recursive: true, force: true }),};diff --git a/test/util-async.test.ts b/test/util-async.test.tsindex 0999c28ec..5995f0461 100644--- a/test/util-async.test.ts+++ b/test/util-async.test.ts@@ -2,7 +2,7 @@ import "./support/auto-fake-sprites.ts";import { test } from "node:test";import assert from "node:assert/strict";-import { sleep, createKeyedQueue } from "../src/util/async.ts";+import { sleep, createKeyedQueue, fetchWithRetry, jitteredBackoffMs, retryAfterMs } from "../src/util/async.ts";test("sleep resolves after roughly the given delay", async () => {const t0 = Date.now();@@ -135,3 +135,158 @@ test("withAbort skips an already cancelled operation and preserves the reason",assert.equal(called, false);assert.equal(await withAbort(async () => 42), 42);});++test("retryAfterMs reads seconds or an HTTP date and caps the wait", () => {+ assert.equal(retryAfterMs(new Headers({ "retry-after": "2" })), 2_000);+ assert.equal(retryAfterMs(new Headers({ "retry-after": "0" })), 0);+ assert.equal(retryAfterMs(new Headers({ "retry-after": "900" })), 30_000);+ const now = Date.parse("2026-09-18T12:00:00Z");+ assert.equal(retryAfterMs(new Headers({ "retry-after": "Fri, 18 Sep 2026 12:00:05 GMT" }), now), 5_000);+ assert.equal(retryAfterMs(new Headers({ "retry-after": "Fri, 18 Sep 2026 11:00:00 GMT" }), now), 0);+ assert.equal(retryAfterMs(new Headers({ "retry-after": "soon" })), undefined);+ assert.equal(retryAfterMs(new Headers()), undefined);+});++test("jitteredBackoffMs grows exponentially, jitters within half its ceiling, and stays bounded", () => {+ for (let attempt = 1; attempt <= 6; attempt++) {+ const ceiling = Math.min(8_000, 500 * 2 ** (attempt - 1));+ for (let i = 0; i < 20; i++) {+ const d = jitteredBackoffMs(attempt);+ assert.ok(d >= ceiling / 2 && d <= ceiling, `attempt ${attempt}: ${d} within [${ceiling / 2}, ${ceiling}]`);+ }+ }+ assert.ok(jitteredBackoffMs(3, { baseDelayMs: 10, maxDelayMs: 25 }) <= 25);+});++const fast = { baseDelayMs: 1, maxDelayMs: 2 };+const reply = (status: number, headers: Record<string, string> = {}) =>+ new Response(`status ${status}`, { status, headers });++test("fetchWithRetry retries 429 and 5xx on idempotent requests, honoring Retry-After", async () => {+ const statuses = [429, 503, 502, 200];+ let calls = 0;+ const t0 = Date.now();+ const res = await fetchWithRetry(+ async () => reply(statuses[calls++]!, calls === 1 ? { "retry-after": "0" } : {}),+ "idempotent",+ fast,+ );+ assert.equal(res.status, 200);+ assert.equal(calls, 4);+ assert.ok(Date.now() - t0 < 1_000);+});++test("fetchWithRetry returns the last transient response once attempts are exhausted", async () => {+ let calls = 0;+ const res = await fetchWithRetry(+ async () => {+ calls++;+ return reply(429, { "retry-after": "0", "x-request-id": `req-${calls}` });+ },+ "idempotent",+ { ...fast, attempts: 3 },+ );+ assert.equal(res.status, 429);+ assert.equal(res.headers.get("x-request-id"), "req-3");+ assert.equal(calls, 3);+});++test("fetchWithRetry never retries client errors or successes", async () => {+ for (const status of [200, 400, 404, 409, 422]) {+ let calls = 0;+ const res = await fetchWithRetry(+ async () => {+ calls++;+ return reply(status);+ },+ "idempotent",+ fast,+ );+ assert.equal(res.status, status);+ assert.equal(calls, 1, `status ${status} is returned to the caller untouched`);+ }+});++test("fetchWithRetry retries connection failures only for idempotent requests, never aborts", async () => {+ let calls = 0;+ const flaky = async () => {+ calls++;+ if (calls === 1) throw new TypeError("fetch failed");+ return reply(200);+ };+ assert.equal((await fetchWithRetry(flaky, "idempotent", fast)).status, 200);+ assert.equal(calls, 2);++ calls = 0;+ await assert.rejects(fetchWithRetry(flaky, "refused", fast), /fetch failed/);+ assert.equal(calls, 1);++ calls = 0;+ const aborted = async () => {+ calls++;+ throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });+ };+ await assert.rejects(fetchWithRetry(aborted, "idempotent", fast), { name: "TimeoutError" });+ assert.equal(calls, 1);+});++test("fetchWithRetry retries throttling but never infers refusal from a server error", async () => {+ for (const [status, expectedCalls] of [+ [429, 2],+ [503, 1],+ [500, 1],+ [502, 1],+ [504, 1],+ ] as const) {+ let calls = 0;+ const res = await fetchWithRetry(+ async () => {+ calls++;+ return reply(calls === 1 ? status : 201, { "retry-after": "0" });+ },+ "refused",+ fast,+ );+ assert.equal(calls, expectedCalls, `status ${status}`);+ assert.equal(res.status, expectedCalls === 2 ? 201 : status);+ }+});++test("fetchWithRetry cancellation interrupts Retry-After without another request", async () => {+ const controller = new AbortController();+ let calls = 0;+ const pending = fetchWithRetry(+ async () => {+ calls++;+ return reply(429, { "retry-after": "30" });+ },+ "idempotent",+ { signal: controller.signal },+ );+ const timeout = setTimeout(() => controller.abort(), 20);+ try {+ await assert.rejects(pending, { name: "AbortError" });+ assert.equal(calls, 1);+ } finally {+ clearTimeout(timeout);+ }+});++test("fetchWithRetry shares a deadline across attempts and backoff", async () => {+ const signals: AbortSignal[] = [];+ await assert.rejects(+ fetchWithRetry(+ async (signal) => {+ signals.push(signal);+ if (signals.length === 1) return reply(503, { "retry-after": "0" });+ return new Promise<Response>((resolve) => setTimeout(() => resolve(reply(200)), 100));+ },+ "idempotent",+ { timeoutMs: 30 },+ ),+ { name: "TimeoutError" },+ );+ assert.equal(signals.length, 2);+ assert.equal(signals[0], signals[1]);+ assert.equal(signals[1]!.aborted, true);+});