Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Retry transient sandbox provider errors and report request ids (#1412)

Regan Bell7832ccf3b179
diff --git a/src/sandbox/agent37-sandbox.ts b/src/sandbox/agent37-sandbox.ts
index 4a9cefcc7..0db15f3fe 100644
--- a/src/sandbox/agent37-sandbox.ts
+++ b/src/sandbox/agent37-sandbox.ts
@@ -1,8 +1,8 @@
import { randomUUID } from "node:crypto";
import type { WorkspaceLayer } from "../types.ts";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
-import { createKeyedQueue, sleep } from "../util/async.ts";
-import { swallowAs, errMessage } from "../util/errors.ts";
+import { createKeyedQueue, fetchWithRetry, sleep } from "../util/async.ts";
+import { httpFailure, swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { nonInteractiveShellPrefix } from "./sandbox-env.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
@@ -109,7 +109,13 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
const scratchKeyByName = new Map<string, string>();
const activeScratch = new Map<string, number>();
- async function api(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<Response> {
+ function send(
+ method: string,
+ path: string,
+ body?: unknown,
+ timeoutMs = 60_000,
+ signal?: AbortSignal,
+ ): Promise<Response> {
return fetchImpl(`${baseUrl}${path}`, {
method,
headers: {
@@ -117,14 +123,21 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
...(body !== undefined ? { "content-type": "application/json" } : {}),
},
...(body !== undefined ? { body: JSON.stringify(body) } : {}),
- signal: AbortSignal.timeout(timeoutMs),
+ signal: signal ?? AbortSignal.timeout(timeoutMs),
});
}
+ function api(method: string, path: string, body?: unknown, timeoutMs?: number): Promise<Response> {
+ const operation = (signal?: AbortSignal) => send(method, path, body, timeoutMs, signal);
+ return method === "GET" || method === "DELETE"
+ ? fetchWithRetry(operation, "idempotent", { timeoutMs })
+ : operation();
+ }
+
async function apiJson<T>(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<T> {
const res = await api(method, path, body, timeoutMs);
if (!res.ok) {
- throw new Error(`agent37 ${method} ${path}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`agent37 ${method} ${path}: ${await httpFailure(res)}`);
}
return (await res.json()) as T;
}
@@ -146,7 +159,7 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
const res = await api("POST", `/v1/instances/${encodeURIComponent(id)}/start`, undefined, START_TIMEOUT_MS);
if (res.ok) continue;
if (res.status !== 400 && res.status !== 409) {
- throw new Error(`agent37 start ${id}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`agent37 start ${id}: ${await httpFailure(res)}`);
}
}
await sleep(READY_POLL_MS);
@@ -154,12 +167,14 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
}
async function createInstance(name: string): Promise<InstanceInfo> {
- const info = await apiJson<InstanceInfo>(
- "POST",
- "/v1/instances",
- { template, name, resources, auto_sleep: true },
- CREATE_TIMEOUT_MS,
+ const res = await fetchWithRetry(
+ (signal) =>
+ send("POST", "/v1/instances", { template, name, resources, auto_sleep: true }, CREATE_TIMEOUT_MS, signal),
+ "refused",
+ { timeoutMs: CREATE_TIMEOUT_MS },
);
+ if (!res.ok) throw new Error(`agent37 create ${name}: ${await httpFailure(res)}`);
+ const info = (await res.json()) as InstanceInfo;
await ensureRunning(info.id);
return info;
}
@@ -181,7 +196,7 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
}
const res = await api("DELETE", `/v1/instances/${encodeURIComponent(found)}`, undefined, 120_000);
if (!res.ok && res.status !== 404) {
- throw new Error(`agent37 delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`agent37 delete ${name}: ${await httpFailure(res)}`);
}
idByName.delete(name);
}
@@ -190,21 +205,21 @@ export function createAgent37Sandbox(workspace: WorkspaceStore, opts: Agent37San
const id = await instanceIdFor(name);
const body = { command: script };
const timeoutMs = timeoutSec * 1000 + 2 * EXIT_GRACE_MS;
- const first = await api("POST", `/v1/instances/${encodeURIComponent(id)}/exec`, body, timeoutMs);
+ const first = await send("POST", `/v1/instances/${encodeURIComponent(id)}/exec`, body, timeoutMs);
let res = first;
if (!first.ok) {
- const detail = (await first.text()).slice(0, 200);
+ const detail = await httpFailure(first);
if (first.status === 404) idByName.delete(name);
const notRunning = first.status === 400 && /running instances/i.test(detail);
const freezing = first.status === 409;
if (first.status !== 404 && !notRunning && !freezing) {
- throw new Error(`agent37 exec ${name}: http ${first.status} ${detail}`);
+ throw new Error(`agent37 exec ${name}: ${detail}`);
}
const retryId = await instanceIdFor(name);
await ensureRunning(retryId);
- res = await api("POST", `/v1/instances/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);
+ res = await send("POST", `/v1/instances/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);
if (!res.ok) {
- throw new Error(`agent37 exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`agent37 exec ${name}: ${await httpFailure(res)}`);
}
}
const parsed = (await res.json()) as InstanceExecResponse;
diff --git a/src/sandbox/aws-microvm-api.ts b/src/sandbox/aws-microvm-api.ts
index 48d16e4d1..a6b61b931 100644
--- a/src/sandbox/aws-microvm-api.ts
+++ b/src/sandbox/aws-microvm-api.ts
@@ -1,8 +1,8 @@
import { SignatureV4 } from "@smithy/signature-v4";
import { Sha256 } from "@aws-crypto/sha256-js";
import { defaultProvider } from "@aws-sdk/credential-provider-node";
-import { sleep } from "../util/async.ts";
-import { errMessage, swallow } from "../util/errors.ts";
+import { fetchWithRetry, sleep } from "../util/async.ts";
+import { errMessage, swallow, withRequestId } from "../util/errors.ts";
type CredentialProvider = () => Promise<{
accessKeyId: string;
@@ -111,11 +111,16 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {
body: payload,
query: {},
});
- const res = await doFetch(`https://${host}${path}`, {
- method,
- headers: signed.headers as Record<string, string>,
- ...(payload ? { body: payload } : {}),
- });
+ const res = await fetchWithRetry(
+ (signal) =>
+ doFetch(`https://${host}${path}`, {
+ method,
+ headers: signed.headers as Record<string, string>,
+ signal,
+ ...(payload ? { body: payload } : {}),
+ }),
+ method === "POST" ? "refused" : "idempotent",
+ );
const text = await res.text();
let json: unknown = undefined;
if (text.trim()) {
@@ -128,7 +133,11 @@ export function createMicrovmApi(opts: AwsMicrovmApiOptions): AwsMicrovmApi {
if (res.status < 200 || res.status >= 300) {
const msg = (json as { message?: string } | undefined)?.message ?? text ?? `HTTP ${res.status}`;
const awsCode = res.headers.get("x-amzn-errortype") ?? undefined;
- throw new AwsApiError(`lambda-microvms ${method} ${path} -> ${res.status}: ${msg}`, res.status, awsCode);
+ throw new AwsApiError(
+ withRequestId(`lambda-microvms ${method} ${path} -> ${res.status}: ${msg}`, res.headers),
+ res.status,
+ awsCode,
+ );
}
return { status: res.status, json: (json ?? {}) as T };
}
diff --git a/src/sandbox/smolmachines-sandbox.ts b/src/sandbox/smolmachines-sandbox.ts
index d6893f35a..0b095b98e 100644
--- a/src/sandbox/smolmachines-sandbox.ts
+++ b/src/sandbox/smolmachines-sandbox.ts
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
-import { sleep } from "../util/async.ts";
-import { swallowAs } from "../util/errors.ts";
+import { fetchWithRetry, sleep } from "../util/async.ts";
+import { httpFailure, swallowAs } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import {
@@ -75,23 +75,35 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
const idByName = new Map<string, string>();
- async function api(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<Response> {
- const res = await fetchImpl(`${baseUrl}${path}`, {
+ function send(
+ method: string,
+ path: string,
+ body?: unknown,
+ timeoutMs = 60_000,
+ signal?: AbortSignal,
+ ): Promise<Response> {
+ return fetchImpl(`${baseUrl}${path}`, {
method,
headers: {
authorization: `Bearer ${opts.token ?? ""}`,
...(body !== undefined ? { "content-type": "application/json" } : {}),
},
...(body !== undefined ? { body: JSON.stringify(body) } : {}),
- signal: AbortSignal.timeout(timeoutMs),
+ signal: signal ?? AbortSignal.timeout(timeoutMs),
});
- return res;
+ }
+
+ function api(method: string, path: string, body?: unknown, timeoutMs?: number): Promise<Response> {
+ const operation = (signal?: AbortSignal) => send(method, path, body, timeoutMs, signal);
+ return method === "GET" || method === "DELETE"
+ ? fetchWithRetry(operation, "idempotent", { timeoutMs })
+ : operation();
}
async function apiJson<T>(method: string, path: string, body?: unknown, timeoutMs = 60_000): Promise<T> {
const res = await api(method, path, body, timeoutMs);
if (!res.ok) {
- throw new Error(`smolmachines ${method} ${path}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines ${method} ${path}: ${await httpFailure(res)}`);
}
return (await res.json()) as T;
}
@@ -128,7 +140,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
return { info: existing, created: false };
}
if (!res.ok) {
- throw new Error(`smolmachines create ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines create ${name}: ${await httpFailure(res)}`);
}
const info = (await res.json()) as MachineInfo;
await startMachine(info.id);
@@ -138,7 +150,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
async function startMachine(id: string): Promise<void> {
const res = await api("POST", `/v1/machines/${encodeURIComponent(id)}/start`);
if (!res.ok && res.status !== 409) {
- throw new Error(`smolmachines start ${id}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines start ${id}: ${await httpFailure(res)}`);
}
await awaitRunning(id);
}
@@ -160,7 +172,7 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
}
const res = await api("DELETE", `/v1/machines/${encodeURIComponent(found)}`);
if (!res.ok && res.status !== 404) {
- throw new Error(`smolmachines delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines delete ${name}: ${await httpFailure(res)}`);
}
idByName.delete(name);
}
@@ -178,19 +190,19 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
...(stdinB64 !== undefined ? { stdin: stdinB64 } : {}),
};
const timeoutMs = timeoutSec * 1000 + 2 * EXIT_GRACE_MS;
- const first = await api("POST", `/v1/machines/${encodeURIComponent(id)}/exec`, body, timeoutMs);
+ const first = await send("POST", `/v1/machines/${encodeURIComponent(id)}/exec`, body, timeoutMs);
let res = first;
if (!first.ok) {
- const detail = (await first.text()).slice(0, 200);
+ const detail = await httpFailure(first);
if (first.status === 404) idByName.delete(name);
if (first.status !== 409 && first.status !== 404 && !/stopped|suspended/i.test(detail)) {
- throw new Error(`smolmachines exec ${name}: http ${first.status} ${detail}`);
+ throw new Error(`smolmachines exec ${name}: ${detail}`);
}
const retryId = await machineIdFor(name);
await startMachine(retryId);
- res = await api("POST", `/v1/machines/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);
+ res = await send("POST", `/v1/machines/${encodeURIComponent(retryId)}/exec`, body, timeoutMs);
if (!res.ok) {
- throw new Error(`smolmachines exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines exec ${name}: ${await httpFailure(res)}`);
}
}
const parsed = (await res.json()) as MachineExecResponse;
@@ -279,26 +291,36 @@ export function createSmolmachinesSandbox(workspace: WorkspaceStore, opts: Smolm
async function writeAbsBytes(name: string, absPath: string, data: Uint8Array): Promise<void> {
const id = await machineIdFor(name);
- const res = await fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {
- method: "PUT",
- headers: { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/octet-stream" },
- body: Buffer.from(data),
- signal: AbortSignal.timeout(FILE_TRANSFER_TIMEOUT_MS),
- });
+ const res = await fetchWithRetry(
+ (signal) =>
+ fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {
+ method: "PUT",
+ headers: { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/octet-stream" },
+ body: Buffer.from(data),
+ signal,
+ }),
+ "idempotent",
+ { timeoutMs: FILE_TRANSFER_TIMEOUT_MS },
+ );
if (!res.ok) {
- throw new Error(`smolmachines write ${absPath}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines write ${absPath}: ${await httpFailure(res)}`);
}
}
async function readAbsBytes(name: string, absPath: string): Promise<Uint8Array | null> {
const id = await machineIdFor(name);
- const res = await fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {
- headers: { authorization: `Bearer ${opts.token ?? ""}` },
- signal: AbortSignal.timeout(FILE_TRANSFER_TIMEOUT_MS),
- });
+ const res = await fetchWithRetry(
+ (signal) =>
+ fetchImpl(`${baseUrl}${filesUrl(id, absPath)}`, {
+ headers: { authorization: `Bearer ${opts.token ?? ""}` },
+ signal,
+ }),
+ "idempotent",
+ { timeoutMs: FILE_TRANSFER_TIMEOUT_MS },
+ );
if (res.status === 404) return null;
if (!res.ok) {
- throw new Error(`smolmachines read ${absPath}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ throw new Error(`smolmachines read ${absPath}: ${await httpFailure(res)}`);
}
return new Uint8Array(await res.arrayBuffer());
}
diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.ts
index 9c64e5f74..15257e47f 100644
--- a/src/sandbox/sprites-sandbox.ts
+++ b/src/sandbox/sprites-sandbox.ts
@@ -2,8 +2,8 @@ import { randomUUID } from "node:crypto";
import { Agent, fetch as undiciFetch } from "undici";
import { SpritesClient } from "@fly/sprites";
import type { WorkspaceStore } from "../workspace/workspace-store.ts";
-import { sleep } from "../util/async.ts";
-import { swallow, swallowAs, errMessage } from "../util/errors.ts";
+import { fetchWithRetry, sleep } from "../util/async.ts";
+import { httpFailure, swallow, swallowAs, errMessage } from "../util/errors.ts";
import { shq } from "../util/shell.ts";
import { createExecProcessSessions, type ExecProcessIo } from "./exec-process-session.ts";
import {
@@ -104,7 +104,7 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
...(body ? { body: Buffer.from(body) } : {}),
signal: AbortSignal.timeout(timeoutSec * 1000 + EXIT_GRACE_MS),
} as RequestInit);
- if (!res.ok) throw new Error(`sprites exec ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ if (!res.ok) throw new Error(`sprites exec ${name}: ${await httpFailure(res)}`);
const raw = Buffer.from(await res.arrayBuffer());
let rc = 0;
const out: Buffer[] = [];
@@ -214,15 +214,21 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
if (egressPolicyByName.get(name) === want) return;
const url = `${baseUrl}/v1/sprites/${encodeURIComponent(name)}/policy/network`;
const headers = { authorization: `Bearer ${opts.token ?? ""}`, "content-type": "application/json" };
- const res = await fetchImpl(url, {
- method: "POST",
- headers,
- body: JSON.stringify({ rules }),
- signal: AbortSignal.timeout(30_000),
+ const res = await fetchWithRetry(
+ (signal) =>
+ fetchImpl(url, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({ rules }),
+ signal,
+ }),
+ "idempotent",
+ { timeoutMs: 30_000 },
+ );
+ if (!res.ok) throw new Error(`sprites egress policy ${name}: ${await httpFailure(res)}`);
+ const check = await fetchWithRetry((signal) => fetchImpl(url, { headers, signal }), "idempotent", {
+ timeoutMs: 30_000,
});
- if (!res.ok)
- throw new Error(`sprites egress policy ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
- const check = await fetchImpl(url, { headers, signal: AbortSignal.timeout(30_000) });
const got = (await check.json().catch(() => null)) as {
rules?: Array<{ domain?: string; action?: string }>;
} | null;
@@ -398,13 +404,17 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
async destroyScope(scopeId: string): Promise<void> {
return base.provisionQueue(scopeId, async () => {
const name = sandboxScopeName(prefix, scopeId);
- const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}`, {
- method: "DELETE",
- headers: { authorization: `Bearer ${opts.token ?? ""}` },
- signal: AbortSignal.timeout(RESTART_TIMEOUT_MS),
- });
- if (!res.ok && res.status !== 404)
- throw new Error(`sprites delete ${name}: http ${res.status} ${(await res.text()).slice(0, 200)}`);
+ const res = await fetchWithRetry(
+ (signal) =>
+ fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}`, {
+ method: "DELETE",
+ headers: { authorization: `Bearer ${opts.token ?? ""}` },
+ signal,
+ }),
+ "idempotent",
+ { timeoutMs: RESTART_TIMEOUT_MS },
+ );
+ if (!res.ok && res.status !== 404) throw new Error(`sprites delete ${name}: ${await httpFailure(res)}`);
ensured.delete(name);
pressureEpisodes.delete(name);
egressPolicyByName.delete(name);
@@ -414,11 +424,16 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
async computerStatus(scopeId: string) {
const name = sandboxScopeName(prefix, scopeId);
const spriteJson = async (path: string): Promise<{ status?: string } | null> => {
- const res = await fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}${path}`, {
- headers: { authorization: `Bearer ${opts.token ?? ""}` },
- signal: AbortSignal.timeout(CHECK_TIMEOUT_MS),
- });
- if (!res.ok) throw new Error(`http ${res.status}`);
+ const res = await fetchWithRetry(
+ (signal) =>
+ fetchImpl(`${baseUrl}/v1/sprites/${encodeURIComponent(name)}${path}`, {
+ headers: { authorization: `Bearer ${opts.token ?? ""}` },
+ signal,
+ }),
+ "idempotent",
+ { timeoutMs: CHECK_TIMEOUT_MS },
+ );
+ if (!res.ok) throw new Error(await httpFailure(res));
return (await res.json().catch(() => null)) as { status?: string } | null;
};
const [machineOut, listedOut] = await Promise.allSettled([spriteJson("/check"), spriteJson("")]);
@@ -458,14 +473,14 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan
});
const plain = await restart(false);
if (plain.ok) return;
- const plainDetail = `http ${plain.status} ${(await plain.text()).slice(0, 200)}`;
+ const plainDetail = await httpFailure(plain);
if (plain.status !== 409 && plain.status !== 502) throw new Error(`sprites restart ${name}: ${plainDetail}`);
let forcedDetail = "";
for (let attempt = 0; attempt < FORCED_RESTART_ATTEMPTS; attempt++) {
if (attempt > 0) await sleep(FORCED_RESTART_RETRY_MS);
const forced = await restart(true);
if (forced.ok) return;
- forcedDetail = `http ${forced.status} ${(await forced.text()).slice(0, 200)}`;
+ forcedDetail = await httpFailure(forced);
if (forced.status !== 409 && forced.status !== 502) break;
}
throw new Error(`sprites restart ${name}: ${plainDetail}; forced retry: ${forcedDetail}`);
diff --git a/src/security/security-screener.ts b/src/security/security-screener.ts
index 7149a543f..79a1cf171 100644
--- a/src/security/security-screener.ts
+++ b/src/security/security-screener.ts
@@ -4,6 +4,7 @@ import { setTimeout as delay } from "node:timers/promises";
import type { ScopeId } from "../types.ts";
import type { SecurityScreenVerdict } from "./security-posture.ts";
import { swallow } from "../util/errors.ts";
+import { retryAfterMs } from "../util/async.ts";
export type SecurityScreenHook = "user_input" | "tool_response";
@@ -193,14 +194,6 @@ function classification(body: string, provider: string): SecurityScreenClassific
};
}
-function retryDelayMs(response: Response, fallback: number): number {
- const retryAfterHeader = response.headers.get("retry-after");
- const retryAfter =
- retryAfterHeader === null || retryAfterHeader.trim() === "" ? Number.NaN : Number(retryAfterHeader);
- if (Number.isFinite(retryAfter) && retryAfter >= 0) return Math.min(retryAfter * 1_000, 30_000);
- return fallback;
-}
-
export function createSecurityScreenProxy(opts: {
provider: string;
endpoint: string;
@@ -255,7 +248,7 @@ export function createSecurityScreenProxy(opts: {
signal: chunkSignal,
});
if (response.status === 429 && attempt < SECURITY_SCREEN_RETRY_MS.length) {
- const waitMs = retryDelayMs(response, SECURITY_SCREEN_RETRY_MS[attempt]!);
+ const waitMs = retryAfterMs(response.headers) ?? SECURITY_SCREEN_RETRY_MS[attempt]!;
await cancelResponseBody(response);
await delay(waitMs, undefined, { signal: chunkSignal });
continue;
diff --git a/src/util/async.ts b/src/util/async.ts
index 6479fec0b..a7b8f3bfe 100644
--- a/src/util/async.ts
+++ b/src/util/async.ts
@@ -1,3 +1,5 @@
+import { setTimeout as delay } from "node:timers/promises";
+
export const sleep = (ms: number, opts?: { unref?: boolean }): Promise<void> =>
new Promise((r) => {
const t = setTimeout(r, ms);
@@ -58,3 +60,78 @@ export async function withAbort<T>(start: () => Promise<T>, signal?: AbortSignal
signal.removeEventListener("abort", onAbort);
}
}
+
+export interface BackoffOptions {
+ attempts?: number;
+ baseDelayMs?: number;
+ maxDelayMs?: number;
+}
+
+const DEFAULT_ATTEMPTS = 4;
+const DEFAULT_BASE_DELAY_MS = 500;
+const DEFAULT_MAX_DELAY_MS = 8_000;
+const RETRY_AFTER_CAP_MS = 30_000;
+const REFUSED_STATUSES: ReadonlySet<number> = new Set([429]);
+const TRANSIENT_STATUSES: ReadonlySet<number> = new Set([429, 500, 502, 503, 504]);
+
+export function jitteredBackoffMs(attempt: number, opts: BackoffOptions = {}): number {
+ const base = opts.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
+ const max = opts.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
+ const ceiling = Math.min(max, base * 2 ** Math.max(0, attempt - 1));
+ return Math.round(ceiling * (0.5 + Math.random() / 2));
+}
+
+export function retryAfterMs(headers: Headers, now = Date.now()): number | undefined {
+ const raw = headers.get("retry-after")?.trim();
+ if (!raw) return undefined;
+ const seconds = Number(raw);
+ if (Number.isFinite(seconds)) return seconds < 0 ? undefined : Math.min(seconds * 1000, RETRY_AFTER_CAP_MS);
+ const at = Date.parse(raw);
+ if (Number.isNaN(at)) return undefined;
+ return Math.min(Math.max(0, at - now), RETRY_AFTER_CAP_MS);
+}
+
+export type RetryClass = "idempotent" | "refused";
+
+function isAbortError(e: unknown): boolean {
+ const name = (e as { name?: unknown } | null)?.name;
+ return name === "AbortError" || name === "TimeoutError";
+}
+
+export interface RetryOptions extends BackoffOptions {
+ signal?: AbortSignal;
+ timeoutMs?: number;
+}
+
+export async function fetchWithRetry(
+ send: (signal: AbortSignal) => Promise<Response>,
+ retry: RetryClass,
+ opts: RetryOptions = {},
+): Promise<Response> {
+ const attempts = opts.attempts ?? DEFAULT_ATTEMPTS;
+ const deadline = AbortSignal.timeout(opts.timeoutMs ?? 60_000);
+ const signal = opts.signal ? AbortSignal.any([opts.signal, deadline]) : deadline;
+ const wait = async (ms: number): Promise<void> => {
+ try {
+ await delay(ms, undefined, { signal });
+ } catch (error) {
+ signal.throwIfAborted();
+ throw error;
+ }
+ };
+ const statuses = retry === "idempotent" ? TRANSIENT_STATUSES : REFUSED_STATUSES;
+ for (let attempt = 1; ; attempt++) {
+ let res: Response;
+ try {
+ res = await withAbort(() => send(signal), signal);
+ } catch (e) {
+ signal.throwIfAborted();
+ if (retry !== "idempotent" || attempt >= attempts || isAbortError(e)) throw e;
+ await wait(jitteredBackoffMs(attempt, opts));
+ continue;
+ }
+ if (!statuses.has(res.status) || attempt >= attempts) return res;
+ await withAbort(() => res.body?.cancel().catch(() => undefined) ?? Promise.resolve(), signal);
+ await wait(retryAfterMs(res.headers) ?? jitteredBackoffMs(attempt, opts));
+ }
+}
diff --git a/src/util/errors.ts b/src/util/errors.ts
index 1eaec7bf2..4e48f046e 100644
--- a/src/util/errors.ts
+++ b/src/util/errors.ts
@@ -84,3 +84,18 @@ export function reportFailureAs<T>(context: string, fallback: T, detail?: string
return fallback;
};
}
+
+const REQUEST_ID_HEADERS = ["x-request-id", "x-amzn-requestid", "fly-request-id"];
+
+export function withRequestId(message: string, headers: Headers): string {
+ for (const name of REQUEST_ID_HEADERS) {
+ const value = headers.get(name);
+ if (value) return `${message} [request id ${value}]`;
+ }
+ return message;
+}
+
+export async function httpFailure(res: Response, bodyChars = 200): Promise<string> {
+ const body = (await res.text().catch(() => "")).slice(0, bodyChars);
+ return withRequestId(`http ${res.status} ${body}`, res.headers);
+}
diff --git a/test/agent37-sandbox.test.ts b/test/agent37-sandbox.test.ts
index b7707a845..74eeec70a 100644
--- a/test/agent37-sandbox.test.ts
+++ b/test/agent37-sandbox.test.ts
@@ -265,3 +265,62 @@ test("profile advertises resident disk and process sessions", () => {
assert.equal(sandbox.profile.processSessions, true);
assert.equal(sandbox.profile.egressEnforcement, "none");
});
+
+test("instance create retries 429 refusals but not an ambiguous 5xx", async () => {
+ fake.failNext(429, {
+ headers: { "retry-after": "0" },
+ match: (c) => c.method === "POST" && c.path === "/v1/instances",
+ });
+ const h = await sandbox.provision(layers);
+ assert.equal(h.coldStart, true);
+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, 2);
+
+ const otherScope = scopeId("personal", "other");
+ fake.failNext(500, {
+ headers: { "x-request-id": "req-create" },
+ match: (c) => c.method === "POST" && c.path === "/v1/instances",
+ });
+ const posts = fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length;
+ await assert.rejects(
+ sandbox.provision([{ scopeId: otherScope, mountPath: "/", mode: "rw" }]),
+ /agent37 create .*: http 500 .*\[request id req-create\]/,
+ );
+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, posts + 1);
+});
+
+for (const failure of ["disconnect", "503"]) {
+ test(`accepted create followed by ${failure} never duplicates the instance`, async () => {
+ let injected = false;
+ const adapter = make({
+ fetchImpl: async (input: string | URL | Request, init?: RequestInit) => {
+ const response = await fake.fetchImpl(input, init);
+ if (!injected && init?.method === "POST" && new URL(String(input)).pathname === "/v1/instances") {
+ injected = true;
+ if (failure === "disconnect") throw new TypeError("connection lost after create");
+ return new Response("upstream unavailable", { status: 503, headers: { "retry-after": "0" } });
+ }
+ return response;
+ },
+ });
+ await assert.rejects(adapter.provision(layers), /connection lost after create|http 503/);
+ assert.equal(fake.names().length, 1);
+ const recovered = await adapter.provision(layers);
+ assert.equal(fake.names().length, 1);
+ assert.equal((await adapter.run(recovered, "echo recovered")).stdout.trim(), "recovered");
+ assert.equal(fake.calls.filter((c) => c.method === "POST" && c.path === "/v1/instances").length, 1);
+ });
+}
+
+test("instance listing retries 429 with Retry-After; exec is never retried and names the request id", async () => {
+ fake.failNext(429, {
+ headers: { "retry-after": "0" },
+ match: (c) => c.method === "GET" && c.path === "/v1/instances",
+ });
+ const h = await sandbox.provision(layers);
+ assert.equal(fake.calls.filter((c) => c.method === "GET" && c.path === "/v1/instances").length, 2);
+
+ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;
+ fake.failNext(502, { headers: { "x-request-id": "req-exec" }, match: (c) => c.path.endsWith("/exec") });
+ await assert.rejects(sandbox.run(h, "echo hi"), /agent37 exec .*: http 502 .*\[request id req-exec\]/);
+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);
+});
diff --git a/test/aws-microvm-api.test.ts b/test/aws-microvm-api.test.ts
index 6cffe702f..30e980733 100644
--- a/test/aws-microvm-api.test.ts
+++ b/test/aws-microvm-api.test.ts
@@ -15,7 +15,9 @@ interface Recorded {
headers: Record<string, string>;
}
-function fakeFetch(reply: (rec: Recorded, n: number) => { status: number; body?: unknown }): {
+function fakeFetch(
+ reply: (rec: Recorded, n: number) => { status: number; body?: unknown; headers?: Record<string, string> },
+): {
fetchImpl: typeof fetch;
calls: Recorded[];
} {
@@ -32,7 +34,7 @@ function fakeFetch(reply: (rec: Recorded, n: number) => { status: number; body?:
calls.push(rec);
const r = reply(rec, calls.length - 1);
const text = r.body === undefined ? "" : JSON.stringify(r.body);
- return new Response(text, { status: r.status, headers: { "content-type": "application/json" } });
+ return new Response(text, { status: r.status, headers: { "content-type": "application/json", ...r.headers } });
}) as unknown as typeof fetch;
return { fetchImpl, calls };
}
@@ -183,3 +185,46 @@ test("readiness reuses only the supplied observation and checks again on the nex
assert.equal(reads, 1);
await assert.rejects(client.ensureRunning("one", "endpoint", { microvmId: "two", state: "RUNNING" }), /another body/);
});
+
+test("control-plane reads retry throttling and outages, and failures carry the AWS request id", async () => {
+ const replies = [
+ { status: 429, body: { message: "Rate exceeded" }, headers: { "retry-after": "0" } },
+ { status: 503, body: { message: "unavailable" }, headers: { "retry-after": "0" } },
+ { status: 200, body: { microvmId: "mvm-1", endpoint: "e", state: "RUNNING" } },
+ ];
+ const { fetchImpl, calls } = fakeFetch((_rec, n) => replies[n]!);
+ const api = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl });
+ assert.equal((await api.getMicrovm("mvm-1")).state, "RUNNING");
+ assert.equal(calls.length, 3);
+
+ const failing = fakeFetch(() => ({
+ status: 500,
+ body: { message: "boom" },
+ headers: { "x-amzn-requestid": "aws-req-1" },
+ }));
+ const api2 = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: failing.fetchImpl });
+ await assert.rejects(api2.getMicrovm("mvm-1"), (e: unknown) => {
+ assert.ok(e instanceof AwsApiError);
+ assert.equal(e.status, 500);
+ assert.match(e.message, /-> 500: boom \[request id aws-req-1\]$/);
+ return true;
+ });
+ assert.equal(failing.calls.length, 4, "an idempotent read is retried until attempts run out");
+});
+
+test("runMicrovm retries a throttled create but never an ambiguous server error", async () => {
+ const throttled = fakeFetch((_rec, n) =>
+ n === 0
+ ? { status: 429, body: { message: "Rate exceeded" }, headers: { "retry-after": "0" } }
+ : { status: 200, body: { microvmId: "mvm-2", endpoint: "e", state: "PENDING" } },
+ );
+ const api = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: throttled.fetchImpl });
+ const run = { imageIdentifier: "img", ingressNetworkConnectors: [], egressNetworkConnectors: [] };
+ assert.equal((await api.runMicrovm(run)).microvmId, "mvm-2");
+ assert.equal(throttled.calls.length, 2);
+
+ const broken = fakeFetch(() => ({ status: 502, body: { message: "bad gateway" } }));
+ const api2 = createMicrovmApi({ region: "us-west-2", credentials: creds, fetchImpl: broken.fetchImpl });
+ await assert.rejects(api2.runMicrovm(run), /-> 502: bad gateway/);
+ assert.equal(broken.calls.length, 1);
+});
diff --git a/test/errors.test.ts b/test/errors.test.ts
index dec636fc6..0e308321b 100644
--- a/test/errors.test.ts
+++ b/test/errors.test.ts
@@ -1,6 +1,14 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { asError, errMessage, errorAlreadyReported, failureCode, reportFailure } from "../src/util/errors.ts";
+import {
+ asError,
+ errMessage,
+ errorAlreadyReported,
+ failureCode,
+ reportFailure,
+ httpFailure,
+ withRequestId,
+} from "../src/util/errors.ts";
import { WorkAdmissionClosed } from "../src/util/admitted-work.ts";
import { errMessage as pluginErrMessage } from "../plugins/chassis/src/errors.ts";
import { runInNewContext } from "node:vm";
@@ -113,3 +121,15 @@ test("reportFailure logs every failure but marks only reportable ones as reporte
"[failed] worker: retry (run=r1): a string throw",
]);
});
+
+test("httpFailure names the status, a clipped body, and the provider request id", async () => {
+ const res = new Response("x".repeat(300), { status: 502, headers: { "x-request-id": "req-7" } });
+ const text = await httpFailure(res);
+ assert.match(text, /^http 502 x{200} \[request id req-7\]$/);
+ assert.equal(await httpFailure(new Response("plain", { status: 429 })), "http 429 plain");
+ assert.equal(
+ withRequestId("lambda -> 429: throttled", new Headers({ "x-amzn-requestid": "aws-1" })),
+ "lambda -> 429: throttled [request id aws-1]",
+ );
+ assert.equal(withRequestId("fine", new Headers()), "fine");
+});
diff --git a/test/sandbox-destroy-scope.test.ts b/test/sandbox-destroy-scope.test.ts
index 4751b7929..f4d5d696c 100644
--- a/test/sandbox-destroy-scope.test.ts
+++ b/test/sandbox-destroy-scope.test.ts
@@ -47,7 +47,7 @@ test("Sprites scope deletion sends only DELETE, retries provider errors, and acc
token: "test",
fetchImpl: async (input, init) => {
calls.push(`${init?.method} ${new URL(String(input)).pathname}`);
- return new Response(null, { status });
+ return new Response(null, { status, headers: { "retry-after": "0" } });
},
});
await assert.rejects(sandbox.destroyScope!("scope"), /503/);
@@ -55,7 +55,7 @@ test("Sprites scope deletion sends only DELETE, retries provider errors, and acc
await sandbox.destroyScope!("scope");
status = 404;
await sandbox.destroyScope!("scope");
- assert.deepEqual(calls, Array(3).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`));
+ assert.deepEqual(calls, Array(6).fill(`DELETE /v1/sprites/${sandboxScopeName("qm", "scope")}`));
});
for (const [name, create, resource] of [
@@ -75,7 +75,7 @@ for (const [name, create, resource] of [
const rows = exists ? [{ id: "existing", name: sandboxScopeName("qm", "scope"), status: "stopped" }] : [];
return Response.json(resource === "instances" ? { data: rows } : rows);
}
- return new Response(null, { status });
+ return new Response(null, { status, headers: { "retry-after": "0" } });
},
});
await assert.rejects(sandbox.destroyScope!("scope"), /503/);
@@ -85,7 +85,7 @@ for (const [name, create, resource] of [
await sandbox.destroyScope!("scope");
assert.deepEqual(calls, [
`GET /v1/${resource}`,
- `DELETE /v1/${resource}/existing`,
+ ...Array(4).fill(`DELETE /v1/${resource}/existing`),
`GET /v1/${resource}`,
`DELETE /v1/${resource}/existing`,
`GET /v1/${resource}`,
diff --git a/test/smolmachines-sandbox.test.ts b/test/smolmachines-sandbox.test.ts
index 7ca38742b..799067656 100644
--- a/test/smolmachines-sandbox.test.ts
+++ b/test/smolmachines-sandbox.test.ts
@@ -225,3 +225,24 @@ test("read and write refuse parent path segments before any provider request", a
[],
);
});
+
+test("control-plane 429 and 5xx are retried with Retry-After honored; exhausted retries name the request id", async () => {
+ fake.failNext(429, { headers: { "retry-after": "0" } });
+ fake.failNext(503);
+ const h = await sandbox.provision(layers);
+ assert.equal(h.coldStart, true);
+ const listing = fake.calls.filter((c) => c.method === "GET" && c.path === "/v1/machines");
+ assert.equal(listing.length, 3, "the machine listing was retried after the 429 and the 503");
+
+ const other = make();
+ for (let i = 1; i <= 4; i++) fake.failNext(429, { headers: { "retry-after": "0", "x-request-id": `req-${i}` } });
+ await assert.rejects(other.provision(layers), /smolmachines GET \/v1\/machines: http 429 .*\[request id req-4\]/);
+});
+
+test("exec is never retried and its failure names the request id", async () => {
+ const h = await sandbox.provision(layers);
+ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;
+ fake.failNext(502, { headers: { "x-request-id": "req-exec" }, match: (c) => c.path.endsWith("/exec") });
+ await assert.rejects(sandbox.run(h, "echo hi"), /smolmachines exec .*: http 502 .*\[request id req-exec\]/);
+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);
+});
diff --git a/test/sprites-sandbox.test.ts b/test/sprites-sandbox.test.ts
index 613e73c72..4a77d32ff 100644
--- a/test/sprites-sandbox.test.ts
+++ b/test/sprites-sandbox.test.ts
@@ -518,3 +518,27 @@ test("every sprites fetch rides one HTTP/1.1 dispatcher, so a bad request fails
assert.ok(optionsKey, "undici Agent must expose its options");
assert.equal(dispatcher[optionsKey]?.allowH2, false);
});
+
+test("control-plane reads retry 429 with Retry-After while exec is never retried", async () => {
+ const h = await sandbox.provision(layers);
+ fake.failNext(429, { headers: { "retry-after": "0" }, match: (c) => c.path.endsWith("/check") });
+ assert.deepEqual(await sandbox.computerStatus!(scope), {
+ machine: "healthy",
+ listed: "warm",
+ provisioned: true,
+ guestResponsive: true,
+ });
+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/check")).length, 2);
+
+ const before = fake.calls.filter((c) => c.path.endsWith("/exec")).length;
+ fake.failNext(502, { headers: { "fly-request-id": "fly-9" }, match: (c) => c.path.endsWith("/exec") });
+ await assert.rejects(sandbox.run(h, "echo hi"), /sprites exec .*: http 502 .*\[request id fly-9\]/);
+ assert.equal(fake.calls.filter((c) => c.path.endsWith("/exec")).length, before + 1);
+
+ for (let i = 1; i <= 4; i++)
+ fake.failNext(503, {
+ headers: { "retry-after": "0", "x-request-id": `req-${i}` },
+ match: (c) => c.method === "DELETE",
+ });
+ await assert.rejects(sandbox.destroyScope!(scope), /sprites delete .*: http 503 .*\[request id req-4\]/);
+});
diff --git a/test/support/fake-agent37.ts b/test/support/fake-agent37.ts
index 0ff06e1cf..ddb580c3f 100644
--- a/test/support/fake-agent37.ts
+++ b/test/support/fake-agent37.ts
@@ -20,6 +20,11 @@ interface FakeInstance {
home: string;
}
+export interface InjectedFailure {
+ headers?: Record<string, string>;
+ match?: (call: { method: string; path: string }) => boolean;
+}
+
export interface FakeInstanceView {
status: string;
template?: string;
@@ -35,6 +40,7 @@ export interface FakeAgent37 {
sleep(name: string, opts?: { freezing?: number }): void;
stop(name: string): void;
fail(name: string): void;
+ failNext(status: number, opts?: InjectedFailure): void;
execScripts(): string[];
cleanup(): void;
}
@@ -48,6 +54,7 @@ export function installFakeAgent37(): FakeAgent37 {
const execScripts: string[] = [];
const calls: Agent37Call[] = [];
let nextId = 1;
+ const injected: Array<InjectedFailure & { status: number }> = [];
const live = (): FakeInstance[] => [...instances.values()].filter((i) => i.status !== "deleted").reverse();
const byName = (name: string): FakeInstance | undefined => live().find((i) => i.name === name);
@@ -131,6 +138,11 @@ export function installFakeAgent37(): FakeAgent37 {
const url = new URL(typeof input === "string" ? input : input.toString());
const method = init?.method ?? "GET";
calls.push({ method, path: url.pathname });
+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));
+ if (at >= 0) {
+ const [next] = injected.splice(at, 1);
+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });
+ }
if (url.pathname === "/v1/instances" && method === "GET") {
return Response.json({ data: live().map(info) });
}
@@ -222,6 +234,9 @@ export function installFakeAgent37(): FakeAgent37 {
const m = byName(name);
if (m) m.status = "failed";
},
+ failNext: (status, opts = {}) => {
+ injected.push({ status, ...opts });
+ },
execScripts: () => [...execScripts],
cleanup: () => rmSync(root, { recursive: true, force: true }),
};
diff --git a/test/support/fake-smolmachines.ts b/test/support/fake-smolmachines.ts
index 652166db4..8b7b88776 100644
--- a/test/support/fake-smolmachines.ts
+++ b/test/support/fake-smolmachines.ts
@@ -18,6 +18,11 @@ interface FakeMachine {
home: string;
}
+export interface InjectedFailure {
+ headers?: Record<string, string>;
+ match?: (call: { method: string; path: string }) => boolean;
+}
+
export interface FakeSmolmachines {
fetchImpl: typeof fetch;
calls: SmolCall[];
@@ -25,6 +30,7 @@ export interface FakeSmolmachines {
names(): string[];
machine(name: string): { state: string; ephemeral: boolean; resources?: Record<string, number> } | null;
stop(name: string): void;
+ failNext(status: number, opts?: InjectedFailure): void;
execScripts(): string[];
reset(): void;
cleanup(): void;
@@ -38,6 +44,7 @@ export function installFakeSmolmachines(): FakeSmolmachines {
const execScripts: string[] = [];
const calls: SmolCall[] = [];
let nextId = 1;
+ const injected: Array<InjectedFailure & { status: number }> = [];
const byName = (name: string): FakeMachine | undefined => [...machines.values()].find((m) => m.name === name);
@@ -115,6 +122,11 @@ export function installFakeSmolmachines(): FakeSmolmachines {
const url = new URL(typeof input === "string" ? input : input.toString());
const method = init?.method ?? "GET";
calls.push({ method, path: url.pathname });
+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));
+ if (at >= 0) {
+ const [next] = injected.splice(at, 1);
+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });
+ }
if (url.pathname === "/v1/machines" && method === "GET") {
return Response.json([...machines.values()].map(info));
}
@@ -191,12 +203,16 @@ export function installFakeSmolmachines(): FakeSmolmachines {
const m = byName(name);
if (m) m.state = "stopped";
},
+ failNext: (status, opts = {}) => {
+ injected.push({ status, ...opts });
+ },
execScripts: () => [...execScripts],
reset: () => {
for (const m of machines.values()) rmSync(m.home, { recursive: true, force: true });
machines.clear();
execScripts.length = 0;
calls.length = 0;
+ injected.length = 0;
},
cleanup: () => rmSync(root, { recursive: true, force: true }),
};
diff --git a/test/support/fake-sprites.ts b/test/support/fake-sprites.ts
index 81bca945f..0f01a25c1 100644
--- a/test/support/fake-sprites.ts
+++ b/test/support/fake-sprites.ts
@@ -17,6 +17,11 @@ export interface SpritesCall {
script?: string;
}
+export interface InjectedFailure {
+ headers?: Record<string, string>;
+ match?: (call: { method: string; path: string }) => boolean;
+}
+
export interface FakeSprites {
client: SpritesClientLike;
fetchImpl: typeof fetch;
@@ -28,6 +33,7 @@ export interface FakeSprites {
execScripts(): string[];
stallAfterRun(name: string): void;
fail502(name: string): void;
+ failNext(status: number, opts?: InjectedFailure): void;
refuseRestart(name: string): void;
refuseForcedRestart(name: string): void;
setPressure(name: string, p: { full10: number; full60: number; load1: number }): void;
@@ -50,6 +56,7 @@ export function installFakeSprites(): FakeSprites {
const refusedRestart = new Set<string>();
const refusedForcedRestart = new Set<string>();
const restarts: string[] = [];
+ const injected: Array<InjectedFailure & { status: number }> = [];
const ensureDir = (name: string): string => {
let s = sprites.get(name);
@@ -114,6 +121,11 @@ export function installFakeSprites(): FakeSprites {
const url = new URL(typeof input === "string" ? input : input.toString());
const method = init?.method ?? "GET";
calls.push({ method, path: url.pathname });
+ const at = injected.findIndex((f) => !f.match || f.match({ method, path: url.pathname }));
+ if (at >= 0) {
+ const [next] = injected.splice(at, 1);
+ return new Response(`injected ${next!.status}`, { status: next!.status, headers: next!.headers ?? {} });
+ }
const health = /^\/v1\/sprites\/([^/]+)\/check$/.exec(url.pathname);
if (health) {
const name = decodeURIComponent(health[1]!);
@@ -211,6 +223,9 @@ export function installFakeSprites(): FakeSprites {
fail502: (name) => {
gateway502.add(name);
},
+ failNext: (status, opts = {}) => {
+ injected.push({ status, ...opts });
+ },
refuseRestart: (name) => {
refusedRestart.add(name);
},
@@ -235,6 +250,7 @@ export function installFakeSprites(): FakeSprites {
refusedRestart.clear();
refusedForcedRestart.clear();
restarts.length = 0;
+ injected.length = 0;
},
cleanup: () => rmSync(root, { recursive: true, force: true }),
};
diff --git a/test/util-async.test.ts b/test/util-async.test.ts
index 0999c28ec..5995f0461 100644
--- a/test/util-async.test.ts
+++ b/test/util-async.test.ts
@@ -2,7 +2,7 @@ import "./support/auto-fake-sprites.ts";
import { test } from "node:test";
import assert from "node:assert/strict";
-import { sleep, createKeyedQueue } from "../src/util/async.ts";
+import { sleep, createKeyedQueue, fetchWithRetry, jitteredBackoffMs, retryAfterMs } from "../src/util/async.ts";
test("sleep resolves after roughly the given delay", async () => {
const t0 = Date.now();
@@ -135,3 +135,158 @@ test("withAbort skips an already cancelled operation and preserves the reason",
assert.equal(called, false);
assert.equal(await withAbort(async () => 42), 42);
});
+
+test("retryAfterMs reads seconds or an HTTP date and caps the wait", () => {
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "2" })), 2_000);
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "0" })), 0);
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "900" })), 30_000);
+ const now = Date.parse("2026-09-18T12:00:00Z");
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "Fri, 18 Sep 2026 12:00:05 GMT" }), now), 5_000);
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "Fri, 18 Sep 2026 11:00:00 GMT" }), now), 0);
+ assert.equal(retryAfterMs(new Headers({ "retry-after": "soon" })), undefined);
+ assert.equal(retryAfterMs(new Headers()), undefined);
+});
+
+test("jitteredBackoffMs grows exponentially, jitters within half its ceiling, and stays bounded", () => {
+ for (let attempt = 1; attempt <= 6; attempt++) {
+ const ceiling = Math.min(8_000, 500 * 2 ** (attempt - 1));
+ for (let i = 0; i < 20; i++) {
+ const d = jitteredBackoffMs(attempt);
+ assert.ok(d >= ceiling / 2 && d <= ceiling, `attempt ${attempt}: ${d} within [${ceiling / 2}, ${ceiling}]`);
+ }
+ }
+ assert.ok(jitteredBackoffMs(3, { baseDelayMs: 10, maxDelayMs: 25 }) <= 25);
+});
+
+const fast = { baseDelayMs: 1, maxDelayMs: 2 };
+const reply = (status: number, headers: Record<string, string> = {}) =>
+ new Response(`status ${status}`, { status, headers });
+
+test("fetchWithRetry retries 429 and 5xx on idempotent requests, honoring Retry-After", async () => {
+ const statuses = [429, 503, 502, 200];
+ let calls = 0;
+ const t0 = Date.now();
+ const res = await fetchWithRetry(
+ async () => reply(statuses[calls++]!, calls === 1 ? { "retry-after": "0" } : {}),
+ "idempotent",
+ fast,
+ );
+ assert.equal(res.status, 200);
+ assert.equal(calls, 4);
+ assert.ok(Date.now() - t0 < 1_000);
+});
+
+test("fetchWithRetry returns the last transient response once attempts are exhausted", async () => {
+ let calls = 0;
+ const res = await fetchWithRetry(
+ async () => {
+ calls++;
+ return reply(429, { "retry-after": "0", "x-request-id": `req-${calls}` });
+ },
+ "idempotent",
+ { ...fast, attempts: 3 },
+ );
+ assert.equal(res.status, 429);
+ assert.equal(res.headers.get("x-request-id"), "req-3");
+ assert.equal(calls, 3);
+});
+
+test("fetchWithRetry never retries client errors or successes", async () => {
+ for (const status of [200, 400, 404, 409, 422]) {
+ let calls = 0;
+ const res = await fetchWithRetry(
+ async () => {
+ calls++;
+ return reply(status);
+ },
+ "idempotent",
+ fast,
+ );
+ assert.equal(res.status, status);
+ assert.equal(calls, 1, `status ${status} is returned to the caller untouched`);
+ }
+});
+
+test("fetchWithRetry retries connection failures only for idempotent requests, never aborts", async () => {
+ let calls = 0;
+ const flaky = async () => {
+ calls++;
+ if (calls === 1) throw new TypeError("fetch failed");
+ return reply(200);
+ };
+ assert.equal((await fetchWithRetry(flaky, "idempotent", fast)).status, 200);
+ assert.equal(calls, 2);
+
+ calls = 0;
+ await assert.rejects(fetchWithRetry(flaky, "refused", fast), /fetch failed/);
+ assert.equal(calls, 1);
+
+ calls = 0;
+ const aborted = async () => {
+ calls++;
+ throw Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" });
+ };
+ await assert.rejects(fetchWithRetry(aborted, "idempotent", fast), { name: "TimeoutError" });
+ assert.equal(calls, 1);
+});
+
+test("fetchWithRetry retries throttling but never infers refusal from a server error", async () => {
+ for (const [status, expectedCalls] of [
+ [429, 2],
+ [503, 1],
+ [500, 1],
+ [502, 1],
+ [504, 1],
+ ] as const) {
+ let calls = 0;
+ const res = await fetchWithRetry(
+ async () => {
+ calls++;
+ return reply(calls === 1 ? status : 201, { "retry-after": "0" });
+ },
+ "refused",
+ fast,
+ );
+ assert.equal(calls, expectedCalls, `status ${status}`);
+ assert.equal(res.status, expectedCalls === 2 ? 201 : status);
+ }
+});
+
+test("fetchWithRetry cancellation interrupts Retry-After without another request", async () => {
+ const controller = new AbortController();
+ let calls = 0;
+ const pending = fetchWithRetry(
+ async () => {
+ calls++;
+ return reply(429, { "retry-after": "30" });
+ },
+ "idempotent",
+ { signal: controller.signal },
+ );
+ const timeout = setTimeout(() => controller.abort(), 20);
+ try {
+ await assert.rejects(pending, { name: "AbortError" });
+ assert.equal(calls, 1);
+ } finally {
+ clearTimeout(timeout);
+ }
+});
+
+test("fetchWithRetry shares a deadline across attempts and backoff", async () => {
+ const signals: AbortSignal[] = [];
+ await assert.rejects(
+ fetchWithRetry(
+ async (signal) => {
+ signals.push(signal);
+ if (signals.length === 1) return reply(503, { "retry-after": "0" });
+ return new Promise<Response>((resolve) => setTimeout(() => resolve(reply(200)), 100));
+ },
+ "idempotent",
+ { timeoutMs: 30 },
+ ),
+ { name: "TimeoutError" },
+ );
+ assert.equal(signals.length, 2);
+ assert.equal(signals[0], signals[1]);
+ assert.equal(signals[1]!.aborted, true);
+});