Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Retry the quarantine list fetch before giving up (#78951)

Nemanja Glumac71fd271cc6cb
diff --git a/release/ci-conductor/src/fetchWithRetry.ts b/release/ci-conductor/src/fetchWithRetry.ts
new file mode 100644
index 00000000000..940bbc9b1f5
--- /dev/null
+++ b/release/ci-conductor/src/fetchWithRetry.ts
@@ -0,0 +1,79 @@
+const RETRYABLE_STATUS = new Set([408, 429, 500, 502, 503, 504]);
+
+export interface FetchWithRetryInit extends RequestInit {
+ /** Number of *retries*, so `3` means up to 4 total requests. */
+ retries?: number;
+ /** First backoff window in ms; doubles each attempt. */
+ baseDelay?: number;
+ /** Upper bound on a single backoff window in ms. */
+ maxDelay?: number;
+ /** Per-attempt timeout in ms. */
+ timeout?: number;
+ /** Decide whether a completed response is worth retrying. */
+ shouldRetry?: (res: Response) => boolean;
+}
+
+/**
+ * fetch() with exponential backoff + jitter.
+ * Retries on network errors, timeouts, and retryable status codes.
+ */
+export async function fetchWithRetry(
+ input: RequestInfo | URL,
+ {
+ retries = 3,
+ baseDelay = 300,
+ maxDelay = 10_000,
+ timeout = 10_000,
+ shouldRetry = (res) => RETRYABLE_STATUS.has(res.status),
+ ...init
+ }: FetchWithRetryInit = {},
+): Promise<Response> {
+ for (let attempt = 0; ; attempt++) {
+ let res: Response | undefined;
+ try {
+ res = await fetch(input, {
+ ...init,
+ signal: init.signal
+ ? AbortSignal.any([init.signal, AbortSignal.timeout(timeout)])
+ : AbortSignal.timeout(timeout),
+ });
+ if (!shouldRetry(res) || attempt === retries) {return res;}
+ res.body?.cancel().catch(() => {}); // don't leak the discarded stream
+ } catch (err) {
+ // caller cancelled, or we're out of attempts
+ if (init.signal?.aborted || attempt === retries) {throw err;}
+ }
+
+ await sleep(retryAfter(res) ?? backoff(attempt, baseDelay, maxDelay), init.signal);
+ }
+}
+
+function backoff(attempt: number, base: number, max: number): number {
+ const ceiling = Math.min(base * 2 ** attempt, max);
+ return ceiling / 2 + Math.random() * (ceiling / 2); // half-jittered
+}
+
+function retryAfter(res: Response | undefined): number | null {
+ const header = res?.headers.get('retry-after');
+ if (!header) {return null;}
+ const seconds = Number(header);
+ const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
+ return ms > 0 ? ms : null;
+}
+
+function sleep(ms: number, signal?: AbortSignal | null): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) {return reject(signal.reason);}
+
+ const onAbort = () => {
+ clearTimeout(timer);
+ reject(signal!.reason);
+ };
+
+ const timer = setTimeout(() => {
+ signal?.removeEventListener('abort', onAbort);
+ resolve();
+ }, ms);
+ signal?.addEventListener('abort', onAbort, { once: true });
+ });
+}
diff --git a/release/ci-conductor/src/quarantine.test.ts b/release/ci-conductor/src/quarantine.test.ts
index 6c35af5f332..2f662c7da62 100644
--- a/release/ci-conductor/src/quarantine.test.ts
+++ b/release/ci-conductor/src/quarantine.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from "bun:test";
+import { afterEach, beforeEach, describe, expect, it, jest } from "bun:test";
import type { NormalizedTest } from "./contract.ts";
import {
@@ -7,6 +7,7 @@ import {
type QuarantineEntry,
checkQuarantineGate,
compareFailedToQuarantine,
+ fetchQuarantineList,
junitFailuresToFailedTests,
matchKey,
} from "./quarantine.ts";
@@ -164,6 +165,160 @@ describe("junitFailuresToFailedTests", () => {
});
});
+describe("fetchQuarantineList", () => {
+ const originalFetch = globalThis.fetch;
+ const originalLog = console.log;
+ const originalError = console.error;
+
+ beforeEach(() => {
+ // The retry loop really sleeps between attempts. Freeze the clock so these
+ // tests exercise the production backoff (no shortened delays) while
+ // deciding for themselves when each sleep ends.
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.clearAllTimers();
+ jest.useRealTimers();
+ globalThis.fetch = originalFetch;
+ console.log = originalLog;
+ console.error = originalError;
+ });
+
+ /**
+ * Serve one canned outcome per call — a `Response` to return or an `Error` to
+ * throw — and mute the gate's logging for the duration of the test.
+ */
+ function stubFetch(outcomes: (Response | Error)[]): () => number {
+ const calls: number[] = [];
+ console.log = () => {};
+ console.error = () => {};
+ globalThis.fetch = (async () => {
+ const outcome = outcomes[calls.length];
+ calls.push(1);
+ if (outcome instanceof Error) {
+ throw outcome;
+ }
+ return outcome;
+ }) as unknown as typeof fetch;
+ return () => calls.length;
+ }
+
+ /**
+ * Hand the event loop a real turn. `useFakeTimers` freezes `setTimeout` but
+ * leaves `setImmediate` alone, so this is what lets the awaits inside the
+ * retry loop actually run while the clock is stopped.
+ */
+ const macrotask = () => new Promise<void>((resolve) => setImmediate(resolve));
+
+ /**
+ * Settle a `fetchQuarantineList` call on the frozen clock: let its pending
+ * awaits run, fire whichever backoff timer that parked it, and repeat until
+ * it resolves. Bounded, so a call that never settles fails loudly instead of
+ * hanging the suite.
+ */
+ async function settle<T>(pending: Promise<T>): Promise<T> {
+ let done = false;
+ void pending.then(
+ () => (done = true),
+ () => (done = true),
+ );
+ for (let pass = 0; pass < 10 && !done; pass++) {
+ await macrotask();
+ if (!done) {
+ jest.runAllTimers();
+ }
+ }
+ if (!done) {
+ throw new Error("fetchQuarantineList never settled");
+ }
+ return pending;
+ }
+
+ const okList = (entries: QuarantineEntry[]) =>
+ new Response(JSON.stringify({ tests: entries }), { status: 200 });
+ const errorStatus = (status: number) => new Response("", { status });
+ const econnreset = () => new Error("The socket connection was closed unexpectedly");
+
+ const fetchList = () =>
+ settle(
+ fetchQuarantineList({
+ baseUrl: "https://ci-conductor.example",
+ suite: "e2e",
+ }),
+ );
+
+ it("returns the list without retrying when the first call succeeds", async () => {
+ const calls = stubFetch([okList([quarantined("a")])]);
+
+ expect(await fetchList()).toEqual([quarantined("a")]);
+ expect(calls()).toBe(1);
+ });
+
+ it("retries a dropped connection and returns the list from a later attempt", async () => {
+ const calls = stubFetch([
+ econnreset(),
+ econnreset(),
+ okList([quarantined("a")]),
+ ]);
+
+ expect(await fetchList()).toEqual([quarantined("a")]);
+ expect(calls()).toBe(3);
+ });
+
+ it("gives up with null once every attempt has failed", async () => {
+ const calls = stubFetch([econnreset(), econnreset(), econnreset()]);
+
+ expect(await fetchList()).toBeNull();
+ expect(calls()).toBe(3);
+ });
+
+ it("retries a 5xx", async () => {
+ const calls = stubFetch([errorStatus(502), okList([])]);
+
+ expect(await fetchList()).toEqual([]);
+ expect(calls()).toBe(2);
+ });
+
+ it("retries a 429", async () => {
+ const calls = stubFetch([errorStatus(429), okList([])]);
+
+ expect(await fetchList()).toEqual([]);
+ expect(calls()).toBe(2);
+ });
+
+ it("does not retry a 4xx — the request itself is wrong", async () => {
+ const calls = stubFetch([errorStatus(401), okList([quarantined("a")])]);
+
+ expect(await fetchList()).toBeNull();
+ expect(calls()).toBe(1);
+ });
+
+ it("waits out the backoff window before retrying, rather than hammering", async () => {
+ const calls = stubFetch([errorStatus(503), okList([])]);
+ const pending = fetchQuarantineList({
+ baseUrl: "https://ci-conductor.example",
+ suite: "e2e",
+ });
+
+ await macrotask();
+ expect(calls()).toBe(1);
+
+ // Half-jittered backoff off a 500ms base puts the first retry somewhere in
+ // [250ms, 500ms) — early enough to keep the gate snappy, late enough to let
+ // a blip pass. Asserting the bounds holds for any jitter draw.
+ jest.advanceTimersByTime(249);
+ await macrotask();
+ expect(calls()).toBe(1);
+
+ jest.advanceTimersByTime(251);
+ await macrotask();
+ expect(calls()).toBe(2);
+
+ expect(await settle(pending)).toEqual([]);
+ });
+});
+
describe("checkQuarantineGate", () => {
const failure: FailedTest = {
test_name: "renders",
@@ -171,7 +326,6 @@ describe("checkQuarantineGate", () => {
file_path: "foo.spec.ts",
};
- // Both branches under test return before any fetch, so no network is touched.
// Capture the gate's log lines so we can assert on the printed verdict.
async function runCapturingLogs(
opts: Parameters<typeof checkQuarantineGate>[0],
@@ -230,6 +384,30 @@ describe("checkQuarantineGate", () => {
expect(result.enforced).toBe(false);
});
+ it("fails the job when the list request fails and there are failures to gate", async () => {
+ const originalFetch = globalThis.fetch;
+ // A 401 fails on the first attempt, so the gate's retry backoff never sleeps.
+ globalThis.fetch = (async () =>
+ new Response("", { status: 401 })) as unknown as typeof fetch;
+ try {
+ const { result, verdict } = await runCapturingLogs({
+ suite: "e2e",
+ failures: [failure],
+ baseUrl: "https://ci-conductor.example",
+ secret: "shh",
+ dryRun: false,
+ });
+ expect(result).toEqual({
+ shouldFail: true,
+ enforced: true,
+ reason: "could not fetch the quarantine list",
+ });
+ expect(verdict).toContain("COULD NOT CHECK");
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+ });
+
it("fails closed AND enforces when it can't check outside dry run", async () => {
const { result } = await runCapturingLogs({
suite: "e2e",
diff --git a/release/ci-conductor/src/quarantine.ts b/release/ci-conductor/src/quarantine.ts
index 1d5ce06ee37..60fc24736a2 100644
--- a/release/ci-conductor/src/quarantine.ts
+++ b/release/ci-conductor/src/quarantine.ts
@@ -2,11 +2,18 @@
// iff every failure is on ci-conductor's quarantine list.
import type { NormalizedTest } from "./contract.ts";
+import { fetchWithRetry } from "./fetchWithRetry.ts";
import { log } from "./util.ts";
// The list endpoint can be slow, but the gate must never hang a CI job.
const REQUEST_TIMEOUT_MS = 15_000;
+// A dropped socket (ECONNRESET) or a 5xx is a blip, not a verdict, so the fetch
+// gets a few tries before the gate gives up. `fetchWithRetry` spaces those tries
+// with jittered exponential backoff, so parallel jobs don't retry in lockstep.
+const MAX_ATTEMPTS = 3;
+const RETRY_BASE_DELAY_MS = 500;
+
/** One quarantined test as served by ci-conductor's `/api/quarantine`. */
export type QuarantineEntry = {
test_name: string;
@@ -104,39 +111,52 @@ export function junitFailuresToFailedTests(
/**
* Fetch the quarantine list for `suite` from ci-conductor. The reporter POSTs to
* `.../webhooks/failed-tests`; the list lives at `.../api/quarantine` on the same
- * host. Returns null (not []) when it can't be retrieved, so the caller can tell
- * "nothing quarantined" from "couldn't check". Never throws. Logs the path and
- * suite only — never the host (public repo) or the secret.
+ * host. Transport errors, timeouts, 429s and 5xx are retried with backoff; a 4xx
+ * is the request's own fault and returns straight away. Returns null (not [])
+ * when it can't be retrieved, so the caller can tell "nothing quarantined" from
+ * "couldn't check". Never throws. Logs the path and suite only — never the host
+ * (public repo) or the secret. The retry policy is fixed rather than injectable:
+ * the tests drive it on fake timers, so there's nothing to shorten for them.
*/
export async function fetchQuarantineList(opts: {
baseUrl: string;
suite: string;
secret?: string;
}): Promise<QuarantineEntry[] | null> {
- const base = opts.baseUrl.replace(/\/+$/, "");
- const url = `${base}/api/quarantine?suite=${encodeURIComponent(opts.suite)}`;
+ const { baseUrl, suite, secret } = opts;
+ const base = baseUrl.replace(/\/+$/, "");
+ const url = `${base}/api/quarantine?suite=${encodeURIComponent(suite)}`;
+ const headers: Record<string, string> = secret
+ ? { "x-internal-secret": secret }
+ : {};
+
try {
- const headers: Record<string, string> = {};
- if (opts.secret) {
- headers["x-internal-secret"] = opts.secret;
- }
- const response = await fetch(url, {
+ const response = await fetchWithRetry(url, {
headers,
- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ retries: MAX_ATTEMPTS - 1,
+ baseDelay: RETRY_BASE_DELAY_MS,
+ timeout: REQUEST_TIMEOUT_MS,
+ // Called once per completed response, so it doubles as the one place a
+ // non-ok status gets logged — including on attempts we then retry.
+ shouldRetry: (res) => {
+ if (!res.ok) {
+ log(
+ `🛑 GET /api/quarantine?suite=${suite} → ${res.status} ${res.statusText}`,
+ );
+ }
+ // A 4xx (bad secret, unknown suite) answers the same way every time.
+ return res.status >= 500 || res.status === 429;
+ },
});
if (!response.ok) {
- log(
- `🛑 GET /api/quarantine?suite=${opts.suite} → ${response.status} ${response.statusText}`,
- );
+ response.body?.cancel().catch(() => {}); // don't leak the unread stream
return null;
}
+ // `json()` is `any`; this names the wire shape we read (and only read).
const body = (await response.json()) as { tests?: QuarantineEntry[] };
return body.tests ?? [];
} catch (error) {
- console.error(
- "[ci-conductor] failed to fetch the quarantine list",
- error,
- );
+ console.error("[ci-conductor] failed to fetch the quarantine list", error);
return null;
}
}
@@ -155,13 +175,14 @@ function finish(opts: {
dryRun: boolean;
// When the gate couldn't reach a real pass/fail decision — e.g. the quarantine
// list was unreachable — show a distinct "couldn't check" verdict rather than a
- // red FAIL. It still fails closed (shouldFail stays true), but it isn't a real
- // gate failure, so it shouldn't read as one in the logs.
+ // red FAIL. It still fails closed (shouldFail stays true): with no list, nothing
+ // excuses the run's failures.
inconclusive?: boolean;
}): GateResult {
const { shouldFail, reason, dryRun, inconclusive = false } = opts;
if (inconclusive) {
log(`⚠️ VERDICT: COULD NOT CHECK — ${reason}.`);
+ log("🚦 no quarantine to apply — the run's failures stand on their own.");
} else if (shouldFail) {
log(`🔴 VERDICT: FAIL — ${reason}.`);
} else {