The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Retry the quarantine list fetch before giving up (#78951)
Nemanja Glumac71fd271cc6cb
diff --git a/release/ci-conductor/src/fetchWithRetry.ts b/release/ci-conductor/src/fetchWithRetry.tsnew file mode 100644index 00000000000..940bbc9b1f5--- /dev/null+++ b/release/ci-conductor/src/fetchWithRetry.ts@@ -0,0 +1,79 @@+const RETRYABLE_STATUS = new Set([408, 429, 500, 502, 503, 504]);++export interface FetchWithRetryInit extends RequestInit {+ /** Number of *retries*, so `3` means up to 4 total requests. */+ retries?: number;+ /** First backoff window in ms; doubles each attempt. */+ baseDelay?: number;+ /** Upper bound on a single backoff window in ms. */+ maxDelay?: number;+ /** Per-attempt timeout in ms. */+ timeout?: number;+ /** Decide whether a completed response is worth retrying. */+ shouldRetry?: (res: Response) => boolean;+}++/**+ * fetch() with exponential backoff + jitter.+ * Retries on network errors, timeouts, and retryable status codes.+ */+export async function fetchWithRetry(+ input: RequestInfo | URL,+ {+ retries = 3,+ baseDelay = 300,+ maxDelay = 10_000,+ timeout = 10_000,+ shouldRetry = (res) => RETRYABLE_STATUS.has(res.status),+ ...init+ }: FetchWithRetryInit = {},+): Promise<Response> {+ for (let attempt = 0; ; attempt++) {+ let res: Response | undefined;+ try {+ res = await fetch(input, {+ ...init,+ signal: init.signal+ ? AbortSignal.any([init.signal, AbortSignal.timeout(timeout)])+ : AbortSignal.timeout(timeout),+ });+ if (!shouldRetry(res) || attempt === retries) {return res;}+ res.body?.cancel().catch(() => {}); // don't leak the discarded stream+ } catch (err) {+ // caller cancelled, or we're out of attempts+ if (init.signal?.aborted || attempt === retries) {throw err;}+ }++ await sleep(retryAfter(res) ?? backoff(attempt, baseDelay, maxDelay), init.signal);+ }+}++function backoff(attempt: number, base: number, max: number): number {+ const ceiling = Math.min(base * 2 ** attempt, max);+ return ceiling / 2 + Math.random() * (ceiling / 2); // half-jittered+}++function retryAfter(res: Response | undefined): number | null {+ const header = res?.headers.get('retry-after');+ if (!header) {return null;}+ const seconds = Number(header);+ const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();+ return ms > 0 ? ms : null;+}++function sleep(ms: number, signal?: AbortSignal | null): Promise<void> {+ return new Promise((resolve, reject) => {+ if (signal?.aborted) {return reject(signal.reason);}++ const onAbort = () => {+ clearTimeout(timer);+ reject(signal!.reason);+ };++ const timer = setTimeout(() => {+ signal?.removeEventListener('abort', onAbort);+ resolve();+ }, ms);+ signal?.addEventListener('abort', onAbort, { once: true });+ });+}diff --git a/release/ci-conductor/src/quarantine.test.ts b/release/ci-conductor/src/quarantine.test.tsindex 6c35af5f332..2f662c7da62 100644--- a/release/ci-conductor/src/quarantine.test.ts+++ b/release/ci-conductor/src/quarantine.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from "bun:test";+import { afterEach, beforeEach, describe, expect, it, jest } from "bun:test";import type { NormalizedTest } from "./contract.ts";import {@@ -7,6 +7,7 @@ import {type QuarantineEntry,checkQuarantineGate,compareFailedToQuarantine,+ fetchQuarantineList,junitFailuresToFailedTests,matchKey,} from "./quarantine.ts";@@ -164,6 +165,160 @@ describe("junitFailuresToFailedTests", () => {});});+describe("fetchQuarantineList", () => {+ const originalFetch = globalThis.fetch;+ const originalLog = console.log;+ const originalError = console.error;++ beforeEach(() => {+ // The retry loop really sleeps between attempts. Freeze the clock so these+ // tests exercise the production backoff (no shortened delays) while+ // deciding for themselves when each sleep ends.+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.clearAllTimers();+ jest.useRealTimers();+ globalThis.fetch = originalFetch;+ console.log = originalLog;+ console.error = originalError;+ });++ /**+ * Serve one canned outcome per call — a `Response` to return or an `Error` to+ * throw — and mute the gate's logging for the duration of the test.+ */+ function stubFetch(outcomes: (Response | Error)[]): () => number {+ const calls: number[] = [];+ console.log = () => {};+ console.error = () => {};+ globalThis.fetch = (async () => {+ const outcome = outcomes[calls.length];+ calls.push(1);+ if (outcome instanceof Error) {+ throw outcome;+ }+ return outcome;+ }) as unknown as typeof fetch;+ return () => calls.length;+ }++ /**+ * Hand the event loop a real turn. `useFakeTimers` freezes `setTimeout` but+ * leaves `setImmediate` alone, so this is what lets the awaits inside the+ * retry loop actually run while the clock is stopped.+ */+ const macrotask = () => new Promise<void>((resolve) => setImmediate(resolve));++ /**+ * Settle a `fetchQuarantineList` call on the frozen clock: let its pending+ * awaits run, fire whichever backoff timer that parked it, and repeat until+ * it resolves. Bounded, so a call that never settles fails loudly instead of+ * hanging the suite.+ */+ async function settle<T>(pending: Promise<T>): Promise<T> {+ let done = false;+ void pending.then(+ () => (done = true),+ () => (done = true),+ );+ for (let pass = 0; pass < 10 && !done; pass++) {+ await macrotask();+ if (!done) {+ jest.runAllTimers();+ }+ }+ if (!done) {+ throw new Error("fetchQuarantineList never settled");+ }+ return pending;+ }++ const okList = (entries: QuarantineEntry[]) =>+ new Response(JSON.stringify({ tests: entries }), { status: 200 });+ const errorStatus = (status: number) => new Response("", { status });+ const econnreset = () => new Error("The socket connection was closed unexpectedly");++ const fetchList = () =>+ settle(+ fetchQuarantineList({+ baseUrl: "https://ci-conductor.example",+ suite: "e2e",+ }),+ );++ it("returns the list without retrying when the first call succeeds", async () => {+ const calls = stubFetch([okList([quarantined("a")])]);++ expect(await fetchList()).toEqual([quarantined("a")]);+ expect(calls()).toBe(1);+ });++ it("retries a dropped connection and returns the list from a later attempt", async () => {+ const calls = stubFetch([+ econnreset(),+ econnreset(),+ okList([quarantined("a")]),+ ]);++ expect(await fetchList()).toEqual([quarantined("a")]);+ expect(calls()).toBe(3);+ });++ it("gives up with null once every attempt has failed", async () => {+ const calls = stubFetch([econnreset(), econnreset(), econnreset()]);++ expect(await fetchList()).toBeNull();+ expect(calls()).toBe(3);+ });++ it("retries a 5xx", async () => {+ const calls = stubFetch([errorStatus(502), okList([])]);++ expect(await fetchList()).toEqual([]);+ expect(calls()).toBe(2);+ });++ it("retries a 429", async () => {+ const calls = stubFetch([errorStatus(429), okList([])]);++ expect(await fetchList()).toEqual([]);+ expect(calls()).toBe(2);+ });++ it("does not retry a 4xx — the request itself is wrong", async () => {+ const calls = stubFetch([errorStatus(401), okList([quarantined("a")])]);++ expect(await fetchList()).toBeNull();+ expect(calls()).toBe(1);+ });++ it("waits out the backoff window before retrying, rather than hammering", async () => {+ const calls = stubFetch([errorStatus(503), okList([])]);+ const pending = fetchQuarantineList({+ baseUrl: "https://ci-conductor.example",+ suite: "e2e",+ });++ await macrotask();+ expect(calls()).toBe(1);++ // Half-jittered backoff off a 500ms base puts the first retry somewhere in+ // [250ms, 500ms) — early enough to keep the gate snappy, late enough to let+ // a blip pass. Asserting the bounds holds for any jitter draw.+ jest.advanceTimersByTime(249);+ await macrotask();+ expect(calls()).toBe(1);++ jest.advanceTimersByTime(251);+ await macrotask();+ expect(calls()).toBe(2);++ expect(await settle(pending)).toEqual([]);+ });+});+describe("checkQuarantineGate", () => {const failure: FailedTest = {test_name: "renders",@@ -171,7 +326,6 @@ describe("checkQuarantineGate", () => {file_path: "foo.spec.ts",};- // Both branches under test return before any fetch, so no network is touched.// Capture the gate's log lines so we can assert on the printed verdict.async function runCapturingLogs(opts: Parameters<typeof checkQuarantineGate>[0],@@ -230,6 +384,30 @@ describe("checkQuarantineGate", () => {expect(result.enforced).toBe(false);});+ it("fails the job when the list request fails and there are failures to gate", async () => {+ const originalFetch = globalThis.fetch;+ // A 401 fails on the first attempt, so the gate's retry backoff never sleeps.+ globalThis.fetch = (async () =>+ new Response("", { status: 401 })) as unknown as typeof fetch;+ try {+ const { result, verdict } = await runCapturingLogs({+ suite: "e2e",+ failures: [failure],+ baseUrl: "https://ci-conductor.example",+ secret: "shh",+ dryRun: false,+ });+ expect(result).toEqual({+ shouldFail: true,+ enforced: true,+ reason: "could not fetch the quarantine list",+ });+ expect(verdict).toContain("COULD NOT CHECK");+ } finally {+ globalThis.fetch = originalFetch;+ }+ });+it("fails closed AND enforces when it can't check outside dry run", async () => {const { result } = await runCapturingLogs({suite: "e2e",diff --git a/release/ci-conductor/src/quarantine.ts b/release/ci-conductor/src/quarantine.tsindex 1d5ce06ee37..60fc24736a2 100644--- a/release/ci-conductor/src/quarantine.ts+++ b/release/ci-conductor/src/quarantine.ts@@ -2,11 +2,18 @@// iff every failure is on ci-conductor's quarantine list.import type { NormalizedTest } from "./contract.ts";+import { fetchWithRetry } from "./fetchWithRetry.ts";import { log } from "./util.ts";// The list endpoint can be slow, but the gate must never hang a CI job.const REQUEST_TIMEOUT_MS = 15_000;+// A dropped socket (ECONNRESET) or a 5xx is a blip, not a verdict, so the fetch+// gets a few tries before the gate gives up. `fetchWithRetry` spaces those tries+// with jittered exponential backoff, so parallel jobs don't retry in lockstep.+const MAX_ATTEMPTS = 3;+const RETRY_BASE_DELAY_MS = 500;+/** One quarantined test as served by ci-conductor's `/api/quarantine`. */export type QuarantineEntry = {test_name: string;@@ -104,39 +111,52 @@ export function junitFailuresToFailedTests(/*** Fetch the quarantine list for `suite` from ci-conductor. The reporter POSTs to* `.../webhooks/failed-tests`; the list lives at `.../api/quarantine` on the same- * host. Returns null (not []) when it can't be retrieved, so the caller can tell- * "nothing quarantined" from "couldn't check". Never throws. Logs the path and- * suite only — never the host (public repo) or the secret.+ * host. Transport errors, timeouts, 429s and 5xx are retried with backoff; a 4xx+ * is the request's own fault and returns straight away. Returns null (not [])+ * when it can't be retrieved, so the caller can tell "nothing quarantined" from+ * "couldn't check". Never throws. Logs the path and suite only — never the host+ * (public repo) or the secret. The retry policy is fixed rather than injectable:+ * the tests drive it on fake timers, so there's nothing to shorten for them.*/export async function fetchQuarantineList(opts: {baseUrl: string;suite: string;secret?: string;}): Promise<QuarantineEntry[] | null> {- const base = opts.baseUrl.replace(/\/+$/, "");- const url = `${base}/api/quarantine?suite=${encodeURIComponent(opts.suite)}`;+ const { baseUrl, suite, secret } = opts;+ const base = baseUrl.replace(/\/+$/, "");+ const url = `${base}/api/quarantine?suite=${encodeURIComponent(suite)}`;+ const headers: Record<string, string> = secret+ ? { "x-internal-secret": secret }+ : {};+try {- const headers: Record<string, string> = {};- if (opts.secret) {- headers["x-internal-secret"] = opts.secret;- }- const response = await fetch(url, {+ const response = await fetchWithRetry(url, {headers,- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ retries: MAX_ATTEMPTS - 1,+ baseDelay: RETRY_BASE_DELAY_MS,+ timeout: REQUEST_TIMEOUT_MS,+ // Called once per completed response, so it doubles as the one place a+ // non-ok status gets logged — including on attempts we then retry.+ shouldRetry: (res) => {+ if (!res.ok) {+ log(+ `🛑 GET /api/quarantine?suite=${suite} → ${res.status} ${res.statusText}`,+ );+ }+ // A 4xx (bad secret, unknown suite) answers the same way every time.+ return res.status >= 500 || res.status === 429;+ },});if (!response.ok) {- log(- `🛑 GET /api/quarantine?suite=${opts.suite} → ${response.status} ${response.statusText}`,- );+ response.body?.cancel().catch(() => {}); // don't leak the unread streamreturn null;}+ // `json()` is `any`; this names the wire shape we read (and only read).const body = (await response.json()) as { tests?: QuarantineEntry[] };return body.tests ?? [];} catch (error) {- console.error(- "[ci-conductor] failed to fetch the quarantine list",- error,- );+ console.error("[ci-conductor] failed to fetch the quarantine list", error);return null;}}@@ -155,13 +175,14 @@ function finish(opts: {dryRun: boolean;// When the gate couldn't reach a real pass/fail decision — e.g. the quarantine// list was unreachable — show a distinct "couldn't check" verdict rather than a- // red FAIL. It still fails closed (shouldFail stays true), but it isn't a real- // gate failure, so it shouldn't read as one in the logs.+ // red FAIL. It still fails closed (shouldFail stays true): with no list, nothing+ // excuses the run's failures.inconclusive?: boolean;}): GateResult {const { shouldFail, reason, dryRun, inconclusive = false } = opts;if (inconclusive) {log(`⚠️ VERDICT: COULD NOT CHECK — ${reason}.`);+ log("🚦 no quarantine to apply — the run's failures stand on their own.");} else if (shouldFail) {log(`🔴 VERDICT: FAIL — ${reason}.`);} else {