Skip to content

The open-source app everyone uses to manage agents at work

fix(ci): retry transient GitHub reads while waiting for source verification (#13429)

Devin Foley08adcc70d5ec
diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs
index 28444873bb..bd40e33cd3 100644
--- a/scripts/cloud-source-verification.mjs
+++ b/scripts/cloud-source-verification.mjs
@@ -61,6 +61,80 @@ export async function readSourceVerification(sha, api) {
return undefined;
}
+// Statuses that say "ask again", not "the answer is no". A release must not be
+// blocked because GitHub returned a gateway error during a 45-minute poll.
+const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
+
+// A rate-limited read also says "ask again", and GitHub reports both primary
+// and secondary rate limits as 403. Only the headers separate that from a
+// token that may not read Actions, which must still fail at once.
+function rateLimited(response) {
+ if (response.status !== 403) return false;
+ const header = (name) => response.headers?.get?.(name) ?? null;
+ return header("retry-after") !== null || header("x-ratelimit-remaining") === "0";
+}
+
+/** Milliseconds from a Retry-After header, when it carries a sane delay. */
+function retryAfterMs(response, capMs) {
+ const value = Number(response?.headers?.get?.("retry-after"));
+ if (!Number.isFinite(value) || value <= 0) return null;
+ return Math.min(value * 1_000, capMs);
+}
+
+/**
+ * The GitHub Actions read used by the polling below, with transient transport
+ * failures retried: network errors, the statuses above, rate-limited 403s, and
+ * a body that fails while it is being read. Any other non-OK status throws on
+ * the first response, because waiting out a wrong token only delays the news.
+ *
+ * `deadlineAt` bounds retries by the caller's own polling deadline, so a read
+ * cannot extend the wait past the timeout it belongs to.
+ */
+export function createActionsReader({
+ token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+ attempts = 4, backoffMs = 1_000, log = console.log,
+ now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000,
+} = {}) {
+ return async (path) => {
+ for (let attempt = 1; ; attempt += 1) {
+ // Retry only while both the attempt budget and the caller's deadline
+ // leave room for the wait this attempt would cost.
+ const waitMs = backoffMs * attempt;
+ const retryable = attempt < attempts && now() + waitMs < deadlineAt();
+ const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs);
+ let response;
+ try {
+ response = await fetchImpl(`https://api.github.com${path}`, {
+ headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },
+ signal: AbortSignal.timeout(30_000), redirect: "error",
+ });
+ } catch (cause) {
+ if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`);
+ log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`);
+ await pause();
+ continue;
+ }
+ if (response.ok) {
+ try {
+ // A 200 whose body is truncated or undecodable is a transport
+ // failure like any other, so it belongs inside the retry.
+ return await response.json();
+ } catch (cause) {
+ if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`);
+ log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`);
+ await pause();
+ continue;
+ }
+ }
+ if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) {
+ throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);
+ }
+ log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`);
+ await pause(response);
+ }
+ };
+}
+
export async function waitForSourceVerification(sha, {
api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
@@ -80,15 +154,12 @@ export async function waitForSourceVerification(sha, {
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
- const api = async (path) => {
- const response = await fetch(`https://api.github.com${path}`, {
- headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },
- signal: AbortSignal.timeout(30_000), redirect: "error",
- });
- if (!response.ok) throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);
- return response.json();
- };
- const proof = await waitForSourceVerification(process.argv[2], { api });
+ // One deadline for both layers: the reader stops retrying when the poll it
+ // serves is out of time, instead of extending the wait past its timeout.
+ const timeoutMs = 45 * 60_000;
+ const deadline = Date.now() + timeoutMs;
+ const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });
+ const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });
const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
console.log(message);
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);
diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs
index d75b2baf2f..4a47abfd16 100644
--- a/scripts/cloud-source-verification.test.mjs
+++ b/scripts/cloud-source-verification.test.mjs
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import test from "node:test";
-import { readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
+import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
const sha = "a".repeat(40);
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
@@ -123,3 +123,111 @@ test("malformed source refs are rejected before any request", async () => {
await assert.rejects(readSourceVerification(ref, () => assert.fail("must not request")), /full lowercase/);
}
});
+
+// A gateway error during the poll must not decide the release. These cover the
+// reader's transport only; the verification semantics above are unchanged.
+function reader({ responses, attempts = 4, ...options }) {
+ const seen = [];
+ const waits = [];
+ const fetchImpl = async () => {
+ const next = responses[seen.length];
+ seen.push(next);
+ if (next instanceof Error) throw next;
+ return {
+ ok: next.status >= 200 && next.status < 300,
+ status: next.status,
+ headers: { get: (name) => next.headers?.[name.toLowerCase()] ?? null },
+ json: async () => {
+ if (next.bodyError) throw next.bodyError;
+ return next.body ?? { ok: true };
+ },
+ };
+ };
+ const api = createActionsReader({
+ token: "t", fetchImpl, attempts, backoffMs: 10,
+ sleep: async (ms) => { waits.push(ms); }, log: () => {}, ...options,
+ });
+ return { api, seen, waits };
+}
+
+test("a transient gateway error is retried instead of failing the release", async () => {
+ const { api, seen, waits } = reader({ responses: [{ status: 502 }, { status: 200, body: { id: 1 } }] });
+ assert.deepEqual(await api("/repos/x"), { id: 1 });
+ assert.equal(seen.length, 2);
+ assert.deepEqual(waits, [10]);
+});
+
+test("every transient status is retried, and the backoff grows", async () => {
+ for (const status of [408, 425, 429, 500, 502, 503, 504]) {
+ const { api, seen, waits } = reader({ responses: [{ status }, { status }, { status: 200, body: { ok: true } }] });
+ await api("/repos/x");
+ assert.equal(seen.length, 3, `status ${status} should be retried`);
+ assert.deepEqual(waits, [10, 20]);
+ }
+});
+
+test("a rate-limited 403 is retried, and a forbidden 403 is not", async () => {
+ // GitHub reports both primary and secondary rate limits as 403; only the
+ // headers tell them apart from a token that may not read Actions.
+ for (const headers of [{ "retry-after": "1" }, { "x-ratelimit-remaining": "0" }]) {
+ const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200, body: { ok: true } }] });
+ await api("/repos/x");
+ assert.equal(seen.length, 2, `403 with ${JSON.stringify(headers)} should be retried`);
+ }
+ for (const headers of [undefined, { "x-ratelimit-remaining": "4999" }]) {
+ const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200 }] });
+ await assert.rejects(api("/repos/x"), /HTTP 403/);
+ assert.equal(seen.length, 1, "a forbidden 403 must fail on the first response");
+ }
+});
+
+test("Retry-After sets the wait, capped so one header cannot stall the poll", async () => {
+ const { api, waits } = reader({ responses: [{ status: 429, headers: { "retry-after": "5" } }, { status: 200 }] });
+ await api("/repos/x");
+ assert.deepEqual(waits, [5_000]);
+
+ const capped = reader({ responses: [{ status: 429, headers: { "retry-after": "86400" } }, { status: 200 }], maxRetryAfterMs: 60_000 });
+ await capped.api("/repos/x");
+ assert.deepEqual(capped.waits, [60_000]);
+});
+
+test("a body that fails while being read is retried", async () => {
+ const { api, seen } = reader({
+ responses: [{ status: 200, bodyError: new Error("terminated") }, { status: 200, body: { id: 7 } }],
+ });
+ assert.deepEqual(await api("/repos/x"), { id: 7 });
+ assert.equal(seen.length, 2);
+});
+
+test("a network failure is retried, and its message survives exhaustion", async () => {
+ const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => new Error("fetch failed")) });
+ await assert.rejects(api("/repos/x"), /GitHub Actions read failed: fetch failed/);
+ assert.equal(seen.length, 4);
+});
+
+test("an authorization failure is not retried", async () => {
+ for (const status of [401, 404, 422]) {
+ const { api, seen } = reader({ responses: [{ status }, { status: 200 }] });
+ await assert.rejects(api("/repos/x"), new RegExp(`HTTP ${status}`));
+ assert.equal(seen.length, 1, `status ${status} must fail on the first response`);
+ }
+});
+
+test("a transient status that never clears fails after its attempt budget", async () => {
+ const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => ({ status: 502 })) });
+ await assert.rejects(api("/repos/x"), /HTTP 502/);
+ assert.equal(seen.length, 4);
+});
+
+test("retries stop at the caller's deadline rather than outliving the poll", async () => {
+ // The wait this attempt would cost does not fit before the deadline, so the
+ // read reports the failure instead of sleeping past the timeout it serves.
+ let clock = 0;
+ const { api, seen, waits } = reader({
+ responses: [{ status: 502 }, { status: 200 }],
+ now: () => clock, deadlineAt: () => 5,
+ });
+ await assert.rejects(api("/repos/x"), /HTTP 502/);
+ assert.equal(seen.length, 1);
+ assert.deepEqual(waits, []);
+});