The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(ci): retry transient GitHub reads while waiting for source verification (#13429)
Devin Foley08adcc70d5ec
diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjsindex 28444873bb..bd40e33cd3 100644--- a/scripts/cloud-source-verification.mjs+++ b/scripts/cloud-source-verification.mjs@@ -61,6 +61,80 @@ export async function readSourceVerification(sha, api) {return undefined;}+// Statuses that say "ask again", not "the answer is no". A release must not be+// blocked because GitHub returned a gateway error during a 45-minute poll.+const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);++// A rate-limited read also says "ask again", and GitHub reports both primary+// and secondary rate limits as 403. Only the headers separate that from a+// token that may not read Actions, which must still fail at once.+function rateLimited(response) {+ if (response.status !== 403) return false;+ const header = (name) => response.headers?.get?.(name) ?? null;+ return header("retry-after") !== null || header("x-ratelimit-remaining") === "0";+}++/** Milliseconds from a Retry-After header, when it carries a sane delay. */+function retryAfterMs(response, capMs) {+ const value = Number(response?.headers?.get?.("retry-after"));+ if (!Number.isFinite(value) || value <= 0) return null;+ return Math.min(value * 1_000, capMs);+}++/**+ * The GitHub Actions read used by the polling below, with transient transport+ * failures retried: network errors, the statuses above, rate-limited 403s, and+ * a body that fails while it is being read. Any other non-OK status throws on+ * the first response, because waiting out a wrong token only delays the news.+ *+ * `deadlineAt` bounds retries by the caller's own polling deadline, so a read+ * cannot extend the wait past the timeout it belongs to.+ */+export function createActionsReader({+ token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),+ attempts = 4, backoffMs = 1_000, log = console.log,+ now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000,+} = {}) {+ return async (path) => {+ for (let attempt = 1; ; attempt += 1) {+ // Retry only while both the attempt budget and the caller's deadline+ // leave room for the wait this attempt would cost.+ const waitMs = backoffMs * attempt;+ const retryable = attempt < attempts && now() + waitMs < deadlineAt();+ const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs);+ let response;+ try {+ response = await fetchImpl(`https://api.github.com${path}`, {+ headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },+ signal: AbortSignal.timeout(30_000), redirect: "error",+ });+ } catch (cause) {+ if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`);+ log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`);+ await pause();+ continue;+ }+ if (response.ok) {+ try {+ // A 200 whose body is truncated or undecodable is a transport+ // failure like any other, so it belongs inside the retry.+ return await response.json();+ } catch (cause) {+ if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`);+ log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`);+ await pause();+ continue;+ }+ }+ if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) {+ throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);+ }+ log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`);+ await pause(response);+ }+ };+}+export async function waitForSourceVerification(sha, {api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,@@ -80,15 +154,12 @@ export async function waitForSourceVerification(sha, {if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {try {if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");- const api = async (path) => {- const response = await fetch(`https://api.github.com${path}`, {- headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },- signal: AbortSignal.timeout(30_000), redirect: "error",- });- if (!response.ok) throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);- return response.json();- };- const proof = await waitForSourceVerification(process.argv[2], { api });+ // One deadline for both layers: the reader stops retrying when the poll it+ // serves is out of time, instead of extending the wait past its timeout.+ const timeoutMs = 45 * 60_000;+ const deadline = Date.now() + timeoutMs;+ const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });+ const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;console.log(message);if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjsindex d75b2baf2f..4a47abfd16 100644--- a/scripts/cloud-source-verification.test.mjs+++ b/scripts/cloud-source-verification.test.mjs@@ -1,6 +1,6 @@import assert from "node:assert/strict";import test from "node:test";-import { readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";+import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";const sha = "a".repeat(40);const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };@@ -123,3 +123,111 @@ test("malformed source refs are rejected before any request", async () => {await assert.rejects(readSourceVerification(ref, () => assert.fail("must not request")), /full lowercase/);}});++// A gateway error during the poll must not decide the release. These cover the+// reader's transport only; the verification semantics above are unchanged.+function reader({ responses, attempts = 4, ...options }) {+ const seen = [];+ const waits = [];+ const fetchImpl = async () => {+ const next = responses[seen.length];+ seen.push(next);+ if (next instanceof Error) throw next;+ return {+ ok: next.status >= 200 && next.status < 300,+ status: next.status,+ headers: { get: (name) => next.headers?.[name.toLowerCase()] ?? null },+ json: async () => {+ if (next.bodyError) throw next.bodyError;+ return next.body ?? { ok: true };+ },+ };+ };+ const api = createActionsReader({+ token: "t", fetchImpl, attempts, backoffMs: 10,+ sleep: async (ms) => { waits.push(ms); }, log: () => {}, ...options,+ });+ return { api, seen, waits };+}++test("a transient gateway error is retried instead of failing the release", async () => {+ const { api, seen, waits } = reader({ responses: [{ status: 502 }, { status: 200, body: { id: 1 } }] });+ assert.deepEqual(await api("/repos/x"), { id: 1 });+ assert.equal(seen.length, 2);+ assert.deepEqual(waits, [10]);+});++test("every transient status is retried, and the backoff grows", async () => {+ for (const status of [408, 425, 429, 500, 502, 503, 504]) {+ const { api, seen, waits } = reader({ responses: [{ status }, { status }, { status: 200, body: { ok: true } }] });+ await api("/repos/x");+ assert.equal(seen.length, 3, `status ${status} should be retried`);+ assert.deepEqual(waits, [10, 20]);+ }+});++test("a rate-limited 403 is retried, and a forbidden 403 is not", async () => {+ // GitHub reports both primary and secondary rate limits as 403; only the+ // headers tell them apart from a token that may not read Actions.+ for (const headers of [{ "retry-after": "1" }, { "x-ratelimit-remaining": "0" }]) {+ const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200, body: { ok: true } }] });+ await api("/repos/x");+ assert.equal(seen.length, 2, `403 with ${JSON.stringify(headers)} should be retried`);+ }+ for (const headers of [undefined, { "x-ratelimit-remaining": "4999" }]) {+ const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200 }] });+ await assert.rejects(api("/repos/x"), /HTTP 403/);+ assert.equal(seen.length, 1, "a forbidden 403 must fail on the first response");+ }+});++test("Retry-After sets the wait, capped so one header cannot stall the poll", async () => {+ const { api, waits } = reader({ responses: [{ status: 429, headers: { "retry-after": "5" } }, { status: 200 }] });+ await api("/repos/x");+ assert.deepEqual(waits, [5_000]);++ const capped = reader({ responses: [{ status: 429, headers: { "retry-after": "86400" } }, { status: 200 }], maxRetryAfterMs: 60_000 });+ await capped.api("/repos/x");+ assert.deepEqual(capped.waits, [60_000]);+});++test("a body that fails while being read is retried", async () => {+ const { api, seen } = reader({+ responses: [{ status: 200, bodyError: new Error("terminated") }, { status: 200, body: { id: 7 } }],+ });+ assert.deepEqual(await api("/repos/x"), { id: 7 });+ assert.equal(seen.length, 2);+});++test("a network failure is retried, and its message survives exhaustion", async () => {+ const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => new Error("fetch failed")) });+ await assert.rejects(api("/repos/x"), /GitHub Actions read failed: fetch failed/);+ assert.equal(seen.length, 4);+});++test("an authorization failure is not retried", async () => {+ for (const status of [401, 404, 422]) {+ const { api, seen } = reader({ responses: [{ status }, { status: 200 }] });+ await assert.rejects(api("/repos/x"), new RegExp(`HTTP ${status}`));+ assert.equal(seen.length, 1, `status ${status} must fail on the first response`);+ }+});++test("a transient status that never clears fails after its attempt budget", async () => {+ const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => ({ status: 502 })) });+ await assert.rejects(api("/repos/x"), /HTTP 502/);+ assert.equal(seen.length, 4);+});++test("retries stop at the caller's deadline rather than outliving the poll", async () => {+ // The wait this attempt would cost does not fit before the deadline, so the+ // read reports the failure instead of sleeping past the timeout it serves.+ let clock = 0;+ const { api, seen, waits } = reader({+ responses: [{ status: 502 }, { status: 200 }],+ now: () => clock, deadlineAt: () => 5,+ });+ await assert.rejects(api("/repos/x"), /HTTP 502/);+ assert.equal(seen.length, 1);+ assert.deepEqual(waits, []);+});