Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

fix(imports): honor bounded Retry-After on hosted scanner reads (#2086)

Michael D'Angelo140f17334918
diff --git a/docs/scanner-imports/semgrep.md b/docs/scanner-imports/semgrep.md
index c3add8c39..9a9e96af0 100644
--- a/docs/scanner-imports/semgrep.md
+++ b/docs/scanner-imports/semgrep.md
@@ -218,6 +218,10 @@ overlapping page. Offset pagination does not provide a point-in-time snapshot.
For a multi-page result the collector also checks whether the first page's
membership has changed. Retry the collection if it reports a change.
+On HTTP 429, the collector retries the same GET once if `Retry-After` specifies
+a wait of 10 seconds or less, in whole seconds or as an HTTP date. Zero or a
+past date triggers an immediate retry. Cancellation stops the wait.
+
A finding that identifies its repository only by name can use the project name,
the Git URL path, or the final path component of either. For any other name,
at least one finding in the collection must pair that name with the project's
diff --git a/docs/scanner-imports/snyk.md b/docs/scanner-imports/snyk.md
index 16d32b003..1a417befb 100644
--- a/docs/scanner-imports/snyk.md
+++ b/docs/scanner-imports/snyk.md
@@ -45,6 +45,10 @@ coordinates, and projects monitored through the CLI or API, read the original
location from `evidence.source_data.issue` in v1 or from the JSON text in
`evidence.source_data.issueJson` in v2.
+On HTTP 429, the collector retries the same GET once if `Retry-After` specifies
+a wait of 10 seconds or less, in whole seconds or as an HTTP date. Zero or a
+past date triggers an immediate retry. Cancellation stops the wait.
+
Use the same source key on later imports. The issue key within the API
origin, organization, project and product identifies the finding. Replaying the
same evidence leaves the version unchanged. Whitespace, object key order and
diff --git a/sdk/typescript/src/hosted-scanner-http.ts b/sdk/typescript/src/hosted-scanner-http.ts
index d24086965..416dcdec1 100644
--- a/sdk/typescript/src/hosted-scanner-http.ts
+++ b/sdk/typescript/src/hosted-scanner-http.ts
@@ -1,3 +1,4 @@
+import { waitForRetry } from "./deduplication/retry.js";
import { canonicalNumber } from "./external-import-contract.js";
import { CodexSecurityError } from "./errors.js";
import { normalizeHostedGitRepositoryUrl } from "./hosted-git-repository.js";
@@ -92,6 +93,56 @@ export function hostedScannerToken(
return token;
}
+function httpDate(hint: string, now: number): number {
+ const normalized = hint.replace("23:59:60", "23:59:59");
+ const leapSecond = normalized === hint ? 0 : 1000;
+ let canonical = normalized;
+ const asctime =
+ /^(\w{3}) (\w{3}) ( \d|\d{2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/u.exec(
+ normalized,
+ );
+ const rfc850 =
+ /^(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\d{2})-(\w{3})-(\d{2}) (\d{2}:\d{2}:\d{2}) GMT$/u.exec(
+ normalized,
+ );
+ if (asctime) {
+ const [, weekday, month, day, time, year] = asctime;
+ canonical = `${weekday}, ${day!.replace(" ", "0")} ${month} ${year} ${time} GMT`;
+ } else if (rfc850) {
+ const [, weekday, day, month, year, time] = rfc850;
+ const cutoff = new Date(now);
+ cutoff.setUTCFullYear(cutoff.getUTCFullYear() + 50);
+ const century = Math.floor(cutoff.getUTCFullYear() / 100) * 100;
+ const dateIn = (century: number) =>
+ `${weekday!.slice(0, 3)}, ${day} ${month} ${century + Number(year)} ${time} GMT`;
+ canonical = dateIn(century);
+ if (Date.parse(canonical) + leapSecond > cutoff.getTime())
+ canonical = dateIn(century - 100);
+ }
+ const date = new Date(canonical);
+ if (leapSecond && new Date(date.getTime() + leapSecond).getUTCDate() !== 1)
+ return NaN;
+ return canonical.length === 29 && date.toUTCString() === canonical
+ ? date.getTime() + leapSecond
+ : NaN;
+}
+
+function retryAfterMs(response: Response): number | undefined {
+ if (response.status !== 429) return;
+ const hint = response.headers.get("retry-after");
+ if (!hint) return;
+ let delay: number;
+ if (/^\d+$/u.test(hint)) {
+ delay = Number(hint) * 1000;
+ } else {
+ const now = Date.now();
+ const serverDate = httpDate(response.headers.get("date") ?? "", now);
+ const reference = Number.isNaN(serverDate) ? now : serverDate;
+ delay = Math.max(0, httpDate(hint, reference) - reference);
+ }
+ return delay <= 10_000 ? delay : undefined;
+}
+
export function hostedScannerHttp(
dependencies: HostedScannerHttpOptions,
callerSignal?: AbortSignal,
@@ -107,15 +158,24 @@ export function hostedScannerHttp(
signal?.throwIfAborted();
if (/[\0\r\n]/u.test(authorization))
throw new Error("Scanner API token cannot contain a NUL or line break.");
- const response = await request(url, {
- method: "GET",
- headers: {
- accept: "application/vnd.api+json, application/json",
- authorization,
- },
- redirect: "error",
- signal,
- });
+ const get = () =>
+ request(new URL(url), {
+ method: "GET",
+ headers: {
+ accept: "application/vnd.api+json, application/json",
+ authorization,
+ },
+ redirect: "error",
+ signal,
+ });
+ let response = await get();
+ const delay = retryAfterMs(response);
+ if (delay !== undefined) {
+ void response.body?.cancel().catch(() => undefined);
+ if (delay > 0) await waitForRetry(delay, signal);
+ signal?.throwIfAborted();
+ response = await get();
+ }
if (!response.ok) {
const retry = response.headers.get("retry-after");
const detail = await response.text();
diff --git a/sdk/typescript/tests-ts/hosted-scanner-http.test.ts b/sdk/typescript/tests-ts/hosted-scanner-http.test.ts
new file mode 100644
index 000000000..d021a6045
--- /dev/null
+++ b/sdk/typescript/tests-ts/hosted-scanner-http.test.ts
@@ -0,0 +1,135 @@
+import { setImmediate } from "node:timers/promises";
+import { afterEach, expect, jest, mock, test } from "bun:test";
+import {
+ hostedScannerHttp,
+ type HostedScannerHttpOptions,
+} from "../src/hosted-scanner-http.js";
+
+const url = new URL(
+ "https://semgrep.dev/api/v1/deployments/example/findings?page=1",
+);
+function get(options: HostedScannerHttpOptions) {
+ return hostedScannerHttp(options).getJson(url, "Bearer synthetic");
+}
+
+const now = Date.UTC(2026, 10, 6, 8, 49, 20, 250);
+afterEach(() => jest.useRealTimers());
+
+function limited(hint: string, cancel: () => void) {
+ return new Response(new ReadableStream({ cancel }), {
+ status: 429,
+ headers: { "Retry-After": hint },
+ });
+}
+
+test.each([
+ ["010", 10_000, null],
+ ["Fri, 06 Nov 2026 08:48:05 GMT", 5_000, "Fri, 06 Nov 2026 08:48:00 GMT"],
+ ["Fri, 06 Nov 2026 08:49:30 GMT", 9_750, null],
+ ["Friday, 06-Nov-26 08:49:30 GMT", 9_750, "unusable date"],
+ ["Fri Nov 6 08:49:30 2026", 9_750, null],
+ ["Sun, 06 Nov 1994 08:49:37 GMT", 0, null],
+ ["Sunday, 06-Nov-94 08:49:37 GMT", 0, null],
+ ["Sat, 31 Dec 2016 23:59:60 GMT", 0, null],
+] as const)(
+ "releases the 429 body and waits for Retry-After %s",
+ async (hint, delay, serverDate) => {
+ jest.useFakeTimers({ now });
+ const released = Promise.withResolvers<void>();
+ const fetch = mock(async () => Response.json({ complete: true }));
+ const response = limited(hint, released.resolve);
+ if (serverDate) response.headers.set("Date", serverDate);
+ fetch.mockResolvedValueOnce(response);
+ const result = get({ fetch });
+ await released.promise;
+ if (delay > 0) {
+ jest.advanceTimersByTime(delay - 1);
+ await setImmediate();
+ expect(fetch).toHaveBeenCalledTimes(1);
+ jest.advanceTimersByTime(1);
+ }
+ expect(await result).toEqual({ complete: true });
+ expect(fetch).toHaveBeenCalledTimes(2);
+ },
+);
+
+test("retries the same URL when a transport rewrites its request path", async () => {
+ const paths: string[] = [];
+ await get({
+ fetch: async (requestUrl) => {
+ requestUrl.pathname = `/proxy${requestUrl.pathname}`;
+ paths.push(requestUrl.pathname);
+ return paths.length === 1
+ ? new Response(null, { status: 429, headers: { "Retry-After": "0" } })
+ : Response.json({});
+ },
+ });
+ expect(paths).toEqual([`/proxy${url.pathname}`, `/proxy${url.pathname}`]);
+});
+
+test.each(["cancel", "timeout"])(
+ "a %s during Retry-After prevents the second GET",
+ async (mode) => {
+ jest.useFakeTimers({ now });
+ const controller = new AbortController();
+ const signal =
+ mode === "timeout" ? AbortSignal.timeout(500) : controller.signal;
+ const released = Promise.withResolvers<void>();
+ const fetch = mock(async () => limited("10", released.resolve));
+ const result = get({
+ fetch,
+ signal,
+ }).catch((error: unknown) => error);
+ await released.promise;
+ if (mode === "timeout") jest.advanceTimersByTime(500);
+ else controller.abort(new Error("collection stopped"));
+ expect(await result).toBe(signal.reason);
+ jest.advanceTimersByTime(10_000);
+ expect(fetch).toHaveBeenCalledTimes(1);
+ },
+);
+
+test("unusable or long hints and other HTTP errors fail on the first response with raw details", async () => {
+ jest.useFakeTimers({ now });
+ const detail = "vendor detail: synthetic-token\n\u001b[31mretry later";
+ for (const [status, hint] of [
+ [429, null],
+ [429, ""],
+ [429, "-1"],
+ [429, "1.5"],
+ [429, "1e0"],
+ [429, "0, 1"],
+ [429, "11"],
+ [429, "9".repeat(400)],
+ [429, "Fri, 06 Nov 2026 08:49:31 GMT"],
+ [429, "Tuesday, 06-Nov-74 08:49:30 GMT"],
+ [429, "2026-11-06"],
+ [429, "Fri, 31 Feb 2023 08:49:30 GMT"],
+ [429, "Fri, 30 Dec 2016 23:59:60 GMT"],
+ [401, "0"],
+ [403, "0"],
+ [503, "0"],
+ [307, "0"],
+ ] as const) {
+ const fetch = mock(
+ async () =>
+ new Response(detail, {
+ status,
+ headers: hint === null ? {} : { "Retry-After": hint },
+ }),
+ );
+ await expect(get({ fetch })).rejects.toThrow(
+ `GET ${url.origin}${url.pathname} failed (HTTP ${status})${hint ? `; Retry-After: ${hint}` : ""}: ${detail}`,
+ );
+ expect(fetch).toHaveBeenCalledTimes(1);
+ }
+});
+
+test("does not retry a fetch rejection", async () => {
+ const error = new TypeError("connection reset");
+ const fetch = mock(async () => {
+ throw error;
+ });
+ await expect(get({ fetch })).rejects.toBe(error);
+ expect(fetch).toHaveBeenCalledTimes(1);
+});
diff --git a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts
index eb1ed7469..edbc22b72 100644
--- a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts
+++ b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts
@@ -708,7 +708,7 @@ test("file and normalized Platform reports keep their Git binding with no accept
}
});
-test("walks the selected project and ref, then rechecks page zero for offset drift", async () => {
+test("retries the throttled page in scope, then rechecks page zero for offset drift", async () => {
const pages: string[] = [];
const report = await collectSemgrepPlatformFindings(options, {
environment,
@@ -733,13 +733,19 @@ test("walks the selected project and ref, then rechecks page zero for offset dri
repository_ids: "77",
dedup: "false",
});
+ expect(init.method).toBe("GET");
pages.push(page);
+ if (pages.length === 2)
+ return new Response("try again", {
+ status: 429,
+ headers: { "Retry-After": "0" },
+ });
return Response.json({
findings: page === "1" ? [{ id: 3001 }] : fullPage,
});
},
});
- expect(pages).toEqual(["0", "1", "0"]);
+ expect(pages).toEqual(["0", "1", "1", "0"]);
expect(report.findingCount).toBe(3001);
expect(JSON.parse(report.findings[3000]!)).toEqual({ id: 3001 });
@@ -809,6 +815,14 @@ test("rejects failed, overlapping or wrong-scope pages without returning a parti
),
).rejects.toThrow(error);
}
+ const limited = () =>
+ new Response("try later", { status: 429, headers: { "Retry-After": "0" } });
+ await expect(
+ collectSemgrepPlatformFindings(
+ options,
+ responses(Response.json({ findings: fullPage }), limited(), limited()),
+ ),
+ ).rejects.toThrow("Retry-After: 0: try later");
});
test.each([null, "[email protected]:example/service.git"])(
diff --git a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts
index ffd178579..4ee91ecd3 100644
--- a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts
+++ b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts
@@ -40,7 +40,8 @@ function transport(...responses: unknown[]) {
"token test-token",
);
if (responses.length === 0) throw new Error("Unexpected request");
- return Response.json(responses.shift());
+ const next = responses.shift();
+ return next instanceof Response ? next : Response.json(next);
},
});
},
@@ -60,7 +61,7 @@ test("the Issues normalizer keeps long issue titles by default", async () => {
});
test.each(["open-source", "code"] as const)(
- "Snyk %s follows scoped pagination, saves original evidence, and replays unchanged in SQLite",
+ "Snyk %s retries the scoped page, saves original evidence, and replays unchanged in SQLite",
async (product) => {
const code = product === "code";
const asset = project(code ? "sast" : "npm");
@@ -83,6 +84,13 @@ test.each(["open-source", "code"] as const)(
next: `?starting_after=${encodeURIComponent(cursor)}`,
},
},
+ new Response("try again", {
+ status: 429,
+ headers: {
+ "Retry-After": "0",
+ Location: "https://other.invalid/issues",
+ },
+ }),
{
data: [record],
links: {
@@ -96,6 +104,7 @@ test.each(["open-source", "code"] as const)(
`/rest/orgs/${options.organizationId}/projects/${options.projectId}`,
path,
path,
+ path,
]);
for (const url of api.calls.slice(1)) {
expect(url.origin).toBe(options.apiOrigin);
@@ -110,6 +119,7 @@ test.each(["open-source", "code"] as const)(
);
}
expect(api.calls[2]!.searchParams.get("starting_after")).toBe(cursor);
+ expect(api.calls[3]!.href).toBe(api.calls[2]!.href);
await using f = await snykLocalFixture({
remote: options.sourceRepositoryUrl,
});
@@ -136,6 +146,22 @@ test.each(["open-source", "code"] as const)(
},
);
+test("reports the last HTTP error when the retried Snyk cursor is still throttled", async () => {
+ const limited = (detail: string) =>
+ new Response(detail, { status: 429, headers: { "Retry-After": "0" } });
+ const api = transport(
+ project(),
+ { data: [], links: { next: "?starting_after=more" } },
+ limited("first failure"),
+ limited("still throttled"),
+ );
+ await expect(api.collect()).rejects.toThrow(
+ "failed (HTTP 429); Retry-After: 0: still throttled",
+ );
+ expect(api.calls).toHaveLength(4);
+ expect(api.calls[3]!.href).toBe(api.calls[2]!.href);
+});
+
test.each([
["uat", "api.eu.snyk.io"],
["sat", "https://api.eu.snyk.io"],