Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(web): keep Drive rate-limit retries inside the caller's deadline and honour HTTP-date Retry-After

Richie McIlroy1c2092cff4f8
diff --git a/apps/web/__tests__/unit/google-drive-storage.test.ts b/apps/web/__tests__/unit/google-drive-storage.test.ts
index 0663d56e3..2367371e0 100644
--- a/apps/web/__tests__/unit/google-drive-storage.test.ts
+++ b/apps/web/__tests__/unit/google-drive-storage.test.ts
@@ -2314,4 +2314,58 @@ describe("Google Drive rate limits", () => {
expect(getGoogleDriveRateLimitDelayMs(0, "600", () => 0)).toBe(16_000);
expect(getGoogleDriveRateLimitDelayMs(1, "soon", () => 0)).toBe(2_000);
});
+
+ it("reads an HTTP-date Retry-After against the current time", () => {
+ const now = Date.parse("Sat, 24 Oct 2026 11:00:00 GMT");
+ expect(
+ getGoogleDriveRateLimitDelayMs(
+ 0,
+ "Sat, 24 Oct 2026 11:00:05 GMT",
+ () => 0.9,
+ now,
+ ),
+ ).toBe(5_000);
+ expect(
+ getGoogleDriveRateLimitDelayMs(
+ 0,
+ "Sat, 24 Oct 2026 11:05:00 GMT",
+ () => 0,
+ now,
+ ),
+ ).toBe(16_000);
+ expect(
+ getGoogleDriveRateLimitDelayMs(
+ 0,
+ "Sat, 24 Oct 2026 10:59:00 GMT",
+ () => 0,
+ now,
+ ),
+ ).toBe(0);
+ });
+
+ it("stops retrying before the waits would outlast a 60 s caller deadline", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockImplementation(async () =>
+ jsonResponse(
+ { error: { code: 429 } },
+ { status: 429, headers: { "Retry-After": "16" } },
+ ),
+ ),
+ );
+ let settled = false;
+ const pending = copy(copyHarness()).then((exit) => {
+ settled = true;
+ return exit;
+ });
+ await vi.advanceTimersByTimeAsync(31_000);
+ expect(settled).toBe(true);
+ const exit = await pending;
+ expect(Exit.isFailure(exit)).toBe(true);
+ expect(fetch).toHaveBeenCalledTimes(2);
+ });
});
diff --git a/packages/web-backend/src/Storage/GoogleDrive.ts b/packages/web-backend/src/Storage/GoogleDrive.ts
index 63d0b43c9..03cfebbc8 100644
--- a/packages/web-backend/src/Storage/GoogleDrive.ts
+++ b/packages/web-backend/src/Storage/GoogleDrive.ts
@@ -503,6 +503,8 @@ const sendDriveRequest = (
const DRIVE_RATE_LIMIT_RETRIES = 4;
const DRIVE_RATE_LIMIT_BASE_DELAY_MS = 1_000;
const DRIVE_RATE_LIMIT_MAX_DELAY_MS = 16_000;
+// Callers such as the source commit give each storage call 60 s, so every retry must still fit inside it.
+const DRIVE_RATE_LIMIT_TOTAL_DELAY_MS = 30_000;
const DRIVE_RATE_LIMIT_REASONS = new Set([
"userRateLimitExceeded",
"rateLimitExceeded",
@@ -530,16 +532,22 @@ export const isGoogleDriveRateLimited = async (response: Response) => {
);
};
+const getRetryAfterMs = (retryAfter: string | null, now: number) => {
+ if (!retryAfter) return undefined;
+ if (/^\d+$/.test(retryAfter.trim())) return Number(retryAfter) * 1_000;
+ const date = Date.parse(retryAfter);
+ return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+};
+
export const getGoogleDriveRateLimitDelayMs = (
attempt: number,
retryAfter: string | null,
random = Math.random,
+ now = Date.now(),
) => {
- const seconds = retryAfter ? Number(retryAfter) : Number.NaN;
const requested =
- Number.isFinite(seconds) && seconds >= 0
- ? seconds * 1_000
- : DRIVE_RATE_LIMIT_BASE_DELAY_MS * 2 ** attempt + random() * 1_000;
+ getRetryAfterMs(retryAfter, now) ??
+ DRIVE_RATE_LIMIT_BASE_DELAY_MS * 2 ** attempt + random() * 1_000;
return Math.min(Math.round(requested), DRIVE_RATE_LIMIT_MAX_DELAY_MS);
};
@@ -551,6 +559,7 @@ const driveFetch = (
) =>
Effect.gen(function* () {
let response = yield* authorizedDriveFetch(config, url, init, tokenStore);
+ let waitedMs = 0;
for (let attempt = 0; attempt < DRIVE_RATE_LIMIT_RETRIES; attempt++) {
const limited = yield* Effect.promise(() =>
isGoogleDriveRateLimited(response),
@@ -560,6 +569,8 @@ const driveFetch = (
attempt,
response.headers.get("retry-after"),
);
+ if (waitedMs + delayMs > DRIVE_RATE_LIMIT_TOTAL_DELAY_MS) break;
+ waitedMs += delayMs;
console.warn(
JSON.stringify({
event: "google_drive_rate_limited",