Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(web): back off and retry Google Drive rate-limit responses instead of failing the request

Richie McIlroy1cc436af18ca
diff --git a/apps/web/__tests__/unit/google-drive-storage.test.ts b/apps/web/__tests__/unit/google-drive-storage.test.ts
index b426c9b57..0663d56e3 100644
--- a/apps/web/__tests__/unit/google-drive-storage.test.ts
+++ b/apps/web/__tests__/unit/google-drive-storage.test.ts
@@ -19,6 +19,7 @@ import {
findGoogleDriveFileByObjectKey,
GOOGLE_DRIVE_FOLDER_MIME_TYPE,
type GoogleDriveTokenStore,
+ getGoogleDriveRateLimitDelayMs,
getGoogleDriveRecordingDownload,
getGoogleDriveRecordingResponse,
syncGoogleDriveVideoNames,
@@ -2193,3 +2194,124 @@ describe("Google Drive title synchronization", () => {
expect(files.get("file-id")?.name).toBe("New title: 测试 🎬.mp4");
});
});
+
+describe("Google Drive rate limits", () => {
+ const copyHarness = () => {
+ const harness = makeRepoHarness([
+ {
+ id: videoId,
+ name: "Rate limited",
+ ownerId: videoOwnerId,
+ storageIntegrationId: integrationId,
+ },
+ ]);
+ harness.putObject(
+ storedObjectInput(".cap-folders/video-1", "folder-id", {
+ contentType: GOOGLE_DRIVE_FOLDER_MIME_TYPE,
+ }),
+ );
+ harness.putObject(
+ storedObjectInput(
+ ".cap-warnings/video-1/DO_NOT_EDIT_OR_DELETE.txt",
+ "warning-id",
+ { contentType: "text/plain" },
+ ),
+ );
+ return harness;
+ };
+ const copy = (harness: RepoHarness) =>
+ Effect.runPromiseExit(
+ copyGoogleDriveFile({
+ repo: harness.repo,
+ config: makeConfig(),
+ sourceFileId: "source-file-id",
+ input: {
+ integrationId,
+ ownerId: integrationOwnerId,
+ videoId,
+ key: resultKey,
+ contentType: "video/mp4",
+ },
+ tokenStore: makeTokenStore(),
+ }),
+ );
+ const copied = () =>
+ jsonResponse({
+ id: "copied-file-id",
+ name: "Rate limited.mp4",
+ mimeType: "video/mp4",
+ size: "1234",
+ });
+ const forbidden = (reason: string) =>
+ jsonResponse(
+ { error: { code: 403, errors: [{ reason, domain: "usageLimits" }] } },
+ { status: 403 },
+ );
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it.each([
+ ["429", () => jsonResponse({ error: { code: 429 } }, { status: 429 })],
+ ["403 userRateLimitExceeded", () => forbidden("userRateLimitExceeded")],
+ ["403 rateLimitExceeded", () => forbidden("rateLimitExceeded")],
+ ])("backs off and retries a %s response", async (_name, limited) => {
+ vi.useFakeTimers();
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockResolvedValueOnce(limited())
+ .mockResolvedValueOnce(limited())
+ .mockResolvedValueOnce(copied()),
+ );
+ const pending = copy(copyHarness());
+ await vi.advanceTimersByTimeAsync(10_000);
+ const exit = await pending;
+ expect(Exit.isSuccess(exit)).toBe(true);
+ expect(fetch).toHaveBeenCalledTimes(3);
+ });
+
+ it.each([
+ "insufficientPermissions",
+ "storageQuotaExceeded",
+ "dailyLimitExceeded",
+ ])("fails a 403 %s at once", async (reason) => {
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(forbidden(reason)));
+ const exit = await copy(copyHarness());
+ expect(Exit.isFailure(exit)).toBe(true);
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("gives up after four retries and reports the rate limit", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(console, "warn").mockImplementation(() => {});
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockImplementation(async () =>
+ jsonResponse({ error: { code: 429 } }, { status: 429 }),
+ ),
+ );
+ const pending = copy(copyHarness());
+ await vi.advanceTimersByTimeAsync(60_000);
+ const exit = await pending;
+ expect(Exit.isFailure(exit)).toBe(true);
+ expect(fetch).toHaveBeenCalledTimes(5);
+ expect(
+ Exit.isFailure(exit) && JSON.stringify(Cause.squash(exit.cause)),
+ ).toContain("429");
+ });
+
+ it("doubles the delay with jitter, honours Retry-After, and caps both", () => {
+ expect(getGoogleDriveRateLimitDelayMs(0, null, () => 0)).toBe(1_000);
+ expect(getGoogleDriveRateLimitDelayMs(3, null, () => 0.5)).toBe(8_500);
+ expect(getGoogleDriveRateLimitDelayMs(6, null, () => 0)).toBe(16_000);
+ expect(getGoogleDriveRateLimitDelayMs(0, "3", () => 0.9)).toBe(3_000);
+ expect(getGoogleDriveRateLimitDelayMs(0, "600", () => 0)).toBe(16_000);
+ expect(getGoogleDriveRateLimitDelayMs(1, "soon", () => 0)).toBe(2_000);
+ });
+});
diff --git a/packages/web-backend/src/Storage/GoogleDrive.ts b/packages/web-backend/src/Storage/GoogleDrive.ts
index ead4134c3..63d0b43c9 100644
--- a/packages/web-backend/src/Storage/GoogleDrive.ts
+++ b/packages/web-backend/src/Storage/GoogleDrive.ts
@@ -500,11 +500,90 @@ const sendDriveRequest = (
catch: (cause) => new Storage.StorageError({ cause }),
});
+const DRIVE_RATE_LIMIT_RETRIES = 4;
+const DRIVE_RATE_LIMIT_BASE_DELAY_MS = 1_000;
+const DRIVE_RATE_LIMIT_MAX_DELAY_MS = 16_000;
+const DRIVE_RATE_LIMIT_REASONS = new Set([
+ "userRateLimitExceeded",
+ "rateLimitExceeded",
+]);
+
+// Drive rejects these before doing any work, so every method can be retried; Google asks for exponential backoff.
+export const isGoogleDriveRateLimited = async (response: Response) => {
+ if (response.status === 429) return true;
+ if (response.status !== 403) return false;
+ const body: unknown = await response
+ .clone()
+ .json()
+ .catch(() => null);
+ const errors =
+ typeof body === "object" && body !== null && "error" in body
+ ? (body.error as { errors?: { reason?: unknown }[] }).errors
+ : undefined;
+ return (
+ Array.isArray(errors) &&
+ errors.some(
+ (error) =>
+ typeof error?.reason === "string" &&
+ DRIVE_RATE_LIMIT_REASONS.has(error.reason),
+ )
+ );
+};
+
+export const getGoogleDriveRateLimitDelayMs = (
+ attempt: number,
+ retryAfter: string | null,
+ random = Math.random,
+) => {
+ const seconds = retryAfter ? Number(retryAfter) : Number.NaN;
+ const requested =
+ Number.isFinite(seconds) && seconds >= 0
+ ? seconds * 1_000
+ : DRIVE_RATE_LIMIT_BASE_DELAY_MS * 2 ** attempt + random() * 1_000;
+ return Math.min(Math.round(requested), DRIVE_RATE_LIMIT_MAX_DELAY_MS);
+};
+
const driveFetch = (
config: GoogleDriveIntegrationConfig,
url: string,
init?: RequestInit,
tokenStore?: GoogleDriveTokenStore,
+) =>
+ Effect.gen(function* () {
+ let response = yield* authorizedDriveFetch(config, url, init, tokenStore);
+ for (let attempt = 0; attempt < DRIVE_RATE_LIMIT_RETRIES; attempt++) {
+ const limited = yield* Effect.promise(() =>
+ isGoogleDriveRateLimited(response),
+ );
+ if (!limited) break;
+ const delayMs = getGoogleDriveRateLimitDelayMs(
+ attempt,
+ response.headers.get("retry-after"),
+ );
+ console.warn(
+ JSON.stringify({
+ event: "google_drive_rate_limited",
+ status: response.status,
+ method: init?.method ?? "GET",
+ attempt: attempt + 1,
+ delayMs,
+ }),
+ );
+ yield* Effect.sleep(delayMs);
+ response = yield* authorizedDriveFetch(config, url, init, tokenStore);
+ }
+ yield* Effect.tryPromise({
+ try: () => assertDriveResponse(response),
+ catch: (cause) => new Storage.StorageError({ cause }),
+ });
+ return response;
+ });
+
+const authorizedDriveFetch = (
+ config: GoogleDriveIntegrationConfig,
+ url: string,
+ init: RequestInit | undefined,
+ tokenStore: GoogleDriveTokenStore | undefined,
) =>
Effect.gen(function* () {
const accessToken = yield* getCachedGoogleDriveAccessToken(
@@ -521,10 +600,6 @@ const driveFetch = (
);
response = yield* sendDriveRequest(refreshedAccessToken, url, init);
}
- yield* Effect.tryPromise({
- try: () => assertDriveResponse(response),
- catch: (cause) => new Storage.StorageError({ cause }),
- });
return response;
});