Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(web): back off and retry Google Drive rate-limit responses instead of failing the request
Richie McIlroy1cc436af18ca
diff --git a/apps/web/__tests__/unit/google-drive-storage.test.ts b/apps/web/__tests__/unit/google-drive-storage.test.tsindex b426c9b57..0663d56e3 100644--- a/apps/web/__tests__/unit/google-drive-storage.test.ts+++ b/apps/web/__tests__/unit/google-drive-storage.test.ts@@ -19,6 +19,7 @@ import {findGoogleDriveFileByObjectKey,GOOGLE_DRIVE_FOLDER_MIME_TYPE,type GoogleDriveTokenStore,+ getGoogleDriveRateLimitDelayMs,getGoogleDriveRecordingDownload,getGoogleDriveRecordingResponse,syncGoogleDriveVideoNames,@@ -2193,3 +2194,124 @@ describe("Google Drive title synchronization", () => {expect(files.get("file-id")?.name).toBe("New title: 测试 🎬.mp4");});});++describe("Google Drive rate limits", () => {+ const copyHarness = () => {+ const harness = makeRepoHarness([+ {+ id: videoId,+ name: "Rate limited",+ ownerId: videoOwnerId,+ storageIntegrationId: integrationId,+ },+ ]);+ harness.putObject(+ storedObjectInput(".cap-folders/video-1", "folder-id", {+ contentType: GOOGLE_DRIVE_FOLDER_MIME_TYPE,+ }),+ );+ harness.putObject(+ storedObjectInput(+ ".cap-warnings/video-1/DO_NOT_EDIT_OR_DELETE.txt",+ "warning-id",+ { contentType: "text/plain" },+ ),+ );+ return harness;+ };+ const copy = (harness: RepoHarness) =>+ Effect.runPromiseExit(+ copyGoogleDriveFile({+ repo: harness.repo,+ config: makeConfig(),+ sourceFileId: "source-file-id",+ input: {+ integrationId,+ ownerId: integrationOwnerId,+ videoId,+ key: resultKey,+ contentType: "video/mp4",+ },+ tokenStore: makeTokenStore(),+ }),+ );+ const copied = () =>+ jsonResponse({+ id: "copied-file-id",+ name: "Rate limited.mp4",+ mimeType: "video/mp4",+ size: "1234",+ });+ const forbidden = (reason: string) =>+ jsonResponse(+ { error: { code: 403, errors: [{ reason, domain: "usageLimits" }] } },+ { status: 403 },+ );++ afterEach(() => {+ vi.useRealTimers();+ });++ it.each([+ ["429", () => jsonResponse({ error: { code: 429 } }, { status: 429 })],+ ["403 userRateLimitExceeded", () => forbidden("userRateLimitExceeded")],+ ["403 rateLimitExceeded", () => forbidden("rateLimitExceeded")],+ ])("backs off and retries a %s response", async (_name, limited) => {+ vi.useFakeTimers();+ vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.stubGlobal(+ "fetch",+ vi+ .fn()+ .mockResolvedValueOnce(limited())+ .mockResolvedValueOnce(limited())+ .mockResolvedValueOnce(copied()),+ );+ const pending = copy(copyHarness());+ await vi.advanceTimersByTimeAsync(10_000);+ const exit = await pending;+ expect(Exit.isSuccess(exit)).toBe(true);+ expect(fetch).toHaveBeenCalledTimes(3);+ });++ it.each([+ "insufficientPermissions",+ "storageQuotaExceeded",+ "dailyLimitExceeded",+ ])("fails a 403 %s at once", async (reason) => {+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(forbidden(reason)));+ const exit = await copy(copyHarness());+ expect(Exit.isFailure(exit)).toBe(true);+ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("gives up after four retries and reports the rate limit", async () => {+ vi.useFakeTimers();+ vi.spyOn(console, "warn").mockImplementation(() => {});+ vi.stubGlobal(+ "fetch",+ vi+ .fn()+ .mockImplementation(async () =>+ jsonResponse({ error: { code: 429 } }, { status: 429 }),+ ),+ );+ const pending = copy(copyHarness());+ await vi.advanceTimersByTimeAsync(60_000);+ const exit = await pending;+ expect(Exit.isFailure(exit)).toBe(true);+ expect(fetch).toHaveBeenCalledTimes(5);+ expect(+ Exit.isFailure(exit) && JSON.stringify(Cause.squash(exit.cause)),+ ).toContain("429");+ });++ it("doubles the delay with jitter, honours Retry-After, and caps both", () => {+ expect(getGoogleDriveRateLimitDelayMs(0, null, () => 0)).toBe(1_000);+ expect(getGoogleDriveRateLimitDelayMs(3, null, () => 0.5)).toBe(8_500);+ expect(getGoogleDriveRateLimitDelayMs(6, null, () => 0)).toBe(16_000);+ expect(getGoogleDriveRateLimitDelayMs(0, "3", () => 0.9)).toBe(3_000);+ expect(getGoogleDriveRateLimitDelayMs(0, "600", () => 0)).toBe(16_000);+ expect(getGoogleDriveRateLimitDelayMs(1, "soon", () => 0)).toBe(2_000);+ });+});diff --git a/packages/web-backend/src/Storage/GoogleDrive.ts b/packages/web-backend/src/Storage/GoogleDrive.tsindex ead4134c3..63d0b43c9 100644--- a/packages/web-backend/src/Storage/GoogleDrive.ts+++ b/packages/web-backend/src/Storage/GoogleDrive.ts@@ -500,11 +500,90 @@ const sendDriveRequest = (catch: (cause) => new Storage.StorageError({ cause }),});+const DRIVE_RATE_LIMIT_RETRIES = 4;+const DRIVE_RATE_LIMIT_BASE_DELAY_MS = 1_000;+const DRIVE_RATE_LIMIT_MAX_DELAY_MS = 16_000;+const DRIVE_RATE_LIMIT_REASONS = new Set([+ "userRateLimitExceeded",+ "rateLimitExceeded",+]);++// Drive rejects these before doing any work, so every method can be retried; Google asks for exponential backoff.+export const isGoogleDriveRateLimited = async (response: Response) => {+ if (response.status === 429) return true;+ if (response.status !== 403) return false;+ const body: unknown = await response+ .clone()+ .json()+ .catch(() => null);+ const errors =+ typeof body === "object" && body !== null && "error" in body+ ? (body.error as { errors?: { reason?: unknown }[] }).errors+ : undefined;+ return (+ Array.isArray(errors) &&+ errors.some(+ (error) =>+ typeof error?.reason === "string" &&+ DRIVE_RATE_LIMIT_REASONS.has(error.reason),+ )+ );+};++export const getGoogleDriveRateLimitDelayMs = (+ attempt: number,+ retryAfter: string | null,+ random = Math.random,+) => {+ const seconds = retryAfter ? Number(retryAfter) : Number.NaN;+ const requested =+ Number.isFinite(seconds) && seconds >= 0+ ? seconds * 1_000+ : DRIVE_RATE_LIMIT_BASE_DELAY_MS * 2 ** attempt + random() * 1_000;+ return Math.min(Math.round(requested), DRIVE_RATE_LIMIT_MAX_DELAY_MS);+};+const driveFetch = (config: GoogleDriveIntegrationConfig,url: string,init?: RequestInit,tokenStore?: GoogleDriveTokenStore,+) =>+ Effect.gen(function* () {+ let response = yield* authorizedDriveFetch(config, url, init, tokenStore);+ for (let attempt = 0; attempt < DRIVE_RATE_LIMIT_RETRIES; attempt++) {+ const limited = yield* Effect.promise(() =>+ isGoogleDriveRateLimited(response),+ );+ if (!limited) break;+ const delayMs = getGoogleDriveRateLimitDelayMs(+ attempt,+ response.headers.get("retry-after"),+ );+ console.warn(+ JSON.stringify({+ event: "google_drive_rate_limited",+ status: response.status,+ method: init?.method ?? "GET",+ attempt: attempt + 1,+ delayMs,+ }),+ );+ yield* Effect.sleep(delayMs);+ response = yield* authorizedDriveFetch(config, url, init, tokenStore);+ }+ yield* Effect.tryPromise({+ try: () => assertDriveResponse(response),+ catch: (cause) => new Storage.StorageError({ cause }),+ });+ return response;+ });++const authorizedDriveFetch = (+ config: GoogleDriveIntegrationConfig,+ url: string,+ init: RequestInit | undefined,+ tokenStore: GoogleDriveTokenStore | undefined,) =>Effect.gen(function* () {const accessToken = yield* getCachedGoogleDriveAccessToken(@@ -521,10 +600,6 @@ const driveFetch = ();response = yield* sendDriveRequest(refreshedAccessToken, url, init);}- yield* Effect.tryPromise({- try: () => assertDriveResponse(response),- catch: (cause) => new Storage.StorageError({ cause }),- });return response;});