Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): send Retry-After on per-minute rate limit 429s (#5025)
Gaurav Chadhaec849f3d60dd
diff --git a/apps/api/src/__tests__/routes/auth-retry-after.routes.test.ts b/apps/api/src/__tests__/routes/auth-retry-after.routes.test.tsnew file mode 100644index 000000000..e68bbbd5d--- /dev/null+++ b/apps/api/src/__tests__/routes/auth-retry-after.routes.test.ts@@ -0,0 +1,57 @@+import express from "express";+import request from "supertest";++vi.mock("../../controllers/auth", () => ({+ authenticateUser: vi.fn(),+}));++import { authenticateUser } from "../../controllers/auth";+import { authMiddleware } from "../../routes/shared";+import { RateLimiterMode } from "../../types";++function appWithAuth() {+ const app = express();+ app.get("/v2/scrape", authMiddleware(RateLimiterMode.Scrape), (_req, res) =>+ res.status(200).json({ success: true }),+ );+ return app;+}++describe("authMiddleware 429s", () => {+ afterEach(() => {+ vi.clearAllMocks();+ });++ it("sends Retry-After and retry_after_seconds when auth reports a reset", async () => {+ vi.mocked(authenticateUser).mockResolvedValue({+ success: false,+ error: "Rate limit exceeded. Please retry after 42s",+ status: 429,+ retryAfterSeconds: 42,+ });++ const response = await request(appWithAuth()).get("/v2/scrape");++ expect(response.status).toBe(429);+ expect(response.headers["retry-after"]).toBe("42");+ expect(response.body).toEqual({+ success: false,+ error: "Rate limit exceeded. Please retry after 42s",+ retry_after_seconds: 42,+ });+ });++ it("omits Retry-After when auth reports no reset", async () => {+ vi.mocked(authenticateUser).mockResolvedValue({+ success: false,+ error: "Unauthorized: Invalid token",+ status: 401,+ });++ const response = await request(appWithAuth()).get("/v2/scrape");++ expect(response.status).toBe(401);+ expect(response.headers["retry-after"]).toBeUndefined();+ expect(response.body.retry_after_seconds).toBeUndefined();+ });+});diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex 5a6293462..156d33c14 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -530,6 +530,50 @@ describe("authenticateUser", () => {expect(consume).not.toHaveBeenCalled();});+ it("returns the per-minute reset as retryAfterSeconds on a keyed 429", async () => {+ config.USE_DB_AUTHENTICATION = true;+ vi.mocked(getValue).mockResolvedValue(null);+ vi.mocked(authCreditUsageChunk).mockResolvedValue([+ {+ api_key: "00000000-0000-4000-8000-000000000000",+ api_key_id: 1,+ team_id: "team-limited",+ org_id: "org-1",+ flags: null,+ },+ ]);+ vi.mocked(redlock.using).mockImplementation(+ async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),+ );+ vi.mocked(getAutumnRateLimiter).mockReturnValue({+ consume: vi.fn().mockRejectedValue({+ msBeforeNext: 41_200,+ consumedPoints: 11,+ remainingPoints: 0,+ }),+ } as never);++ const auth = await authenticateUser(+ {+ headers: {+ authorization: "Bearer 00000000-0000-4000-8000-000000000000",+ },+ socket: { remoteAddress: "127.0.0.1" },+ },+ {},+ RateLimiterMode.Scrape,+ );++ expect(auth).toEqual(+ expect.objectContaining({+ success: false,+ status: 429,+ retryAfterSeconds: 42,+ error: expect.stringContaining("retry after 42s"),+ }),+ );+ });+it("accepts a signed MCP delegation through the managed credential purpose without caching", async () => {config.USE_DB_AUTHENTICATION = true;config.MCP_DELEGATED_CREDENTIAL_SECRET = "mcp-delegation-secret";diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 3c96bb48b..a1fee9ca7 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -1106,7 +1106,7 @@ async function supaAuthenticateUser(rateLimiterRes,});- const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1;+ const secs = Math.ceil(rateLimiterRes.msBeforeNext / 1000) || 1;const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);// We can only send a rate limit email every 7 days, send notification already has the date in between checking@@ -1120,6 +1120,7 @@ async function supaAuthenticateUser(success: false,error: `Rate limit exceeded. Consumed (req/min): ${rateLimiterRes.consumedPoints}, Remaining (req/min): ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,status: 429,+ retryAfterSeconds: secs,};}diff --git a/apps/api/src/lib/keyless.conversion.test.ts b/apps/api/src/lib/keyless.conversion.test.tsindex fa3386eb5..a1c1c46ca 100644--- a/apps/api/src/lib/keyless.conversion.test.ts+++ b/apps/api/src/lib/keyless.conversion.test.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, it, vi } from "vitest";vi.mock("../services/rate-limiter", () => ({- redisRateLimitClient: { ttl: vi.fn() },+ redisRateLimitClient: { pttl: vi.fn() },}));import { config } from "../config";import { logger } from "./logger";@@ -55,7 +55,7 @@ describe("keyless conversion cohort telemetry", () => {it("adds the cohort to reservation-limit exhaustion telemetry", async () => {config.KEYLESS_CONVERSION_HMAC_SECRET = "b".repeat(32);- vi.spyOn(redisRateLimitClient, "ttl").mockResolvedValue(42);+ vi.spyOn(redisRateLimitClient, "pttl").mockResolvedValue(41_200);const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);await keylessLimitBody("preview_keyless_203.0.113.8", "search");@@ -65,6 +65,7 @@ describe("keyless conversion cohort telemetry", () => {expect.objectContaining({event: "keyless_exhausted",reason: "credits",+ retryAfterSeconds: 42,conversionCohort: keylessConversionCohort("203.0.113.8"),}),);diff --git a/apps/api/src/lib/keyless.signup-prompt.test.ts b/apps/api/src/lib/keyless.signup-prompt.test.tsindex 33aaf3c38..99474b003 100644--- a/apps/api/src/lib/keyless.signup-prompt.test.ts+++ b/apps/api/src/lib/keyless.signup-prompt.test.ts@@ -9,7 +9,7 @@ vi.mock("ioredis", () => ({},}));vi.mock("../services/rate-limiter", () => ({- redisRateLimitClient: { ttl: vi.fn().mockResolvedValue(-1), on: vi.fn() },+ redisRateLimitClient: { pttl: vi.fn().mockResolvedValue(-1), on: vi.fn() },}));vi.mock("./keyless-prompt-analytics", () => ({trackKeylessPromptShown: vi.fn(),diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 1e9167f1d..b034b1052 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -151,8 +151,8 @@ type KeylessConsumeResult = {retryAfterSeconds?: number;};-function positiveRedisTtl(ttl: number): number | undefined {- return ttl > 0 ? ttl : undefined;+function positiveRedisTtl(pttl: number): number | undefined {+ return pttl > 0 ? Math.ceil(pttl / 1000) : undefined;}/**@@ -162,7 +162,7 @@ function positiveRedisTtl(ttl: number): number | undefined {*/async function retryAfterSecondsFor(key: string): Promise<number | undefined> {try {- return positiveRedisTtl(await redisRateLimitClient.ttl(key));+ return positiveRedisTtl(await redisRateLimitClient.pttl(key));} catch {return undefined;}@@ -259,7 +259,7 @@ export async function keylessLimitBody(let retryAfterSeconds: number | undefined;try {retryAfterSeconds = ip- ? positiveRedisTtl(await redisRateLimitClient.ttl(creditsKey(ip)))+ ? positiveRedisTtl(await redisRateLimitClient.pttl(creditsKey(ip))): undefined;} catch {// The reservation already proved the limit; missing TTL must not turn itsdiff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 55c930cb6..2c153622b 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -317,6 +317,9 @@ export function authMiddleware(if (auth.status === 401 || auth.agentAuthDiscovery) {applyAgentAuthDiscoveryHeader(res);}+ if (auth.retryAfterSeconds) {+ res.setHeader("Retry-After", String(auth.retryAfterSeconds));+ }return res.status(auth.status).json({success: false,error: auth.error,