Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): send Retry-After on per-minute rate limit 429s (#5025)

Gaurav Chadhaec849f3d60dd
diff --git a/apps/api/src/__tests__/routes/auth-retry-after.routes.test.ts b/apps/api/src/__tests__/routes/auth-retry-after.routes.test.ts
new file mode 100644
index 000000000..e68bbbd5d
--- /dev/null
+++ b/apps/api/src/__tests__/routes/auth-retry-after.routes.test.ts
@@ -0,0 +1,57 @@
+import express from "express";
+import request from "supertest";
+
+vi.mock("../../controllers/auth", () => ({
+ authenticateUser: vi.fn(),
+}));
+
+import { authenticateUser } from "../../controllers/auth";
+import { authMiddleware } from "../../routes/shared";
+import { RateLimiterMode } from "../../types";
+
+function appWithAuth() {
+ const app = express();
+ app.get("/v2/scrape", authMiddleware(RateLimiterMode.Scrape), (_req, res) =>
+ res.status(200).json({ success: true }),
+ );
+ return app;
+}
+
+describe("authMiddleware 429s", () => {
+ afterEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("sends Retry-After and retry_after_seconds when auth reports a reset", async () => {
+ vi.mocked(authenticateUser).mockResolvedValue({
+ success: false,
+ error: "Rate limit exceeded. Please retry after 42s",
+ status: 429,
+ retryAfterSeconds: 42,
+ });
+
+ const response = await request(appWithAuth()).get("/v2/scrape");
+
+ expect(response.status).toBe(429);
+ expect(response.headers["retry-after"]).toBe("42");
+ expect(response.body).toEqual({
+ success: false,
+ error: "Rate limit exceeded. Please retry after 42s",
+ retry_after_seconds: 42,
+ });
+ });
+
+ it("omits Retry-After when auth reports no reset", async () => {
+ vi.mocked(authenticateUser).mockResolvedValue({
+ success: false,
+ error: "Unauthorized: Invalid token",
+ status: 401,
+ });
+
+ const response = await request(appWithAuth()).get("/v2/scrape");
+
+ expect(response.status).toBe(401);
+ expect(response.headers["retry-after"]).toBeUndefined();
+ expect(response.body.retry_after_seconds).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.ts
index 5a6293462..156d33c14 100644
--- a/apps/api/src/controllers/__tests__/auth.test.ts
+++ b/apps/api/src/controllers/__tests__/auth.test.ts
@@ -530,6 +530,50 @@ describe("authenticateUser", () => {
expect(consume).not.toHaveBeenCalled();
});
+ it("returns the per-minute reset as retryAfterSeconds on a keyed 429", async () => {
+ config.USE_DB_AUTHENTICATION = true;
+ vi.mocked(getValue).mockResolvedValue(null);
+ vi.mocked(authCreditUsageChunk).mockResolvedValue([
+ {
+ api_key: "00000000-0000-4000-8000-000000000000",
+ api_key_id: 1,
+ team_id: "team-limited",
+ org_id: "org-1",
+ flags: null,
+ },
+ ]);
+ vi.mocked(redlock.using).mockImplementation(
+ async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),
+ );
+ vi.mocked(getAutumnRateLimiter).mockReturnValue({
+ consume: vi.fn().mockRejectedValue({
+ msBeforeNext: 41_200,
+ consumedPoints: 11,
+ remainingPoints: 0,
+ }),
+ } as never);
+
+ const auth = await authenticateUser(
+ {
+ headers: {
+ authorization: "Bearer 00000000-0000-4000-8000-000000000000",
+ },
+ socket: { remoteAddress: "127.0.0.1" },
+ },
+ {},
+ RateLimiterMode.Scrape,
+ );
+
+ expect(auth).toEqual(
+ expect.objectContaining({
+ success: false,
+ status: 429,
+ retryAfterSeconds: 42,
+ error: expect.stringContaining("retry after 42s"),
+ }),
+ );
+ });
+
it("accepts a signed MCP delegation through the managed credential purpose without caching", async () => {
config.USE_DB_AUTHENTICATION = true;
config.MCP_DELEGATED_CREDENTIAL_SECRET = "mcp-delegation-secret";
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 3c96bb48b..a1fee9ca7 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -1106,7 +1106,7 @@ async function supaAuthenticateUser(
rateLimiterRes,
});
- const secs = Math.round(rateLimiterRes.msBeforeNext / 1000) || 1;
+ const secs = Math.ceil(rateLimiterRes.msBeforeNext / 1000) || 1;
const retryDate = new Date(Date.now() + rateLimiterRes.msBeforeNext);
// We can only send a rate limit email every 7 days, send notification already has the date in between checking
@@ -1120,6 +1120,7 @@ async function supaAuthenticateUser(
success: false,
error: `Rate limit exceeded. Consumed (req/min): ${rateLimiterRes.consumedPoints}, Remaining (req/min): ${rateLimiterRes.remainingPoints}. Upgrade your plan at https://firecrawl.dev/pricing for increased rate limits or please retry after ${secs}s, resets at ${retryDate}`,
status: 429,
+ retryAfterSeconds: secs,
};
}
diff --git a/apps/api/src/lib/keyless.conversion.test.ts b/apps/api/src/lib/keyless.conversion.test.ts
index fa3386eb5..a1c1c46ca 100644
--- a/apps/api/src/lib/keyless.conversion.test.ts
+++ b/apps/api/src/lib/keyless.conversion.test.ts
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from "vitest";
vi.mock("../services/rate-limiter", () => ({
- redisRateLimitClient: { ttl: vi.fn() },
+ redisRateLimitClient: { pttl: vi.fn() },
}));
import { config } from "../config";
import { logger } from "./logger";
@@ -55,7 +55,7 @@ describe("keyless conversion cohort telemetry", () => {
it("adds the cohort to reservation-limit exhaustion telemetry", async () => {
config.KEYLESS_CONVERSION_HMAC_SECRET = "b".repeat(32);
- vi.spyOn(redisRateLimitClient, "ttl").mockResolvedValue(42);
+ vi.spyOn(redisRateLimitClient, "pttl").mockResolvedValue(41_200);
const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
await keylessLimitBody("preview_keyless_203.0.113.8", "search");
@@ -65,6 +65,7 @@ describe("keyless conversion cohort telemetry", () => {
expect.objectContaining({
event: "keyless_exhausted",
reason: "credits",
+ retryAfterSeconds: 42,
conversionCohort: keylessConversionCohort("203.0.113.8"),
}),
);
diff --git a/apps/api/src/lib/keyless.signup-prompt.test.ts b/apps/api/src/lib/keyless.signup-prompt.test.ts
index 33aaf3c38..99474b003 100644
--- a/apps/api/src/lib/keyless.signup-prompt.test.ts
+++ b/apps/api/src/lib/keyless.signup-prompt.test.ts
@@ -9,7 +9,7 @@ vi.mock("ioredis", () => ({
},
}));
vi.mock("../services/rate-limiter", () => ({
- redisRateLimitClient: { ttl: vi.fn().mockResolvedValue(-1), on: vi.fn() },
+ redisRateLimitClient: { pttl: vi.fn().mockResolvedValue(-1), on: vi.fn() },
}));
vi.mock("./keyless-prompt-analytics", () => ({
trackKeylessPromptShown: vi.fn(),
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index 1e9167f1d..b034b1052 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -151,8 +151,8 @@ type KeylessConsumeResult = {
retryAfterSeconds?: number;
};
-function positiveRedisTtl(ttl: number): number | undefined {
- return ttl > 0 ? ttl : undefined;
+function positiveRedisTtl(pttl: number): number | undefined {
+ return pttl > 0 ? Math.ceil(pttl / 1000) : undefined;
}
/**
@@ -162,7 +162,7 @@ function positiveRedisTtl(ttl: number): number | undefined {
*/
async function retryAfterSecondsFor(key: string): Promise<number | undefined> {
try {
- return positiveRedisTtl(await redisRateLimitClient.ttl(key));
+ return positiveRedisTtl(await redisRateLimitClient.pttl(key));
} catch {
return undefined;
}
@@ -259,7 +259,7 @@ export async function keylessLimitBody(
let retryAfterSeconds: number | undefined;
try {
retryAfterSeconds = ip
- ? positiveRedisTtl(await redisRateLimitClient.ttl(creditsKey(ip)))
+ ? positiveRedisTtl(await redisRateLimitClient.pttl(creditsKey(ip)))
: undefined;
} catch {
// The reservation already proved the limit; missing TTL must not turn its
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 55c930cb6..2c153622b 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -317,6 +317,9 @@ export function authMiddleware(
if (auth.status === 401 || auth.agentAuthDiscovery) {
applyAgentAuthDiscoveryHeader(res);
}
+ if (auth.retryAfterSeconds) {
+ res.setHeader("Retry-After", String(auth.retryAfterSeconds));
+ }
return res.status(auth.status).json({
success: false,
error: auth.error,