Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(keyless): preserve quota recovery details (#4211)

Himadri Mishraea4bad0fe028
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index f7a2308f6..893f63256 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -252,6 +252,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
});
expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.reason).toBe("credits");
expect(blocked.body.error).toContain("keyless free tier rate limit");
expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
expect(blocked.body.error).toContain(
@@ -296,6 +297,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
});
expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.reason).toBe("credits");
expect(blocked.body.error).toContain("keyless free tier rate limit");
expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");
expect(blocked.body.error).toContain(
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index da064b371..cd789a197 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -515,15 +515,18 @@ async function handleKeylessAuth(
logger.warn("Keyless request blocked", {
...baseLog,
blocked: true,
+ event: "keyless_exhausted",
reason: result.reason,
+ retryAfterSeconds: result.retryAfterSeconds,
});
return {
success: false,
error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
status: 429,
- // Out of free quota — emit the OAuth-discovery header so agents can find
- // the key/signup flow at the moment they actually need a key.
+ // Direct API callers receive discovery metadata; MCP maps this in-band.
agentAuthDiscovery: true,
+ keylessReason: result.reason,
+ retryAfterSeconds: result.retryAfterSeconds,
};
}
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 8a6ece1db..c5292a2b9 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -29,8 +29,8 @@ import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
- KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -156,10 +156,9 @@ export async function scrapeController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json({
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- });
+ return res.status(429).json(
+ await keylessLimitBody(req.auth.team_id, "v1_scrape"),
+ );
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 129b83e48..fea667c0e 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -34,8 +34,8 @@ import {
import { fromV1ScrapeOptions } from "../v2/types";
import { getSearchForcedKind } from "../../lib/zdr-helpers";
import {
- KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -212,10 +212,9 @@ export async function searchController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json({
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- });
+ return res.status(429).json(
+ await keylessLimitBody(req.auth.team_id, "v1_search"),
+ );
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
index cbe5f22d0..fa61ec6f5 100644
--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
@@ -3,6 +3,13 @@ import { vi } from "vitest";
const mockLogRequest = vi.fn();
const mockLogSearch = vi.fn();
const mockLogResearchEndpoint = vi.fn();
+const mockReserveKeylessCredits = vi.fn().mockResolvedValue({ ok: true });
+const mockKeylessLimitBody = vi.fn().mockResolvedValue({
+ success: false,
+ error: "keyless limit reached",
+ reason: "credits",
+});
+const mockProjectSearchTotalCredits = vi.fn(() => 0);
vi.mock("../../../services/logging/log_job", () => ({
logRequest: (...args: any[]) => mockLogRequest(...args),
@@ -23,12 +30,13 @@ vi.mock("../../../services/billing/credit_billing", () => ({
vi.mock("../../../lib/keyless", () => ({
KEYLESS_FREE_TIER_LIMIT_MESSAGE: "keyless limit reached",
adjustKeylessCredits: vi.fn().mockResolvedValue(undefined),
+ keylessLimitBody: (...args: any[]) => mockKeylessLimitBody(...args),
logKeylessCreditUsage: vi.fn().mockResolvedValue(undefined),
- reserveKeylessCredits: vi.fn().mockResolvedValue({ ok: true }),
+ reserveKeylessCredits: (...args: any[]) => mockReserveKeylessCredits(...args),
}));
vi.mock("../../../lib/keyless-credit-projection", () => ({
- projectSearchTotalCredits: () => 0,
+ projectSearchTotalCredits: () => mockProjectSearchTotalCredits(),
}));
vi.mock("../../../lib/threat-protection/request", () => ({
@@ -120,9 +128,34 @@ beforeEach(() => {
mockLogSearch.mockResolvedValue(undefined);
mockLogResearchEndpoint.mockResolvedValue(undefined);
mockExecuteSearch.mockResolvedValue(executeResult());
+ mockReserveKeylessCredits.mockResolvedValue({ ok: true });
+ mockKeylessLimitBody.mockResolvedValue({
+ success: false,
+ error: "keyless limit reached",
+ reason: "credits",
+ });
+ mockProjectSearchTotalCredits.mockReturnValue(0);
});
describe("developer category code_searches ledger", () => {
+ it("returns the structured keyless 429 when projected credits cannot be reserved", async () => {
+ mockProjectSearchTotalCredits.mockReturnValue(2);
+ mockReserveKeylessCredits.mockResolvedValue({ ok: false });
+ const req = makeReq({ query: "http client", categories: ["developer"] });
+ const res = makeRes();
+
+ await searchController(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(429);
+ expect(mockKeylessLimitBody).toHaveBeenCalledWith(TEAM_ID, "v2_search");
+ expect(res.json).toHaveBeenCalledWith({
+ success: false,
+ error: "keyless limit reached",
+ reason: "credits",
+ });
+ expect(mockExecuteSearch).not.toHaveBeenCalled();
+ });
+
it("writes exactly one code_searches row for a developer category search", async () => {
const req = makeReq({
query: "vector database client",
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 9ba6795d2..eae34cff1 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -27,8 +27,8 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
- KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -378,10 +378,9 @@ export async function parseController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json({
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- });
+ return res.status(429).json(
+ await keylessLimitBody(req.auth.team_id, "v2_parse"),
+ );
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 9c8454631..5751c7127 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -49,6 +49,7 @@ import {
KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
keylessTeamUuid,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -558,10 +559,7 @@ async function createSessionForScrape(
if (!reservation.ok) {
return {
status: 429,
- body: {
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- },
+ body: await keylessLimitBody(req.auth.team_id, "v2_browser"),
error: true,
};
}
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index ef33aecb9..e56da6a8e 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -30,8 +30,8 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
- KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -186,10 +186,9 @@ export async function scrapeController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json({
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- });
+ return res.status(429).json(
+ await keylessLimitBody(req.auth.team_id, "v2_scrape"),
+ );
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 38df521f6..3f1d6c5ac 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -8,8 +8,8 @@ import {
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import {
- KEYLESS_FREE_TIER_LIMIT_MESSAGE,
adjustKeylessCredits,
+ keylessLimitBody,
logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
@@ -235,10 +235,9 @@ export async function searchController(
);
if (!reservation.ok) {
applyAgentAuthDiscoveryHeader(res);
- return res.status(429).json({
- success: false,
- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
- });
+ return res.status(429).json(
+ await keylessLimitBody(req.auth.team_id, "v2_search"),
+ );
}
reservedKeylessCredits = projectedKeylessCredits;
}
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index bf04f3af4..a7aeb0543 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -5,6 +5,7 @@ import { db } from "../db/connection";
import * as schema from "../db/schema";
import { redisRateLimitClient } from "../services/rate-limiter";
import { isKeylessIpSuspicious } from "./spur";
+import { logger } from "./logger";
// Keyless free tier: scrape, search, and interact can be used without an API key
// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO
@@ -86,13 +87,69 @@ export function isKeylessIpEligible(ip: string): boolean {
const requestsKey = (ip: string) => `keyless_requests:${ip}`;
const creditsKey = (ip: string) => `keyless_credits:${ip}`;
+export type KeylessQuotaReason = "requests" | "credits";
+
type KeylessConsumeResult = {
ok: boolean;
- reason?: "requests" | "credits";
+ reason?: KeylessQuotaReason;
requestsUsed: number;
creditsUsed: number;
+ retryAfterSeconds?: number;
};
+function positiveRedisTtl(ttl: number): number | undefined {
+ return ttl > 0 ? ttl : undefined;
+}
+
+/**
+ * Quota denials remain valid when Redis cannot provide the optional TTL hint.
+ * Keep that secondary lookup from changing a controlled 429 into a generic
+ * authentication failure.
+ */
+async function retryAfterSecondsFor(key: string): Promise<number | undefined> {
+ try {
+ return positiveRedisTtl(await redisRateLimitClient.ttl(key));
+ } catch {
+ return undefined;
+ }
+}
+
+/** Structured response for projected-credit reservation exhaustion. */
+export async function keylessLimitBody(
+ teamId: string,
+ mode: string,
+): Promise<{
+ success: false;
+ error: string;
+ reason: "credits";
+ retry_after_seconds?: number;
+}> {
+ const ip = keylessIpFromTeamId(teamId);
+ let retryAfterSeconds: number | undefined;
+ try {
+ retryAfterSeconds = ip
+ ? positiveRedisTtl(await redisRateLimitClient.ttl(creditsKey(ip)))
+ : undefined;
+ } catch {
+ // The reservation already proved the limit; missing TTL must not turn its
+ // controlled 429 into a server error.
+ }
+ logger.warn("Keyless request blocked", {
+ canonicalLog: "keyless/consume",
+ event: "keyless_exhausted",
+ blocked: true,
+ reason: "credits",
+ mode,
+ retryAfterSeconds,
+ });
+ return {
+ success: false,
+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,
+ reason: "credits",
+ ...(retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {}),
+ };
+}
+
type KeylessCreditReservationResult = {
ok: boolean;
creditsUsed: number;
@@ -122,10 +179,22 @@ export async function consumeKeylessRequest(
);
if (requestsUsed > requestLimit) {
- return { ok: false, reason: "requests", requestsUsed, creditsUsed };
+ return {
+ ok: false,
+ reason: "requests",
+ requestsUsed,
+ creditsUsed,
+ retryAfterSeconds: await retryAfterSecondsFor(rKey),
+ };
}
if (creditsUsed >= creditLimit) {
- return { ok: false, reason: "credits", requestsUsed, creditsUsed };
+ return {
+ ok: false,
+ reason: "credits",
+ requestsUsed,
+ creditsUsed,
+ retryAfterSeconds: await retryAfterSecondsFor(creditsKey(ip)),
+ };
}
return { ok: true, requestsUsed, creditsUsed };
}
@@ -219,7 +288,11 @@ return next
*/
export async function checkKeylessEligibility(
ip: string,
-): Promise<{ eligible: boolean; reason?: string }> {
+): Promise<{
+ eligible: boolean;
+ reason?: KeylessQuotaReason | "disabled" | "ineligible_ip" | "suspicious" | "error";
+ retryAfterSeconds?: number;
+}> {
if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };
if (!ip || !isKeylessIpEligible(ip)) {
return { eligible: false, reason: "ineligible_ip" };
@@ -236,14 +309,23 @@ export async function checkKeylessEligibility(
10,
);
if (requestsUsed >= (KEYLESS_REQUESTS_PER_DAY ?? 0)) {
- return { eligible: false, reason: "requests" };
+ return {
+ eligible: false,
+ reason: "requests",
+ retryAfterSeconds: await retryAfterSecondsFor(requestsKey(ip)),
+ };
}
+ const creditKey = creditsKey(ip);
const creditsUsed = parseInt(
- (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",
+ (await redisRateLimitClient.get(creditKey)) ?? "0",
10,
);
if (creditsUsed >= (KEYLESS_CREDITS_PER_DAY ?? 0)) {
- return { eligible: false, reason: "credits" };
+ return {
+ eligible: false,
+ reason: "credits",
+ retryAfterSeconds: await retryAfterSecondsFor(creditKey),
+ };
}
return { eligible: true };
} catch {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index d3e78aae9..9079c1f0c 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -248,7 +248,14 @@ export function authMiddleware(
}
return res
.status(auth.status)
- .json({ success: false, error: auth.error });
+ .json({
+ success: false,
+ error: auth.error,
+ ...(auth.keylessReason ? { reason: auth.keylessReason } : {}),
+ ...(auth.retryAfterSeconds
+ ? { retry_after_seconds: auth.retryAfterSeconds }
+ : {}),
+ });
} else {
return;
}
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index efacc556b..80d3a3dbc 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -177,6 +177,9 @@ export type AuthResponse =
// When true, send the agent OAuth-discovery WWW-Authenticate header even on
// non-401 responses (e.g. keyless cap 429s) so agents can find the key flow.
agentAuthDiscovery?: boolean;
+ // Machine-readable keyless quota details for trusted MCP recovery.
+ keylessReason?: "requests" | "credits";
+ retryAfterSeconds?: number;
};
export enum NotificationType {