Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(keyless): preserve quota recovery details (#4211)
Himadri Mishraea4bad0fe028
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex f7a2308f6..893f63256 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -252,6 +252,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);+ expect(blocked.body.reason).toBe("credits");expect(blocked.body.error).toContain("keyless free tier rate limit");expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");expect(blocked.body.error).toContain(@@ -296,6 +297,7 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {});expect(blocked.statusCode).toBe(429);+ expect(blocked.body.reason).toBe("credits");expect(blocked.body.error).toContain("keyless free tier rate limit");expect(blocked.body.error).toContain("https://www.firecrawl.dev/signin");expect(blocked.body.error).toContain(diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex da064b371..cd789a197 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -515,15 +515,18 @@ async function handleKeylessAuth(logger.warn("Keyless request blocked", {...baseLog,blocked: true,+ event: "keyless_exhausted",reason: result.reason,+ retryAfterSeconds: result.retryAfterSeconds,});return {success: false,error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,status: 429,- // Out of free quota — emit the OAuth-discovery header so agents can find- // the key/signup flow at the moment they actually need a key.+ // Direct API callers receive discovery metadata; MCP maps this in-band.agentAuthDiscovery: true,+ keylessReason: result.reason,+ retryAfterSeconds: result.retryAfterSeconds,};}diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 8a6ece1db..c5292a2b9 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -29,8 +29,8 @@ import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -156,10 +156,9 @@ export async function scrapeController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json({- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- });+ return res.status(429).json(+ await keylessLimitBody(req.auth.team_id, "v1_scrape"),+ );}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 129b83e48..fea667c0e 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -34,8 +34,8 @@ import {import { fromV1ScrapeOptions } from "../v2/types";import { getSearchForcedKind } from "../../lib/zdr-helpers";import {- KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -212,10 +212,9 @@ export async function searchController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json({- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- });+ return res.status(429).json(+ await keylessLimitBody(req.auth.team_id, "v1_search"),+ );}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex cbe5f22d0..fa61ec6f5 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -3,6 +3,13 @@ import { vi } from "vitest";const mockLogRequest = vi.fn();const mockLogSearch = vi.fn();const mockLogResearchEndpoint = vi.fn();+const mockReserveKeylessCredits = vi.fn().mockResolvedValue({ ok: true });+const mockKeylessLimitBody = vi.fn().mockResolvedValue({+ success: false,+ error: "keyless limit reached",+ reason: "credits",+});+const mockProjectSearchTotalCredits = vi.fn(() => 0);vi.mock("../../../services/logging/log_job", () => ({logRequest: (...args: any[]) => mockLogRequest(...args),@@ -23,12 +30,13 @@ vi.mock("../../../services/billing/credit_billing", () => ({vi.mock("../../../lib/keyless", () => ({KEYLESS_FREE_TIER_LIMIT_MESSAGE: "keyless limit reached",adjustKeylessCredits: vi.fn().mockResolvedValue(undefined),+ keylessLimitBody: (...args: any[]) => mockKeylessLimitBody(...args),logKeylessCreditUsage: vi.fn().mockResolvedValue(undefined),- reserveKeylessCredits: vi.fn().mockResolvedValue({ ok: true }),+ reserveKeylessCredits: (...args: any[]) => mockReserveKeylessCredits(...args),}));vi.mock("../../../lib/keyless-credit-projection", () => ({- projectSearchTotalCredits: () => 0,+ projectSearchTotalCredits: () => mockProjectSearchTotalCredits(),}));vi.mock("../../../lib/threat-protection/request", () => ({@@ -120,9 +128,34 @@ beforeEach(() => {mockLogSearch.mockResolvedValue(undefined);mockLogResearchEndpoint.mockResolvedValue(undefined);mockExecuteSearch.mockResolvedValue(executeResult());+ mockReserveKeylessCredits.mockResolvedValue({ ok: true });+ mockKeylessLimitBody.mockResolvedValue({+ success: false,+ error: "keyless limit reached",+ reason: "credits",+ });+ mockProjectSearchTotalCredits.mockReturnValue(0);});describe("developer category code_searches ledger", () => {+ it("returns the structured keyless 429 when projected credits cannot be reserved", async () => {+ mockProjectSearchTotalCredits.mockReturnValue(2);+ mockReserveKeylessCredits.mockResolvedValue({ ok: false });+ const req = makeReq({ query: "http client", categories: ["developer"] });+ const res = makeRes();++ await searchController(req, res);++ expect(res.status).toHaveBeenCalledWith(429);+ expect(mockKeylessLimitBody).toHaveBeenCalledWith(TEAM_ID, "v2_search");+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ error: "keyless limit reached",+ reason: "credits",+ });+ expect(mockExecuteSearch).not.toHaveBeenCalled();+ });+it("writes exactly one code_searches row for a developer category search", async () => {const req = makeReq({query: "vector database client",diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 9ba6795d2..eae34cff1 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -27,8 +27,8 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -378,10 +378,9 @@ export async function parseController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json({- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- });+ return res.status(429).json(+ await keylessLimitBody(req.auth.team_id, "v2_parse"),+ );}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 9c8454631..5751c7127 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -49,6 +49,7 @@ import {KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,keylessTeamUuid,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -558,10 +559,7 @@ async function createSessionForScrape(if (!reservation.ok) {return {status: 429,- body: {- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- },+ body: await keylessLimitBody(req.auth.team_id, "v2_browser"),error: true,};}diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex ef33aecb9..e56da6a8e 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -30,8 +30,8 @@ import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {- KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -186,10 +186,9 @@ export async function scrapeController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json({- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- });+ return res.status(429).json(+ await keylessLimitBody(req.auth.team_id, "v2_scrape"),+ );}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 38df521f6..3f1d6c5ac 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -8,8 +8,8 @@ import {} from "./types";import { billTeam } from "../../services/billing/credit_billing";import {- KEYLESS_FREE_TIER_LIMIT_MESSAGE,adjustKeylessCredits,+ keylessLimitBody,logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";@@ -235,10 +235,9 @@ export async function searchController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json({- success: false,- error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,- });+ return res.status(429).json(+ await keylessLimitBody(req.auth.team_id, "v2_search"),+ );}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex bf04f3af4..a7aeb0543 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -5,6 +5,7 @@ import { db } from "../db/connection";import * as schema from "../db/schema";import { redisRateLimitClient } from "../services/rate-limiter";import { isKeylessIpSuspicious } from "./spur";+import { logger } from "./logger";// Keyless free tier: scrape, search, and interact can be used without an API key// from the official MCP server, CLI, or SDKs. It's gated per-IP/day by TWO@@ -86,13 +87,69 @@ export function isKeylessIpEligible(ip: string): boolean {const requestsKey = (ip: string) => `keyless_requests:${ip}`;const creditsKey = (ip: string) => `keyless_credits:${ip}`;+export type KeylessQuotaReason = "requests" | "credits";+type KeylessConsumeResult = {ok: boolean;- reason?: "requests" | "credits";+ reason?: KeylessQuotaReason;requestsUsed: number;creditsUsed: number;+ retryAfterSeconds?: number;};+function positiveRedisTtl(ttl: number): number | undefined {+ return ttl > 0 ? ttl : undefined;+}++/**+ * Quota denials remain valid when Redis cannot provide the optional TTL hint.+ * Keep that secondary lookup from changing a controlled 429 into a generic+ * authentication failure.+ */+async function retryAfterSecondsFor(key: string): Promise<number | undefined> {+ try {+ return positiveRedisTtl(await redisRateLimitClient.ttl(key));+ } catch {+ return undefined;+ }+}++/** Structured response for projected-credit reservation exhaustion. */+export async function keylessLimitBody(+ teamId: string,+ mode: string,+): Promise<{+ success: false;+ error: string;+ reason: "credits";+ retry_after_seconds?: number;+}> {+ const ip = keylessIpFromTeamId(teamId);+ let retryAfterSeconds: number | undefined;+ try {+ retryAfterSeconds = ip+ ? positiveRedisTtl(await redisRateLimitClient.ttl(creditsKey(ip)))+ : undefined;+ } catch {+ // The reservation already proved the limit; missing TTL must not turn its+ // controlled 429 into a server error.+ }+ logger.warn("Keyless request blocked", {+ canonicalLog: "keyless/consume",+ event: "keyless_exhausted",+ blocked: true,+ reason: "credits",+ mode,+ retryAfterSeconds,+ });+ return {+ success: false,+ error: KEYLESS_FREE_TIER_LIMIT_MESSAGE,+ reason: "credits",+ ...(retryAfterSeconds ? { retry_after_seconds: retryAfterSeconds } : {}),+ };+}+type KeylessCreditReservationResult = {ok: boolean;creditsUsed: number;@@ -122,10 +179,22 @@ export async function consumeKeylessRequest();if (requestsUsed > requestLimit) {- return { ok: false, reason: "requests", requestsUsed, creditsUsed };+ return {+ ok: false,+ reason: "requests",+ requestsUsed,+ creditsUsed,+ retryAfterSeconds: await retryAfterSecondsFor(rKey),+ };}if (creditsUsed >= creditLimit) {- return { ok: false, reason: "credits", requestsUsed, creditsUsed };+ return {+ ok: false,+ reason: "credits",+ requestsUsed,+ creditsUsed,+ retryAfterSeconds: await retryAfterSecondsFor(creditsKey(ip)),+ };}return { ok: true, requestsUsed, creditsUsed };}@@ -219,7 +288,11 @@ return next*/export async function checkKeylessEligibility(ip: string,-): Promise<{ eligible: boolean; reason?: string }> {+): Promise<{+ eligible: boolean;+ reason?: KeylessQuotaReason | "disabled" | "ineligible_ip" | "suspicious" | "error";+ retryAfterSeconds?: number;+}> {if (!isKeylessConfigured()) return { eligible: false, reason: "disabled" };if (!ip || !isKeylessIpEligible(ip)) {return { eligible: false, reason: "ineligible_ip" };@@ -236,14 +309,23 @@ export async function checkKeylessEligibility(10,);if (requestsUsed >= (KEYLESS_REQUESTS_PER_DAY ?? 0)) {- return { eligible: false, reason: "requests" };+ return {+ eligible: false,+ reason: "requests",+ retryAfterSeconds: await retryAfterSecondsFor(requestsKey(ip)),+ };}+ const creditKey = creditsKey(ip);const creditsUsed = parseInt(- (await redisRateLimitClient.get(creditsKey(ip))) ?? "0",+ (await redisRateLimitClient.get(creditKey)) ?? "0",10,);if (creditsUsed >= (KEYLESS_CREDITS_PER_DAY ?? 0)) {- return { eligible: false, reason: "credits" };+ return {+ eligible: false,+ reason: "credits",+ retryAfterSeconds: await retryAfterSecondsFor(creditKey),+ };}return { eligible: true };} catch {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex d3e78aae9..9079c1f0c 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -248,7 +248,14 @@ export function authMiddleware(}return res.status(auth.status)- .json({ success: false, error: auth.error });+ .json({+ success: false,+ error: auth.error,+ ...(auth.keylessReason ? { reason: auth.keylessReason } : {}),+ ...(auth.retryAfterSeconds+ ? { retry_after_seconds: auth.retryAfterSeconds }+ : {}),+ });} else {return;}diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex efacc556b..80d3a3dbc 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -177,6 +177,9 @@ export type AuthResponse =// When true, send the agent OAuth-discovery WWW-Authenticate header even on// non-401 responses (e.g. keyless cap 429s) so agents can find the key flow.agentAuthDiscovery?: boolean;+ // Machine-readable keyless quota details for trusted MCP recovery.+ keylessReason?: "requests" | "credits";+ retryAfterSeconds?: number;};export enum NotificationType {