Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): share each index's per-minute limit across every entry point (#5589)
Kris Oei6c044dbf8350
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex a06b054df..dfce42515 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -28,6 +28,7 @@ import {reportKeylessPromptShown,} from "../lib/keyless";import { keylessSignupSurface } from "../lib/keyless-signup-link";+import { consumeIndexRateLimit } from "../lib/index-daily-limit";import { isKeylessIpSuspicious } from "../lib/spur";import { verifyWorldIdCredential } from "../lib/world-id";import { keylessAuthTotal } from "../lib/keyless-metrics";@@ -781,6 +782,31 @@ async function handleKeylessAuth(};}+ if (modeLabel === "gov" || modeLabel === "research") {+ const perMinute = await consumeIndexRateLimit(+ modeLabel,+ teamId,+ 1,+ undefined,+ 1,+ );+ if (!perMinute.ok) {+ keylessAuthTotal.inc({ mode, outcome: "index_rate_limit" });+ logger.warn("Keyless request blocked", {+ ...baseLog,+ blocked: true,+ reason: "index_rate_limit",+ retryAfterSeconds: perMinute.retryAfterSeconds,+ });+ return {+ success: false,+ error: perMinute.error,+ status: 429,+ retryAfterSeconds: perMinute.retryAfterSeconds,+ };+ }+ }+logger.debug("Keyless request consumed", { ...baseLog, blocked: false });keylessAuthTotal.inc({ mode, outcome: "allowed" });diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex b35f1f34a..35b075f3c 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -325,7 +325,8 @@ export async function providerScrapeController(retryAfterSeconds?: unknown;} | null;if (- refused?.code === "index_daily_limit" &&+ (refused?.code === "index_daily_limit" ||+ refused?.code === "index_rate_limit") &&typeof refused.retryAfterSeconds === "number")res.setHeader("Retry-After", String(refused.retryAfterSeconds));diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 9d0669c06..06ba0b523 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -57,6 +57,7 @@ import {import { wantsDeveloperCategory } from "../../search/developer";import { wantsGovCategory } from "../../search/gov";import {+ consumeIndexRateLimit,indexDailyLimitRefusal,reserveIndexCalls,} from "../../lib/index-daily-limit";@@ -467,6 +468,24 @@ async function searchControllerInner(timeMs: waited,});+ if (govSearch) {+ const perMinute = await consumeIndexRateLimit(+ "gov",+ req.auth.team_id,+ req.acuc?.rate_limit_multiplier ?? 1,+ req.acuc?.flags,+ 1,+ );+ if (!perMinute.ok) {+ res.setHeader("Retry-After", String(perMinute.retryAfterSeconds));+ return res.status(429).json({+ success: false,+ error: perMinute.error,+ retry_after_seconds: perMinute.retryAfterSeconds,+ });+ }+ }+if (projectedKeylessCredits > 0) {const reservation = await reserveKeylessCredits(req.auth.team_id,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 8790128d7..e93d0a8b3 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1660,6 +1660,7 @@ export type ErrorResponse = {login_url?: string;replacement?: string;documentation_url?: string;+ retry_after_seconds?: number;};export type ScrapeResponse =diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.tsindex 01d03a1c4..3c4c2a586 100644--- a/apps/api/src/lib/index-daily-limit.ts+++ b/apps/api/src/lib/index-daily-limit.ts@@ -1,13 +1,21 @@import { config } from "../config";-import { redisRateLimitClient } from "../services/rate-limiter";+import type { TeamFlags } from "../controllers/v1/types";+import {+ getAutumnRateLimiter,+ redisRateLimitClient,+} from "../services/rate-limiter";+import { RateLimiterMode } from "../types";import { isKeylessTeamId } from "./keyless";import { logger } from "./logger";// Daily call budgets for the free Firecrawl indexes, per team and per UTC day,// sized by plan. Every route into an index reserves from the same counter, so a// team cannot raise its budget by switching between the direct endpoint, the-// search category and the Alexandria provider. A new route into an index (for-// example the `research` category on /search) must reserve here as well.+// search category and the Alexandria provider. The per-minute limit is shared+// the same way: it is the direct endpoint's limiter, which every other route+// and keyless auth take from through consumeIndexRateLimit. A new route into an+// index (for example the `research` category on /search) must reserve here and+// take from that limiter as well.export type IndexName = "gov" | "research";@@ -29,6 +37,11 @@ const DISPLAY_NAMES: Record<IndexName, string> = {const UPGRADE_URL = "https://firecrawl.dev/pricing";+const RATE_LIMITER_MODES: Record<IndexName, RateLimiterMode> = {+ gov: RateLimiterMode.GovSearch,+ research: RateLimiterMode.Research,+};+const BASE_CALLS_PER_DAY = 1000;// Thresholds are the Autumn `rate_limits` grants: 500 is Growth, 1000 is@@ -167,6 +180,82 @@ return 1}}+/**+ * Takes the calls from the team's per-minute limit for the index where auth+ * has not counted them: a route that reaches the index through another+ * endpoint, or a keyless request, which auth does not rate limit. One script+ * checks and counts on the index endpoint's own limiter key, so every route+ * shares one count and window, and a refused batch takes nothing. Like the+ * daily check, it allows the calls when Redis fails or does not answer in+ * time.+ */+export async function consumeIndexRateLimit(+ index: IndexName,+ teamId: string,+ rateLimitMultiplier: number,+ flags: TeamFlags | null | undefined,+ calls: number,+): Promise<+ { ok: true } | { ok: false; error: string; retryAfterSeconds: number }+> {+ if (calls <= 0) return { ok: true };+ const limiter = getAutumnRateLimiter(+ RATE_LIMITER_MODES[index],+ rateLimitMultiplier,+ flags ?? undefined,+ );+ try {+ const [admitted, ttl] = (await withTimeout(+ redisRateLimitClient.eval(+ `+local current = tonumber(redis.call("GET", KEYS[1]) or "0")+local calls = tonumber(ARGV[1])+local ttl = redis.call("PTTL", KEYS[1])+if ttl == -1 then+ redis.call("EXPIRE", KEYS[1], ARGV[3])+ ttl = tonumber(ARGV[3]) * 1000+end+if current + calls > tonumber(ARGV[2]) then+ return {0, ttl}+end+if ttl < 0 then+ redis.call("SET", KEYS[1], current + calls, "EX", ARGV[3])+else+ redis.call("INCRBY", KEYS[1], calls)+end+return {1, ttl}+`,+ 1,+ limiter.getKey(teamId),+ calls,+ limiter.points,+ limiter.duration,+ ),+ )) as [number, number];+ if (admitted === 1) return { ok: true };+ const retryAfterSeconds = Math.max(+ 1,+ Math.ceil((ttl >= 0 ? ttl : limiter.duration * 1000) / 1000),+ );+ const needs =+ calls > 1+ ? ` and this request needs ${calls.toLocaleString("en-US")}`+ : "";+ return {+ ok: false,+ retryAfterSeconds,+ error: `${DISPLAY_NAMES[index]} rate limit reached: your team can make ${limiter.points.toLocaleString("en-US")} calls a minute across every endpoint that queries the index${needs}. Retry in ${retryAfterSeconds}s.`,+ };+ } catch (error) {+ logger.warn("Index rate limit check failed; allowing request", {+ index,+ teamId,+ error,+ });+ return { ok: true };+ }+}+/** The caller-facing fields of a refusal, shared by every route into an index. */export function indexDailyLimitRefusal(index: IndexName,diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex a6d9f76aa..93efebba8 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -38,8 +38,9 @@ import {} from "../lib/exchange";import { getExchangeAccessForRequestBody } from "../lib/exchange-request";import { isFreeCatalogueSearch } from "../search/alexandria";-import { getScrapeZDR } from "../lib/zdr-helpers";-import { isLockdownZeroDataRetention } from "../lib/safe-mode";+import { wantsGovCategory } from "../search/gov";+import { getScrapeZDR, getThreatProtection } from "../lib/zdr-helpers";+import { getSafeMode, isLockdownZeroDataRetention } from "../lib/safe-mode";import {agentInteropStatus,isAgentInteropSecretValid,@@ -131,15 +132,25 @@ export function checkCreditsMiddleware(// If verified, fall through to normal credit check (key is now on real account)}- // Catalogue discovery is free; provider execution reserves its own credits.+ // Catalogue discovery and Government Index results are free; provider+ // execution reserves its own credits. A gov search still pays to scrape+ // its results, and for threat scans where threat protection or Safe Mode+ // can turn them on.const sources = (req.body as any)?.sources;const categories = (req.body as any)?.categories;- const freeCatalogue =- req.path === "/search" && isFreeCatalogueSearch(sources, categories);+ const formats = (req.body as any)?.scrapeOptions?.formats;+ const freeGovSearch =+ wantsGovCategory(categories) &&+ !(Array.isArray(formats) && formats.length > 0) &&+ getThreatProtection(req.acuc?.flags) === "disabled" &&+ !getSafeMode(req.acuc?.flags);+ const freeSearch =+ req.path === "/search" &&+ (isFreeCatalogueSearch(sources, categories) || freeGovSearch);if ((req.path === "/scrape" &&(req.body as any)?.alexandria !== undefined) ||- freeCatalogue+ freeSearch) {req.account = { remainingCredits: Infinity };return next();diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 00f434a2b..70ee46159 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -4,6 +4,7 @@ import { config } from "../../config";import type { TeamFlags } from "../../controllers/v2/types";import { reportExchangeUsageBilling } from "../../lib/exchange";import {+ consumeIndexRateLimit,indexDailyLimitRefusal,reserveIndexCalls,type IndexName,@@ -323,11 +324,28 @@ export async function retrieveProviders(input: {if (denied) return refuse(denied);for (const [index, provider] of INDEX_PROVIDERS) {+ const calls = input.calls.filter(+ call => call.provider === provider,+ ).length;+ const perMinute = await consumeIndexRateLimit(+ index,+ input.teamId,+ input.rateLimitMultiplier,+ input.flags,+ calls,+ );+ if (!perMinute.ok)+ return refuse(+ refusal(429, perMinute.error, {+ code: "index_rate_limit",+ retryAfterSeconds: perMinute.retryAfterSeconds,+ }),+ );const indexBudget = await reserveIndexCalls(index,input.teamId,input.rateLimitMultiplier,- input.calls.filter(call => call.provider === provider).length,+ calls,);if (!indexBudget.ok) {const { error, ...limitFields } = indexDailyLimitRefusal(