Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): share each index's per-minute limit across every entry point (#5589)

Kris Oei6c044dbf8350
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index a06b054df..dfce42515 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -28,6 +28,7 @@ import {
reportKeylessPromptShown,
} from "../lib/keyless";
import { keylessSignupSurface } from "../lib/keyless-signup-link";
+import { consumeIndexRateLimit } from "../lib/index-daily-limit";
import { isKeylessIpSuspicious } from "../lib/spur";
import { verifyWorldIdCredential } from "../lib/world-id";
import { keylessAuthTotal } from "../lib/keyless-metrics";
@@ -781,6 +782,31 @@ async function handleKeylessAuth(
};
}
+ if (modeLabel === "gov" || modeLabel === "research") {
+ const perMinute = await consumeIndexRateLimit(
+ modeLabel,
+ teamId,
+ 1,
+ undefined,
+ 1,
+ );
+ if (!perMinute.ok) {
+ keylessAuthTotal.inc({ mode, outcome: "index_rate_limit" });
+ logger.warn("Keyless request blocked", {
+ ...baseLog,
+ blocked: true,
+ reason: "index_rate_limit",
+ retryAfterSeconds: perMinute.retryAfterSeconds,
+ });
+ return {
+ success: false,
+ error: perMinute.error,
+ status: 429,
+ retryAfterSeconds: perMinute.retryAfterSeconds,
+ };
+ }
+ }
+
logger.debug("Keyless request consumed", { ...baseLog, blocked: false });
keylessAuthTotal.inc({ mode, outcome: "allowed" });
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index b35f1f34a..35b075f3c 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -325,7 +325,8 @@ export async function providerScrapeController(
retryAfterSeconds?: unknown;
} | null;
if (
- refused?.code === "index_daily_limit" &&
+ (refused?.code === "index_daily_limit" ||
+ refused?.code === "index_rate_limit") &&
typeof refused.retryAfterSeconds === "number"
)
res.setHeader("Retry-After", String(refused.retryAfterSeconds));
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 9d0669c06..06ba0b523 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -57,6 +57,7 @@ import {
import { wantsDeveloperCategory } from "../../search/developer";
import { wantsGovCategory } from "../../search/gov";
import {
+ consumeIndexRateLimit,
indexDailyLimitRefusal,
reserveIndexCalls,
} from "../../lib/index-daily-limit";
@@ -467,6 +468,24 @@ async function searchControllerInner(
timeMs: waited,
});
+ if (govSearch) {
+ const perMinute = await consumeIndexRateLimit(
+ "gov",
+ req.auth.team_id,
+ req.acuc?.rate_limit_multiplier ?? 1,
+ req.acuc?.flags,
+ 1,
+ );
+ if (!perMinute.ok) {
+ res.setHeader("Retry-After", String(perMinute.retryAfterSeconds));
+ return res.status(429).json({
+ success: false,
+ error: perMinute.error,
+ retry_after_seconds: perMinute.retryAfterSeconds,
+ });
+ }
+ }
+
if (projectedKeylessCredits > 0) {
const reservation = await reserveKeylessCredits(
req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 8790128d7..e93d0a8b3 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1660,6 +1660,7 @@ export type ErrorResponse = {
login_url?: string;
replacement?: string;
documentation_url?: string;
+ retry_after_seconds?: number;
};
export type ScrapeResponse =
diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.ts
index 01d03a1c4..3c4c2a586 100644
--- a/apps/api/src/lib/index-daily-limit.ts
+++ b/apps/api/src/lib/index-daily-limit.ts
@@ -1,13 +1,21 @@
import { config } from "../config";
-import { redisRateLimitClient } from "../services/rate-limiter";
+import type { TeamFlags } from "../controllers/v1/types";
+import {
+ getAutumnRateLimiter,
+ redisRateLimitClient,
+} from "../services/rate-limiter";
+import { RateLimiterMode } from "../types";
import { isKeylessTeamId } from "./keyless";
import { logger } from "./logger";
// Daily call budgets for the free Firecrawl indexes, per team and per UTC day,
// sized by plan. Every route into an index reserves from the same counter, so a
// team cannot raise its budget by switching between the direct endpoint, the
-// search category and the Alexandria provider. A new route into an index (for
-// example the `research` category on /search) must reserve here as well.
+// search category and the Alexandria provider. The per-minute limit is shared
+// the same way: it is the direct endpoint's limiter, which every other route
+// and keyless auth take from through consumeIndexRateLimit. A new route into an
+// index (for example the `research` category on /search) must reserve here and
+// take from that limiter as well.
export type IndexName = "gov" | "research";
@@ -29,6 +37,11 @@ const DISPLAY_NAMES: Record<IndexName, string> = {
const UPGRADE_URL = "https://firecrawl.dev/pricing";
+const RATE_LIMITER_MODES: Record<IndexName, RateLimiterMode> = {
+ gov: RateLimiterMode.GovSearch,
+ research: RateLimiterMode.Research,
+};
+
const BASE_CALLS_PER_DAY = 1000;
// Thresholds are the Autumn `rate_limits` grants: 500 is Growth, 1000 is
@@ -167,6 +180,82 @@ return 1
}
}
+/**
+ * Takes the calls from the team's per-minute limit for the index where auth
+ * has not counted them: a route that reaches the index through another
+ * endpoint, or a keyless request, which auth does not rate limit. One script
+ * checks and counts on the index endpoint's own limiter key, so every route
+ * shares one count and window, and a refused batch takes nothing. Like the
+ * daily check, it allows the calls when Redis fails or does not answer in
+ * time.
+ */
+export async function consumeIndexRateLimit(
+ index: IndexName,
+ teamId: string,
+ rateLimitMultiplier: number,
+ flags: TeamFlags | null | undefined,
+ calls: number,
+): Promise<
+ { ok: true } | { ok: false; error: string; retryAfterSeconds: number }
+> {
+ if (calls <= 0) return { ok: true };
+ const limiter = getAutumnRateLimiter(
+ RATE_LIMITER_MODES[index],
+ rateLimitMultiplier,
+ flags ?? undefined,
+ );
+ try {
+ const [admitted, ttl] = (await withTimeout(
+ redisRateLimitClient.eval(
+ `
+local current = tonumber(redis.call("GET", KEYS[1]) or "0")
+local calls = tonumber(ARGV[1])
+local ttl = redis.call("PTTL", KEYS[1])
+if ttl == -1 then
+ redis.call("EXPIRE", KEYS[1], ARGV[3])
+ ttl = tonumber(ARGV[3]) * 1000
+end
+if current + calls > tonumber(ARGV[2]) then
+ return {0, ttl}
+end
+if ttl < 0 then
+ redis.call("SET", KEYS[1], current + calls, "EX", ARGV[3])
+else
+ redis.call("INCRBY", KEYS[1], calls)
+end
+return {1, ttl}
+`,
+ 1,
+ limiter.getKey(teamId),
+ calls,
+ limiter.points,
+ limiter.duration,
+ ),
+ )) as [number, number];
+ if (admitted === 1) return { ok: true };
+ const retryAfterSeconds = Math.max(
+ 1,
+ Math.ceil((ttl >= 0 ? ttl : limiter.duration * 1000) / 1000),
+ );
+ const needs =
+ calls > 1
+ ? ` and this request needs ${calls.toLocaleString("en-US")}`
+ : "";
+ return {
+ ok: false,
+ retryAfterSeconds,
+ error: `${DISPLAY_NAMES[index]} rate limit reached: your team can make ${limiter.points.toLocaleString("en-US")} calls a minute across every endpoint that queries the index${needs}. Retry in ${retryAfterSeconds}s.`,
+ };
+ } catch (error) {
+ logger.warn("Index rate limit check failed; allowing request", {
+ index,
+ teamId,
+ error,
+ });
+ return { ok: true };
+ }
+}
+
/** The caller-facing fields of a refusal, shared by every route into an index. */
export function indexDailyLimitRefusal(
index: IndexName,
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index a6d9f76aa..93efebba8 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -38,8 +38,9 @@ import {
} from "../lib/exchange";
import { getExchangeAccessForRequestBody } from "../lib/exchange-request";
import { isFreeCatalogueSearch } from "../search/alexandria";
-import { getScrapeZDR } from "../lib/zdr-helpers";
-import { isLockdownZeroDataRetention } from "../lib/safe-mode";
+import { wantsGovCategory } from "../search/gov";
+import { getScrapeZDR, getThreatProtection } from "../lib/zdr-helpers";
+import { getSafeMode, isLockdownZeroDataRetention } from "../lib/safe-mode";
import {
agentInteropStatus,
isAgentInteropSecretValid,
@@ -131,15 +132,25 @@ export function checkCreditsMiddleware(
// If verified, fall through to normal credit check (key is now on real account)
}
- // Catalogue discovery is free; provider execution reserves its own credits.
+ // Catalogue discovery and Government Index results are free; provider
+ // execution reserves its own credits. A gov search still pays to scrape
+ // its results, and for threat scans where threat protection or Safe Mode
+ // can turn them on.
const sources = (req.body as any)?.sources;
const categories = (req.body as any)?.categories;
- const freeCatalogue =
- req.path === "/search" && isFreeCatalogueSearch(sources, categories);
+ const formats = (req.body as any)?.scrapeOptions?.formats;
+ const freeGovSearch =
+ wantsGovCategory(categories) &&
+ !(Array.isArray(formats) && formats.length > 0) &&
+ getThreatProtection(req.acuc?.flags) === "disabled" &&
+ !getSafeMode(req.acuc?.flags);
+ const freeSearch =
+ req.path === "/search" &&
+ (isFreeCatalogueSearch(sources, categories) || freeGovSearch);
if (
(req.path === "/scrape" &&
(req.body as any)?.alexandria !== undefined) ||
- freeCatalogue
+ freeSearch
) {
req.account = { remainingCredits: Infinity };
return next();
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
index 00f434a2b..70ee46159 100644
--- a/apps/api/src/services/alexandria/retrieve.ts
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -4,6 +4,7 @@ import { config } from "../../config";
import type { TeamFlags } from "../../controllers/v2/types";
import { reportExchangeUsageBilling } from "../../lib/exchange";
import {
+ consumeIndexRateLimit,
indexDailyLimitRefusal,
reserveIndexCalls,
type IndexName,
@@ -323,11 +324,28 @@ export async function retrieveProviders(input: {
if (denied) return refuse(denied);
for (const [index, provider] of INDEX_PROVIDERS) {
+ const calls = input.calls.filter(
+ call => call.provider === provider,
+ ).length;
+ const perMinute = await consumeIndexRateLimit(
+ index,
+ input.teamId,
+ input.rateLimitMultiplier,
+ input.flags,
+ calls,
+ );
+ if (!perMinute.ok)
+ return refuse(
+ refusal(429, perMinute.error, {
+ code: "index_rate_limit",
+ retryAfterSeconds: perMinute.retryAfterSeconds,
+ }),
+ );
const indexBudget = await reserveIndexCalls(
index,
input.teamId,
input.rateLimitMultiplier,
- input.calls.filter(call => call.provider === provider).length,
+ calls,
);
if (!indexBudget.ok) {
const { error, ...limitFields } = indexDailyLimitRefusal(