Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): plan-tiered daily limits for the gov and research indexes (#5160)
Kris Oei5442955a643a
diff --git a/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.tsindex cd8cf4ab9..2017c4eb2 100644--- a/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts+++ b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts@@ -52,6 +52,13 @@ vi.mock("../../lib/keyless", () => ({chargeKeylessCredits: async () => {},}));+// These tests cover the deprecation notice, not the daily index limit.+// The limiter needs Redis, so here it admits every call.+vi.mock("../../lib/index-daily-limit", () => ({+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),+ indexDailyLimitRefusal: vi.fn(),+}));+vi.mock("../../services/billing/credit_billing", () => ({billTeam: async () => {},}));diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 47647c391..2d8b7cd22 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -124,10 +124,6 @@ const configSchema = z.object({// ExpressEXPRESS_TRUST_PROXY: z.coerce.number().optional(),- GOV_INDEX_CALLS_PER_DAY: emptyStringAsDefault(- z.coerce.number().int().nonnegative().default(500),- ),-// Keyless free tier (scrape/search/interact without an API key, per-IP/day).// Non-negative integers; 0 means "enabled but no budget", unset means "off".KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),diff --git a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts b/apps/api/src/controllers/v2/__tests__/gov-search.test.tsindex 9cf4f69e8..6b80beaf4 100644--- a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts+++ b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts@@ -26,6 +26,13 @@ vi.mock("../../../lib/keyless", async importOriginal => {return { ...actual, chargeKeylessCredits: mocks.chargeKeylessCredits };});+// These tests cover routing, billing and retention, not the daily index limit.+// The limiter needs Redis, so here it admits every call.+vi.mock("../../../lib/index-daily-limit", () => ({+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),+ indexDailyLimitRefusal: vi.fn(),+}));+vi.mock("../../../lib/logger", () => ({logger: {info: vi.fn(),diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.tsindex 36fd47f70..cf165b45a 100644--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts@@ -23,6 +23,13 @@ vi.mock("../../../lib/keyless", () => ({chargeKeylessCredits: vi.fn().mockResolvedValue(undefined),}));+// These tests cover routing, billing and retention, not the daily index limit.+// The limiter needs Redis, so here it admits every call.+vi.mock("../../../lib/index-daily-limit", () => ({+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),+ indexDailyLimitRefusal: vi.fn(),+}));+vi.mock("../../../lib/logger", () => ({logger: {info: vi.fn(),diff --git a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.tsindex b1ee34745..9a961c6b3 100644--- a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts@@ -24,6 +24,13 @@ vi.mock("../../../lib/keyless", () => ({chargeKeylessCredits: mocks.chargeKeylessCredits,}));+// These tests cover routing, billing and retention, not the daily index limit.+// The limiter needs Redis, so here it admits every call.+vi.mock("../../../lib/index-daily-limit", () => ({+ reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),+ indexDailyLimitRefusal: vi.fn(),+}));+vi.mock("../../../lib/logger", () => ({logger: {info: vi.fn(),diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 99941d29f..e5d4863d4 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -9,8 +9,9 @@ import {} from "../../lib/research-upstream";import { chargeKeylessCredits } from "../../lib/keyless";import {- govIndexDailyLimitMessage,- reserveGovIndexCalls,+ indexDailyLimitRefusal,+ reserveIndexCalls,+ type IndexName,} from "../../lib/index-daily-limit";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";@@ -47,6 +48,15 @@ const SIMILAR_PAPERS_TIMEOUT_MS = 10_000;const GITHUB_SEARCH_TIMEOUT_MS = 12_000;const PAPER_READ_TIMEOUT_MS = 120_000;+// The paper endpoints, all served by the Research Index and limited by its+// daily budget. GitHub search is billed per search and stays outside it.+const RESEARCH_INDEX_KINDS = new Set<ResearchRequestKind>([+ "research_paper_search",+ "research_related_papers",+ "research_paper_read",+ "research_paper_inspect",+]);+const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];@@ -270,18 +280,10 @@ function creditsFor(body: any,req: RequestWithAuth<any, any, any>,) {- // Make certain index accesses free (AI/ML & life-sciences indices and the- // gov index). Research paper and gov endpoints should have no credit cost.- // Explicit exceptions remain billable: GitHub search and developer/code- // search.- const freeResearchKinds = new Set([- "research_paper_search",- "research_related_papers",- "research_paper_read",- "research_paper_inspect",- "gov_search",- ]);- if (freeResearchKinds.has(config.kind)) return 0;+ // The Research Index and the Government Index are free. GitHub search and+ // developer/code search remain billable.+ if (config.kind === "gov_search" || RESEARCH_INDEX_KINDS.has(config.kind))+ return 0;if (config.billAs === "scrape") return 1;const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);@@ -395,19 +397,30 @@ function createResearchController(api_key_id: authedReq.acuc?.api_key_id ?? null,});- let releaseGovCall: (() => Promise<void>) | undefined;- if (endpoint.kind === "gov_search") {- const indexBudget = await reserveGovIndexCalls(authedReq.auth.team_id, 1);+ const index: IndexName | null =+ endpoint.kind === "gov_search"+ ? "gov"+ : RESEARCH_INDEX_KINDS.has(endpoint.kind)+ ? "research"+ : null;+ let releaseIndexCall: (() => Promise<void>) | undefined;+ if (index) {+ const indexBudget = await reserveIndexCalls(+ index,+ authedReq.auth.team_id,+ authedReq.acuc?.rate_limit_multiplier ?? 1,+ 1,+ );if (!indexBudget.ok) {chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));return res.status(429).json({success: false,code: "INDEX_DAILY_LIMIT",- error: govIndexDailyLimitMessage(indexBudget),+ ...indexDailyLimitRefusal(index, authedReq.auth.team_id, indexBudget),});}- releaseGovCall = indexBudget.release;+ releaseIndexCall = indexBudget.release;}let statusCode = 500;@@ -424,7 +437,7 @@ function createResearchController(zeroDataRetention,);if (!upstream) {- if (releaseGovCall) await releaseGovCall();+ if (releaseIndexCall) await releaseIndexCall();statusCode = 404;error = "Research service is not configured";return res.status(404).end();diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 3023fa64a..fe60d4314 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -179,6 +179,7 @@ export async function providerScrapeController(zeroDataRetention,clientIp: req.ip,billingCountry: await billingCountryForTeam(req.auth.team_id),+ rateLimitMultiplier: req.acuc.rate_limit_multiplier,});} catch (error) {logger.error("Provider scrape unavailable", {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 0ec134673..924cb977a 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -53,8 +53,8 @@ import {import { wantsDeveloperCategory } from "../../search/developer";import { wantsGovCategory } from "../../search/gov";import {- govIndexDailyLimitMessage,- reserveGovIndexCalls,+ indexDailyLimitRefusal,+ reserveIndexCalls,} from "../../lib/index-daily-limit";import { requestOrigin } from "../../lib/request-origin";import { isAgentInteropSecretValid } from "../../lib/agent-interop";@@ -423,7 +423,12 @@ async function searchControllerInner(}if (wantsGovCategory(categories)) {- const indexBudget = await reserveGovIndexCalls(req.auth.team_id, 1);+ const indexBudget = await reserveIndexCalls(+ "gov",+ req.auth.team_id,+ req.acuc?.rate_limit_multiplier ?? 1,+ 1,+ );if (!indexBudget.ok) {if (reservedKeylessCredits > 0) {await adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits);@@ -433,7 +438,7 @@ async function searchControllerInner(return res.status(429).json({success: false,code: "INDEX_DAILY_LIMIT",- error: govIndexDailyLimitMessage(indexBudget),+ ...indexDailyLimitRefusal("gov", req.auth.team_id, indexBudget),});}}diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.tsindex eb633e916..028c8a3f3 100644--- a/apps/api/src/lib/index-daily-limit.ts+++ b/apps/api/src/lib/index-daily-limit.ts@@ -1,8 +1,16 @@-import { config } from "../config";import { redisRateLimitClient } from "../services/rate-limiter";+import { isKeylessTeamId } from "./keyless";import { logger } from "./logger";-type GovIndexDailyReservation =+// Daily call budgets for the free Firecrawl indexes, per team and per UTC day,+// sized by plan. Every route into an index reserves from the same counter, so a+// team cannot raise its budget by switching between the direct endpoint, the+// search category and the Alexandria provider. A new route into an index (for+// example the `research` category on /search) must reserve here as well.++export type IndexName = "gov" | "research";++type IndexDailyReservation =| { ok: true; release: () => Promise<void> }| {ok: false;@@ -10,8 +18,29 @@ type GovIndexDailyReservation =limit: number;requested: number;retryAfterSeconds: number;+ resetsAt: Date;};+const DISPLAY_NAMES: Record<IndexName, string> = {+ gov: "Government Index",+ research: "Research Index",+};++const UPGRADE_URL = "https://firecrawl.dev/pricing";++const BASE_CALLS_PER_DAY = 1000;++// Thresholds are the Autumn `rate_limits` grants: 500 is Growth, 1000 is+// Scale, and Enterprise grants more. A wrong threshold refuses or unlimits real+// teams, not just their queue order. `PLAN_PRIORITY_TIERS` in+// services/rate-limiter.ts keys on the same grants, and the Alexandria catalogue+// in firecrawl/exchange (`usageLimits` on the index providers) and the public+// rate-limits docs repeat these numbers; change them together.+const PLAN_TIERS: { minMultiplier: number; callsPerDay: number | null }[] = [+ { minMultiplier: 1000, callsPerDay: null },+ { minMultiplier: 500, callsPerDay: 2000 },+];+const REDIS_TIMEOUT_MS = 500;function withTimeout<T>(promise: Promise<T>): Promise<T> {@@ -27,14 +56,23 @@ function withTimeout<T>(promise: Promise<T>): Promise<T> {]).finally(() => clearTimeout(timer));}-export async function reserveGovIndexCalls(+/** The team's daily calls per index for its rate-limit multiplier; null is unlimited. */+function indexCallsPerDay(rateLimitMultiplier: number): number | null {+ const tier = PLAN_TIERS.find(t => rateLimitMultiplier >= t.minMultiplier);+ return tier ? tier.callsPerDay : BASE_CALLS_PER_DAY;+}++export async function reserveIndexCalls(+ index: IndexName,teamId: string,+ rateLimitMultiplier: number,calls: number,now = new Date(),-): Promise<GovIndexDailyReservation> {- if (calls <= 0) return { ok: true, release: async () => {} };+): Promise<IndexDailyReservation> {+ const limit = indexCallsPerDay(rateLimitMultiplier);+ if (calls <= 0 || limit === null)+ return { ok: true, release: async () => {} };- const limit = config.GOV_INDEX_CALLS_PER_DAY;const day = now.toISOString().slice(0, 10);const resetAt = Math.floor(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1) /@@ -44,7 +82,9 @@ export async function reserveGovIndexCalls(1,Math.ceil(resetAt - now.getTime() / 1000),);- const key = `index-daily-calls:gov:${teamId}:${day}`;+ // FROZEN: `index-daily-calls:gov:` holds live counters in production.+ // Renaming the key resets every team's count for the day.+ const key = `index-daily-calls:${index}:${teamId}:${day}`;const pending = Promise.resolve().then(() =>redisRateLimitClient.eval(`@@ -84,7 +124,8 @@ return 1),);} catch (error) {- logger.warn("Government index daily limit release failed", {+ logger.warn("Index daily limit release failed", {+ index,teamId,error,});@@ -100,9 +141,11 @@ return 1limit,requested: calls,retryAfterSeconds,+ resetsAt: new Date(resetAt * 1000),};} catch (error) {- logger.warn("Government index daily limit check failed; allowing request", {+ logger.warn("Index daily limit check failed; allowing request", {+ index,teamId,error,});@@ -120,8 +163,33 @@ return 1}}-export function govIndexDailyLimitMessage(- reservation: Extract<GovIndexDailyReservation, { ok: false }>,-): string {- return `Daily government index limit reached: this team has used ${reservation.used} of ${reservation.limit} government index calls today and this request needs ${reservation.requested}. The limit resets at 00:00 UTC, in ${reservation.retryAfterSeconds}s. Contact [email protected] for bulk access.`;+/** The caller-facing fields of a refusal, shared by every route into an index. */+export function indexDailyLimitRefusal(+ index: IndexName,+ teamId: string,+ reservation: Extract<IndexDailyReservation, { ok: false }>,+) {+ const name = DISPLAY_NAMES[index];+ const totalMinutes = Math.max(+ 1,+ Math.ceil(reservation.retryAfterSeconds / 60),+ );+ const hours = Math.floor(totalMinutes / 60);+ const minutes = totalMinutes % 60;+ const resetsIn = hours > 0 ? `${hours}h ${minutes}m` : `${minutes}m`;+ const limit = reservation.limit.toLocaleString("en-US");+ const needs =+ reservation.requested > 1+ ? ` and this request needs ${reservation.requested.toLocaleString("en-US")}`+ : "";+ const error = isKeylessTeamId(teamId)+ ? `Daily ${name} limit reached: requests without an API key are limited to ${limit} a day${needs}. The limit resets at 00:00 UTC, in ${resetsIn}. Add an API key, and upgrade to Scale or Enterprise for unlimited calls: ${UPGRADE_URL}`+ : `Daily ${name} limit reached: your team has used ${reservation.used.toLocaleString("en-US")} of its ${limit} calls today${needs}. The limit resets at 00:00 UTC, in ${resetsIn}. Upgrade to Scale or Enterprise for unlimited ${name} calls: ${UPGRADE_URL}`;+ return {+ error,+ limit: reservation.limit,+ used: reservation.used,+ resetsAt: reservation.resetsAt.toISOString(),+ upgradeUrl: UPGRADE_URL,+ };}diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 0d779b740..72d23e319 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -125,6 +125,10 @@ export function keylessExhaustionTelemetry(ip: string): Record<string, string> {export { keylessTeamId, keylessTeamUuid };+export function isKeylessTeamId(teamId: string): boolean {+ return teamId.startsWith(KEYLESS_TEAM_PREFIX);+}+// The keyless identity (quota bucket) of a team: a client IP or a World ID// identity. Quota keys take it as is; prompts take keylessPromptIp.function keylessIpFromTeamId(teamId: string): string | null {diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsindex f8ae29574..0d8620955 100644--- a/apps/api/src/services/alexandria/retrieve.test.ts+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -69,6 +69,7 @@ const run = (overrides: Record<string, unknown> = {}) =>requestId: "request-1",scrapeId: "scrape-1",timeoutMs: 50000,+ rateLimitMultiplier: 1,...overrides,});const executions = () =>diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex fc18705f2..33586922d 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -4,8 +4,9 @@ import { config } from "../../config";import type { TeamFlags } from "../../controllers/v2/types";import { reportExchangeUsageBilling } from "../../lib/exchange";import {- govIndexDailyLimitMessage,- reserveGovIndexCalls,+ indexDailyLimitRefusal,+ reserveIndexCalls,+ type IndexName,} from "../../lib/index-daily-limit";import { logger } from "../../lib/logger";import {@@ -34,6 +35,13 @@ const RETENTION_SECONDS = 7 * 86400;const GRACE_MS = 30_000;const HOLD_GRACE_MS = 15 * 60_000;+// The providers served by the free Firecrawl indexes, each reserving from that+// index's daily budget. Matched on the exact provider slug.+const INDEX_PROVIDERS: [IndexName, string][] = [+ ["gov", "firecrawl-gov-index"],+ ["research", "firecrawl-research-index"],+];+// One record per charge id (team + x-request-id), written before the provider// call so a crash is found as unresolved instead of re-executed, and rewritten// with the response so replays never execute twice.@@ -106,6 +114,8 @@ export async function retrieveProviders(input: {zeroDataRetention?: boolean;clientIp?: string;billingCountry?: string;+ /** The team's Autumn rate-limit multiplier, which sizes its index budgets. */+ rateLimitMultiplier: number;}): Promise<ProviderRetrieval> {const notExecuted = (response: ExchangeResponse): ProviderRetrieval => ({...response,@@ -254,9 +264,12 @@ export async function retrieveProviders(input: {apiKeyId: input.apiKeyId,};const remaining = () => Math.max(1, record.deadline - Date.now());- let releaseGovCalls: (() => Promise<void>) | undefined;+ const indexReleases: (() => Promise<void>)[] = [];+ const releaseIndexCalls = async () => {+ for (const release of indexReleases) await release();+ };const refuse = async (response: ExchangeResponse, executed = false) => {- if (!executed && releaseGovCalls) await releaseGovCalls();+ if (!executed) await releaseIndexCalls();await redisRateLimitClient.del(key);return { ...response, executed, scrapeId: input.scrapeId };};@@ -264,6 +277,32 @@ export async function retrieveProviders(input: {let lockId: string | undefined;let maximumCredits: number;let refundable = false;+ const finalize = async (credits: number) => {+ if (!lockId) return true;+ const settled = await autumnService.finalizeCreditsLock({+ // Omitted without an org: the settle goes straight to Autumn, the route+ // an unnameable org already took.+ team:+ input.orgId !== null+ ? { teamId: input.teamId, orgId: input.orgId }+ : undefined,+ lockId,+ action: credits > 0 ? "confirm" : "release",+ ...(credits > 0 ? { overrideValue: credits } : {}),+ heldValue: maximumCredits,+ featureId,+ properties,+ });+ if (!settled)+ logger.error("Provider credits were not settled; hold expires unbilled", {+ chargeId: id,+ teamId: input.teamId,+ lockId,+ credits,+ });+ return settled;+ };+try {const denied = await authorizeProviders(input.teamId,@@ -275,19 +314,28 @@ export async function retrieveProviders(input: {);if (denied) return refuse(denied);- const govCalls = input.calls.filter(- call => call.provider === "firecrawl-gov-index",- ).length;- if (govCalls > 0) {- const indexBudget = await reserveGovIndexCalls(input.teamId, govCalls);- if (!indexBudget.ok)+ for (const [index, provider] of INDEX_PROVIDERS) {+ const indexBudget = await reserveIndexCalls(+ index,+ input.teamId,+ input.rateLimitMultiplier,+ input.calls.filter(call => call.provider === provider).length,+ );+ if (!indexBudget.ok) {+ const { error, ...limitFields } = indexDailyLimitRefusal(+ index,+ input.teamId,+ indexBudget,+ );return refuse(- refusal(429, govIndexDailyLimitMessage(indexBudget), {+ refusal(429, error, {code: "index_daily_limit",retryAfterSeconds: indexBudget.retryAfterSeconds,+ ...limitFields,}),);- releaseGovCalls = indexBudget.release;+ }+ indexReleases.push(indexBudget.release);}const quote = await exchangeRequest({@@ -369,13 +417,18 @@ export async function retrieveProviders(input: {);lockId = hold.lockId;}++ // The record is written before the provider runs, so a failed write means+ // nothing ran: the catch below hands back the index calls and the hold.+ record.lockId = lockId;+ record.maximumCredits = maximumCredits;+ await write(record);} catch (error) {- if (releaseGovCalls) await releaseGovCalls();+ if (lockId) await finalize(0).catch(() => false);+ await releaseIndexCalls();await redisRateLimitClient.del(key).catch(() => {});throw error;}- record.lockId = lockId;- record.maximumCredits = maximumCredits;const fail = async (reason: string): Promise<ProviderRetrieval> => {logger.error("Provider request outcome is uncertain; needs reconciliation",@@ -390,34 +443,8 @@ export async function retrieveProviders(input: {await write({ ...record, failure: reason }).catch(() => {});return { ...unresolved(id), executed: true, scrapeId: input.scrapeId };};- const finalize = async (credits: number) => {- if (!lockId) return true;- const settled = await autumnService.finalizeCreditsLock({- // Omitted without an org: the settle goes straight to Autumn, the route- // an unnameable org already took.- team:- input.orgId !== null- ? { teamId: input.teamId, orgId: input.orgId }- : undefined,- lockId,- action: credits > 0 ? "confirm" : "release",- ...(credits > 0 ? { overrideValue: credits } : {}),- heldValue: maximumCredits,- featureId,- properties,- });- if (!settled)- logger.error("Provider credits were not settled; hold expires unbilled", {- chargeId: id,- teamId: input.teamId,- lockId,- credits,- });- return settled;- };try {- await write(record);const response = await exchangeRequest({teamId: input.teamId,path: "/v1/retrieve",