Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(gov): enforce daily index call limits (#5144)
Kris Oeib24ee937fe86
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a25b5fb45..44d37c1ca 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -99,6 +99,10 @@ const configSchema = z.object({// ExpressEXPRESS_TRUST_PROXY: z.coerce.number().optional(),+ GOV_INDEX_CALLS_PER_DAY: emptyStringAsDefault(+ z.coerce.number().int().nonnegative().default(500),+ ),+// Keyless free tier (scrape/search/interact without an API key, per-IP/day).// Non-negative integers; 0 means "enabled but no budget", unset means "off".KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 372adace1..fb29390b6 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -8,6 +8,10 @@ import {fetchResearchUpstream,} from "../../lib/research-upstream";import { chargeKeylessCredits } from "../../lib/keyless";+import {+ govIndexDailyLimitMessage,+ reserveGovIndexCalls,+} from "../../lib/index-daily-limit";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";import {@@ -353,6 +357,21 @@ function createResearchController(api_key_id: authedReq.acuc?.api_key_id ?? null,});+ let releaseGovCall: (() => Promise<void>) | undefined;+ if (endpoint.kind === "gov_search") {+ const indexBudget = await reserveGovIndexCalls(authedReq.auth.team_id, 1);+ if (!indexBudget.ok) {+ chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});+ res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));+ return res.status(429).json({+ success: false,+ code: "INDEX_DAILY_LIMIT",+ error: govIndexDailyLimitMessage(indexBudget),+ });+ }+ releaseGovCall = indexBudget.release;+ }+let statusCode = 500;let responseBody: any = null;let error: string | undefined;@@ -366,6 +385,7 @@ function createResearchController(queryKeys,);if (!upstream) {+ if (releaseGovCall) await releaseGovCall();statusCode = 404;error = "Research service is not configured";return res.status(404).end();diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex de78d1403..e3476ad10 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -214,7 +214,18 @@ export async function providerScrapeController();}- if (result.status !== 200) return res.status(result.status).json(result.body);+ if (result.status !== 200) {+ const refused = result.body as {+ code?: unknown;+ retryAfterSeconds?: unknown;+ } | null;+ if (+ refused?.code === "index_daily_limit" &&+ typeof refused.retryAfterSeconds === "number"+ )+ res.setHeader("Retry-After", String(refused.retryAfterSeconds));+ return res.status(result.status).json(result.body);+ }const answer = answerSchema.parse(result.body);if (legacy) {if (legacyBody && "requests" in legacyBody) return res.json(answer);diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 08fcc544a..a06550cc9 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -52,6 +52,10 @@ import {} from "../../lib/key-restriction";import { wantsDeveloperCategory } from "../../search/developer";import { wantsGovCategory } from "../../search/gov";+import {+ govIndexDailyLimitMessage,+ reserveGovIndexCalls,+} from "../../lib/index-daily-limit";import { requestOrigin } from "../../lib/request-origin";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { applyNotice, type Notice } from "../../lib/deprecations";@@ -409,6 +413,18 @@ async function searchControllerInner(reservedKeylessCredits = projectedKeylessCredits;}+ if (wantsGovCategory(categories)) {+ const indexBudget = await reserveGovIndexCalls(req.auth.team_id, 1);+ if (!indexBudget.ok) {+ res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));+ return res.status(429).json({+ success: false,+ code: "INDEX_DAILY_LIMIT",+ error: govIndexDailyLimitMessage(indexBudget),+ });+ }+ }+const result = await executeSearch({query: req.body.query,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 1d679061c..006ac7eb8 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -103,6 +103,7 @@ const errorMap: Record<ErrorCodes, any> = {// Zod errorsBAD_REQUEST: null,BAD_REQUEST_INVALID_JSON: null,+ INDEX_DAILY_LIMIT: null,PARSE_UNSUPPORTED_OPTIONS: null,// Agent thread rejections are API-level, never transported through workers.diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex e8460ddd8..9c0fe10f6 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -35,6 +35,7 @@ export type ErrorCodes =| "SCRAPE_ACTIONS_NOT_SUPPORTED"| "SCRAPE_BRANDING_NOT_SUPPORTED"| "AGENT_INDEX_ONLY"+ | "INDEX_DAILY_LIMIT"| "SCRAPE_AUDIO_UNSUPPORTED_URL"| "SCRAPE_VIDEO_UNSUPPORTED_URL"| "SCRAPE_MEDIA_ACCESS_DENIED"diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.tsnew file mode 100644index 000000000..a0cba2647--- /dev/null+++ b/apps/api/src/lib/index-daily-limit.ts@@ -0,0 +1,127 @@+import { config } from "../config";+import { redisRateLimitClient } from "../services/rate-limiter";+import { logger } from "./logger";++export type GovIndexDailyReservation =+ | { ok: true; release: () => Promise<void> }+ | {+ ok: false;+ used: number;+ limit: number;+ requested: number;+ retryAfterSeconds: number;+ };++const REDIS_TIMEOUT_MS = 500;++function withTimeout<T>(promise: Promise<T>): Promise<T> {+ let timer: NodeJS.Timeout;+ return Promise.race([+ promise,+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error("Redis did not answer in time")),+ REDIS_TIMEOUT_MS,+ );+ }),+ ]).finally(() => clearTimeout(timer));+}++export async function reserveGovIndexCalls(+ teamId: string,+ calls: number,+ now = new Date(),+): Promise<GovIndexDailyReservation> {+ if (calls <= 0) return { ok: true, release: async () => {} };++ const limit = config.GOV_INDEX_CALLS_PER_DAY;+ const day = now.toISOString().slice(0, 10);+ const resetAt = Math.floor(+ Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1) /+ 1000,+ );+ const retryAfterSeconds = Math.max(+ 1,+ Math.ceil(resetAt - now.getTime() / 1000),+ );+ const key = `index-daily-calls:gov:${teamId}:${day}`;+ const pending = Promise.resolve().then(() =>+ redisRateLimitClient.eval(+ `+local current = tonumber(redis.call("GET", KEYS[1]) or "0")+local calls = tonumber(ARGV[1])+if current + calls > tonumber(ARGV[2]) then+ return {0, current}+end+redis.call("INCRBY", KEYS[1], calls)+redis.call("EXPIREAT", KEYS[1], ARGV[3])+return {1, current}+`,+ 1,+ key,+ calls,+ limit,+ resetAt,+ ),+ ) as Promise<[number, number]>;++ let releasePromise: Promise<void> | undefined;+ const release = () =>+ (releasePromise ??= (async () => {+ try {+ await withTimeout(+ redisRateLimitClient.eval(+ `+local current = tonumber(redis.call("GET", KEYS[1]) or "0")+if current > 0 then+ redis.call("DECRBY", KEYS[1], math.min(current, tonumber(ARGV[1])))+end+return 1+`,+ 1,+ key,+ calls,+ ),+ );+ } catch (error) {+ logger.warn("Government index daily limit release failed", {+ teamId,+ error,+ });+ }+ })());++ try {+ const [admitted, used] = await withTimeout(pending);+ if (admitted === 1) return { ok: true, release };+ return {+ ok: false,+ used: Number(used),+ limit,+ requested: calls,+ retryAfterSeconds,+ };+ } catch (error) {+ logger.warn("Government index daily limit check failed; allowing request", {+ teamId,+ error,+ });+ let releaseScheduled = false;+ return {+ ok: true,+ release: async () => {+ if (releaseScheduled) return;+ releaseScheduled = true;+ void pending+ .then(([admitted]) => (admitted === 1 ? release() : undefined))+ .catch(() => {});+ },+ };+ }+}++export function govIndexDailyLimitMessage(+ reservation: Extract<GovIndexDailyReservation, { ok: false }>,+): string {+ return `Daily government index limit reached: this team has used ${reservation.used} of ${reservation.limit} government index calls today and this request needs ${reservation.requested}. The limit resets at 00:00 UTC, in ${reservation.retryAfterSeconds}s. Contact [email protected] for bulk access.`;+}diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 82105e972..483f79322 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -3,6 +3,10 @@ import { z } from "zod";import { config } from "../../config";import type { TeamFlags } from "../../controllers/v2/types";import { reportExchangeUsageBilling } from "../../lib/exchange";+import {+ govIndexDailyLimitMessage,+ reserveGovIndexCalls,+} from "../../lib/index-daily-limit";import { logger } from "../../lib/logger";import {autumnService,@@ -223,7 +227,9 @@ export async function retrieveProviders(input: {apiKeyId: input.apiKeyId,};const remaining = () => Math.max(1, record.deadline - Date.now());+ let releaseGovCalls: (() => Promise<void>) | undefined;const refuse = async (response: ExchangeResponse, executed = false) => {+ if (!executed && releaseGovCalls) await releaseGovCalls();await redisRateLimitClient.del(key);return { ...response, executed, scrapeId: input.scrapeId };};@@ -240,6 +246,21 @@ export async function retrieveProviders(input: {);if (denied) return refuse(denied);+ const govCalls = input.calls.filter(+ call => call.provider === "firecrawl-gov-index",+ ).length;+ if (govCalls > 0) {+ const indexBudget = await reserveGovIndexCalls(input.teamId, govCalls);+ if (!indexBudget.ok)+ return refuse(+ refusal(429, govIndexDailyLimitMessage(indexBudget), {+ code: "index_daily_limit",+ retryAfterSeconds: indexBudget.retryAfterSeconds,+ }),+ );+ releaseGovCalls = indexBudget.release;+ }+const quote = await exchangeRequest({teamId: input.teamId,path: "/v1/retrieve/quote",@@ -317,6 +338,7 @@ export async function retrieveProviders(input: {lockId = hold.lockId;}} catch (error) {+ if (releaseGovCalls) await releaseGovCalls();await redisRateLimitClient.del(key).catch(() => {});throw error;}