Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(gov): enforce daily index call limits (#5144)

Kris Oeib24ee937fe86
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a25b5fb45..44d37c1ca 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -99,6 +99,10 @@ const configSchema = z.object({
// Express
EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
+ GOV_INDEX_CALLS_PER_DAY: emptyStringAsDefault(
+ z.coerce.number().int().nonnegative().default(500),
+ ),
+
// Keyless free tier (scrape/search/interact without an API key, per-IP/day).
// Non-negative integers; 0 means "enabled but no budget", unset means "off".
KEYLESS_CREDITS_PER_DAY: z.coerce.number().int().nonnegative().optional(),
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 372adace1..fb29390b6 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -8,6 +8,10 @@ import {
fetchResearchUpstream,
} from "../../lib/research-upstream";
import { chargeKeylessCredits } from "../../lib/keyless";
+import {
+ govIndexDailyLimitMessage,
+ reserveGovIndexCalls,
+} from "../../lib/index-daily-limit";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";
import {
@@ -353,6 +357,21 @@ function createResearchController(
api_key_id: authedReq.acuc?.api_key_id ?? null,
});
+ let releaseGovCall: (() => Promise<void>) | undefined;
+ if (endpoint.kind === "gov_search") {
+ const indexBudget = await reserveGovIndexCalls(authedReq.auth.team_id, 1);
+ if (!indexBudget.ok) {
+ chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});
+ res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));
+ return res.status(429).json({
+ success: false,
+ code: "INDEX_DAILY_LIMIT",
+ error: govIndexDailyLimitMessage(indexBudget),
+ });
+ }
+ releaseGovCall = indexBudget.release;
+ }
+
let statusCode = 500;
let responseBody: any = null;
let error: string | undefined;
@@ -366,6 +385,7 @@ function createResearchController(
queryKeys,
);
if (!upstream) {
+ if (releaseGovCall) await releaseGovCall();
statusCode = 404;
error = "Research service is not configured";
return res.status(404).end();
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index de78d1403..e3476ad10 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -214,7 +214,18 @@ export async function providerScrapeController(
);
}
- if (result.status !== 200) return res.status(result.status).json(result.body);
+ if (result.status !== 200) {
+ const refused = result.body as {
+ code?: unknown;
+ retryAfterSeconds?: unknown;
+ } | null;
+ if (
+ refused?.code === "index_daily_limit" &&
+ typeof refused.retryAfterSeconds === "number"
+ )
+ res.setHeader("Retry-After", String(refused.retryAfterSeconds));
+ return res.status(result.status).json(result.body);
+ }
const answer = answerSchema.parse(result.body);
if (legacy) {
if (legacyBody && "requests" in legacyBody) return res.json(answer);
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 08fcc544a..a06550cc9 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -52,6 +52,10 @@ import {
} from "../../lib/key-restriction";
import { wantsDeveloperCategory } from "../../search/developer";
import { wantsGovCategory } from "../../search/gov";
+import {
+ govIndexDailyLimitMessage,
+ reserveGovIndexCalls,
+} from "../../lib/index-daily-limit";
import { requestOrigin } from "../../lib/request-origin";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
import { applyNotice, type Notice } from "../../lib/deprecations";
@@ -409,6 +413,18 @@ async function searchControllerInner(
reservedKeylessCredits = projectedKeylessCredits;
}
+ if (wantsGovCategory(categories)) {
+ const indexBudget = await reserveGovIndexCalls(req.auth.team_id, 1);
+ if (!indexBudget.ok) {
+ res.setHeader("Retry-After", String(indexBudget.retryAfterSeconds));
+ return res.status(429).json({
+ success: false,
+ code: "INDEX_DAILY_LIMIT",
+ error: govIndexDailyLimitMessage(indexBudget),
+ });
+ }
+ }
+
const result = await executeSearch(
{
query: req.body.query,
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 1d679061c..006ac7eb8 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -103,6 +103,7 @@ const errorMap: Record<ErrorCodes, any> = {
// Zod errors
BAD_REQUEST: null,
BAD_REQUEST_INVALID_JSON: null,
+ INDEX_DAILY_LIMIT: null,
PARSE_UNSUPPORTED_OPTIONS: null,
// Agent thread rejections are API-level, never transported through workers.
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index e8460ddd8..9c0fe10f6 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -35,6 +35,7 @@ export type ErrorCodes =
| "SCRAPE_ACTIONS_NOT_SUPPORTED"
| "SCRAPE_BRANDING_NOT_SUPPORTED"
| "AGENT_INDEX_ONLY"
+ | "INDEX_DAILY_LIMIT"
| "SCRAPE_AUDIO_UNSUPPORTED_URL"
| "SCRAPE_VIDEO_UNSUPPORTED_URL"
| "SCRAPE_MEDIA_ACCESS_DENIED"
diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.ts
new file mode 100644
index 000000000..a0cba2647
--- /dev/null
+++ b/apps/api/src/lib/index-daily-limit.ts
@@ -0,0 +1,127 @@
+import { config } from "../config";
+import { redisRateLimitClient } from "../services/rate-limiter";
+import { logger } from "./logger";
+
+export type GovIndexDailyReservation =
+ | { ok: true; release: () => Promise<void> }
+ | {
+ ok: false;
+ used: number;
+ limit: number;
+ requested: number;
+ retryAfterSeconds: number;
+ };
+
+const REDIS_TIMEOUT_MS = 500;
+
+function withTimeout<T>(promise: Promise<T>): Promise<T> {
+ let timer: NodeJS.Timeout;
+ return Promise.race([
+ promise,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error("Redis did not answer in time")),
+ REDIS_TIMEOUT_MS,
+ );
+ }),
+ ]).finally(() => clearTimeout(timer));
+}
+
+export async function reserveGovIndexCalls(
+ teamId: string,
+ calls: number,
+ now = new Date(),
+): Promise<GovIndexDailyReservation> {
+ if (calls <= 0) return { ok: true, release: async () => {} };
+
+ const limit = config.GOV_INDEX_CALLS_PER_DAY;
+ const day = now.toISOString().slice(0, 10);
+ const resetAt = Math.floor(
+ Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1) /
+ 1000,
+ );
+ const retryAfterSeconds = Math.max(
+ 1,
+ Math.ceil(resetAt - now.getTime() / 1000),
+ );
+ const key = `index-daily-calls:gov:${teamId}:${day}`;
+ const pending = Promise.resolve().then(() =>
+ redisRateLimitClient.eval(
+ `
+local current = tonumber(redis.call("GET", KEYS[1]) or "0")
+local calls = tonumber(ARGV[1])
+if current + calls > tonumber(ARGV[2]) then
+ return {0, current}
+end
+redis.call("INCRBY", KEYS[1], calls)
+redis.call("EXPIREAT", KEYS[1], ARGV[3])
+return {1, current}
+`,
+ 1,
+ key,
+ calls,
+ limit,
+ resetAt,
+ ),
+ ) as Promise<[number, number]>;
+
+ let releasePromise: Promise<void> | undefined;
+ const release = () =>
+ (releasePromise ??= (async () => {
+ try {
+ await withTimeout(
+ redisRateLimitClient.eval(
+ `
+local current = tonumber(redis.call("GET", KEYS[1]) or "0")
+if current > 0 then
+ redis.call("DECRBY", KEYS[1], math.min(current, tonumber(ARGV[1])))
+end
+return 1
+`,
+ 1,
+ key,
+ calls,
+ ),
+ );
+ } catch (error) {
+ logger.warn("Government index daily limit release failed", {
+ teamId,
+ error,
+ });
+ }
+ })());
+
+ try {
+ const [admitted, used] = await withTimeout(pending);
+ if (admitted === 1) return { ok: true, release };
+ return {
+ ok: false,
+ used: Number(used),
+ limit,
+ requested: calls,
+ retryAfterSeconds,
+ };
+ } catch (error) {
+ logger.warn("Government index daily limit check failed; allowing request", {
+ teamId,
+ error,
+ });
+ let releaseScheduled = false;
+ return {
+ ok: true,
+ release: async () => {
+ if (releaseScheduled) return;
+ releaseScheduled = true;
+ void pending
+ .then(([admitted]) => (admitted === 1 ? release() : undefined))
+ .catch(() => {});
+ },
+ };
+ }
+}
+
+export function govIndexDailyLimitMessage(
+ reservation: Extract<GovIndexDailyReservation, { ok: false }>,
+): string {
+ return `Daily government index limit reached: this team has used ${reservation.used} of ${reservation.limit} government index calls today and this request needs ${reservation.requested}. The limit resets at 00:00 UTC, in ${reservation.retryAfterSeconds}s. Contact [email protected] for bulk access.`;
+}
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
index 82105e972..483f79322 100644
--- a/apps/api/src/services/alexandria/retrieve.ts
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -3,6 +3,10 @@ import { z } from "zod";
import { config } from "../../config";
import type { TeamFlags } from "../../controllers/v2/types";
import { reportExchangeUsageBilling } from "../../lib/exchange";
+import {
+ govIndexDailyLimitMessage,
+ reserveGovIndexCalls,
+} from "../../lib/index-daily-limit";
import { logger } from "../../lib/logger";
import {
autumnService,
@@ -223,7 +227,9 @@ export async function retrieveProviders(input: {
apiKeyId: input.apiKeyId,
};
const remaining = () => Math.max(1, record.deadline - Date.now());
+ let releaseGovCalls: (() => Promise<void>) | undefined;
const refuse = async (response: ExchangeResponse, executed = false) => {
+ if (!executed && releaseGovCalls) await releaseGovCalls();
await redisRateLimitClient.del(key);
return { ...response, executed, scrapeId: input.scrapeId };
};
@@ -240,6 +246,21 @@ export async function retrieveProviders(input: {
);
if (denied) return refuse(denied);
+ const govCalls = input.calls.filter(
+ call => call.provider === "firecrawl-gov-index",
+ ).length;
+ if (govCalls > 0) {
+ const indexBudget = await reserveGovIndexCalls(input.teamId, govCalls);
+ if (!indexBudget.ok)
+ return refuse(
+ refusal(429, govIndexDailyLimitMessage(indexBudget), {
+ code: "index_daily_limit",
+ retryAfterSeconds: indexBudget.retryAfterSeconds,
+ }),
+ );
+ releaseGovCalls = indexBudget.release;
+ }
+
const quote = await exchangeRequest({
teamId: input.teamId,
path: "/v1/retrieve/quote",
@@ -317,6 +338,7 @@ export async function retrieveProviders(input: {
lockId = hold.lockId;
}
} catch (error) {
+ if (releaseGovCalls) await releaseGovCalls();
await redisRateLimitClient.del(key).catch(() => {});
throw error;
}