Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): per-team overrides for the Government Index daily limit (#5417)
Erik Engervall2b68c4c56efc
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 72acadb95..d214e52eb 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -78,6 +78,26 @@ const researchKeylessDisabled = z.preprocess(.default([...RESEARCH_PAPER_OPERATIONS]),);+const teamDailyLimits = z.preprocess(+ value =>+ value === undefined+ ? []+ : typeof value === "string"+ ? value+ .split(",")+ .map(entry => entry.trim().toLowerCase())+ .filter(Boolean)+ .map(entry => entry.split(":").map(part => part.trim()))+ : value,+ z+ .array(z.tuple([z.uuid(), z.coerce.number().int().positive()]))+ .refine(+ pairs => new Set(pairs.map(([teamId]) => teamId)).size === pairs.length,+ "each team id may appear only once",+ )+ .transform(pairs => Object.fromEntries(pairs)),+);+const containsLoneSurrogate = (value: string): boolean => {for (let index = 0; index < value.length; index++) {const codeUnit = value.charCodeAt(index);@@ -118,6 +138,9 @@ const configSchema = z.object({RESEARCH_PROXY_URL: z.string().url().optional(),SEARCH_PLATFORM_URL: z.string().url().optional(),RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,+ // Per-team Government Index daily limits as teamId:limit,teamId:limit. A+ // listed team's limit replaces its plan limit.+ GOV_INDEX_CALLS_PER_DAY_OVERRIDES: teamDailyLimits,LABS_SEARCH_URL: z.string().url().optional(),LABS_SEARCH_SECRET: z.string().optional(),diff --git a/apps/api/src/lib/__tests__/index-daily-limit.test.ts b/apps/api/src/lib/__tests__/index-daily-limit.test.tsnew file mode 100644index 000000000..c96e634e2--- /dev/null+++ b/apps/api/src/lib/__tests__/index-daily-limit.test.ts@@ -0,0 +1,70 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const OVERRIDDEN_TEAM = "00000000-0000-4000-8000-000000000001";+const OTHER_TEAM = "00000000-0000-4000-8000-000000000002";++// config parses the overrides at import, so set them first.+const { redis } = vi.hoisted(() => {+ process.env.GOV_INDEX_CALLS_PER_DAY_OVERRIDES =+ " 00000000-0000-4000-8000-000000000001:20000 ,";+ return { redis: { eval: vi.fn() } };+});++vi.mock("../../services/rate-limiter", () => ({ redisRateLimitClient: redis }));+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));++import { config } from "../../config";+import {+ indexDailyLimitRefusal,+ reserveIndexCalls,+} from "../index-daily-limit";++const NOW = new Date("2026-10-27T12:00:00Z");++function limitPassedToRedis() {+ return redis.eval.mock.calls[0][4];+}++beforeEach(() => {+ redis.eval.mockReset();+ redis.eval.mockResolvedValue([1, 0]);+});++describe("Government Index per-team overrides", () => {+ it("parses the configured list", () => {+ expect(config.GOV_INDEX_CALLS_PER_DAY_OVERRIDES).toEqual({+ [OVERRIDDEN_TEAM]: 20000,+ });+ });++ it("uses the override for a listed team", async () => {+ await reserveIndexCalls("gov", OVERRIDDEN_TEAM, 1, 51, NOW);+ expect(limitPassedToRedis()).toBe(20000);+ });++ it("keeps the plan limit for other teams", async () => {+ await reserveIndexCalls("gov", OTHER_TEAM, 1, 51, NOW);+ expect(limitPassedToRedis()).toBe(1000);+ });++ it("does not apply to the Research Index", async () => {+ await reserveIndexCalls("research", OVERRIDDEN_TEAM, 1, 1, NOW);+ expect(limitPassedToRedis()).toBe(1000);+ });++ it("reports the override in the refusal", async () => {+ redis.eval.mockResolvedValue([0, 19990]);+ const reservation = await reserveIndexCalls(+ "gov",+ OVERRIDDEN_TEAM,+ 1,+ 51,+ NOW,+ );+ expect(reservation.ok).toBe(false);+ if (reservation.ok) return;+ const refusal = indexDailyLimitRefusal("gov", OVERRIDDEN_TEAM, reservation);+ expect(refusal.limit).toBe(20000);+ expect(refusal.error).toContain("19,990 of its 20,000 calls today");+ });+});diff --git a/apps/api/src/lib/index-daily-limit.ts b/apps/api/src/lib/index-daily-limit.tsindex 028c8a3f3..01d03a1c4 100644--- a/apps/api/src/lib/index-daily-limit.ts+++ b/apps/api/src/lib/index-daily-limit.ts@@ -1,3 +1,4 @@+import { config } from "../config";import { redisRateLimitClient } from "../services/rate-limiter";import { isKeylessTeamId } from "./keyless";import { logger } from "./logger";@@ -69,7 +70,10 @@ export async function reserveIndexCalls(calls: number,now = new Date(),): Promise<IndexDailyReservation> {- const limit = indexCallsPerDay(rateLimitMultiplier);+ const limit =+ (index === "gov"+ ? config.GOV_INDEX_CALLS_PER_DAY_OVERRIDES?.[teamId]+ : undefined) ?? indexCallsPerDay(rateLimitMultiplier);if (calls <= 0 || limit === null)return { ok: true, release: async () => {} };