Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Record keyless client IPs for zero-credit operations
Claude06506155cf05
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 0c6febe7a..524efb6c6 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -1,14 +1,16 @@import { config } from "../../../config";-import { describeIf, TEST_PRODUCTION } from "../lib";+import { describeIf, itIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";-import { and, desc, eq } from "drizzle-orm";+import { and, desc, eq, gt } from "drizzle-orm";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";+import { redisRateLimitClient } from "../../../services/rate-limiter";const HAS_RESEARCH = !!config.RESEARCH_PROXY_URL;const KEYLESS_ENABLED =process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&process.env.KEYLESS_CREDITS_PER_DAY !== undefined;+const KEYLESS_PROXY_SECRET = process.env.KEYLESS_PROXY_SECRET;const KEYLESS_DISABLED_OPERATIONS = config.RESEARCH_KEYLESS_DISABLED;const KEYLESS_SEARCH_ENABLED =@@ -33,6 +35,59 @@ async function waitForSingleRow<T>(return null;}+// Keyless requests are keyed on the client IP. When the trusted-proxy secret is+// configured we forward a unique documentation-range (RFC 5737) IP so each test+// owns an isolated per-IP bucket; otherwise we fall back to the loopback IP the+// server actually keyed on, recovered from the limiter keyspace the same way+// keyless.test.ts does.+function keylessHeaders(forwardedIp: string): Record<string, string> {+ return KEYLESS_PROXY_SECRET+ ? {+ "x-firecrawl-keyless-secret": KEYLESS_PROXY_SECRET,+ "x-firecrawl-keyless-ip": forwardedIp,+ }+ : {};+}++async function resolveKeylessIp(forwardedIp: string): Promise<string> {+ if (KEYLESS_PROXY_SECRET) return forwardedIp;+ const keys = await redisRateLimitClient.keys("keyless_requests:*");+ expect(keys.length).toBeGreaterThan(0);+ return keys[0].slice("keyless_requests:".length);+}++async function maxKeylessUsageId(): Promise<number | undefined> {+ const rows = await db+ .select({ id: schema.keyless_credit_usage.id })+ .from(schema.keyless_credit_usage)+ .orderBy(desc(schema.keyless_credit_usage.id))+ .limit(1);+ return rows[0]?.id;+}++// The audit insert is fire-and-forget in the controller, so callers snapshot the+// latest id first and then poll for a newer row for their IP.+async function waitForKeylessUsageRow(ip: string, afterId: number | undefined) {+ return waitForSingleRow<typeof schema.keyless_credit_usage.$inferSelect>(+ async () => {+ const rows = await db+ .select()+ .from(schema.keyless_credit_usage)+ .where(+ and(+ eq(schema.keyless_credit_usage.ip, ip),+ afterId !== undefined+ ? gt(schema.keyless_credit_usage.id, afterId)+ : undefined,+ ),+ )+ .orderBy(desc(schema.keyless_credit_usage.id))+ .limit(1);+ return rows[0] ?? null;+ },+ );+}+describeIf(HAS_RESEARCH)("Research API", () => {it("serves paper search from the canonical mount", async () => {const identity = await idmux({@@ -213,6 +268,75 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).not.toBe(401);}, 120000);++ // The Research Index paper endpoints cost 0 credits. Until now the whole+ // keyless usage log was skipped for zero-credit operations, so these+ // requests left no client IP anywhere: during a distributed+ // corpus-harvest incident only 3.0% of the traffic had a resolvable IP.+ // The IP must be recorded even though nothing is billed.+ itIf(config.USE_DB_AUTHENTICATION === true)(+ "records the keyless client IP for a zero-credit paper search",+ async () => {+ const forwardedIp = "198.51.100.11";+ const beforeMaxId = await maxKeylessUsageId();++ const res = await researchRaw(+ "/v2/search/research/papers",+ { query: "sparse attention", k: 1 },+ undefined,+ keylessHeaders(forwardedIp),+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);++ const ip = await resolveKeylessIp(forwardedIp);+ const row = await waitForKeylessUsageRow(ip, beforeMaxId);++ expect(row).not.toBeNull();+ expect(row!.ip).toBe(ip);+ // Observability only: the row exists, but nothing was charged.+ expect(row!.credits_used).toBe(0);++ // The keyless *credit* budget must not be drawn down by a free+ // operation. Only meaningful on the isolated forwarded IP β the+ // loopback bucket is shared with the other keyless snips.+ if (KEYLESS_PROXY_SECRET) {+ const creditsUsed = Number(+ (await redisRateLimitClient.get(`keyless_credits:${ip}`)) ?? "0",+ );+ expect(creditsUsed).toBe(0);+ }+ },+ 120000,+ );++ // Failure path: ID enumeration β the exact shape of the incident β mostly+ // produces upstream misses, so a miss must record the IP too.+ itIf(config.USE_DB_AUTHENTICATION === true)(+ "records the keyless client IP when a paper lookup misses",+ async () => {+ const forwardedIp = "198.51.100.12";+ const beforeMaxId = await maxKeylessUsageId();++ const res = await researchRaw(+ "/v2/search/research/papers/0000.00000",+ undefined,+ undefined,+ keylessHeaders(forwardedIp),+ );++ expect(res.statusCode).not.toBe(401);+ expect(res.statusCode).toBeGreaterThanOrEqual(400);++ const ip = await resolveKeylessIp(forwardedIp);+ const row = await waitForKeylessUsageRow(ip, beforeMaxId);++ expect(row).not.toBeNull();+ expect(row!.credits_used).toBe(0);+ },+ 120000,+ );});describeIf(KEYLESS_INSPECT_DISABLED)("keyless research disabled", () => {diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 8d4e230eb..ecfee6341 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -345,7 +345,6 @@ function createResearchController(credits,});});- chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {});}} else {error =@@ -374,6 +373,16 @@ function createResearchController(return res.status(502).end();} finally {const timeTaken = (Date.now() - started) / 1000;++ // No-op for keyed teams. For keyless callers this both charges the per-IP+ // daily credit budget and records the client IP in `keyless_credit_usage`.+ // It runs on every outcome, not just billable successes: the paper+ // endpoints cost 0 credits and ID enumeration mostly produces upstream+ // misses, so gating this on `credits > 0` left those requests with no IP+ // recorded anywhere. `credits` is still 0 on every non-2xx path, so+ // nothing extra is charged.+ chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {});+logResearchEndpoint({table: endpoint.table,id: jobId,diff --git a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts b/apps/api/src/lib/__tests__/keyless-usage-log.test.tsnew file mode 100644index 000000000..3fe9cc0bb--- /dev/null+++ b/apps/api/src/lib/__tests__/keyless-usage-log.test.ts@@ -0,0 +1,134 @@+// The keyless usage log is the only place a keyless caller's IP is persisted β+// `requests.team_id` collapses all keyless traffic onto one preview team. The+// end-to-end coverage for the zero-credit Research Index endpoints lives in+// snips/v2/research.test.ts, but that suite only runs where a research upstream+// is configured. These tests pin the library-level contract everywhere: a+// zero-credit keyless request still writes a row, and it still charges nothing.+const { dbInsert, insertValues, redisIncrby, redisExpire } = vi.hoisted(() => {+ const insertValues = vi.fn().mockResolvedValue(undefined);+ return {+ insertValues,+ dbInsert: vi.fn(() => ({ values: insertValues })),+ redisIncrby: vi.fn().mockResolvedValue(1),+ redisExpire: vi.fn().mockResolvedValue(1),+ };+});++vi.mock("../../db/connection", () => ({+ db: { insert: dbInsert },+ dbRr: { select: vi.fn() },+ dbIndex: { select: vi.fn() },+}));++vi.mock("../../services/rate-limiter", () => ({+ redisRateLimitClient: {+ incrby: redisIncrby,+ expire: redisExpire,+ get: vi.fn().mockResolvedValue(null),+ incr: vi.fn().mockResolvedValue(1),+ ttl: vi.fn().mockResolvedValue(-1),+ eval: vi.fn().mockResolvedValue([1, 1]),+ },+}));++import { config } from "../../config";+import {+ chargeKeylessCredits,+ keylessTeamId,+ keylessTeamUuid,+ logKeylessCreditUsage,+} from "../keyless";++const IP = "203.0.113.99";+const KEYLESS_TEAM = keylessTeamId(IP);++let previousDbAuth: boolean | undefined;++beforeEach(() => {+ vi.clearAllMocks();+ insertValues.mockResolvedValue(undefined);+ previousDbAuth = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = true;+});++afterEach(() => {+ config.USE_DB_AUTHENTICATION = previousDbAuth;+});++describe("logKeylessCreditUsage", () => {+ it("records the client IP for a zero-credit keyless operation", async () => {+ await logKeylessCreditUsage(KEYLESS_TEAM, 0);++ expect(dbInsert).toHaveBeenCalledTimes(1);+ expect(insertValues).toHaveBeenCalledWith({+ team_id: keylessTeamUuid(KEYLESS_TEAM),+ ip: IP,+ credits_used: 0,+ });+ });++ it("still records the actual credits for a billable keyless operation", async () => {+ await logKeylessCreditUsage(KEYLESS_TEAM, 2.1);++ expect(insertValues).toHaveBeenCalledWith(+ expect.objectContaining({ ip: IP, credits_used: 3 }),+ );+ });++ it("never writes a negative row for a reconciliation refund", async () => {+ await logKeylessCreditUsage(KEYLESS_TEAM, -5);++ expect(insertValues).toHaveBeenCalledWith(+ expect.objectContaining({ ip: IP, credits_used: 0 }),+ );+ });++ it("writes nothing for a team that is not keyless", async () => {+ await logKeylessCreditUsage("some-real-team-id", 0);++ expect(dbInsert).not.toHaveBeenCalled();+ });++ it("writes nothing when DB auth is off", async () => {+ config.USE_DB_AUTHENTICATION = false;++ await logKeylessCreditUsage(KEYLESS_TEAM, 0);++ expect(dbInsert).not.toHaveBeenCalled();+ });++ it("swallows insert failures so the request is unaffected", async () => {+ insertValues.mockRejectedValue(new Error("db down"));++ await expect(+ logKeylessCreditUsage(KEYLESS_TEAM, 0),+ ).resolves.toBeUndefined();+ });+});++describe("chargeKeylessCredits", () => {+ it("records a zero-credit request without drawing down the credit budget", async () => {+ await chargeKeylessCredits(KEYLESS_TEAM, 0);++ expect(redisIncrby).not.toHaveBeenCalled();+ expect(insertValues).toHaveBeenCalledWith(+ expect.objectContaining({ ip: IP, credits_used: 0 }),+ );+ });++ it("charges the credit budget for a billable request", async () => {+ await chargeKeylessCredits(KEYLESS_TEAM, 4);++ expect(redisIncrby).toHaveBeenCalledWith(`keyless_credits:${IP}`, 4);+ expect(insertValues).toHaveBeenCalledWith(+ expect.objectContaining({ ip: IP, credits_used: 4 }),+ );+ });++ it("does nothing at all for a team that is not keyless", async () => {+ await chargeKeylessCredits("some-real-team-id", 5);++ expect(redisIncrby).not.toHaveBeenCalled();+ expect(dbInsert).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 3ff9a53fb..1ada0d83a 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -368,24 +368,34 @@ export async function checkKeylessEligibility(ip: string): Promise<{}/**- * Append a row to `keyless_credit_usage` recording the actual credits a completed- * keyless request consumed (per-IP keyless team UUID + raw IP), for abuse- * monitoring. No-op for non-keyless teams, non-positive credits, or when DB auth- * is off. Best-effort β never throws.+ * Append a row to `keyless_credit_usage` recording a completed keyless request:+ * the per-IP keyless team UUID, the raw IP, and the credits it consumed β the+ * only place a keyless caller's IP is persisted (`requests.team_id` collapses+ * all keyless traffic onto one preview team).+ *+ * Zero-credit operations are recorded too. The free Research Index paper+ * endpoints bill nothing, so skipping them meant their client IP was never+ * written anywhere and abuse on them was invisible. A `credits_used = 0` row+ * cannot move any credit total; it only makes the request countable.+ *+ * No-op for non-keyless teams or when DB auth is off. Best-effort β never+ * throws.*/export async function logKeylessCreditUsage(teamId: string,credits: number,): Promise<void> {const ip = keylessIpFromTeamId(teamId);- if (!ip || !Number.isFinite(credits) || credits <= 0) return;+ if (!ip || !Number.isFinite(credits)) return;const teamUuid = keylessTeamUuid(teamId);if (config.USE_DB_AUTHENTICATION !== true || !teamUuid) return;try {await db.insert(schema.keyless_credit_usage).values({team_id: teamUuid,ip,- credits_used: Math.ceil(credits),+ // The column records consumption, so a negative reconciliation delta must+ // not land as a negative row.+ credits_used: Math.max(0, Math.ceil(credits)),});} catch {// Logging is best-effort.@@ -394,26 +404,31 @@ export async function logKeylessCreditUsage(/*** Add the actual credits a completed request consumed to the IP's daily credit- * counter. No-op for non-keyless teams. Best-effort; never throws. Used by the- * worker for the non-reserved path; the controllers reserve up front and call- * `logKeylessCreditUsage` directly at reconciliation.+ * counter, and record the request in `keyless_credit_usage`. The counter is only+ * touched for positive credits, so a zero-credit operation becomes observable+ * without drawing down any budget. No-op for non-keyless teams. Best-effort;+ * never throws. Used by the worker for the non-reserved path; the controllers+ * reserve up front and call `logKeylessCreditUsage` directly at reconciliation.*/export async function chargeKeylessCredits(teamId: string,credits: number,): Promise<void> {const ip = keylessIpFromTeamId(teamId);- if (!ip || !Number.isFinite(credits) || credits <= 0) return;- const inc = Math.ceil(credits);- try {- const key = creditsKey(ip);- const total = await redisRateLimitClient.incrby(key, inc);- if (total === inc) {- await redisRateLimitClient.expire(key, DAY_SECONDS);+ if (!ip || !Number.isFinite(credits)) return;++ if (credits > 0) {+ const inc = Math.ceil(credits);+ try {+ const key = creditsKey(ip);+ const total = await redisRateLimitClient.incrby(key, inc);+ if (total === inc) {+ await redisRateLimitClient.expire(key, DAY_SECONDS);+ }+ } catch {+ // Counter is best-effort; a missed charge just means the IP gets a few+ // extra free credits today.}- } catch {- // Counter is best-effort; a missed charge just means the IP gets a few- // extra free credits today.}// Log the usage to keyless_credit_usage for abuse monitoring. Best-effort.