Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor: log zero-credit keyless usage until the DB migration lands
Karan Lokchandani5cc38c058a37
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 524efb6c6..11e9119b8 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -1,7 +1,7 @@import { config } from "../../../config";import { describeIf, itIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";-import { and, desc, eq, gt } from "drizzle-orm";+import { and, desc, eq } from "drizzle-orm";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";import { redisRateLimitClient } from "../../../services/rate-limiter";@@ -56,36 +56,16 @@ async function resolveKeylessIp(forwardedIp: string): Promise<string> {return keys[0].slice("keyless_requests:".length);}-async function maxKeylessUsageId(): Promise<number | undefined> {+// Zero-credit keyless usage is recorded as a canonical log line for now (the+// durable keyless_credit_usage row waits on the firecrawl-db migration), so+// these tests assert no row is written and no credit budget moves. The+// library-level log-line contract is pinned in lib/__tests__/keyless-usage-log.+async function keylessUsageRowCount(ip: string): Promise<number> {const rows = await db.select({ id: schema.keyless_credit_usage.id }).from(schema.keyless_credit_usage)- .orderBy(desc(schema.keyless_credit_usage.id))- .limit(1);- return rows[0]?.id;-}--// The audit insert is fire-and-forget in the controller, so callers snapshot the-// latest id first and then poll for a newer row for their IP.-async function waitForKeylessUsageRow(ip: string, afterId: number | undefined) {- return waitForSingleRow<typeof schema.keyless_credit_usage.$inferSelect>(- async () => {- const rows = await db- .select()- .from(schema.keyless_credit_usage)- .where(- and(- eq(schema.keyless_credit_usage.ip, ip),- afterId !== undefined- ? gt(schema.keyless_credit_usage.id, afterId)- : undefined,- ),- )- .orderBy(desc(schema.keyless_credit_usage.id))- .limit(1);- return rows[0] ?? null;- },- );+ .where(eq(schema.keyless_credit_usage.ip, ip));+ return rows.length;}describeIf(HAS_RESEARCH)("Research API", () => {@@ -269,16 +249,16 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).not.toBe(401);}, 120000);- // The Research Index paper endpoints cost 0 credits. Until now the whole- // keyless usage log was skipped for zero-credit operations, so these- // requests left no client IP anywhere: during a distributed- // corpus-harvest incident only 3.0% of the traffic had a resolvable IP.- // The IP must be recorded even though nothing is billed.+ // The Research Index paper endpoints cost 0 credits. The zero-credit+ // charge path must run for them (it emits the canonical keyless/usage log+ // line carrying the client IP β during a distributed corpus-harvest+ // incident only 3.0% of the traffic had a resolvable IP) without touching+ // either durable store: no keyless_credit_usage row (that lands with the+ // firecrawl-db migration) and no credit-budget draw-down.itIf(config.USE_DB_AUTHENTICATION === true)(- "records the keyless client IP for a zero-credit paper search",+ "a zero-credit paper search writes no usage row and charges nothing",async () => {const forwardedIp = "198.51.100.11";- const beforeMaxId = await maxKeylessUsageId();const res = await researchRaw("/v2/search/research/papers",@@ -291,12 +271,10 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.body.success).toBe(true);const ip = await resolveKeylessIp(forwardedIp);- const row = await waitForKeylessUsageRow(ip, beforeMaxId);-- expect(row).not.toBeNull();- expect(row!.ip).toBe(ip);- // Observability only: the row exists, but nothing was charged.- expect(row!.credits_used).toBe(0);+ // Give the fire-and-forget charge path time to run before asserting+ // it stayed out of the DB.+ await sleep(2000);+ expect(await keylessUsageRowCount(ip)).toBe(0);// The keyless *credit* budget must not be drawn down by a free// operation. Only meaningful on the isolated forwarded IP β the@@ -312,12 +290,12 @@ describeIf(HAS_RESEARCH)("Research API", () => {);// Failure path: ID enumeration β the exact shape of the incident β mostly- // produces upstream misses, so a miss must record the IP too.+ // produces upstream misses. The charge path must run there too (credits+ // are 0 on every non-2xx path) and stay equally free of DB writes.itIf(config.USE_DB_AUTHENTICATION === true)(- "records the keyless client IP when a paper lookup misses",+ "a keyless paper lookup miss also writes no usage row",async () => {const forwardedIp = "198.51.100.12";- const beforeMaxId = await maxKeylessUsageId();const res = await researchRaw("/v2/search/research/papers/0000.00000",@@ -330,10 +308,8 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).toBeGreaterThanOrEqual(400);const ip = await resolveKeylessIp(forwardedIp);- const row = await waitForKeylessUsageRow(ip, beforeMaxId);-- expect(row).not.toBeNull();- expect(row!.credits_used).toBe(0);+ await sleep(2000);+ expect(await keylessUsageRowCount(ip)).toBe(0);},120000,);diff --git a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts b/apps/api/src/lib/__tests__/keyless-usage-log.test.tsindex 3fe9cc0bb..ebc75bc66 100644--- a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts+++ b/apps/api/src/lib/__tests__/keyless-usage-log.test.ts@@ -38,6 +38,7 @@ import {keylessTeamUuid,logKeylessCreditUsage,} from "../keyless";+import { logger } from "../logger";const IP = "203.0.113.99";const KEYLESS_TEAM = keylessTeamId(IP);@@ -56,15 +57,24 @@ afterEach(() => {});describe("logKeylessCreditUsage", () => {- it("records the client IP for a zero-credit keyless operation", async () => {+ // Zero-credit operations are recorded as a canonical log line for now; the+ // durable `keyless_credit_usage` row is deferred to the firecrawl-db+ // migration (high write volume needs its own review).+ it("logs the client IP for a zero-credit keyless operation without a DB write", async () => {+ const info = vi.spyOn(logger, "info").mockImplementation(() => logger);+await logKeylessCreditUsage(KEYLESS_TEAM, 0);- expect(dbInsert).toHaveBeenCalledTimes(1);- expect(insertValues).toHaveBeenCalledWith({- team_id: keylessTeamUuid(KEYLESS_TEAM),- ip: IP,- credits_used: 0,- });+ expect(dbInsert).not.toHaveBeenCalled();+ expect(info).toHaveBeenCalledWith(+ "Keyless zero-credit usage",+ expect.objectContaining({+ canonicalLog: "keyless/usage",+ ip: IP,+ teamId: keylessTeamUuid(KEYLESS_TEAM),+ creditsUsed: 0,+ }),+ );});it("still records the actual credits for a billable keyless operation", async () => {@@ -75,24 +85,31 @@ describe("logKeylessCreditUsage", () => {);});- it("never writes a negative row for a reconciliation refund", async () => {+ it("treats a reconciliation refund as zero-credit: log line, no negative row", async () => {+ const info = vi.spyOn(logger, "info").mockImplementation(() => logger);+await logKeylessCreditUsage(KEYLESS_TEAM, -5);- expect(insertValues).toHaveBeenCalledWith(- expect.objectContaining({ ip: IP, credits_used: 0 }),+ expect(dbInsert).not.toHaveBeenCalled();+ expect(info).toHaveBeenCalledWith(+ "Keyless zero-credit usage",+ expect.objectContaining({ creditsUsed: 0 }),);});- it("writes nothing for a team that is not keyless", async () => {+ it("writes and logs nothing for a team that is not keyless", async () => {+ const info = vi.spyOn(logger, "info").mockImplementation(() => logger);+await logKeylessCreditUsage("some-real-team-id", 0);expect(dbInsert).not.toHaveBeenCalled();+ expect(info).not.toHaveBeenCalled();});- it("writes nothing when DB auth is off", async () => {+ it("writes no billable row when DB auth is off", async () => {config.USE_DB_AUTHENTICATION = false;- await logKeylessCreditUsage(KEYLESS_TEAM, 0);+ await logKeylessCreditUsage(KEYLESS_TEAM, 3);expect(dbInsert).not.toHaveBeenCalled();});@@ -101,18 +118,22 @@ describe("logKeylessCreditUsage", () => {insertValues.mockRejectedValue(new Error("db down"));await expect(- logKeylessCreditUsage(KEYLESS_TEAM, 0),+ logKeylessCreditUsage(KEYLESS_TEAM, 3),).resolves.toBeUndefined();});});describe("chargeKeylessCredits", () => {it("records a zero-credit request without drawing down the credit budget", async () => {+ const info = vi.spyOn(logger, "info").mockImplementation(() => logger);+await chargeKeylessCredits(KEYLESS_TEAM, 0);expect(redisIncrby).not.toHaveBeenCalled();- expect(insertValues).toHaveBeenCalledWith(- expect.objectContaining({ ip: IP, credits_used: 0 }),+ expect(dbInsert).not.toHaveBeenCalled();+ expect(info).toHaveBeenCalledWith(+ "Keyless zero-credit usage",+ expect.objectContaining({ ip: IP, creditsUsed: 0 }),);});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 1ada0d83a..26d8caee3 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -368,18 +368,22 @@ export async function checkKeylessEligibility(ip: string): Promise<{}/**- * Append a row to `keyless_credit_usage` recording a completed keyless request:- * the per-IP keyless team UUID, the raw IP, and the credits it consumed β the- * only place a keyless caller's IP is persisted (`requests.team_id` collapses- * all keyless traffic onto one preview team).+ * Record a completed keyless request β the only place a keyless caller's IP is+ * persisted (`requests.team_id` collapses all keyless traffic onto one preview+ * team).*- * Zero-credit operations are recorded too. The free Research Index paper- * endpoints bill nothing, so skipping them meant their client IP was never- * written anywhere and abuse on them was invisible. A `credits_used = 0` row- * cannot move any credit total; it only makes the request countable.+ * Billable requests append a row to `keyless_credit_usage` exactly as before.*- * No-op for non-keyless teams or when DB auth is off. Best-effort β never- * throws.+ * Zero-credit operations are recorded too, but as a canonical log line rather+ * than a DB row: the free Research Index paper endpoints bill nothing, so+ * skipping them meant their client IP was never written anywhere and abuse on+ * them was invisible (during the corpus-harvest incident only 3.0% of the+ * traffic had a resolvable IP). Zero-credit volume is orders of magnitude+ * higher than billable volume, so the durable row for it is deferred until the+ * companion firecrawl-db migration lands; until then the log line carries the+ * same fields.+ *+ * No-op for non-keyless teams. Best-effort β never throws.*/export async function logKeylessCreditUsage(teamId: string,@@ -388,14 +392,30 @@ export async function logKeylessCreditUsage(const ip = keylessIpFromTeamId(teamId);if (!ip || !Number.isFinite(credits)) return;const teamUuid = keylessTeamUuid(teamId);- if (config.USE_DB_AUTHENTICATION !== true || !teamUuid) return;+ if (!teamUuid) return;++ // The column records consumption, so a negative reconciliation delta must+ // not land as a negative value.+ const creditsUsed = Math.max(0, Math.ceil(credits));++ if (creditsUsed <= 0) {+ // TODO(firecrawl-db): switch to a `keyless_credit_usage` row once the+ // zero-credit usage migration is merged.+ logger.info("Keyless zero-credit usage", {+ canonicalLog: "keyless/usage",+ ip,+ teamId: teamUuid,+ creditsUsed: 0,+ });+ return;+ }++ if (config.USE_DB_AUTHENTICATION !== true) return;try {await db.insert(schema.keyless_credit_usage).values({team_id: teamUuid,ip,- // The column records consumption, so a negative reconciliation delta must- // not land as a negative row.- credits_used: Math.max(0, Math.ceil(credits)),+ credits_used: creditsUsed,});} catch {// Logging is best-effort.