Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
address review round 2: gate zero-credit log behind DB auth, surface IP in message body, isolate snips row assertions
Karan Lokchandani96a2b46082f8
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex eacb69369..6d46b49e7 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -1,7 +1,7 @@import { config } from "../../../config";import { describeIf, itIf, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchRaw } from "./lib";-import { and, desc, eq } from "drizzle-orm";+import { and, desc, eq, gt } from "drizzle-orm";import { db } from "../../../db/connection";import * as schema from "../../../db/schema";import { redisRateLimitClient } from "../../../services/rate-limiter";@@ -75,11 +75,31 @@ async function resolveKeylessIp(// durable keyless_credit_usage row waits on the firecrawl-db migration), so// these tests assert no row is written and no credit budget moves. The// library-level log-line contract is pinned in lib/__tests__/keyless-usage-log.-async function keylessUsageRowCount(ip: string): Promise<number> {+// Row assertions are isolated against rows accumulated by earlier runs (the+// pre-log-only version of these tests wrote durable rows for these very IPs):+// snapshot the max id up front and only count rows past it.+async function maxKeylessUsageRowId(): Promise<number> {const rows = await db.select({ id: schema.keyless_credit_usage.id }).from(schema.keyless_credit_usage)- .where(eq(schema.keyless_credit_usage.ip, ip));+ .orderBy(desc(schema.keyless_credit_usage.id))+ .limit(1);+ return rows[0]?.id ?? 0;+}++async function keylessUsageRowsAfter(+ ip: string,+ afterId: number,+): Promise<number> {+ const rows = await db+ .select({ id: schema.keyless_credit_usage.id })+ .from(schema.keyless_credit_usage)+ .where(+ and(+ eq(schema.keyless_credit_usage.ip, ip),+ gt(schema.keyless_credit_usage.id, afterId),+ ),+ );return rows.length;}@@ -275,6 +295,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {async () => {const forwardedIp = "198.51.100.11";const before = await snapshotKeylessCounts();+ const afterId = await maxKeylessUsageRowId();const res = await researchRaw("/v2/search/research/papers",@@ -290,7 +311,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {// Give the fire-and-forget charge path time to run before asserting// it stayed out of the DB.await sleep(2000);- expect(await keylessUsageRowCount(ip)).toBe(0);+ expect(await keylessUsageRowsAfter(ip, afterId)).toBe(0);// The keyless *credit* budget must not be drawn down by a free// operation. Only meaningful on the isolated forwarded IP — the@@ -313,6 +334,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {async () => {const forwardedIp = "198.51.100.12";const before = await snapshotKeylessCounts();+ const afterId = await maxKeylessUsageRowId();const res = await researchRaw("/v2/search/research/papers/0000.00000",@@ -326,7 +348,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {const ip = await resolveKeylessIp(forwardedIp, before);await sleep(2000);- expect(await keylessUsageRowCount(ip)).toBe(0);+ expect(await keylessUsageRowsAfter(ip, afterId)).toBe(0);},120000,);diff --git a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts b/apps/api/src/lib/__tests__/keyless-usage-log.test.tsindex 932422b12..461481334 100644--- a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts+++ b/apps/api/src/lib/__tests__/keyless-usage-log.test.ts@@ -69,8 +69,11 @@ describe("logKeylessCreditUsage", () => {await logKeylessCreditUsage(KEYLESS_TEAM, 0);expect(dbInsert).not.toHaveBeenCalled();+ // The IP rides in the message body (console transports only serialize+ // metadata for warn/error) AND in the structured fields (the Cloud+ // Logging query contract).expect(info).toHaveBeenCalledWith(- "Keyless zero-credit usage",+ expect.stringContaining(`Keyless zero-credit usage ip=${IP}`),expect.objectContaining({canonicalLog: "keyless/usage",ip: IP,@@ -80,6 +83,16 @@ describe("logKeylessCreditUsage", () => {);});+ it("logs no zero-credit line when DB auth is off (same gate as billable rows)", async () => {+ config.USE_DB_AUTHENTICATION = false;+ const info = vi.spyOn(logger, "info").mockImplementation(() => logger);++ await logKeylessCreditUsage(KEYLESS_TEAM, 0);++ expect(dbInsert).not.toHaveBeenCalled();+ expect(info).not.toHaveBeenCalled();+ });+it("still records the actual credits for a billable keyless operation", async () => {await logKeylessCreditUsage(KEYLESS_TEAM, 2.1);@@ -95,7 +108,7 @@ describe("logKeylessCreditUsage", () => {expect(dbInsert).not.toHaveBeenCalled();expect(info).toHaveBeenCalledWith(- "Keyless zero-credit usage",+ expect.stringContaining("Keyless zero-credit usage"),expect.objectContaining({ creditsUsed: 0 }),);});@@ -135,7 +148,7 @@ describe("chargeKeylessCredits", () => {expect(redisIncrby).not.toHaveBeenCalled();expect(dbInsert).not.toHaveBeenCalled();expect(info).toHaveBeenCalledWith(- "Keyless zero-credit usage",+ expect.stringContaining("Keyless zero-credit usage"),expect.objectContaining({ ip: IP, creditsUsed: 0 }),);});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 26d8caee3..07c551b20 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -398,10 +398,17 @@ export async function logKeylessCreditUsage(// not land as a negative value.const creditsUsed = Math.max(0, Math.ceil(credits));+ // Both branches sit behind the same gate as every other keyless usage+ // record: self-hosted deployments without DB auth track nothing.+ if (config.USE_DB_AUTHENTICATION !== true) return;+if (creditsUsed <= 0) {// TODO(firecrawl-db): switch to a `keyless_credit_usage` row once the- // zero-credit usage migration is merged.- logger.info("Keyless zero-credit usage", {+ // zero-credit usage migration is merged. The IP is repeated in the+ // message body because the console transport only serializes metadata+ // for warn/error lines; the structured fields are the contract for+ // Cloud Logging queries.+ logger.info(`Keyless zero-credit usage ip=${ip} team=${teamUuid}`, {canonicalLog: "keyless/usage",ip,teamId: teamUuid,@@ -410,7 +417,6 @@ export async function logKeylessCreditUsage(return;}- if (config.USE_DB_AUTHENTICATION !== true) return;try {await db.insert(schema.keyless_credit_usage).values({team_id: teamUuid,