Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
address review: charge zero-credit usage on validation failures, deterministic keyless IP resolution in snips, refresh stale comments
Karan Lokchandani5a54acff8ed1
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 11e9119b8..eacb69369 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -37,9 +37,9 @@ async function waitForSingleRow<T>(// Keyless requests are keyed on the client IP. When the trusted-proxy secret is// configured we forward a unique documentation-range (RFC 5737) IP so each test-// owns an isolated per-IP bucket; otherwise we fall back to the loopback IP the-// server actually keyed on, recovered from the limiter keyspace the same way-// keyless.test.ts does.+// owns an isolated per-IP bucket; otherwise we fall back to the IP the server+// actually keyed on, identified deterministically by diffing the limiter+// keyspace counters around the request rather than picking an arbitrary key.function keylessHeaders(forwardedIp: string): Record<string, string> {return KEYLESS_PROXY_SECRET? {@@ -49,11 +49,26 @@ function keylessHeaders(forwardedIp: string): Record<string, string> {: {};}-async function resolveKeylessIp(forwardedIp: string): Promise<string> {- if (KEYLESS_PROXY_SECRET) return forwardedIp;+async function snapshotKeylessCounts(): Promise<Map<string, number>> {const keys = await redisRateLimitClient.keys("keyless_requests:*");- expect(keys.length).toBeGreaterThan(0);- return keys[0].slice("keyless_requests:".length);+ const counts = new Map<string, number>();+ for (const key of keys) {+ counts.set(key, Number((await redisRateLimitClient.get(key)) ?? "0"));+ }+ return counts;+}++async function resolveKeylessIp(+ forwardedIp: string,+ before: Map<string, number>,+): Promise<string> {+ if (KEYLESS_PROXY_SECRET) return forwardedIp;+ const after = await snapshotKeylessCounts();+ const bumped = [...after.entries()]+ .filter(([key, count]) => count > (before.get(key) ?? 0))+ .map(([key]) => key.slice("keyless_requests:".length));+ expect(bumped.length).toBe(1);+ return bumped[0];}// Zero-credit keyless usage is recorded as a canonical log line for now (the@@ -259,6 +274,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {"a zero-credit paper search writes no usage row and charges nothing",async () => {const forwardedIp = "198.51.100.11";+ const before = await snapshotKeylessCounts();const res = await researchRaw("/v2/search/research/papers",@@ -270,7 +286,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).toBe(200);expect(res.body.success).toBe(true);- const ip = await resolveKeylessIp(forwardedIp);+ const ip = await resolveKeylessIp(forwardedIp, before);// Give the fire-and-forget charge path time to run before asserting// it stayed out of the DB.await sleep(2000);@@ -296,6 +312,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {"a keyless paper lookup miss also writes no usage row",async () => {const forwardedIp = "198.51.100.12";+ const before = await snapshotKeylessCounts();const res = await researchRaw("/v2/search/research/papers/0000.00000",@@ -307,7 +324,7 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.statusCode).not.toBe(401);expect(res.statusCode).toBeGreaterThanOrEqual(400);- const ip = await resolveKeylessIp(forwardedIp);+ const ip = await resolveKeylessIp(forwardedIp, before);await sleep(2000);expect(await keylessUsageRowCount(ip)).toBe(0);},diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex ecfee6341..5a65ae7de 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -273,6 +273,10 @@ function createResearchController(const parsed = schema.safeParse(source);if (!parsed.success) {logger.warn("Invalid research query", { error: parsed.error.issues });+ // The outer finally below never runs for validation failures, but a+ // malformed keyless request must still leave the canonical keyless/usage+ // trail — ID-enumeration abuse produces plenty of 400s.+ chargeKeylessCredits(authedReq.auth.team_id, 0).catch(() => {});return researchError(res,400,@@ -374,13 +378,15 @@ function createResearchController(} finally {const timeTaken = (Date.now() - started) / 1000;- // No-op for keyed teams. For keyless callers this both charges the per-IP- // daily credit budget and records the client IP in `keyless_credit_usage`.- // It runs on every outcome, not just billable successes: the paper- // endpoints cost 0 credits and ID enumeration mostly produces upstream- // misses, so gating this on `credits > 0` left those requests with no IP- // recorded anywhere. `credits` is still 0 on every non-2xx path, so- // nothing extra is charged.+ // No-op for keyed teams. For keyless callers: billable credits are added+ // to the per-IP daily budget and land as a `keyless_credit_usage` row;+ // zero-credit outcomes emit the canonical `keyless/usage` log line+ // instead (the durable zero-credit row waits on the firecrawl-db+ // migration). It runs on every outcome, not just billable successes: the+ // paper endpoints cost 0 credits and ID enumeration mostly produces+ // upstream misses, so gating this on `credits > 0` left those requests+ // with no IP recorded anywhere. `credits` is still 0 on every non-2xx+ // path, so nothing extra is charged.chargeKeylessCredits(authedReq.auth.team_id, credits).catch(() => {});logResearchEndpoint({diff --git a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts b/apps/api/src/lib/__tests__/keyless-usage-log.test.tsindex ebc75bc66..932422b12 100644--- a/apps/api/src/lib/__tests__/keyless-usage-log.test.ts+++ b/apps/api/src/lib/__tests__/keyless-usage-log.test.ts@@ -3,7 +3,10 @@// end-to-end coverage for the zero-credit Research Index endpoints lives in// snips/v2/research.test.ts, but that suite only runs where a research upstream// is configured. These tests pin the library-level contract everywhere: a-// zero-credit keyless request still writes a row, and it still charges nothing.+// zero-credit keyless request emits the canonical `keyless/usage` log line+// carrying the client IP (the durable `keyless_credit_usage` row is deferred+// to the firecrawl-db migration), writes no DB row, and charges nothing;+// billable requests keep writing rows exactly as before.const { dbInsert, insertValues, redisIncrby, redisExpire } = vi.hoisted(() => {const insertValues = vi.fn().mockResolvedValue(undefined);return {