Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/keyless): restore keyless_credit_usage audit logging after reserve refactor
mogerycec5a26916c3
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex d79c91507..fa8fb776e 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -8,6 +8,9 @@ import {scrapeTimeout,} from "../lib";import { redisRateLimitClient } from "../../../services/rate-limiter";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";+import { and, desc, eq, gt } from "drizzle-orm";import request from "supertest";// The keyless tier is disabled unless both limits are configured. The harness@@ -359,6 +362,63 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {scrapeTimeout,);+ it(+ "writes a keyless_credit_usage audit row for a successful keyless scrape (200)",+ async () => {+ // The audit insert is best-effort and fire-and-forget in the controller,+ // so capture the latest id first, then poll for a newer row afterwards.+ const beforeMaxId = (+ await db+ .select({ id: schema.keyless_credit_usage.id })+ .from(schema.keyless_credit_usage)+ .orderBy(desc(schema.keyless_credit_usage.id))+ .limit(1)+ )[0]?.id;++ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ formats: ["markdown"],+ });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ const creditsUsed = response.body.data.metadata.creditsUsed;+ expect(creditsUsed).toBeGreaterThan(0);++ const ip = await currentKeylessIp();++ let row: typeof schema.keyless_credit_usage.$inferSelect | undefined;+ for (let i = 0; i < 20; i++) {+ const rows = await db+ .select()+ .from(schema.keyless_credit_usage)+ .where(+ and(+ eq(schema.keyless_credit_usage.ip, ip),+ beforeMaxId !== undefined+ ? gt(schema.keyless_credit_usage.id, beforeMaxId)+ : undefined,+ ),+ )+ .orderBy(desc(schema.keyless_credit_usage.id))+ .limit(1);+ if (rows.length > 0) {+ row = rows[0];+ break;+ }+ await new Promise(resolve => setTimeout(resolve, 250));+ }++ expect(row).toBeDefined();+ expect(row!.credits_used).toBe(creditsUsed);+ },+ scrapeTimeout,+ );+it("rejects projected keyless search with scrape options before search (429)",async () => {diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 3a1452c59..93c22669f 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -26,6 +26,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { projectScrapeCredits } from "../../lib/keyless-credit-projection";@@ -306,10 +307,14 @@ export async function scrapeController(if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {reconciledKeylessCredits = true;+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;adjustKeylessCredits(req.auth.team_id,- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ actualKeylessCredits - reservedKeylessCredits,).catch(() => {});+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(+ () => {},+ );}const totalRequestTime = new Date().getTime() - middlewareStartTime;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 0b4f37a19..394ed6a73 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -30,6 +30,7 @@ import { getSearchForcedKind } from "../../lib/zdr-helpers";import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";@@ -254,6 +255,9 @@ export async function searchController(req.auth.team_id,result.totalCredits - reservedKeylessCredits,).catch(() => {});+ logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch(+ () => {},+ );}const endTime = new Date().getTime();diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex a39da3297..85fc0117f 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -29,6 +29,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { projectScrapeCredits } from "../../lib/keyless-credit-projection";@@ -656,10 +657,14 @@ export async function parseController(if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {reconciledKeylessCredits = true;+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;adjustKeylessCredits(req.auth.team_id,- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ actualKeylessCredits - reservedKeylessCredits,).catch(() => {});+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(+ () => {},+ );}const totalRequestTime = new Date().getTime() - middlewareStartTime;diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 6bb067646..c54cbc110 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -49,6 +49,7 @@ import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,keylessTeamUuid,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";@@ -482,6 +483,7 @@ export async function scrapeStopInteractiveBrowserController(adjustKeylessCredits(req.auth.team_id, creditsBilled - reservedCredits).catch(() => {},);+ logKeylessCreditUsage(req.auth.team_id, creditsBilled).catch(() => {});logger.info("Browser session destroyed", {sessionDurationMs: durationMs,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b99bbf2b0..d779a9005 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -27,6 +27,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { projectScrapeCredits } from "../../lib/keyless-credit-projection";@@ -447,10 +448,14 @@ export async function scrapeController(if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {reconciledKeylessCredits = true;+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;adjustKeylessCredits(req.auth.team_id,- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ actualKeylessCredits - reservedKeylessCredits,).catch(() => {});+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(+ () => {},+ );}const totalRequestTime = new Date().getTime() - middlewareStartTime;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex b661eef42..05331a269 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -10,6 +10,7 @@ import { billTeam } from "../../services/billing/credit_billing";import {KEYLESS_CREDITS_MESSAGE,adjustKeylessCredits,+ logKeylessCreditUsage,reserveKeylessCredits,} from "../../lib/keyless";import { v7 as uuidv7 } from "uuid";@@ -212,6 +213,9 @@ export async function searchController(req.auth.team_id,result.totalCredits - reservedKeylessCredits,).catch(() => {});+ logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch(+ () => {},+ );}const endTime = new Date().getTime();diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex 9b63063c7..665ef558f 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -249,9 +249,36 @@ export async function checkKeylessEligibility(}}+/**+ * Append a row to `keyless_credit_usage` recording the actual credits a completed+ * keyless request consumed (per-IP keyless team UUID + raw IP), for abuse+ * monitoring. No-op for non-keyless teams, non-positive credits, or when DB auth+ * is off. Best-effort — never throws.+ */+export async function logKeylessCreditUsage(+ teamId: string,+ credits: number,+): Promise<void> {+ const ip = keylessIpFromTeamId(teamId);+ if (!ip || !Number.isFinite(credits) || credits <= 0) return;+ const teamUuid = keylessTeamUuid(teamId);+ if (config.USE_DB_AUTHENTICATION !== true || !teamUuid) return;+ try {+ await db.insert(schema.keyless_credit_usage).values({+ team_id: teamUuid,+ ip,+ credits_used: Math.ceil(credits),+ });+ } catch {+ // Logging is best-effort.+ }+}+/*** Add the actual credits a completed request consumed to the IP's daily credit- * counter. No-op for non-keyless teams. Best-effort; never throws.+ * counter. No-op for non-keyless teams. Best-effort; never throws. Used by the+ * worker for the non-reserved path; the controllers reserve up front and call+ * `logKeylessCreditUsage` directly at reconciliation.*/export async function chargeKeylessCredits(teamId: string,@@ -271,18 +298,6 @@ export async function chargeKeylessCredits(// extra free credits today.}- // Log the usage to keyless_credit_usage (per-IP keyless team UUID + raw IP)- // for abuse monitoring. Best-effort — never block the request.- const teamUuid = keylessTeamUuid(teamId);- if (config.USE_DB_AUTHENTICATION === true && teamUuid) {- try {- await db.insert(schema.keyless_credit_usage).values({- team_id: teamUuid,- ip,- credits_used: inc,- });- } catch {- // Logging is best-effort.- }- }+ // Log the usage to keyless_credit_usage for abuse monitoring. Best-effort.+ await logKeylessCreditUsage(teamId, credits);}