Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/keyless): restore keyless_credit_usage audit logging after reserve refactor

mogerycec5a26916c3
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index d79c91507..fa8fb776e 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -8,6 +8,9 @@ import {
scrapeTimeout,
} from "../lib";
import { redisRateLimitClient } from "../../../services/rate-limiter";
+import { db } from "../../../db/connection";
+import * as schema from "../../../db/schema";
+import { and, desc, eq, gt } from "drizzle-orm";
import request from "supertest";
// The keyless tier is disabled unless both limits are configured. The harness
@@ -359,6 +362,63 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
scrapeTimeout,
);
+ it(
+ "writes a keyless_credit_usage audit row for a successful keyless scrape (200)",
+ async () => {
+ // The audit insert is best-effort and fire-and-forget in the controller,
+ // so capture the latest id first, then poll for a newer row afterwards.
+ const beforeMaxId = (
+ await db
+ .select({ id: schema.keyless_credit_usage.id })
+ .from(schema.keyless_credit_usage)
+ .orderBy(desc(schema.keyless_credit_usage.id))
+ .limit(1)
+ )[0]?.id;
+
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ formats: ["markdown"],
+ });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ const creditsUsed = response.body.data.metadata.creditsUsed;
+ expect(creditsUsed).toBeGreaterThan(0);
+
+ const ip = await currentKeylessIp();
+
+ let row: typeof schema.keyless_credit_usage.$inferSelect | undefined;
+ for (let i = 0; i < 20; i++) {
+ const rows = await db
+ .select()
+ .from(schema.keyless_credit_usage)
+ .where(
+ and(
+ eq(schema.keyless_credit_usage.ip, ip),
+ beforeMaxId !== undefined
+ ? gt(schema.keyless_credit_usage.id, beforeMaxId)
+ : undefined,
+ ),
+ )
+ .orderBy(desc(schema.keyless_credit_usage.id))
+ .limit(1);
+ if (rows.length > 0) {
+ row = rows[0];
+ break;
+ }
+ await new Promise(resolve => setTimeout(resolve, 250));
+ }
+
+ expect(row).toBeDefined();
+ expect(row!.credits_used).toBe(creditsUsed);
+ },
+ scrapeTimeout,
+ );
+
it(
"rejects projected keyless search with scrape options before search (429)",
async () => {
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 3a1452c59..93c22669f 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -26,6 +26,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
@@ -306,10 +307,14 @@ export async function scrapeController(
if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
reconciledKeylessCredits = true;
+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
adjustKeylessCredits(
req.auth.team_id,
- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ actualKeylessCredits - reservedKeylessCredits,
).catch(() => {});
+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
+ () => {},
+ );
}
const totalRequestTime = new Date().getTime() - middlewareStartTime;
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 0b4f37a19..394ed6a73 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -30,6 +30,7 @@ import { getSearchForcedKind } from "../../lib/zdr-helpers";
import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
@@ -254,6 +255,9 @@ export async function searchController(
req.auth.team_id,
result.totalCredits - reservedKeylessCredits,
).catch(() => {});
+ logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch(
+ () => {},
+ );
}
const endTime = new Date().getTime();
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index a39da3297..85fc0117f 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -29,6 +29,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
@@ -656,10 +657,14 @@ export async function parseController(
if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
reconciledKeylessCredits = true;
+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
adjustKeylessCredits(
req.auth.team_id,
- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ actualKeylessCredits - reservedKeylessCredits,
).catch(() => {});
+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
+ () => {},
+ );
}
const totalRequestTime = new Date().getTime() - middlewareStartTime;
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 6bb067646..c54cbc110 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -49,6 +49,7 @@ import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
keylessTeamUuid,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
@@ -482,6 +483,7 @@ export async function scrapeStopInteractiveBrowserController(
adjustKeylessCredits(req.auth.team_id, creditsBilled - reservedCredits).catch(
() => {},
);
+ logKeylessCreditUsage(req.auth.team_id, creditsBilled).catch(() => {});
logger.info("Browser session destroyed", {
sessionDurationMs: durationMs,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index b99bbf2b0..d779a9005 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -27,6 +27,7 @@ import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
@@ -447,10 +448,14 @@ export async function scrapeController(
if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
reconciledKeylessCredits = true;
+ const actualKeylessCredits = doc?.metadata?.creditsUsed ?? 0;
adjustKeylessCredits(
req.auth.team_id,
- (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ actualKeylessCredits - reservedKeylessCredits,
).catch(() => {});
+ logKeylessCreditUsage(req.auth.team_id, actualKeylessCredits).catch(
+ () => {},
+ );
}
const totalRequestTime = new Date().getTime() - middlewareStartTime;
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index b661eef42..05331a269 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -10,6 +10,7 @@ import { billTeam } from "../../services/billing/credit_billing";
import {
KEYLESS_CREDITS_MESSAGE,
adjustKeylessCredits,
+ logKeylessCreditUsage,
reserveKeylessCredits,
} from "../../lib/keyless";
import { v7 as uuidv7 } from "uuid";
@@ -212,6 +213,9 @@ export async function searchController(
req.auth.team_id,
result.totalCredits - reservedKeylessCredits,
).catch(() => {});
+ logKeylessCreditUsage(req.auth.team_id, result.totalCredits).catch(
+ () => {},
+ );
}
const endTime = new Date().getTime();
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index 9b63063c7..665ef558f 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -249,9 +249,36 @@ export async function checkKeylessEligibility(
}
}
+/**
+ * Append a row to `keyless_credit_usage` recording the actual credits a completed
+ * keyless request consumed (per-IP keyless team UUID + raw IP), for abuse
+ * monitoring. No-op for non-keyless teams, non-positive credits, or when DB auth
+ * is off. Best-effort — never throws.
+ */
+export async function logKeylessCreditUsage(
+ teamId: string,
+ credits: number,
+): Promise<void> {
+ const ip = keylessIpFromTeamId(teamId);
+ if (!ip || !Number.isFinite(credits) || credits <= 0) return;
+ const teamUuid = keylessTeamUuid(teamId);
+ if (config.USE_DB_AUTHENTICATION !== true || !teamUuid) return;
+ try {
+ await db.insert(schema.keyless_credit_usage).values({
+ team_id: teamUuid,
+ ip,
+ credits_used: Math.ceil(credits),
+ });
+ } catch {
+ // Logging is best-effort.
+ }
+}
+
/**
* Add the actual credits a completed request consumed to the IP's daily credit
- * counter. No-op for non-keyless teams. Best-effort; never throws.
+ * counter. No-op for non-keyless teams. Best-effort; never throws. Used by the
+ * worker for the non-reserved path; the controllers reserve up front and call
+ * `logKeylessCreditUsage` directly at reconciliation.
*/
export async function chargeKeylessCredits(
teamId: string,
@@ -271,18 +298,6 @@ export async function chargeKeylessCredits(
// extra free credits today.
}
- // Log the usage to keyless_credit_usage (per-IP keyless team UUID + raw IP)
- // for abuse monitoring. Best-effort — never block the request.
- const teamUuid = keylessTeamUuid(teamId);
- if (config.USE_DB_AUTHENTICATION === true && teamUuid) {
- try {
- await db.insert(schema.keyless_credit_usage).values({
- team_id: teamUuid,
- ip,
- credits_used: inc,
- });
- } catch {
- // Logging is best-effort.
- }
- }
+ // Log the usage to keyless_credit_usage for abuse monitoring. Best-effort.
+ await logKeylessCreditUsage(teamId, credits);
}