Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
[codex] Reserve keyless credits atomically (#3796)
Gergő Móriczef29f7f25854
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.tsindex 22e73a4c7..d79c91507 100644--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts@@ -22,9 +22,8 @@ const KEYLESS_ENABLED =Number.isFinite(KEYLESS_CREDITS_PER_DAY) &&KEYLESS_CREDITS_PER_DAY > 0;-// Keyless free tier: scrape, parse, search, and interact work without an API key-// from the official MCP server (origin "mcp*"), CLI (integration "cli"), or SDKs-// (origin "<lang>-sdk@..."). Gated per-IP/day by a request cap AND a credit cap.+// Keyless free tier: scrape, parse, search, and interact work without an API key.+// Gated per-IP/day by a request cap AND a credit cap.// These tests are the only ones that exercise keyless mode, so they exclusively// own the `keyless_*` rate-limit keyspace — we flush it before each test (and run// sequentially) for isolation. `req.ip` is loopback under supertest, so the@@ -203,6 +202,254 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {expect(blocked.body.error).toContain("unauthenticated credits");});+ it(+ "rejects projected multi-credit keyless parse before parsing (429)",+ async () => {+ await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v2/parse")+ .field(+ "options",+ JSON.stringify({+ origin: "mcp",+ proxy: "basic",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { title: { type: "string" } },+ },+ },+ ],+ }),+ )+ .attach("file", Buffer.from("<h1>Blocked parse</h1>"), {+ filename: "blocked.html",+ contentType: "text/html",+ });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "rejects projected multi-credit keyless scrape before scraping (429)",+ async () => {+ await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ proxy: "basic",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { title: { type: "string" } },+ },+ },+ ],+ });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "rejects projected multi-credit v1 keyless scrape before scraping (429)",+ async () => {+ await request(TEST_API_URL)+ .post("/v1/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v1/scrape")+ .set("Content-Type", "application/json")+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ proxy: "basic",+ formats: ["json"],+ jsonOptions: {+ schema: {+ type: "object",+ properties: { title: { type: "string" } },+ },+ },+ });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "reconciles keyless scrape reservation to actual credits (200)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "mcp",+ proxy: "basic",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { title: { type: "string" } },+ },+ },+ ],+ });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.data.metadata.creditsUsed).toBe(5);++ const ip = await currentKeylessIp();+ const creditsUsed = Number(+ await redisRateLimitClient.get(`keyless_credits:${ip}`),+ );+ expect(creditsUsed).toBe(response.body.data.metadata.creditsUsed);+ },+ scrapeTimeout,+ );++ it(+ "rejects projected keyless search with scrape options before search (429)",+ async () => {+ await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v2/search")+ .set("Content-Type", "application/json")+ .send({+ query: "firecrawl",+ limit: 1,+ origin: "mcp",+ scrapeOptions: {+ proxy: "basic",+ formats: ["markdown"],+ },+ });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "rejects projected v1 keyless search with scrape options before search (429)",+ async () => {+ await request(TEST_API_URL)+ .post("/v1/scrape")+ .set("Content-Type", "application/json")+ .send({ origin: "mcp" });+ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post("/v1/search")+ .set("Content-Type", "application/json")+ .send({+ query: "firecrawl",+ limit: 1,+ origin: "mcp",+ scrapeOptions: {+ proxy: "basic",+ formats: ["markdown"],+ },+ });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );++ it(+ "reconciles keyless search reservation to actual credits (200)",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/search")+ .set("Content-Type", "application/json")+ .send({ query: "firecrawl", limit: 1, origin: "mcp" });++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.creditsUsed).toBe("number");++ const ip = await currentKeylessIp();+ const creditsUsed = Number(+ await redisRateLimitClient.get(`keyless_credits:${ip}`),+ );+ expect(creditsUsed).toBe(response.body.creditsUsed);+ },+ scrapeTimeout,+ );+itIf(!!process.env.KEYLESS_PROXY_SECRET)("rate-limits per forwarded client IP when the proxy secret is provided",async () => {@@ -311,6 +558,43 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {scrapeTimeout,);+ itIf(!!config.BROWSER_SERVICE_URL)(+ "rejects projected keyless interact session creation before browser work (429)",+ async () => {+ const scrapeResponse = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({+ url: TEST_SUITE_WEBSITE,+ origin: "website-keyless-interact-test",+ proxy: "basic",+ formats: ["markdown"],+ });++ expect(scrapeResponse.statusCode).toBe(200);+ expect(scrapeResponse.body.success).toBe(true);+ expect(typeof scrapeResponse.body.scrape_id).toBe("string");++ const ip = await currentKeylessIp();+ await redisRateLimitClient.set(+ `keyless_credits:${ip}`,+ String(KEYLESS_CREDITS_PER_DAY - 1),+ );++ const blocked = await request(TEST_API_URL)+ .post(`/v2/scrape/${scrapeResponse.body.scrape_id}/interact`)+ .set("Content-Type", "application/json")+ .send({ code: "return 1;", origin: "mcp" });++ expect(blocked.statusCode).toBe(429);+ expect(blocked.body.error).toContain("unauthenticated credits");+ expect(blocked.headers["www-authenticate"]).toContain(+ "resource_metadata",+ );+ },+ scrapeTimeout,+ );+it("allows keyless scrape from an mcp origin (200)",async () => {diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex f07a6c891..81eaeb915 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -9,6 +9,7 @@ import { getAgentSponsorStatus } from "../services/agent-sponsor";import { getRedisConnection } from "../services/queue-service";import { getRateLimiter } from "../services/rate-limiter";import {+ KEYLESS_CREDITS_MESSAGE,consumeKeylessRequest,isKeylessConfigured,isKeylessIpEligible,@@ -471,8 +472,6 @@ export async function clearACUCTeam(team_id: string): Promise<void> {const KEYLESS_REQUESTS_MESSAGE = `You've reached today's limit of free, unauthenticated requests to Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;-const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;-const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 4b07896d9..3a1452c59 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -23,6 +23,13 @@ import { logRequest } from "../../services/logging/log_job";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ reserveKeylessCredits,+} from "../../lib/keyless";+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -96,6 +103,30 @@ export async function scrapeController(req.body.timeout,req.auth.team_id,);+ const projectedKeylessCredits = !isDirectToBullMQ+ ? projectScrapeCredits(+ scrapeOptions,+ req.acuc?.flags ?? null,+ zeroDataRetention ?? false,+ )+ : 0;+ let reservedKeylessCredits = 0;+ let reconciledKeylessCredits = false;++ if (projectedKeylessCredits > 0) {+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ projectedKeylessCredits,+ );+ if (!reservation.ok) {+ applyAgentAuthDiscoveryHeader(res);+ return res.status(429).json({+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ });+ }+ reservedKeylessCredits = projectedKeylessCredits;+ }const totalWait =(req.body.waitFor ?? 0) +@@ -171,6 +202,7 @@ export async function scrapeController(zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,+ keylessReserved: reservedKeylessCredits > 0,logRequestPromise: logRequestPromise,},};@@ -180,6 +212,13 @@ export async function scrapeController(},);} catch (e) {+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(+ () => {},+ );+ }+const timeoutErr =e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";@@ -265,6 +304,14 @@ export async function scrapeController(}}+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(+ req.auth.team_id,+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ ).catch(() => {});+ }+const totalRequestTime = new Date().getTime() - middlewareStartTime;const controllerTime = new Date().getTime() - controllerStartTime;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 692fd3482..0b4f37a19 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -27,6 +27,13 @@ import {} from "../../search/transform";import { fromV1ScrapeOptions } from "../v2/types";import { getSearchForcedKind } from "../../lib/zdr-helpers";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ reserveKeylessCredits,+} from "../../lib/keyless";+import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";// Used for deep researchexport async function searchAndScrapeSearchResult(@@ -108,6 +115,8 @@ export async function searchController(const isSearchPreview =config.SEARCH_PREVIEW_TOKEN !== undefined &&config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ let reservedKeylessCredits = 0;+ let reconciledKeylessCredits = false;try {req.body = searchRequestSchema.parse(req.body);@@ -141,6 +150,32 @@ export async function searchController(const shouldScrape =req.body.scrapeOptions.formats &&req.body.scrapeOptions.formats.length > 0;+ const projectedKeylessCredits = !isSearchPreview+ ? projectSearchTotalCredits(+ {+ limit: req.body.limit,+ enterprise: teamEnterprise,+ scrapeOptions: shouldScrape ? scrapeOptions : undefined,+ },+ req.acuc?.flags ?? null,+ zeroDataRetention,+ )+ : 0;++ if (projectedKeylessCredits > 0) {+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ projectedKeylessCredits,+ );+ if (!reservation.ok) {+ applyAgentAuthDiscoveryHeader(res);+ return res.status(429).json({+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ });+ }+ reservedKeylessCredits = projectedKeylessCredits;+ }// Execute search using v2 logicconst result = await executeSearch(@@ -168,6 +203,7 @@ export async function searchController(bypassBilling: false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ keylessReserved: reservedKeylessCredits > 0,},logger,);@@ -212,6 +248,14 @@ export async function searchController(});}+ if (reservedKeylessCredits > 0) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(+ req.auth.team_id,+ result.totalCredits - reservedKeylessCredits,+ ).catch(() => {});+ }+const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;@@ -255,6 +299,13 @@ export async function searchController(return res.status(200).json(responseData);} catch (error) {+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(+ () => {},+ );+ }+if (error instanceof z.ZodError) {logger.warn("Invalid request body", { error: error.issues });return res.status(400).json({diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 34539c11e..a39da3297 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -26,7 +26,13 @@ import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";-import { chargeKeylessCredits } from "../../lib/keyless";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ reserveKeylessCredits,+} from "../../lib/keyless";+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import path from "node:path";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -349,6 +355,34 @@ export async function parseController(const agentRequestId = req.body.__agentInterop?.requestId ?? null;const boostConcurrency =req.body.__agentInterop?.boostConcurrency ?? false;+ const isDirectToBullMQ =+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ const projectedKeylessCredits =+ shouldBill && !isDirectToBullMQ+ ? projectScrapeCredits(+ req.body,+ req.acuc?.flags ?? null,+ zeroDataRetention,+ )+ : 0;+ let reservedKeylessCredits = 0;+ let reconciledKeylessCredits = false;++ if (projectedKeylessCredits > 0) {+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ projectedKeylessCredits,+ );+ if (!reservation.ok) {+ applyAgentAuthDiscoveryHeader(res);+ return res.status(429).json({+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ });+ }+ reservedKeylessCredits = projectedKeylessCredits;+ }const logger = _logger.child({method: "parseController",@@ -395,10 +429,6 @@ export async function parseController(const origin = req.body.origin;const timeout = req.body.timeout;- const isDirectToBullMQ =- config.SEARCH_PREVIEW_TOKEN !== undefined &&- config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;-const totalWait = 0;let lockTime: number | null = null;@@ -494,6 +524,7 @@ export async function parseController(zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,+ keylessReserved: reservedKeylessCredits > 0,requestId: agentRequestId ?? undefined,},};@@ -512,6 +543,13 @@ export async function parseController(},);} catch (e) {+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(+ () => {},+ );+ }+const timeoutErr =e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";@@ -616,6 +654,14 @@ export async function parseController(}}+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(+ req.auth.team_id,+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ ).catch(() => {});+ }+const totalRequestTime = new Date().getTime() - middlewareStartTime;const controllerTime = new Date().getTime() - controllerStartTime;@@ -663,11 +709,6 @@ export async function parseController(concurrencyQueueDurationMs: lockTime || undefined,});- chargeKeylessCredits(- req.auth.team_id,- doc?.metadata?.creditsUsed ?? 0,- ).catch(() => {});-return res.status(200).json({success: true,data: {diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 8056e9d1b..6bb067646 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -45,7 +45,12 @@ import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";import { getScrapeZDR } from "../../lib/zdr-helpers";import { RequestWithAuth, ScrapeOptions } from "./types";import { billTeam } from "../../services/billing/credit_billing";-import { chargeKeylessCredits, keylessTeamUuid } from "../../lib/keyless";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ keylessTeamUuid,+ reserveKeylessCredits,+} from "../../lib/keyless";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { logRequest } from "../../services/logging/log_job";import { integrationSchema } from "../../utils/integration";@@ -56,6 +61,7 @@ import {calculateBrowserSessionCredits,} from "../../lib/browser-billing";import { autumnService } from "../../services/autumn/autumn.service";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";// ---------------------------------------------------------------------------// Schemas@@ -200,6 +206,12 @@ export async function scrapeInteractController((scrape.options as ScrapeOptions).profile,);if ("error" in created) {+ if (+ created.status === 429 &&+ created.body.error === KEYLESS_CREDITS_MESSAGE+ ) {+ applyAgentAuthDiscoveryHeader(res);+ }return res.status(created.status).json(created.body);}session = created.session;@@ -463,9 +475,13 @@ export async function scrapeStopInteractiveBrowserController(});});- // Charge the keyless free tier's per-IP daily credit budget (no-op for- // non-keyless teams).- chargeKeylessCredits(req.auth.team_id, creditsBilled).catch(() => {});+ const reservedCredits = calculateBrowserSessionCredits(+ session.ttl_total * 1000,+ BROWSER_CREDITS_PER_HOUR,+ );+ adjustKeylessCredits(req.auth.team_id, creditsBilled - reservedCredits).catch(+ () => {},+ );logger.info("Browser session destroyed", {sessionDurationMs: durationMs,@@ -525,6 +541,22 @@ async function createSessionForScrape(// Credit check (uses base rate — actual billing may be higher if prompts are used)const estimatedCredits = calculateBrowserSessionCredits(ttl * 1000);+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ estimatedCredits,+ );+ if (!reservation.ok) {+ return {+ status: 429,+ body: {+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ },+ error: true,+ };+ }+ const keylessReserved = estimatedCredits;+const autumnResult = await autumnService.checkCredits({teamId: req.auth.team_id,value: estimatedCredits,@@ -532,6 +564,7 @@ async function createSessionForScrape(});if (autumnResult !== null && !autumnResult.allowed) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});return {status: 402,body: {@@ -546,6 +579,7 @@ async function createSessionForScrape(const concurrencyLimit = req.acuc?.concurrency ?? 2;const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);if (activeCount >= concurrencyLimit) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});return {status: 429,body: {@@ -587,6 +621,9 @@ async function createSessionForScrape(break;} catch (err) {if (err instanceof BrowserServiceError && err.status === 409) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(+ () => {},+ );return {status: 409,body: {@@ -610,6 +647,7 @@ async function createSessionForScrape(}if (!svcResponse) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});logger.error("Failed to create browser session after all retries", {error: lastCreateError,});@@ -695,6 +733,7 @@ async function createSessionForScrape();}} catch (err) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});logger.error("Failed to initialize scrape browser session context", {error: err,});@@ -753,6 +792,7 @@ async function createSessionForScrape(return { session };} catch (err) {+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});logger.error("Failed to persist browser session, cleaning up", {error: err,});diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 2b6df4743..b99bbf2b0 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -24,6 +24,13 @@ import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ reserveKeylessCredits,+} from "../../lib/keyless";+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -111,6 +118,34 @@ export async function scrapeController(const agentRequestId = req.body.__agentInterop?.requestId ?? null;const boostConcurrency =req.body.__agentInterop?.boostConcurrency ?? false;+ const isDirectToBullMQ =+ config.SEARCH_PREVIEW_TOKEN !== undefined &&+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;+ const projectedKeylessCredits =+ shouldBill && !isDirectToBullMQ+ ? projectScrapeCredits(+ req.body,+ req.acuc?.flags ?? null,+ zeroDataRetention,+ )+ : 0;+ let reservedKeylessCredits = 0;+ let reconciledKeylessCredits = false;++ if (projectedKeylessCredits > 0) {+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ projectedKeylessCredits,+ );+ if (!reservation.ok) {+ applyAgentAuthDiscoveryHeader(res);+ return res.status(429).json({+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ });+ }+ reservedKeylessCredits = projectedKeylessCredits;+ }const logger = _logger.child({method: "scrapeController",@@ -159,10 +194,6 @@ export async function scrapeController(const origin = req.body.origin;const timeout = req.body.timeout;- const isDirectToBullMQ =- config.SEARCH_PREVIEW_TOKEN !== undefined &&- config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;-const totalWait =(req.body.waitFor ?? 0) +(req.body.actions ?? []).reduce(@@ -261,6 +292,7 @@ export async function scrapeController(zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,+ keylessReserved: reservedKeylessCredits > 0,requestId: agentRequestId ?? undefined,logRequestPromise: logRequestPromise,},@@ -280,6 +312,13 @@ export async function scrapeController(},);} catch (e) {+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(+ () => {},+ );+ }+const timeoutErr =e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";@@ -406,6 +445,14 @@ export async function scrapeController(}}+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(+ req.auth.team_id,+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,+ ).catch(() => {});+ }+const totalRequestTime = new Date().getTime() - middlewareStartTime;const controllerTime = new Date().getTime() - controllerStartTime;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 1db7b506f..b661eef42 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -7,7 +7,11 @@ import {searchRequestSchema,} from "./types";import { billTeam } from "../../services/billing/credit_billing";-import { chargeKeylessCredits } from "../../lib/keyless";+import {+ KEYLESS_CREDITS_MESSAGE,+ adjustKeylessCredits,+ reserveKeylessCredits,+} from "../../lib/keyless";import { v7 as uuidv7 } from "uuid";import { logSearch, logRequest } from "../../services/logging/log_job";import { logger as _logger } from "../../lib/logger";@@ -21,6 +25,8 @@ import {import { executeSearch } from "../../search/execute";import type { BillingMetadata } from "../../services/billing/types";import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";+import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -48,6 +54,8 @@ export async function searchController(config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;let zeroDataRetention = teamForcedKind !== null;+ let reservedKeylessCredits = 0;+ let reconciledKeylessCredits = false;try {req.body = searchRequestSchema.parse(req.body);@@ -121,6 +129,33 @@ export async function searchController(});}+ const projectedKeylessCredits =+ !isSearchPreview && shouldBill+ ? projectSearchTotalCredits(+ {+ limit: req.body.limit,+ enterprise: req.body.enterprise,+ scrapeOptions: req.body.scrapeOptions,+ },+ req.acuc?.flags ?? null,+ zeroDataRetention,+ )+ : 0;+ if (projectedKeylessCredits > 0) {+ const reservation = await reserveKeylessCredits(+ req.auth.team_id,+ projectedKeylessCredits,+ );+ if (!reservation.ok) {+ applyAgentAuthDiscoveryHeader(res);+ return res.status(429).json({+ success: false,+ error: KEYLESS_CREDITS_MESSAGE,+ });+ }+ reservedKeylessCredits = projectedKeylessCredits;+ }+const result = await executeSearch({query: req.body.query,@@ -151,6 +186,7 @@ export async function searchController(zeroDataRetention,billing,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ keylessReserved: reservedKeylessCredits > 0,},logger,);@@ -170,12 +206,13 @@ export async function searchController();}- // Charge the keyless free tier's per-IP daily credit budget for search- // credits (no-op for non-keyless teams; scrape jobs within search charge- // themselves via the scrape worker).- chargeKeylessCredits(req.auth.team_id, result.searchCredits).catch(- () => {},- );+ if (reservedKeylessCredits > 0) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(+ req.auth.team_id,+ result.totalCredits - reservedKeylessCredits,+ ).catch(() => {});+ }const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;@@ -223,6 +260,13 @@ export async function searchController(id: jobId,});} catch (error) {+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {+ reconciledKeylessCredits = true;+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(+ () => {},+ );+ }+if (error instanceof z.ZodError) {logger.warn("Invalid request body", { error: error.issues });return res.status(400).json({diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.tsnew file mode 100644index 000000000..425c8139e--- /dev/null+++ b/apps/api/src/lib/keyless-credit-projection.ts@@ -0,0 +1,95 @@+import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";+import { hasFormatOfType } from "./format-utils";++export function projectScrapeCredits(+ options: ScrapeOptions,+ flags: TeamFlags,+ zeroDataRetention: boolean,+): number {+ let credits = 1;++ if (options.lockdown) {+ credits += 4;+ }++ const changeTrackingFormat = hasFormatOfType(+ options.formats,+ "changeTracking",+ );+ if (+ hasFormatOfType(options.formats, "json") ||+ changeTrackingFormat?.modes?.includes("json")+ ) {+ credits = 5;+ }++ if (hasFormatOfType(options.formats, "deterministicJson")) {+ credits = 10;+ }++ if (+ hasFormatOfType(options.formats, "question") ||+ hasFormatOfType(options.formats, "query")+ ) {+ credits += 4;+ }++ if (hasFormatOfType(options.formats, "highlights")) {+ credits += 4;+ }++ if (hasFormatOfType(options.formats, "audio")) {+ credits += 4;+ }++ if (hasFormatOfType(options.formats, "video")) {+ credits += 4;+ }++ if (zeroDataRetention && !options.lockdown) {+ credits += flags?.zdrCost ?? 1;+ }++ if (options.redactPII) {+ credits += 4;+ }++ if (+ options.proxy === "stealth" ||+ options.proxy === "enhanced" ||+ options.proxy === "auto"+ ) {+ credits += 4;+ }++ return credits;+}++function projectSearchCredits(+ limit: number,+ enterprise: ("default" | "anon" | "zdr")[] | undefined,+): number {+ const creditsPerTenResults = enterprise?.includes("zdr") ? 10 : 2;+ return Math.ceil(limit / 10) * creditsPerTenResults;+}++export function projectSearchTotalCredits(+ params: {+ limit: number;+ enterprise?: ("default" | "anon" | "zdr")[];+ scrapeOptions?: ScrapeOptions;+ },+ flags: TeamFlags,+ zeroDataRetention: boolean,+): number {+ const searchCredits = projectSearchCredits(params.limit, params.enterprise);+ const shouldScrape =+ params.scrapeOptions?.formats && params.scrapeOptions.formats.length > 0;+ if (!shouldScrape || !params.scrapeOptions) return searchCredits;++ return (+ searchCredits ++ params.limit *+ projectScrapeCredits(params.scrapeOptions, flags, zeroDataRetention)+ );+}diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex a634a6a53..9b63063c7 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -17,6 +17,8 @@ import { isKeylessIpSuspicious } from "./spur";const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;+export const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;+// The tier is "configured" when BOTH limits are set — even to 0. Unset means the// feature is off (callers get a plain Unauthorized); 0 means it's on but the// budget is exhausted (callers get the 429 cap message).@@ -87,6 +89,12 @@ type KeylessConsumeResult = {creditsUsed: number;};+type KeylessCreditReservationResult = {+ ok: boolean;+ creditsUsed: number;+ limit: number;+};+/*** Consume one request from the per-IP daily request budget and check the credit* budget (credits are charged after the request completes, in@@ -118,6 +126,88 @@ export async function consumeKeylessRequest(return { ok: true, requestsUsed, creditsUsed };}+/**+ * Atomically reserve projected credits from the keyless per-IP daily credit+ * budget. No-op for non-keyless teams.+ */+export async function reserveKeylessCredits(+ teamId: string,+ projectedCredits: number,+): Promise<KeylessCreditReservationResult> {+ const ip = keylessIpFromTeamId(teamId);+ const limit = KEYLESS_CREDITS_PER_DAY ?? 0;+ if (!ip || !Number.isFinite(projectedCredits) || projectedCredits <= 0) {+ return { ok: true, creditsUsed: 0, limit };+ }++ const projected = Math.ceil(projectedCredits);+ const key = creditsKey(ip);+ const result = (await redisRateLimitClient.eval(+ `+local current = tonumber(redis.call("GET", KEYS[1]) or "0")+local projected = tonumber(ARGV[1])+local limit = tonumber(ARGV[2])+local ttl = tonumber(ARGV[3])+if current + projected > limit then+ return {0, current}+end+local total = redis.call("INCRBY", KEYS[1], projected)+if total == projected then+ redis.call("EXPIRE", KEYS[1], ttl)+end+return {1, total}+`,+ 1,+ key,+ projected,+ limit,+ DAY_SECONDS,+ )) as [number, number];++ return {+ ok: result[0] === 1,+ creditsUsed: Number(result[1] ?? 0),+ limit,+ };+}++/**+ * Reconcile a prior keyless reservation to actual credits. Delta may be+ * negative; the counter is clamped at zero to tolerate retries or races.+ */+export async function adjustKeylessCredits(+ teamId: string,+ deltaCredits: number,+): Promise<number | null> {+ const ip = keylessIpFromTeamId(teamId);+ if (!ip || !Number.isFinite(deltaCredits) || deltaCredits === 0) return null;++ const delta =+ deltaCredits > 0+ ? Math.ceil(deltaCredits)+ : -Math.ceil(Math.abs(deltaCredits));+ const key = creditsKey(ip);+ const total = (await redisRateLimitClient.eval(+ `+local current = tonumber(redis.call("GET", KEYS[1]) or "0")+local delta = tonumber(ARGV[1])+local ttl = tonumber(ARGV[2])+local next = current + delta+if next < 0 then+ next = 0+end+redis.call("SET", KEYS[1], next, "EX", ttl)+return next+`,+ 1,+ key,+ delta,+ DAY_SECONDS,+ )) as number;++ return Number(total);+}+/*** Read-only check of whether an IP could currently use the keyless tier (no* consumption). Used by the hosted MCP to decide, at connect time, whether todiff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex be84f45c7..88261c13f 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -47,6 +47,7 @@ interface SearchContext {zeroDataRetention?: boolean;billing?: BillingMetadata;agentIndexOnly?: boolean;+ keylessReserved?: boolean;}interface SearchExecuteResult {@@ -169,6 +170,7 @@ export async function executeSearch(requestId,billing,agentIndexOnly: context.agentIndexOnly,+ keylessReserved: context.keylessReserved,};const allDocsWithCostTracking = await scrapeSearchResults(diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex eb391eb75..1c29fcc51 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -39,6 +39,7 @@ interface ScrapeSearchOptions {requestId?: string;billing?: BillingMetadata;agentIndexOnly?: boolean;+ keylessReserved?: boolean;}async function scrapeSearchResultDirect(@@ -87,6 +88,7 @@ async function scrapeSearchResultDirect(startTime: Date.now(),zeroDataRetention,apiKeyId: options.apiKeyId,+ keylessReserved: options.keylessReserved ?? false,requestId: options.requestId,billing: options.billing,},diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex dcf1d7467..3f8e9c4e0 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -139,9 +139,11 @@ async function billScrapeJob(unsupportedFeatures,);- // Charge the keyless free tier's per-IP daily credit budget (no-op for- // non-keyless teams).- await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);+ // Charge the keyless free tier's per-IP daily credit budget unless this+ // request reserved credits in the controller and will reconcile there.+ if (!job.data.keylessReserved) {+ await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);+ }if (job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 2c8c0f30d..c5812296d 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -18,6 +18,7 @@ type ScrapeJobCommon = {team_id: string;zeroDataRetention: boolean;billing?: BillingMetadata;+ keylessReserved?: boolean;traceContext?: SerializedTraceContext;skipNuq?: boolean;requestId?: string;