Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

[codex] Reserve keyless credits atomically (#3796)

Gergő Móriczef29f7f25854
diff --git a/apps/api/src/__tests__/snips/v2/keyless.test.ts b/apps/api/src/__tests__/snips/v2/keyless.test.ts
index 22e73a4c7..d79c91507 100644
--- a/apps/api/src/__tests__/snips/v2/keyless.test.ts
+++ b/apps/api/src/__tests__/snips/v2/keyless.test.ts
@@ -22,9 +22,8 @@ const KEYLESS_ENABLED =
Number.isFinite(KEYLESS_CREDITS_PER_DAY) &&
KEYLESS_CREDITS_PER_DAY > 0;
-// Keyless free tier: scrape, parse, search, and interact work without an API key
-// from the official MCP server (origin "mcp*"), CLI (integration "cli"), or SDKs
-// (origin "<lang>-sdk@..."). Gated per-IP/day by a request cap AND a credit cap.
+// Keyless free tier: scrape, parse, search, and interact work without an API key.
+// Gated per-IP/day by a request cap AND a credit cap.
// These tests are the only ones that exercise keyless mode, so they exclusively
// own the `keyless_*` rate-limit keyspace — we flush it before each test (and run
// sequentially) for isolation. `req.ip` is loopback under supertest, so the
@@ -203,6 +202,254 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
expect(blocked.body.error).toContain("unauthenticated credits");
});
+ it(
+ "rejects projected multi-credit keyless parse before parsing (429)",
+ async () => {
+ await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/parse")
+ .field(
+ "options",
+ JSON.stringify({
+ origin: "mcp",
+ proxy: "basic",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { title: { type: "string" } },
+ },
+ },
+ ],
+ }),
+ )
+ .attach("file", Buffer.from("<h1>Blocked parse</h1>"), {
+ filename: "blocked.html",
+ contentType: "text/html",
+ });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "rejects projected multi-credit keyless scrape before scraping (429)",
+ async () => {
+ await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ proxy: "basic",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { title: { type: "string" } },
+ },
+ },
+ ],
+ });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "rejects projected multi-credit v1 keyless scrape before scraping (429)",
+ async () => {
+ await request(TEST_API_URL)
+ .post("/v1/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v1/scrape")
+ .set("Content-Type", "application/json")
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ proxy: "basic",
+ formats: ["json"],
+ jsonOptions: {
+ schema: {
+ type: "object",
+ properties: { title: { type: "string" } },
+ },
+ },
+ });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "reconciles keyless scrape reservation to actual credits (200)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "mcp",
+ proxy: "basic",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { title: { type: "string" } },
+ },
+ },
+ ],
+ });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data.metadata.creditsUsed).toBe(5);
+
+ const ip = await currentKeylessIp();
+ const creditsUsed = Number(
+ await redisRateLimitClient.get(`keyless_credits:${ip}`),
+ );
+ expect(creditsUsed).toBe(response.body.data.metadata.creditsUsed);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "rejects projected keyless search with scrape options before search (429)",
+ async () => {
+ await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v2/search")
+ .set("Content-Type", "application/json")
+ .send({
+ query: "firecrawl",
+ limit: 1,
+ origin: "mcp",
+ scrapeOptions: {
+ proxy: "basic",
+ formats: ["markdown"],
+ },
+ });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "rejects projected v1 keyless search with scrape options before search (429)",
+ async () => {
+ await request(TEST_API_URL)
+ .post("/v1/scrape")
+ .set("Content-Type", "application/json")
+ .send({ origin: "mcp" });
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post("/v1/search")
+ .set("Content-Type", "application/json")
+ .send({
+ query: "firecrawl",
+ limit: 1,
+ origin: "mcp",
+ scrapeOptions: {
+ proxy: "basic",
+ formats: ["markdown"],
+ },
+ });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "reconciles keyless search reservation to actual credits (200)",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/search")
+ .set("Content-Type", "application/json")
+ .send({ query: "firecrawl", limit: 1, origin: "mcp" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(typeof response.body.creditsUsed).toBe("number");
+
+ const ip = await currentKeylessIp();
+ const creditsUsed = Number(
+ await redisRateLimitClient.get(`keyless_credits:${ip}`),
+ );
+ expect(creditsUsed).toBe(response.body.creditsUsed);
+ },
+ scrapeTimeout,
+ );
+
itIf(!!process.env.KEYLESS_PROXY_SECRET)(
"rate-limits per forwarded client IP when the proxy secret is provided",
async () => {
@@ -311,6 +558,43 @@ describeIf(KEYLESS_ENABLED)("Keyless free tier", () => {
scrapeTimeout,
);
+ itIf(!!config.BROWSER_SERVICE_URL)(
+ "rejects projected keyless interact session creation before browser work (429)",
+ async () => {
+ const scrapeResponse = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({
+ url: TEST_SUITE_WEBSITE,
+ origin: "website-keyless-interact-test",
+ proxy: "basic",
+ formats: ["markdown"],
+ });
+
+ expect(scrapeResponse.statusCode).toBe(200);
+ expect(scrapeResponse.body.success).toBe(true);
+ expect(typeof scrapeResponse.body.scrape_id).toBe("string");
+
+ const ip = await currentKeylessIp();
+ await redisRateLimitClient.set(
+ `keyless_credits:${ip}`,
+ String(KEYLESS_CREDITS_PER_DAY - 1),
+ );
+
+ const blocked = await request(TEST_API_URL)
+ .post(`/v2/scrape/${scrapeResponse.body.scrape_id}/interact`)
+ .set("Content-Type", "application/json")
+ .send({ code: "return 1;", origin: "mcp" });
+
+ expect(blocked.statusCode).toBe(429);
+ expect(blocked.body.error).toContain("unauthenticated credits");
+ expect(blocked.headers["www-authenticate"]).toContain(
+ "resource_metadata",
+ );
+ },
+ scrapeTimeout,
+ );
+
it(
"allows keyless scrape from an mcp origin (200)",
async () => {
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index f07a6c891..81eaeb915 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -9,6 +9,7 @@ import { getAgentSponsorStatus } from "../services/agent-sponsor";
import { getRedisConnection } from "../services/queue-service";
import { getRateLimiter } from "../services/rate-limiter";
import {
+ KEYLESS_CREDITS_MESSAGE,
consumeKeylessRequest,
isKeylessConfigured,
isKeylessIpEligible,
@@ -471,8 +472,6 @@ export async function clearACUCTeam(team_id: string): Promise<void> {
const KEYLESS_REQUESTS_MESSAGE = `You've reached today's limit of free, unauthenticated requests to Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
-const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
-
const KEYLESS_ENDPOINT_NOT_AVAILABLE_MESSAGE = `This endpoint is not available without an API key. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
const KEYLESS_SUSPICIOUS_IP_MESSAGE = `Unfortunately, your IP address looks suspicious, so Firecrawl can't be used without an API key from here. Sign up for a free API key at https://firecrawl.dev for 1000 credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 4b07896d9..3a1452c59 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -23,6 +23,13 @@ import { logRequest } from "../../services/logging/log_job";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -96,6 +103,30 @@ export async function scrapeController(
req.body.timeout,
req.auth.team_id,
);
+ const projectedKeylessCredits = !isDirectToBullMQ
+ ? projectScrapeCredits(
+ scrapeOptions,
+ req.acuc?.flags ?? null,
+ zeroDataRetention ?? false,
+ )
+ : 0;
+ let reservedKeylessCredits = 0;
+ let reconciledKeylessCredits = false;
+
+ if (projectedKeylessCredits > 0) {
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ projectedKeylessCredits,
+ );
+ if (!reservation.ok) {
+ applyAgentAuthDiscoveryHeader(res);
+ return res.status(429).json({
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ });
+ }
+ reservedKeylessCredits = projectedKeylessCredits;
+ }
const totalWait =
(req.body.waitFor ?? 0) +
@@ -171,6 +202,7 @@ export async function scrapeController(
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
concurrencyLimited: limited,
+ keylessReserved: reservedKeylessCredits > 0,
logRequestPromise: logRequestPromise,
},
};
@@ -180,6 +212,13 @@ export async function scrapeController(
},
);
} catch (e) {
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
+ () => {},
+ );
+ }
+
const timeoutErr =
e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";
@@ -265,6 +304,14 @@ export async function scrapeController(
}
}
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(
+ req.auth.team_id,
+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ ).catch(() => {});
+ }
+
const totalRequestTime = new Date().getTime() - middlewareStartTime;
const controllerTime = new Date().getTime() - controllerStartTime;
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 692fd3482..0b4f37a19 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -27,6 +27,13 @@ import {
} from "../../search/transform";
import { fromV1ScrapeOptions } from "../v2/types";
import { getSearchForcedKind } from "../../lib/zdr-helpers";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
+import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
// Used for deep research
export async function searchAndScrapeSearchResult(
@@ -108,6 +115,8 @@ export async function searchController(
const isSearchPreview =
config.SEARCH_PREVIEW_TOKEN !== undefined &&
config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ let reservedKeylessCredits = 0;
+ let reconciledKeylessCredits = false;
try {
req.body = searchRequestSchema.parse(req.body);
@@ -141,6 +150,32 @@ export async function searchController(
const shouldScrape =
req.body.scrapeOptions.formats &&
req.body.scrapeOptions.formats.length > 0;
+ const projectedKeylessCredits = !isSearchPreview
+ ? projectSearchTotalCredits(
+ {
+ limit: req.body.limit,
+ enterprise: teamEnterprise,
+ scrapeOptions: shouldScrape ? scrapeOptions : undefined,
+ },
+ req.acuc?.flags ?? null,
+ zeroDataRetention,
+ )
+ : 0;
+
+ if (projectedKeylessCredits > 0) {
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ projectedKeylessCredits,
+ );
+ if (!reservation.ok) {
+ applyAgentAuthDiscoveryHeader(res);
+ return res.status(429).json({
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ });
+ }
+ reservedKeylessCredits = projectedKeylessCredits;
+ }
// Execute search using v2 logic
const result = await executeSearch(
@@ -168,6 +203,7 @@ export async function searchController(
bypassBilling: false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ keylessReserved: reservedKeylessCredits > 0,
},
logger,
);
@@ -212,6 +248,14 @@ export async function searchController(
});
}
+ if (reservedKeylessCredits > 0) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(
+ req.auth.team_id,
+ result.totalCredits - reservedKeylessCredits,
+ ).catch(() => {});
+ }
+
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
@@ -255,6 +299,13 @@ export async function searchController(
return res.status(200).json(responseData);
} catch (error) {
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
+ () => {},
+ );
+ }
+
if (error instanceof z.ZodError) {
logger.warn("Invalid request body", { error: error.issues });
return res.status(400).json({
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 34539c11e..a39da3297 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -26,7 +26,13 @@ import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
-import { chargeKeylessCredits } from "../../lib/keyless";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import path from "node:path";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
@@ -349,6 +355,34 @@ export async function parseController(
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
const boostConcurrency =
req.body.__agentInterop?.boostConcurrency ?? false;
+ const isDirectToBullMQ =
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ const projectedKeylessCredits =
+ shouldBill && !isDirectToBullMQ
+ ? projectScrapeCredits(
+ req.body,
+ req.acuc?.flags ?? null,
+ zeroDataRetention,
+ )
+ : 0;
+ let reservedKeylessCredits = 0;
+ let reconciledKeylessCredits = false;
+
+ if (projectedKeylessCredits > 0) {
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ projectedKeylessCredits,
+ );
+ if (!reservation.ok) {
+ applyAgentAuthDiscoveryHeader(res);
+ return res.status(429).json({
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ });
+ }
+ reservedKeylessCredits = projectedKeylessCredits;
+ }
const logger = _logger.child({
method: "parseController",
@@ -395,10 +429,6 @@ export async function parseController(
const origin = req.body.origin;
const timeout = req.body.timeout;
- const isDirectToBullMQ =
- config.SEARCH_PREVIEW_TOKEN !== undefined &&
- config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
-
const totalWait = 0;
let lockTime: number | null = null;
@@ -494,6 +524,7 @@ export async function parseController(
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
concurrencyLimited: limited,
+ keylessReserved: reservedKeylessCredits > 0,
requestId: agentRequestId ?? undefined,
},
};
@@ -512,6 +543,13 @@ export async function parseController(
},
);
} catch (e) {
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
+ () => {},
+ );
+ }
+
const timeoutErr =
e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";
@@ -616,6 +654,14 @@ export async function parseController(
}
}
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(
+ req.auth.team_id,
+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ ).catch(() => {});
+ }
+
const totalRequestTime = new Date().getTime() - middlewareStartTime;
const controllerTime = new Date().getTime() - controllerStartTime;
@@ -663,11 +709,6 @@ export async function parseController(
concurrencyQueueDurationMs: lockTime || undefined,
});
- chargeKeylessCredits(
- req.auth.team_id,
- doc?.metadata?.creditsUsed ?? 0,
- ).catch(() => {});
-
return res.status(200).json({
success: true,
data: {
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 8056e9d1b..6bb067646 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -45,7 +45,12 @@ import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { RequestWithAuth, ScrapeOptions } from "./types";
import { billTeam } from "../../services/billing/credit_billing";
-import { chargeKeylessCredits, keylessTeamUuid } from "../../lib/keyless";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ keylessTeamUuid,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { logRequest } from "../../services/logging/log_job";
import { integrationSchema } from "../../utils/integration";
@@ -56,6 +61,7 @@ import {
calculateBrowserSessionCredits,
} from "../../lib/browser-billing";
import { autumnService } from "../../services/autumn/autumn.service";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
// ---------------------------------------------------------------------------
// Schemas
@@ -200,6 +206,12 @@ export async function scrapeInteractController(
(scrape.options as ScrapeOptions).profile,
);
if ("error" in created) {
+ if (
+ created.status === 429 &&
+ created.body.error === KEYLESS_CREDITS_MESSAGE
+ ) {
+ applyAgentAuthDiscoveryHeader(res);
+ }
return res.status(created.status).json(created.body);
}
session = created.session;
@@ -463,9 +475,13 @@ export async function scrapeStopInteractiveBrowserController(
});
});
- // Charge the keyless free tier's per-IP daily credit budget (no-op for
- // non-keyless teams).
- chargeKeylessCredits(req.auth.team_id, creditsBilled).catch(() => {});
+ const reservedCredits = calculateBrowserSessionCredits(
+ session.ttl_total * 1000,
+ BROWSER_CREDITS_PER_HOUR,
+ );
+ adjustKeylessCredits(req.auth.team_id, creditsBilled - reservedCredits).catch(
+ () => {},
+ );
logger.info("Browser session destroyed", {
sessionDurationMs: durationMs,
@@ -525,6 +541,22 @@ async function createSessionForScrape(
// Credit check (uses base rate — actual billing may be higher if prompts are used)
const estimatedCredits = calculateBrowserSessionCredits(ttl * 1000);
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ estimatedCredits,
+ );
+ if (!reservation.ok) {
+ return {
+ status: 429,
+ body: {
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ },
+ error: true,
+ };
+ }
+ const keylessReserved = estimatedCredits;
+
const autumnResult = await autumnService.checkCredits({
teamId: req.auth.team_id,
value: estimatedCredits,
@@ -532,6 +564,7 @@ async function createSessionForScrape(
});
if (autumnResult !== null && !autumnResult.allowed) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});
return {
status: 402,
body: {
@@ -546,6 +579,7 @@ async function createSessionForScrape(
const concurrencyLimit = req.acuc?.concurrency ?? 2;
const activeCount = await getCombinedTeamActiveCount(req.auth.team_id);
if (activeCount >= concurrencyLimit) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});
return {
status: 429,
body: {
@@ -587,6 +621,9 @@ async function createSessionForScrape(
break;
} catch (err) {
if (err instanceof BrowserServiceError && err.status === 409) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(
+ () => {},
+ );
return {
status: 409,
body: {
@@ -610,6 +647,7 @@ async function createSessionForScrape(
}
if (!svcResponse) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});
logger.error("Failed to create browser session after all retries", {
error: lastCreateError,
});
@@ -695,6 +733,7 @@ async function createSessionForScrape(
);
}
} catch (err) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});
logger.error("Failed to initialize scrape browser session context", {
error: err,
});
@@ -753,6 +792,7 @@ async function createSessionForScrape(
return { session };
} catch (err) {
+ adjustKeylessCredits(req.auth.team_id, -keylessReserved).catch(() => {});
logger.error("Failed to persist browser session, cleaning up", {
error: err,
});
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 2b6df4743..b99bbf2b0 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -24,6 +24,13 @@ import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
+import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
@@ -111,6 +118,34 @@ export async function scrapeController(
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
const boostConcurrency =
req.body.__agentInterop?.boostConcurrency ?? false;
+ const isDirectToBullMQ =
+ config.SEARCH_PREVIEW_TOKEN !== undefined &&
+ config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+ const projectedKeylessCredits =
+ shouldBill && !isDirectToBullMQ
+ ? projectScrapeCredits(
+ req.body,
+ req.acuc?.flags ?? null,
+ zeroDataRetention,
+ )
+ : 0;
+ let reservedKeylessCredits = 0;
+ let reconciledKeylessCredits = false;
+
+ if (projectedKeylessCredits > 0) {
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ projectedKeylessCredits,
+ );
+ if (!reservation.ok) {
+ applyAgentAuthDiscoveryHeader(res);
+ return res.status(429).json({
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ });
+ }
+ reservedKeylessCredits = projectedKeylessCredits;
+ }
const logger = _logger.child({
method: "scrapeController",
@@ -159,10 +194,6 @@ export async function scrapeController(
const origin = req.body.origin;
const timeout = req.body.timeout;
- const isDirectToBullMQ =
- config.SEARCH_PREVIEW_TOKEN !== undefined &&
- config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
-
const totalWait =
(req.body.waitFor ?? 0) +
(req.body.actions ?? []).reduce(
@@ -261,6 +292,7 @@ export async function scrapeController(
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
concurrencyLimited: limited,
+ keylessReserved: reservedKeylessCredits > 0,
requestId: agentRequestId ?? undefined,
logRequestPromise: logRequestPromise,
},
@@ -280,6 +312,13 @@ export async function scrapeController(
},
);
} catch (e) {
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
+ () => {},
+ );
+ }
+
const timeoutErr =
e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";
@@ -406,6 +445,14 @@ export async function scrapeController(
}
}
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(
+ req.auth.team_id,
+ (doc?.metadata?.creditsUsed ?? 0) - reservedKeylessCredits,
+ ).catch(() => {});
+ }
+
const totalRequestTime = new Date().getTime() - middlewareStartTime;
const controllerTime = new Date().getTime() - controllerStartTime;
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 1db7b506f..b661eef42 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -7,7 +7,11 @@ import {
searchRequestSchema,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
-import { chargeKeylessCredits } from "../../lib/keyless";
+import {
+ KEYLESS_CREDITS_MESSAGE,
+ adjustKeylessCredits,
+ reserveKeylessCredits,
+} from "../../lib/keyless";
import { v7 as uuidv7 } from "uuid";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { logger as _logger } from "../../lib/logger";
@@ -21,6 +25,8 @@ import {
import { executeSearch } from "../../search/execute";
import type { BillingMetadata } from "../../services/billing/types";
import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";
+import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
export async function searchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
@@ -48,6 +54,8 @@ export async function searchController(
config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
let zeroDataRetention = teamForcedKind !== null;
+ let reservedKeylessCredits = 0;
+ let reconciledKeylessCredits = false;
try {
req.body = searchRequestSchema.parse(req.body);
@@ -121,6 +129,33 @@ export async function searchController(
});
}
+ const projectedKeylessCredits =
+ !isSearchPreview && shouldBill
+ ? projectSearchTotalCredits(
+ {
+ limit: req.body.limit,
+ enterprise: req.body.enterprise,
+ scrapeOptions: req.body.scrapeOptions,
+ },
+ req.acuc?.flags ?? null,
+ zeroDataRetention,
+ )
+ : 0;
+ if (projectedKeylessCredits > 0) {
+ const reservation = await reserveKeylessCredits(
+ req.auth.team_id,
+ projectedKeylessCredits,
+ );
+ if (!reservation.ok) {
+ applyAgentAuthDiscoveryHeader(res);
+ return res.status(429).json({
+ success: false,
+ error: KEYLESS_CREDITS_MESSAGE,
+ });
+ }
+ reservedKeylessCredits = projectedKeylessCredits;
+ }
+
const result = await executeSearch(
{
query: req.body.query,
@@ -151,6 +186,7 @@ export async function searchController(
zeroDataRetention,
billing,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ keylessReserved: reservedKeylessCredits > 0,
},
logger,
);
@@ -170,12 +206,13 @@ export async function searchController(
);
}
- // Charge the keyless free tier's per-IP daily credit budget for search
- // credits (no-op for non-keyless teams; scrape jobs within search charge
- // themselves via the scrape worker).
- chargeKeylessCredits(req.auth.team_id, result.searchCredits).catch(
- () => {},
- );
+ if (reservedKeylessCredits > 0) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(
+ req.auth.team_id,
+ result.totalCredits - reservedKeylessCredits,
+ ).catch(() => {});
+ }
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
@@ -223,6 +260,13 @@ export async function searchController(
id: jobId,
});
} catch (error) {
+ if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
+ reconciledKeylessCredits = true;
+ adjustKeylessCredits(req.auth.team_id, -reservedKeylessCredits).catch(
+ () => {},
+ );
+ }
+
if (error instanceof z.ZodError) {
logger.warn("Invalid request body", { error: error.issues });
return res.status(400).json({
diff --git a/apps/api/src/lib/keyless-credit-projection.ts b/apps/api/src/lib/keyless-credit-projection.ts
new file mode 100644
index 000000000..425c8139e
--- /dev/null
+++ b/apps/api/src/lib/keyless-credit-projection.ts
@@ -0,0 +1,95 @@
+import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";
+import { hasFormatOfType } from "./format-utils";
+
+export function projectScrapeCredits(
+ options: ScrapeOptions,
+ flags: TeamFlags,
+ zeroDataRetention: boolean,
+): number {
+ let credits = 1;
+
+ if (options.lockdown) {
+ credits += 4;
+ }
+
+ const changeTrackingFormat = hasFormatOfType(
+ options.formats,
+ "changeTracking",
+ );
+ if (
+ hasFormatOfType(options.formats, "json") ||
+ changeTrackingFormat?.modes?.includes("json")
+ ) {
+ credits = 5;
+ }
+
+ if (hasFormatOfType(options.formats, "deterministicJson")) {
+ credits = 10;
+ }
+
+ if (
+ hasFormatOfType(options.formats, "question") ||
+ hasFormatOfType(options.formats, "query")
+ ) {
+ credits += 4;
+ }
+
+ if (hasFormatOfType(options.formats, "highlights")) {
+ credits += 4;
+ }
+
+ if (hasFormatOfType(options.formats, "audio")) {
+ credits += 4;
+ }
+
+ if (hasFormatOfType(options.formats, "video")) {
+ credits += 4;
+ }
+
+ if (zeroDataRetention && !options.lockdown) {
+ credits += flags?.zdrCost ?? 1;
+ }
+
+ if (options.redactPII) {
+ credits += 4;
+ }
+
+ if (
+ options.proxy === "stealth" ||
+ options.proxy === "enhanced" ||
+ options.proxy === "auto"
+ ) {
+ credits += 4;
+ }
+
+ return credits;
+}
+
+function projectSearchCredits(
+ limit: number,
+ enterprise: ("default" | "anon" | "zdr")[] | undefined,
+): number {
+ const creditsPerTenResults = enterprise?.includes("zdr") ? 10 : 2;
+ return Math.ceil(limit / 10) * creditsPerTenResults;
+}
+
+export function projectSearchTotalCredits(
+ params: {
+ limit: number;
+ enterprise?: ("default" | "anon" | "zdr")[];
+ scrapeOptions?: ScrapeOptions;
+ },
+ flags: TeamFlags,
+ zeroDataRetention: boolean,
+): number {
+ const searchCredits = projectSearchCredits(params.limit, params.enterprise);
+ const shouldScrape =
+ params.scrapeOptions?.formats && params.scrapeOptions.formats.length > 0;
+ if (!shouldScrape || !params.scrapeOptions) return searchCredits;
+
+ return (
+ searchCredits +
+ params.limit *
+ projectScrapeCredits(params.scrapeOptions, flags, zeroDataRetention)
+ );
+}
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index a634a6a53..9b63063c7 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -17,6 +17,8 @@ import { isKeylessIpSuspicious } from "./spur";
const KEYLESS_REQUESTS_PER_DAY = config.KEYLESS_REQUESTS_PER_DAY;
const KEYLESS_CREDITS_PER_DAY = config.KEYLESS_CREDITS_PER_DAY;
+export const KEYLESS_CREDITS_MESSAGE = `You've reached today's limit of free, unauthenticated credits for Firecrawl. Sign up for a free API key at https://firecrawl.dev for 1000 more credits and higher rate limits for free. (If you're an agent, you can also use https://firecrawl.dev/auth.md)`;
+
// The tier is "configured" when BOTH limits are set — even to 0. Unset means the
// feature is off (callers get a plain Unauthorized); 0 means it's on but the
// budget is exhausted (callers get the 429 cap message).
@@ -87,6 +89,12 @@ type KeylessConsumeResult = {
creditsUsed: number;
};
+type KeylessCreditReservationResult = {
+ ok: boolean;
+ creditsUsed: number;
+ limit: number;
+};
+
/**
* Consume one request from the per-IP daily request budget and check the credit
* budget (credits are charged after the request completes, in
@@ -118,6 +126,88 @@ export async function consumeKeylessRequest(
return { ok: true, requestsUsed, creditsUsed };
}
+/**
+ * Atomically reserve projected credits from the keyless per-IP daily credit
+ * budget. No-op for non-keyless teams.
+ */
+export async function reserveKeylessCredits(
+ teamId: string,
+ projectedCredits: number,
+): Promise<KeylessCreditReservationResult> {
+ const ip = keylessIpFromTeamId(teamId);
+ const limit = KEYLESS_CREDITS_PER_DAY ?? 0;
+ if (!ip || !Number.isFinite(projectedCredits) || projectedCredits <= 0) {
+ return { ok: true, creditsUsed: 0, limit };
+ }
+
+ const projected = Math.ceil(projectedCredits);
+ const key = creditsKey(ip);
+ const result = (await redisRateLimitClient.eval(
+ `
+local current = tonumber(redis.call("GET", KEYS[1]) or "0")
+local projected = tonumber(ARGV[1])
+local limit = tonumber(ARGV[2])
+local ttl = tonumber(ARGV[3])
+if current + projected > limit then
+ return {0, current}
+end
+local total = redis.call("INCRBY", KEYS[1], projected)
+if total == projected then
+ redis.call("EXPIRE", KEYS[1], ttl)
+end
+return {1, total}
+`,
+ 1,
+ key,
+ projected,
+ limit,
+ DAY_SECONDS,
+ )) as [number, number];
+
+ return {
+ ok: result[0] === 1,
+ creditsUsed: Number(result[1] ?? 0),
+ limit,
+ };
+}
+
+/**
+ * Reconcile a prior keyless reservation to actual credits. Delta may be
+ * negative; the counter is clamped at zero to tolerate retries or races.
+ */
+export async function adjustKeylessCredits(
+ teamId: string,
+ deltaCredits: number,
+): Promise<number | null> {
+ const ip = keylessIpFromTeamId(teamId);
+ if (!ip || !Number.isFinite(deltaCredits) || deltaCredits === 0) return null;
+
+ const delta =
+ deltaCredits > 0
+ ? Math.ceil(deltaCredits)
+ : -Math.ceil(Math.abs(deltaCredits));
+ const key = creditsKey(ip);
+ const total = (await redisRateLimitClient.eval(
+ `
+local current = tonumber(redis.call("GET", KEYS[1]) or "0")
+local delta = tonumber(ARGV[1])
+local ttl = tonumber(ARGV[2])
+local next = current + delta
+if next < 0 then
+ next = 0
+end
+redis.call("SET", KEYS[1], next, "EX", ttl)
+return next
+`,
+ 1,
+ key,
+ delta,
+ DAY_SECONDS,
+ )) as number;
+
+ return Number(total);
+}
+
/**
* Read-only check of whether an IP could currently use the keyless tier (no
* consumption). Used by the hosted MCP to decide, at connect time, whether to
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index be84f45c7..88261c13f 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -47,6 +47,7 @@ interface SearchContext {
zeroDataRetention?: boolean;
billing?: BillingMetadata;
agentIndexOnly?: boolean;
+ keylessReserved?: boolean;
}
interface SearchExecuteResult {
@@ -169,6 +170,7 @@ export async function executeSearch(
requestId,
billing,
agentIndexOnly: context.agentIndexOnly,
+ keylessReserved: context.keylessReserved,
};
const allDocsWithCostTracking = await scrapeSearchResults(
diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.ts
index eb391eb75..1c29fcc51 100644
--- a/apps/api/src/search/scrape.ts
+++ b/apps/api/src/search/scrape.ts
@@ -39,6 +39,7 @@ interface ScrapeSearchOptions {
requestId?: string;
billing?: BillingMetadata;
agentIndexOnly?: boolean;
+ keylessReserved?: boolean;
}
async function scrapeSearchResultDirect(
@@ -87,6 +88,7 @@ async function scrapeSearchResultDirect(
startTime: Date.now(),
zeroDataRetention,
apiKeyId: options.apiKeyId,
+ keylessReserved: options.keylessReserved ?? false,
requestId: options.requestId,
billing: options.billing,
},
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index dcf1d7467..3f8e9c4e0 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -139,9 +139,11 @@ async function billScrapeJob(
unsupportedFeatures,
);
- // Charge the keyless free tier's per-IP daily credit budget (no-op for
- // non-keyless teams).
- await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);
+ // Charge the keyless free tier's per-IP daily credit budget unless this
+ // request reserved credits in the controller and will reconcile there.
+ if (!job.data.keylessReserved) {
+ await chargeKeylessCredits(job.data.team_id, creditsToBeBilled);
+ }
if (
job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 2c8c0f30d..c5812296d 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -18,6 +18,7 @@ type ScrapeJobCommon = {
team_id: string;
zeroDataRetention: boolean;
billing?: BillingMetadata;
+ keylessReserved?: boolean;
traceContext?: SerializedTraceContext;
skipNuq?: boolean;
requestId?: string;