Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add agent browser interop billing (#4314)
Gergő Móricz14f46a554061
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 8998cfd6d..1d8c894b6 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -16,6 +16,20 @@ const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>z.preprocess(value => (value === "" ? undefined : value), schema);+const containsLoneSurrogate = (value: string): boolean => {+ for (let index = 0; index < value.length; index++) {+ const codeUnit = value.charCodeAt(index);+ if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {+ const nextCodeUnit = value.charCodeAt(index + 1);+ if (!(nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff)) return true;+ index++;+ } else if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {+ return true;+ }+ }+ return false;+};+/* Schema */const configSchema = z.object({// Application@@ -388,7 +402,15 @@ const configSchema = z.object({DISABLE_MONITORING: z.stringbool().default(false),EXTRACT_V3_BETA_URL: z.string().optional(),- AGENT_INTEROP_SECRET: z.string().optional(),+ AGENT_INTEROP_SECRET: z+ .string()+ .refine(value => value.trim().length > 0, {+ error: "AGENT_INTEROP_SECRET must not be blank",+ })+ .refine(value => !containsLoneSurrogate(value), {+ error: "AGENT_INTEROP_SECRET must not contain lone surrogates",+ })+ .optional(),// Wikipedia Enterprise APIWIKIPEDIA_ENTERPRISE_USERNAME: z.string().optional(),diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex e4ac42355..080cea700 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -43,6 +43,7 @@ import {resolveThreatProtection,} from "../../lib/threat-protection/request";import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";@@ -102,7 +103,7 @@ export async function batchScrapeController(if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&- req.body.__agentInterop.auth !== config.AGENT_INTEROP_SECRET+ !isAgentInteropSecretValid(req.body.__agentInterop.auth)) {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 8b351f31c..43cece723 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -34,6 +34,7 @@ import {calculateBrowserSessionCredits,} from "../../lib/browser-billing";import { autumnService } from "../../services/autumn/autumn.service";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";// ---------------------------------------------------------------------------// Zod schemas@@ -51,6 +52,13 @@ const browserCreateRequestSchema = z.object({saveChanges: z.boolean().default(true),}).optional(),+ __agentInterop: z+ .object({+ auth: z.string(),+ requestId: z.string().uuid(),+ shouldBill: z.boolean(),+ })+ .optional(),});type BrowserCreateRequest = z.infer<typeof browserCreateRequestSchema>;@@ -214,6 +222,25 @@ export async function browserCreateController(req.body = browserCreateRequestSchema.parse(req.body);+ if (+ req.body.__agentInterop &&+ config.AGENT_INTEROP_SECRET &&+ !isAgentInteropSecretValid(req.body.__agentInterop.auth)+ ) {+ return res.status(403).json({+ success: false,+ error: "Invalid agent interop.",+ });+ } else if (req.body.__agentInterop && !config.AGENT_INTEROP_SECRET) {+ return res.status(403).json({+ success: false,+ error: "Agent interop is not enabled.",+ });+ }++ const shouldBill = req.body.__agentInterop?.shouldBill ?? true;+ const agentRequestId = req.body.__agentInterop?.requestId ?? null;+const {ttl,activityTtl,@@ -234,26 +261,28 @@ export async function browserCreateController(logger.info("Creating browser session", { ttl, activityTtl });// 0a. Check if team has enough credits for the full TTL- const estimatedCredits = calculateBrowserSessionCredits(ttl * 1000);- const autumnResult = await autumnService.checkCredits({- teamId: req.auth.team_id,- value: estimatedCredits,- properties: {- source: "browserCreate",- path: req.path,- apiKeyId: req.acuc?.api_key_id ?? null,- },- });-- if (autumnResult !== null && !autumnResult.allowed) {- logger.warn("Insufficient credits for browser session TTL", {- estimatedCredits,- ttl,- });- return res.status(402).json({- success: false,- error: `Insufficient credits for a ${ttl}s browser session (requires ~${estimatedCredits} credits). For more credits, you can upgrade your plan at https://firecrawl.dev/pricing.`,+ if (shouldBill) {+ const estimatedCredits = calculateBrowserSessionCredits(ttl * 1000);+ const autumnResult = await autumnService.checkCredits({+ teamId: req.auth.team_id,+ value: estimatedCredits,+ properties: {+ source: "browserCreate",+ path: req.path,+ apiKeyId: req.acuc?.api_key_id ?? null,+ },});++ if (autumnResult !== null && !autumnResult.allowed) {+ logger.warn("Insufficient credits for browser session TTL", {+ estimatedCredits,+ ttl,+ });+ return res.status(402).json({+ success: false,+ error: `Insufficient credits for a ${ttl}s browser session (requires ~${estimatedCredits} credits). For more credits, you can upgrade your plan at https://firecrawl.dev/pricing.`,+ });+ }}// 0b. Enforce concurrency limit (shared pool with scrape/crawl/interact)@@ -343,20 +372,24 @@ export async function browserCreateController(// 2. Persist session in Supabasetry {- await logRequest({- id: sessionId,- kind: "browser",- api_version: "v2",- team_id: req.auth.team_id,- target_hint: "Browser session",- origin: "api",- integration: integration ?? null,- zeroDataRetention: false,- api_key_id: req.acuc!.api_key_id,- });+ if (!agentRequestId) {+ await logRequest({+ id: sessionId,+ kind: "browser",+ api_version: "v2",+ team_id: req.auth.team_id,+ target_hint: "Browser session",+ origin: "api",+ integration: integration ?? null,+ zeroDataRetention: false,+ api_key_id: req.acuc!.api_key_id,+ });+ }await insertBrowserSession({id: sessionId,team_id: req.auth.team_id,+ request_id: agentRequestId ?? sessionId,+ should_bill: shouldBill,browser_id: svcResponse.sessionId,workspace_id: "",context_id: "",@@ -600,28 +633,30 @@ export async function browserDeleteController(const rate = usedPrompt? INTERACT_CREDITS_PER_HOUR: BROWSER_CREDITS_PER_HOUR;- const creditsBilled = calculateBrowserSessionCredits(durationMs, rate);+ const creditsBilled = session.should_bill+ ? calculateBrowserSessionCredits(durationMs, rate)+ : 0;clearBrowserSessionPromptFlag(session.id).catch(() => {});- updateBrowserSessionCreditsUsed(session.id, creditsBilled).catch(error => {- logger.error("Failed to update credits_used on browser session", {- error,- sessionId: session.id,- creditsBilled,- });- });-- billTeam(req.auth.team_id, creditsBilled, req.acuc?.api_key_id ?? null, {- endpoint: usedPrompt ? "interact" : "browser",- jobId: session.id,- }).catch(error => {- logger.error("Failed to bill team for browser session", {- error,- creditsBilled,- durationMs,+ await updateBrowserSessionCreditsUsed(session.id, creditsBilled);++ if (session.should_bill) {+ const agentRequestId =+ session.request_id && session.request_id !== session.id+ ? session.request_id+ : null;+ billTeam(req.auth.team_id, creditsBilled, req.acuc?.api_key_id ?? null, {+ endpoint: agentRequestId ? "agent" : usedPrompt ? "interact" : "browser",+ jobId: agentRequestId ?? session.id,+ }).catch(error => {+ logger.error("Failed to bill team for browser session", {+ error,+ creditsBilled,+ durationMs,+ });});- });+ }logger.info("Browser session destroyed", {sessionDurationMs: durationMs,@@ -630,6 +665,8 @@ export async function browserDeleteController(return res.status(200).json({success: true,+ sessionDurationMs: durationMs,+ creditsBilled,});}@@ -738,35 +775,32 @@ export async function browserWebhookDestroyedController(const rate = usedPrompt? INTERACT_CREDITS_PER_HOUR: BROWSER_CREDITS_PER_HOUR;- const creditsBilled = calculateBrowserSessionCredits(durationMs, rate);+ const creditsBilled = session.should_bill+ ? calculateBrowserSessionCredits(durationMs, rate)+ : 0;clearBrowserSessionPromptFlag(session.id).catch(() => {});- updateBrowserSessionCreditsUsed(session.id, creditsBilled).catch(error => {- logger.error(- "Failed to update credits_used on browser session via webhook",- {+ await updateBrowserSessionCreditsUsed(session.id, creditsBilled);++ if (session.should_bill) {+ const agentRequestId =+ session.request_id && session.request_id !== session.id+ ? session.request_id+ : null;+ billTeam(session.team_id, creditsBilled, null, {+ endpoint: agentRequestId ? "agent" : usedPrompt ? "interact" : "browser",+ jobId: agentRequestId ?? session.id,+ }).catch(error => {+ logger.error("Failed to bill team for browser session via webhook", {error,+ teamId: session.team_id,sessionId: session.id,creditsBilled,- },- );- });-- billTeam(- session.team_id,- creditsBilled,- null, // api_key_id not available in webhook context- { endpoint: usedPrompt ? "interact" : "browser", jobId: session.id },- ).catch(error => {- logger.error("Failed to bill team for browser session via webhook", {- error,- teamId: session.team_id,- sessionId: session.id,- creditsBilled,- durationMs,+ durationMs,+ });});- });+ }logger.info("Session marked as destroyed via webhook", {sessionId: session.id,diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 3ccc60735..8ee5d7a00 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -40,6 +40,7 @@ import {DOCUMENT_EXTENSIONS,documentExtensionFromContentType,} from "../../lib/document-formats";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;export const SUPPORTED_PARSE_FILE_TYPES =@@ -325,7 +326,7 @@ export async function parseController(if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&- req.body.__agentInterop.auth !== config.AGENT_INTEROP_SECRET+ !isAgentInteropSecretValid(req.body.__agentInterop.auth)) {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 5751c7127..145cfb820 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -804,6 +804,8 @@ async function createSessionForScrape(const session = await insertBrowserSession({id: sessionId,team_id: req.auth.team_id,+ request_id: sessionId,+ should_bill: true,scrape_id: scrapeId,browser_id: svcResponse.sessionId,workspace_id: "",diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex e56da6a8e..0b8aaaa9d 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -39,6 +39,7 @@ import { projectScrapeCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -148,7 +149,7 @@ export async function scrapeController(if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&- req.body.__agentInterop.auth !== config.AGENT_INTEROP_SECRET+ !isAgentInteropSecretValid(req.body.__agentInterop.auth)) {return res.status(403).json({success: false,@@ -186,9 +187,9 @@ export async function scrapeController();if (!reservation.ok) {applyAgentAuthDiscoveryHeader(res);- return res.status(429).json(- await keylessLimitBody(req.auth.team_id, "v2_scrape"),- );+ return res+ .status(429)+ .json(await keylessLimitBody(req.auth.team_id, "v2_scrape"));}reservedKeylessCredits = projectedKeylessCredits;}diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 47dfb6fe8..4370619d5 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -41,6 +41,7 @@ import {} from "../../lib/key-restriction";import { wantsDeveloperCategory } from "../../search/developer";import { requestOrigin } from "../../lib/request-origin";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -111,7 +112,7 @@ export async function searchController(if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&- req.body.__agentInterop.auth !== config.AGENT_INTEROP_SECRET+ !isAgentInteropSecretValid(req.body.__agentInterop.auth)) {return res.status(403).json({success: false,diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex c9320e741..97a17a381 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -127,6 +127,8 @@ export const browser_session_activities = pgTable(export const browser_sessions = pgTable("browser_sessions", {id: uuid("id").notNull(),team_id: text("team_id").notNull(),+ request_id: uuid("request_id"),+ should_bill: boolean("should_bill").notNull().default(true),browser_id: text("browser_id").notNull(),workspace_id: text("workspace_id").notNull(),context_id: text("context_id").notNull(),diff --git a/apps/api/src/lib/agent-interop.ts b/apps/api/src/lib/agent-interop.tsnew file mode 100644index 000000000..f5c7795ce--- /dev/null+++ b/apps/api/src/lib/agent-interop.ts@@ -0,0 +1,20 @@+import { timingSafeEqual } from "node:crypto";+import { config } from "../config";++export function isAgentInteropSecretValid(provided: unknown): boolean {+ const expected = config.AGENT_INTEROP_SECRET;+ if (+ typeof provided !== "string" ||+ !expected ||+ expected.trim().length === 0+ ) {+ return false;+ }++ const providedBuffer = Buffer.from(provided, "utf16le");+ const expectedBuffer = Buffer.from(expected, "utf16le");+ return (+ providedBuffer.length === expectedBuffer.length &&+ timingSafeEqual(providedBuffer, expectedBuffer)+ );+}diff --git a/apps/api/src/lib/browser-sessions.ts b/apps/api/src/lib/browser-sessions.tsindex 05aa3dcd2..a9d4efb38 100644--- a/apps/api/src/lib/browser-sessions.ts+++ b/apps/api/src/lib/browser-sessions.ts@@ -15,6 +15,8 @@ type BrowserSessionStatus = "active" | "destroyed" | "error";interface BrowserSessionRow {id: string;team_id: string;+ request_id: string | null;+ should_bill: boolean;scrape_id?: string | null; // linked scrape job id for /scrape/:jobId/interact sessionsbrowser_id: string; // browser service sessionIdworkspace_id: string; // unused (legacy), stored as ""diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 9079c1f0c..7e88dfd31 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -32,6 +32,7 @@ import { getTeamBalance } from "../services/autumn/usage";import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";import { getExchangeAccessForRequestBody } from "../lib/exchange-request";import { getScrapeZDR } from "../lib/zdr-helpers";+import { isAgentInteropSecretValid } from "../lib/agent-interop";export function checkCreditsMiddleware(_minimum?: number,@@ -45,7 +46,7 @@ export function checkCreditsMiddleware(req.body &&(req.body as any).__agentInterop &&(req.body as any).__agentInterop.auth &&- (req.body as any).__agentInterop.auth === config.AGENT_INTEROP_SECRET &&+ isAgentInteropSecretValid((req.body as any).__agentInterop.auth) &&(req.body as any).__agentInterop.shouldBill === false) {return next();@@ -246,16 +247,14 @@ export function authMiddleware(if (auth.status === 401 || auth.agentAuthDiscovery) {applyAgentAuthDiscoveryHeader(res);}- return res- .status(auth.status)- .json({- success: false,- error: auth.error,- ...(auth.keylessReason ? { reason: auth.keylessReason } : {}),- ...(auth.retryAfterSeconds- ? { retry_after_seconds: auth.retryAfterSeconds }- : {}),- });+ return res.status(auth.status).json({+ success: false,+ error: auth.error,+ ...(auth.keylessReason ? { reason: auth.keylessReason } : {}),+ ...(auth.retryAfterSeconds+ ? { retry_after_seconds: auth.retryAfterSeconds }+ : {}),+ });} else {return;}