Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(extract): port to billing credits (#2482)
Gergő Móriczeebe41623e4b
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 23137225b..fa37b1767 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -465,7 +465,7 @@ async function supaAuthenticateUser(};}- chunk = await getACUC(normalizedApi, false, true, mode);+ chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);if (chunk === null) {return {diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex 3e7a80183..abfa06775 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -128,7 +128,11 @@ export async function extractStatusController(error: (() => {if (typeof extract?.error === "string") return extract.error;if (extract?.error && typeof extract.error === "object") {- return typeof extract.error.message === "string" ? extract.error.message : (typeof extract.error.error === "string" ? extract.error.error : JSON.stringify(extract.error));+ return typeof extract.error.message === "string"+ ? extract.error.message+ : typeof extract.error.error === "string"+ ? extract.error.error+ : JSON.stringify(extract.error);}return undefined;})(),@@ -139,5 +143,6 @@ export async function extractStatusController(costTracking: extract?.showCostTracking ? extract.costTracking : undefined,sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,+ creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,});}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex d9fe3294c..ce06468f5 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1095,6 +1095,7 @@ export interface ExtractResponse {[key: string]: string[];};tokensUsed?: number;+ creditsUsed?: number;}export type CrawlResponse =diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 80604a589..3c55c14e6 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -128,7 +128,11 @@ export async function extractStatusController(error: (() => {if (typeof extract?.error === "string") return extract.error;if (extract?.error && typeof extract.error === "object") {- return extract.error.message || extract.error.error || JSON.stringify(extract.error);+ return (+ extract.error.message ||+ extract.error.error ||+ JSON.stringify(extract.error)+ );}return undefined;})(),@@ -139,5 +143,6 @@ export async function extractStatusController(costTracking: extract?.showCostTracking ? extract.costTracking : undefined,sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,+ creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,});}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 43e4a1a13..093b2f1bc 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1004,6 +1004,7 @@ export interface ExtractResponse {[key: string]: string[];};tokensUsed?: number;+ creditsUsed?: number;}export type CrawlResponse =diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex c161666de..3b153e403 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -40,6 +40,7 @@ type StoredExtract = {};sessionIds?: string[];tokensBilled?: number;+ creditsBilled?: number;zeroDataRetention?: boolean;};diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex c2f377a9c..c9b730146 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -133,6 +133,7 @@ export async function performExtraction(});const tokens_billed = 300 + calculateThinkingCost(costTracking);+ const creditsToBill = Math.ceil(tokens_billed / 15);logExtract({id: extractId,request_id: extractId,@@ -140,24 +141,19 @@ export async function performExtraction(team_id: teamId,options: request,model_kind: "fire-1",- credits_cost: Math.ceil(tokens_billed / 15),+ credits_cost: creditsToBill,is_successful: false,error: "No search results found",cost_tracking: costTracking.toJSON(),});- await billTeam(- teamId,- subId,- tokens_billed,- apiKeyId,- logger,- true,- ).catch(error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- });+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ },+ );return {success: false,@@ -670,6 +666,7 @@ export async function performExtraction(multiEntitySchema: JSON.stringify(multiEntitySchema),});const tokens_billed = 300 + calculateThinkingCost(costTracking);+ const creditsToBill = Math.ceil(tokens_billed / 15);logExtract({id: extractId,request_id: extractId,@@ -677,23 +674,18 @@ export async function performExtraction(team_id: teamId,options: request,model_kind: "fire-1",- credits_cost: Math.ceil(tokens_billed / 15),+ credits_cost: creditsToBill,is_successful: false,error: "Failed to transform array to object",cost_tracking: costTracking.toJSON(),});- await billTeam(- teamId,- subId,- tokens_billed,- apiKeyId,- logger,- true,- ).catch(error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- });+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ error => {+ logger.error(+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,+ );+ },+ );return {success: false,error: getErrorContactMessage(),@@ -781,30 +773,26 @@ export async function performExtraction(logger.debug("Scrapes finished.", { docCount: validResults.length });} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);+ const creditsToBill = Math.ceil(tokens_billed / 15);logExtract({id: extractId,request_id: extractId,urls: request.urls || [],team_id: teamId,options: request,- credits_cost: Math.ceil(tokens_billed / 15),+ credits_cost: creditsToBill,is_successful: false,error: "Failed to scrape documents",model_kind: "fire-1",cost_tracking: costTracking.toJSON(),});- await billTeam(- teamId,- subId,- tokens_billed,- apiKeyId,- logger,- true,- ).catch(error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- });+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ error => {+ logger.error(+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,+ );+ },+ );return {success: false,error: error.message,@@ -820,25 +808,21 @@ export async function performExtraction("All provided URLs are either invalid, unsupported or failed to be scraped.";logger.error(errorMessage);const tokens_billed = 300 + calculateThinkingCost(costTracking);- await billTeam(- teamId,- subId,- tokens_billed,- apiKeyId,- logger,- true,- ).catch(error => {- logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,- );- });+ const creditsToBill = Math.ceil(tokens_billed / 15);+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ error => {+ logger.error(+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,+ );+ },+ );logExtract({id: extractId,request_id: extractId,urls: request.urls || [],team_id: teamId,options: request,- credits_cost: Math.ceil(tokens_billed / 15),+ credits_cost: creditsToBill,is_successful: false,error: errorMessage,model_kind: "fire-1",@@ -1009,11 +993,13 @@ export async function performExtraction(tokensToBill = 1;}+ const creditsToBill = Math.ceil(tokensToBill / 15);+// Bill team for usage- await billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch(+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {logger.error(- `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,);},);@@ -1025,7 +1011,7 @@ export async function performExtraction(urls: request.urls || [],team_id: teamId,options: request,- credits_cost: Math.ceil(tokensToBill / 15),+ credits_cost: creditsToBill,is_successful: true,result: finalResult ?? {},model_kind: "fire-1",@@ -1036,6 +1022,7 @@ export async function performExtraction(llmUsage,sources,tokensBilled: tokensToBill,+ creditsBilled: creditsToBill,// costTracking,}).catch(error => {logger.error(@@ -1078,10 +1065,11 @@ export async function performExtraction(};} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);- await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch(+ const creditsToBill = Math.ceil(tokens_billed / 15);+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {logger.error(- `Failed to bill team ${teamId} for thinking tokens: ${error}`,+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,);},);@@ -1091,7 +1079,7 @@ export async function performExtraction(urls: request.urls || [],team_id: teamId,options: request,- credits_cost: Math.ceil(tokens_billed / 15),+ credits_cost: creditsToBill,is_successful: false,error:error instanceof Error ? error.message : "An unexpected error occurred",diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 63936aa32..1b70bc4ab 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -866,10 +866,12 @@ export async function performExtraction_F0(tokensToBill = 1;}+ const creditsToBill = Math.ceil(tokensToBill / 15);+// Bill team for usage- billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch(error => {+ billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {logger.error(- `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,);});@@ -881,7 +883,7 @@ export async function performExtraction_F0(team_id: teamId,options: request,model_kind: "fire-0",- credits_cost: Math.ceil(tokensToBill / 15),+ credits_cost: creditsToBill,is_successful: true,result: finalResult ?? {},cost_tracking: {@@ -908,6 +910,7 @@ export async function performExtraction_F0(llmUsage,sources,tokensBilled: tokensToBill,+ creditsBilled: creditsToBill,}).catch(error => {logger.error(`Failed to update extract ${extractId} status to completed: ${error}`,diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 1ddea4ef5..f4a20f896 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -213,7 +213,7 @@ v1Router.post("/extract",authMiddleware(RateLimiterMode.Extract),countryCheck,- checkCreditsMiddleware(1),+ checkCreditsMiddleware(20),wrap(extractController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 099b095ef..f088fb632 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -286,7 +286,7 @@ v2Router.post("/extract",authMiddleware(RateLimiterMode.Extract),countryCheck,- checkCreditsMiddleware(1),+ checkCreditsMiddleware(20),blocklistMiddleware,wrap(extractController),);diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex f6402965b..618a29f93 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -10,9 +10,6 @@ import { getValue, setValue } from "../redis";import { queueBillingOperation } from "./batch_billing";import type { Logger } from "winston";-// Deprecated, done via rpc-const FREE_CREDITS = 500;-/*** If you do not know the subscription_id in the current context, pass subscription_id as undefined.*/@@ -22,7 +19,6 @@ export async function billTeam(credits: number,api_key_id: number | null,logger?: Logger,- is_extract: boolean = false,) {// Maintain the withAuth wrapper for authenticationreturn withAuth(@@ -32,7 +28,6 @@ export async function billTeam(credits: number,api_key_id: number | null,logger: Logger | undefined,- is_extract: boolean,) => {// Within the authenticated context, queue the billing operationreturn queueBillingOperation(@@ -40,11 +35,11 @@ export async function billTeam(subscription_id,credits,api_key_id,- is_extract,+ false,);},{ success: true, message: "No DB, bypassed." },- )(team_id, subscription_id, credits, api_key_id, logger, is_extract);+ )(team_id, subscription_id, credits, api_key_id, logger);}type CheckTeamCreditsResponse = {diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 8b74e4f13..46cf1e906 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.8.1",+ "version": "4.8.2","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 4022ba8d9..0fc9799f0 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -360,6 +360,7 @@ export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {error?: string;warning?: string;sources?: string[];+ creditsUsed?: number;}/**diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex badb39fd1..2556b63c7 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -529,6 +529,7 @@ export interface ExtractResponse {warning?: string;sources?: Record<string, unknown>;expiresAt?: string;+ creditsUsed?: number;}export interface AgentOptions {diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 2d8182dca..dfea7024c 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.10.0"+__version__ = "4.10.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex 5f39fa988..797cb8b6b 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -339,6 +339,7 @@ class V1ExtractResponse(pydantic.BaseModel, Generic[T]):error: Optional[str] = Nonewarning: Optional[str] = Nonesources: Optional[Dict[Any, Any]] = None+ creditsUsed: Optional[int] = Noneclass V1SearchParams(pydantic.BaseModel):query: strdiff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.pyindex 87e4712db..96d7804ae 100644--- a/apps/python-sdk/firecrawl/v2/methods/extract.py+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py@@ -53,6 +53,17 @@ def _prepare_extract_request(return body+def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:+ out = dict(payload)+ if "expiresAt" in out and "expires_at" not in out:+ out["expires_at"] = out["expiresAt"]+ if "creditsUsed" in out and "credits_used" not in out:+ out["credits_used"] = out["creditsUsed"]+ if "tokensUsed" in out and "tokens_used" not in out:+ out["tokens_used"] = out["tokensUsed"]+ return out++def start_extract(client: HttpClient,urls: Optional[List[str]],@@ -84,14 +95,16 @@ def start_extract(resp = client.post("/v2/extract", body)if not resp.ok:handle_response_error(resp, "extract")- return ExtractResponse(**resp.json())+ payload = _normalize_extract_response_payload(resp.json())+ return ExtractResponse(**payload)def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:resp = client.get(f"/v2/extract/{job_id}")if not resp.ok:handle_response_error(resp, "extract-status")- return ExtractResponse(**resp.json())+ payload = _normalize_extract_response_payload(resp.json())+ return ExtractResponse(**payload)def wait_extract(@@ -145,4 +158,4 @@ def extract(job_id = getattr(started, "id", None)if not job_id:return started- return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)\ No newline at end of file+ return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex afdfd270d..e99139633 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -771,7 +771,8 @@ class ExtractResponse(BaseModel):warning: Optional[str] = Nonesources: Optional[Dict[str, Any]] = Noneexpires_at: Optional[datetime] = None-+ credits_used: Optional[int] = None+ tokens_used: Optional[int] = None# Usage/limits typesclass ConcurrencyCheck(BaseModel):