Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(extract): port to billing credits (#2482)

Gergő Móriczeebe41623e4b
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 23137225b..fa37b1767 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -465,7 +465,7 @@ async function supaAuthenticateUser(
};
}
- chunk = await getACUC(normalizedApi, false, true, mode);
+ chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);
if (chunk === null) {
return {
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index 3e7a80183..abfa06775 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -128,7 +128,11 @@ export async function extractStatusController(
error: (() => {
if (typeof extract?.error === "string") return extract.error;
if (extract?.error && typeof extract.error === "object") {
- return typeof extract.error.message === "string" ? extract.error.message : (typeof extract.error.error === "string" ? extract.error.error : JSON.stringify(extract.error));
+ return typeof extract.error.message === "string"
+ ? extract.error.message
+ : typeof extract.error.error === "string"
+ ? extract.error.error
+ : JSON.stringify(extract.error);
}
return undefined;
})(),
@@ -139,5 +143,6 @@ export async function extractStatusController(
costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,
+ creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,
});
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index d9fe3294c..ce06468f5 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1095,6 +1095,7 @@ export interface ExtractResponse {
[key: string]: string[];
};
tokensUsed?: number;
+ creditsUsed?: number;
}
export type CrawlResponse =
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 80604a589..3c55c14e6 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -128,7 +128,11 @@ export async function extractStatusController(
error: (() => {
if (typeof extract?.error === "string") return extract.error;
if (extract?.error && typeof extract.error === "object") {
- return extract.error.message || extract.error.error || JSON.stringify(extract.error);
+ return (
+ extract.error.message ||
+ extract.error.error ||
+ JSON.stringify(extract.error)
+ );
}
return undefined;
})(),
@@ -139,5 +143,6 @@ export async function extractStatusController(
costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,
+ creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,
});
}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 43e4a1a13..093b2f1bc 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1004,6 +1004,7 @@ export interface ExtractResponse {
[key: string]: string[];
};
tokensUsed?: number;
+ creditsUsed?: number;
}
export type CrawlResponse =
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index c161666de..3b153e403 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -40,6 +40,7 @@ type StoredExtract = {
};
sessionIds?: string[];
tokensBilled?: number;
+ creditsBilled?: number;
zeroDataRetention?: boolean;
};
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index c2f377a9c..c9b730146 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -133,6 +133,7 @@ export async function performExtraction(
});
const tokens_billed = 300 + calculateThinkingCost(costTracking);
+ const creditsToBill = Math.ceil(tokens_billed / 15);
logExtract({
id: extractId,
request_id: extractId,
@@ -140,24 +141,19 @@ export async function performExtraction(
team_id: teamId,
options: request,
model_kind: "fire-1",
- credits_cost: Math.ceil(tokens_billed / 15),
+ credits_cost: creditsToBill,
is_successful: false,
error: "No search results found",
cost_tracking: costTracking.toJSON(),
});
- await billTeam(
- teamId,
- subId,
- tokens_billed,
- apiKeyId,
- logger,
- true,
- ).catch(error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- });
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ },
+ );
return {
success: false,
@@ -670,6 +666,7 @@ export async function performExtraction(
multiEntitySchema: JSON.stringify(multiEntitySchema),
});
const tokens_billed = 300 + calculateThinkingCost(costTracking);
+ const creditsToBill = Math.ceil(tokens_billed / 15);
logExtract({
id: extractId,
request_id: extractId,
@@ -677,23 +674,18 @@ export async function performExtraction(
team_id: teamId,
options: request,
model_kind: "fire-1",
- credits_cost: Math.ceil(tokens_billed / 15),
+ credits_cost: creditsToBill,
is_successful: false,
error: "Failed to transform array to object",
cost_tracking: costTracking.toJSON(),
});
- await billTeam(
- teamId,
- subId,
- tokens_billed,
- apiKeyId,
- logger,
- true,
- ).catch(error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- });
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ error => {
+ logger.error(
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
+ );
+ },
+ );
return {
success: false,
error: getErrorContactMessage(),
@@ -781,30 +773,26 @@ export async function performExtraction(
logger.debug("Scrapes finished.", { docCount: validResults.length });
} catch (error) {
const tokens_billed = 300 + calculateThinkingCost(costTracking);
+ const creditsToBill = Math.ceil(tokens_billed / 15);
logExtract({
id: extractId,
request_id: extractId,
urls: request.urls || [],
team_id: teamId,
options: request,
- credits_cost: Math.ceil(tokens_billed / 15),
+ credits_cost: creditsToBill,
is_successful: false,
error: "Failed to scrape documents",
model_kind: "fire-1",
cost_tracking: costTracking.toJSON(),
});
- await billTeam(
- teamId,
- subId,
- tokens_billed,
- apiKeyId,
- logger,
- true,
- ).catch(error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- });
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ error => {
+ logger.error(
+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,
+ );
+ },
+ );
return {
success: false,
error: error.message,
@@ -820,25 +808,21 @@ export async function performExtraction(
"All provided URLs are either invalid, unsupported or failed to be scraped.";
logger.error(errorMessage);
const tokens_billed = 300 + calculateThinkingCost(costTracking);
- await billTeam(
- teamId,
- subId,
- tokens_billed,
- apiKeyId,
- logger,
- true,
- ).catch(error => {
- logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
- );
- });
+ const creditsToBill = Math.ceil(tokens_billed / 15);
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ error => {
+ logger.error(
+ `Failed to bill team ${teamId} for thinking tokens: ${error}`,
+ );
+ },
+ );
logExtract({
id: extractId,
request_id: extractId,
urls: request.urls || [],
team_id: teamId,
options: request,
- credits_cost: Math.ceil(tokens_billed / 15),
+ credits_cost: creditsToBill,
is_successful: false,
error: errorMessage,
model_kind: "fire-1",
@@ -1009,11 +993,13 @@ export async function performExtraction(
tokensToBill = 1;
}
+ const creditsToBill = Math.ceil(tokensToBill / 15);
+
// Bill team for usage
- await billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch(
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
error => {
logger.error(
- `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
);
},
);
@@ -1025,7 +1011,7 @@ export async function performExtraction(
urls: request.urls || [],
team_id: teamId,
options: request,
- credits_cost: Math.ceil(tokensToBill / 15),
+ credits_cost: creditsToBill,
is_successful: true,
result: finalResult ?? {},
model_kind: "fire-1",
@@ -1036,6 +1022,7 @@ export async function performExtraction(
llmUsage,
sources,
tokensBilled: tokensToBill,
+ creditsBilled: creditsToBill,
// costTracking,
}).catch(error => {
logger.error(
@@ -1078,10 +1065,11 @@ export async function performExtraction(
};
} catch (error) {
const tokens_billed = 300 + calculateThinkingCost(costTracking);
- await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch(
+ const creditsToBill = Math.ceil(tokens_billed / 15);
+ await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
error => {
logger.error(
- `Failed to bill team ${teamId} for thinking tokens: ${error}`,
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
);
},
);
@@ -1091,7 +1079,7 @@ export async function performExtraction(
urls: request.urls || [],
team_id: teamId,
options: request,
- credits_cost: Math.ceil(tokens_billed / 15),
+ credits_cost: creditsToBill,
is_successful: false,
error:
error instanceof Error ? error.message : "An unexpected error occurred",
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 63936aa32..1b70bc4ab 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -866,10 +866,12 @@ export async function performExtraction_F0(
tokensToBill = 1;
}
+ const creditsToBill = Math.ceil(tokensToBill / 15);
+
// Bill team for usage
- billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch(error => {
+ billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {
logger.error(
- `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,
+ `Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
);
});
@@ -881,7 +883,7 @@ export async function performExtraction_F0(
team_id: teamId,
options: request,
model_kind: "fire-0",
- credits_cost: Math.ceil(tokensToBill / 15),
+ credits_cost: creditsToBill,
is_successful: true,
result: finalResult ?? {},
cost_tracking: {
@@ -908,6 +910,7 @@ export async function performExtraction_F0(
llmUsage,
sources,
tokensBilled: tokensToBill,
+ creditsBilled: creditsToBill,
}).catch(error => {
logger.error(
`Failed to update extract ${extractId} status to completed: ${error}`,
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 1ddea4ef5..f4a20f896 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -213,7 +213,7 @@ v1Router.post(
"/extract",
authMiddleware(RateLimiterMode.Extract),
countryCheck,
- checkCreditsMiddleware(1),
+ checkCreditsMiddleware(20),
wrap(extractController),
);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 099b095ef..f088fb632 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -286,7 +286,7 @@ v2Router.post(
"/extract",
authMiddleware(RateLimiterMode.Extract),
countryCheck,
- checkCreditsMiddleware(1),
+ checkCreditsMiddleware(20),
blocklistMiddleware,
wrap(extractController),
);
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index f6402965b..618a29f93 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -10,9 +10,6 @@ import { getValue, setValue } from "../redis";
import { queueBillingOperation } from "./batch_billing";
import type { Logger } from "winston";
-// Deprecated, done via rpc
-const FREE_CREDITS = 500;
-
/**
* If you do not know the subscription_id in the current context, pass subscription_id as undefined.
*/
@@ -22,7 +19,6 @@ export async function billTeam(
credits: number,
api_key_id: number | null,
logger?: Logger,
- is_extract: boolean = false,
) {
// Maintain the withAuth wrapper for authentication
return withAuth(
@@ -32,7 +28,6 @@ export async function billTeam(
credits: number,
api_key_id: number | null,
logger: Logger | undefined,
- is_extract: boolean,
) => {
// Within the authenticated context, queue the billing operation
return queueBillingOperation(
@@ -40,11 +35,11 @@ export async function billTeam(
subscription_id,
credits,
api_key_id,
- is_extract,
+ false,
);
},
{ success: true, message: "No DB, bypassed." },
- )(team_id, subscription_id, credits, api_key_id, logger, is_extract);
+ )(team_id, subscription_id, credits, api_key_id, logger);
}
type CheckTeamCreditsResponse = {
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 8b74e4f13..46cf1e906 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.8.1",
+ "version": "4.8.2",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 4022ba8d9..0fc9799f0 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -360,6 +360,7 @@ export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {
error?: string;
warning?: string;
sources?: string[];
+ creditsUsed?: number;
}
/**
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index badb39fd1..2556b63c7 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -529,6 +529,7 @@ export interface ExtractResponse {
warning?: string;
sources?: Record<string, unknown>;
expiresAt?: string;
+ creditsUsed?: number;
}
export interface AgentOptions {
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 2d8182dca..dfea7024c 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.10.0"
+__version__ = "4.10.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index 5f39fa988..797cb8b6b 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -339,6 +339,7 @@ class V1ExtractResponse(pydantic.BaseModel, Generic[T]):
error: Optional[str] = None
warning: Optional[str] = None
sources: Optional[Dict[Any, Any]] = None
+ creditsUsed: Optional[int] = None
class V1SearchParams(pydantic.BaseModel):
query: str
diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.py
index 87e4712db..96d7804ae 100644
--- a/apps/python-sdk/firecrawl/v2/methods/extract.py
+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py
@@ -53,6 +53,17 @@ def _prepare_extract_request(
return body
+def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
+ out = dict(payload)
+ if "expiresAt" in out and "expires_at" not in out:
+ out["expires_at"] = out["expiresAt"]
+ if "creditsUsed" in out and "credits_used" not in out:
+ out["credits_used"] = out["creditsUsed"]
+ if "tokensUsed" in out and "tokens_used" not in out:
+ out["tokens_used"] = out["tokensUsed"]
+ return out
+
+
def start_extract(
client: HttpClient,
urls: Optional[List[str]],
@@ -84,14 +95,16 @@ def start_extract(
resp = client.post("/v2/extract", body)
if not resp.ok:
handle_response_error(resp, "extract")
- return ExtractResponse(**resp.json())
+ payload = _normalize_extract_response_payload(resp.json())
+ return ExtractResponse(**payload)
def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
resp = client.get(f"/v2/extract/{job_id}")
if not resp.ok:
handle_response_error(resp, "extract-status")
- return ExtractResponse(**resp.json())
+ payload = _normalize_extract_response_payload(resp.json())
+ return ExtractResponse(**payload)
def wait_extract(
@@ -145,4 +158,4 @@ def extract(
job_id = getattr(started, "id", None)
if not job_id:
return started
- return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
\ No newline at end of file
+ return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index afdfd270d..e99139633 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -771,7 +771,8 @@ class ExtractResponse(BaseModel):
warning: Optional[str] = None
sources: Optional[Dict[str, Any]] = None
expires_at: Optional[datetime] = None
-
+ credits_used: Optional[int] = None
+ tokens_used: Optional[int] = None
# Usage/limits types
class ConcurrencyCheck(BaseModel):