Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(cost-tracking): add model tracking and more costs

Gergő Móricz7df557e59cbd
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index ae3df2614..523c9c8c1 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -72,6 +72,7 @@ export class CostTracking {
type: "smartScrape" | "other",
metadata: Record<string, any>,
cost: number,
+ model: string,
tokens?: {
input: number,
output: number,
diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.ts
index 9696bee2e..6591a41b1 100644
--- a/apps/api/src/lib/extract/url-processor.ts
+++ b/apps/api/src/lib/extract/url-processor.ts
@@ -28,6 +28,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Cost
module: "extract",
method: "generateBasicCompletion",
},
+ model: "openai/gpt-4o",
cost: calculateCost("openai/gpt-4o", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),
tokens: {
input: result.usage?.promptTokens ?? 0,
@@ -54,6 +55,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Cost
module: "extract",
method: "generateBasicCompletion",
},
+ model: "openai/gpt-4o-mini",
cost: calculateCost("openai/gpt-4o-mini", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),
tokens: {
input: result.usage?.promptTokens ?? 0,
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
index 8458e5060..97e5b666d 100644
--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
@@ -144,6 +144,7 @@ export async function smartScrape({
costTracking.addCall({
type: "smartScrape",
cost: response.tokenUsage,
+ model: "firecrawl/smart-scrape",
metadata: {
module: "smartScrape",
method: "smartScrape",
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 4ff1f2545..64edea235 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -187,6 +187,8 @@ export function calculateCost(
) {
const modelCosts = {
"openai/o3-mini": { input_cost: 1.1, output_cost: 4.4 },
+ "openai/gpt-4o-mini": { input_cost: 0.15, output_cost: 0.6 },
+ "openai/gpt-4o": { input_cost: 2.5, output_cost: 10 },
"google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },
"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },
"google/gemini-2.0-flash-thinking-exp:free": {
@@ -289,6 +291,7 @@ export async function generateCompletions({
...costTrackingOptions.metadata,
gcDetails: "no-object",
},
+ model: currentModel.modelId,
cost: calculateCost(
currentModel.modelId,
result.usage?.promptTokens ?? 0,
@@ -340,6 +343,7 @@ export async function generateCompletions({
...costTrackingOptions.metadata,
gcDetails: "no-object fallback",
},
+ model: currentModel.modelId,
cost: calculateCost(
currentModel.modelId,
result.usage?.promptTokens ?? 0,
@@ -454,6 +458,7 @@ export async function generateCompletions({
repairUsage?.promptTokens ?? 0,
repairUsage?.completionTokens ?? 0,
),
+ model: currentModel.modelId,
tokens: {
input: repairUsage?.promptTokens ?? 0,
output: repairUsage?.completionTokens ?? 0,
@@ -513,6 +518,7 @@ export async function generateCompletions({
input: result.usage?.promptTokens ?? 0,
output: result.usage?.completionTokens ?? 0,
},
+ model: currentModel.modelId,
cost: calculateCost(
currentModel.modelId,
result.usage?.promptTokens ?? 0,
@@ -547,6 +553,7 @@ export async function generateCompletions({
input: result.usage?.promptTokens ?? 0,
output: result.usage?.completionTokens ?? 0,
},
+ model: currentModel.modelId,
cost: calculateCost(
currentModel.modelId,
result.usage?.promptTokens ?? 0,