Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(cost-tracking): add model tracking and more costs
Gergő Móricz7df557e59cbd
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex ae3df2614..523c9c8c1 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -72,6 +72,7 @@ export class CostTracking {type: "smartScrape" | "other",metadata: Record<string, any>,cost: number,+ model: string,tokens?: {input: number,output: number,diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.tsindex 9696bee2e..6591a41b1 100644--- a/apps/api/src/lib/extract/url-processor.ts+++ b/apps/api/src/lib/extract/url-processor.ts@@ -28,6 +28,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Costmodule: "extract",method: "generateBasicCompletion",},+ model: "openai/gpt-4o",cost: calculateCost("openai/gpt-4o", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),tokens: {input: result.usage?.promptTokens ?? 0,@@ -54,6 +55,7 @@ export async function generateBasicCompletion(prompt: string, costTracking: Costmodule: "extract",method: "generateBasicCompletion",},+ model: "openai/gpt-4o-mini",cost: calculateCost("openai/gpt-4o-mini", result.usage?.promptTokens ?? 0, result.usage?.completionTokens ?? 0),tokens: {input: result.usage?.promptTokens ?? 0,diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.tsindex 8458e5060..97e5b666d 100644--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts@@ -144,6 +144,7 @@ export async function smartScrape({costTracking.addCall({type: "smartScrape",cost: response.tokenUsage,+ model: "firecrawl/smart-scrape",metadata: {module: "smartScrape",method: "smartScrape",diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 4ff1f2545..64edea235 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -187,6 +187,8 @@ export function calculateCost() {const modelCosts = {"openai/o3-mini": { input_cost: 1.1, output_cost: 4.4 },+ "openai/gpt-4o-mini": { input_cost: 0.15, output_cost: 0.6 },+ "openai/gpt-4o": { input_cost: 2.5, output_cost: 10 },"google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },"google/gemini-2.0-flash-thinking-exp:free": {@@ -289,6 +291,7 @@ export async function generateCompletions({...costTrackingOptions.metadata,gcDetails: "no-object",},+ model: currentModel.modelId,cost: calculateCost(currentModel.modelId,result.usage?.promptTokens ?? 0,@@ -340,6 +343,7 @@ export async function generateCompletions({...costTrackingOptions.metadata,gcDetails: "no-object fallback",},+ model: currentModel.modelId,cost: calculateCost(currentModel.modelId,result.usage?.promptTokens ?? 0,@@ -454,6 +458,7 @@ export async function generateCompletions({repairUsage?.promptTokens ?? 0,repairUsage?.completionTokens ?? 0,),+ model: currentModel.modelId,tokens: {input: repairUsage?.promptTokens ?? 0,output: repairUsage?.completionTokens ?? 0,@@ -513,6 +518,7 @@ export async function generateCompletions({input: result.usage?.promptTokens ?? 0,output: result.usage?.completionTokens ?? 0,},+ model: currentModel.modelId,cost: calculateCost(currentModel.modelId,result.usage?.promptTokens ?? 0,@@ -547,6 +553,7 @@ export async function generateCompletions({input: result.usage?.promptTokens ?? 0,output: result.usage?.completionTokens ?? 0,},+ model: currentModel.modelId,cost: calculateCost(currentModel.modelId,result.usage?.promptTokens ?? 0,