Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(cost-tracking): record branding LLM calls and price all models in use (#4818)
Abimael Martellf5d456e735f4
diff --git a/apps/api/src/__tests__/lib/branding/llm-cost-tracking.test.ts b/apps/api/src/__tests__/lib/branding/llm-cost-tracking.test.tsnew file mode 100644index 000000000..398b30308--- /dev/null+++ b/apps/api/src/__tests__/lib/branding/llm-cost-tracking.test.ts@@ -0,0 +1,147 @@+import { vi, describe, it, expect, beforeEach } from "vitest";+import type { Mock } from "vitest";+import { generateObject, NoObjectGeneratedError } from "ai";++vi.mock("ai", async importOriginal => ({+ ...(await importOriginal<typeof import("ai")>()),+ generateObject: vi.fn(),+}));+vi.mock("../../../lib/generic-ai", () => ({+ getModel: vi.fn(),+}));++import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";+import { CostTracking } from "../../../lib/cost-tracking";+import { getModel } from "../../../lib/generic-ai";+import { logger } from "../../../lib/logger";++// A screenshot always selects gpt-4o, independent of the prompt's length.+const run = (costTracking: CostTracking) =>+ enhanceBrandingWithLLM({+ jsAnalysis: {},+ buttons: [],+ url: "https://example.com",+ screenshot: "data:image/png;base64,AAAA",+ costTracking,+ logger,+ });++const parseFailure = (usage: { inputTokens: number; outputTokens: number }) =>+ new NoObjectGeneratedError({+ message: "No object generated: could not parse the response.",+ text: "not json",+ response: {} as any,+ usage: usage as any,+ finishReason: "stop",+ });++beforeEach(() => {+ (generateObject as Mock).mockReset();+ (getModel as Mock)+ .mockReset()+ .mockImplementation((name: string) => ({ modelId: name }));+});++describe("branding LLM cost tracking", () => {+ it("records a successful call with its model, tokens and cost", async () => {+ (generateObject as Mock).mockResolvedValueOnce({+ object: {+ cleanedFonts: [],+ buttonClassification: {+ primaryButtonIndex: -1,+ primaryButtonReasoning: "none",+ secondaryButtonIndex: -1,+ secondaryButtonReasoning: "none",+ confidence: 0.5,+ },+ colorRoles: {+ primaryColor: "#000000",+ accentColor: "#ffffff",+ backgroundColor: "#ffffff",+ textPrimary: "#000000",+ confidence: 0.5,+ },+ },+ usage: { inputTokens: 1000, outputTokens: 200 },+ });+ const costTracking = new CostTracking();++ await run(costTracking);++ expect(costTracking.calls).toHaveLength(1);+ const call = costTracking.calls[0];+ expect(call.model).toBe("gpt-4o");+ expect(call.metadata).toEqual({+ module: "branding",+ method: "enhanceBrandingWithLLM",+ });+ expect(call.tokens).toEqual({ input: 1000, output: 200 });+ expect(call.cost).toBeCloseTo((1000 * 2.5 + 200 * 10) / 1_000_000, 12);+ });++ it("records the model getModel resolved, not the one requested", async () => {+ // e.g. a MODEL_NAME override.+ (getModel as Mock).mockReturnValueOnce({ modelId: "gpt-4.1" });+ (generateObject as Mock).mockResolvedValueOnce({+ object: {},+ usage: { inputTokens: 1000, outputTokens: 200 },+ });+ const costTracking = new CostTracking();++ await run(costTracking);++ expect(costTracking.calls[0].model).toBe("gpt-4.1");+ expect(costTracking.calls[0].cost).toBeCloseTo(+ (1000 * 2 + 200 * 8) / 1_000_000,+ 12,+ );+ });++ it("records a call whose output failed to parse, using the error's usage", async () => {+ (generateObject as Mock).mockRejectedValueOnce(+ parseFailure({ inputTokens: 500, outputTokens: 50 }),+ );+ const costTracking = new CostTracking();++ const result = await run(costTracking);++ expect(result.buttonClassification.primaryButtonReasoning).toBe(+ "LLM failed",+ );+ expect(costTracking.calls).toHaveLength(1);+ expect(costTracking.calls[0].tokens).toEqual({ input: 500, output: 50 });+ expect(costTracking.calls[0].cost).toBeGreaterThan(0);+ });++ it("records nothing when the request itself failed", async () => {+ (generateObject as Mock).mockRejectedValueOnce(+ new Error("connection reset"),+ );+ const costTracking = new CostTracking();++ await run(costTracking);++ expect(costTracking.calls).toHaveLength(0);+ });++ it("propagates a cost limit hit while recording a parse failure", async () => {+ (generateObject as Mock).mockRejectedValueOnce(+ parseFailure({ inputTokens: 1_000_000, outputTokens: 0 }),+ );++ await expect(run(new CostTracking(0.01))).rejects.toThrow(+ "Cost limit exceeded",+ );+ });++ it("propagates a cost limit instead of treating it as an LLM failure", async () => {+ (generateObject as Mock).mockResolvedValueOnce({+ object: {},+ usage: { inputTokens: 1_000_000, outputTokens: 0 },+ });++ await expect(run(new CostTracking(0.01))).rejects.toThrow(+ "Cost limit exceeded",+ );+ });+});diff --git a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.tsindex 67118ccca..aa89c87d6 100644--- a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts+++ b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts@@ -1,6 +1,7 @@import { vi, describe, it, expect } from "vitest";-vi.mock("ai", () => ({+vi.mock("ai", async importOriginal => ({+ ...(await importOriginal<typeof import("ai")>()),generateObject: vi.fn().mockRejectedValue(new Error("connection reset")),}));vi.mock("../../../lib/generic-ai", () => ({@@ -9,6 +10,7 @@ vi.mock("../../../lib/generic-ai", () => ({import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";import { mergeBrandingResults } from "../../../lib/branding/merge";+import { CostTracking } from "../../../lib/cost-tracking";import { logger } from "../../../lib/logger";const CANDIDATES = [@@ -38,6 +40,7 @@ describe("LLM failure fallback", () => {buttons: [],logoCandidates: CANDIDATES,url: "https://example.com",+ costTracking: new CostTracking(),logger,});diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.tsindex d76e55479..848d7e260 100644--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts@@ -1,3 +1,6 @@+import { eq } from "drizzle-orm";+import { db } from "../../../db/connection";+import * as schema from "../../../db/schema";import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";import { scrape, scrapeTimeout, idmux, Identity } from "./lib";@@ -35,6 +38,83 @@ describe("Branding declared-logo fallback", () => {);});+type CostTrackingCall = {+ model: string;+ cost: number;+ metadata: Record<string, unknown>;+ tokens?: { input: number; output: number };+};++// The scrape row is written when the job finishes; give the insert a moment.+async function getCostTrackingCalls(+ scrapeId: string,+): Promise<CostTrackingCall[]> {+ for (let attempt = 0; attempt < 10; attempt++) {+ const rows = await db+ .select({ cost_tracking: schema.scrapes.cost_tracking })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId))+ .limit(1);+ if (rows.length === 1) {+ const costTracking = rows[0].cost_tracking as {+ calls?: CostTrackingCall[];+ } | null;+ return costTracking?.calls ?? [];+ }+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ throw new Error(`No scrapes row for ${scrapeId}`);+}++const isBrandingCall = (call: CostTrackingCall) =>+ call.metadata?.module === "branding" &&+ call.metadata?.method === "enhanceBrandingWithLLM";++describe("Branding cost tracking", () => {+ concurrentIf(TEST_PRODUCTION)(+ "records the branding LLM call with its model and cost",+ async () => {+ const response = await scrape(+ {+ url: "https://firecrawl-test-site.vercel.app/branding-declared-only",+ formats: ["branding"],+ timeout: scrapeTimeout,+ },+ identity,+ );+ expect(response.branding).toBeDefined();++ const calls = await getCostTrackingCalls(response.metadata.scrapeId!);+ const brandingCalls = calls.filter(isBrandingCall);++ expect(brandingCalls).toHaveLength(1);+ expect(brandingCalls[0].model).toMatch(/^gpt-4o/);+ expect(brandingCalls[0].tokens?.input).toBeGreaterThan(0);+ expect(brandingCalls[0].cost).toBeGreaterThan(0);+ },+ scrapeTimeout + 15000,+ );++ concurrentIf(TEST_PRODUCTION)(+ "records no branding call when branding is not requested",+ async () => {+ const response = await scrape(+ {+ url: "https://firecrawl-test-site.vercel.app/branding-declared-only",+ formats: ["markdown"],+ timeout: scrapeTimeout,+ },+ identity,+ );++ const calls = await getCostTrackingCalls(response.metadata.scrapeId!);++ expect(calls.filter(isBrandingCall)).toHaveLength(0);+ },+ scrapeTimeout + 15000,+ );+});+// TODO: fix this test// Need to run on fire-enginedescribe.skip("Branding extraction", () => {diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex 7af7fb6bd..9f2211c32 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -1,6 +1,8 @@-import { generateObject } from "ai";+import { generateObject, LanguageModelUsage, NoObjectGeneratedError } from "ai";import { config } from "../../config";+import { calculateCost } from "../../scraper/scrapeURL/transformers/llmExtract";+import { CostLimitExceededError } from "../cost-tracking";import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";import { buildBrandingPrompt } from "./prompt";import { BrandingLLMInput } from "./types";@@ -77,6 +79,22 @@ export function unwrapSchemaShapedAnswer(text: string): string | null {return value === UNRESOLVED ? null : JSON.stringify(value);}+function recordBrandingCall(+ input: BrandingLLMInput,+ modelName: string,+ usage: LanguageModelUsage | undefined,+) {+ const inputTokens = usage?.inputTokens ?? 0;+ const outputTokens = usage?.outputTokens ?? 0;+ input.costTracking.addCall({+ type: "other",+ metadata: { module: "branding", method: "enhanceBrandingWithLLM" },+ model: modelName,+ cost: calculateCost(modelName, inputTokens, outputTokens),+ tokens: { input: inputTokens, output: outputTokens },+ });+}+function isDebugBrandingEnabled(input: BrandingLLMInput): boolean {return (config.DEBUG_BRANDING === true || input.teamFlags?.debugBranding === true@@ -108,6 +126,9 @@ export async function enhanceBrandingWithLLM(const modelName = isComplexCase ? "gpt-4o" : "gpt-4o-mini";const model = getModel(modelName);+ // getModel honors a MODEL_NAME override; record the model that actually ran.+ const modelId =+ (typeof model === "string" ? model : model.modelId) || modelName;if (isDebugBrandingEnabled(input)) {const logoCandidates = input.logoCandidates || [];@@ -204,6 +225,8 @@ export async function enhanceBrandingWithLLM(},});+ recordBrandingCall(input, modelId, result.usage);+if (isDebugBrandingEnabled(input)) {const reasoningPreview = result.reasoning? result.reasoning.length > 1000@@ -248,6 +271,16 @@ export async function enhanceBrandingWithLLM(}return resultObject;} catch (error) {+ if (error instanceof CostLimitExceededError) {+ throw error;+ }++ // The model still ran (and billed) when its output failed to parse or+ // validate.+ if (NoObjectGeneratedError.isInstance(error)) {+ recordBrandingCall(input, modelId, error.usage);+ }+// Refusal: API returned content type "refusal" (e.g. "I can't assist with that") but the SDK// expects "output_text", so it throws before we get a result. Treat as soft failure, not a bug.const message = error instanceof Error ? error.message : String(error);diff --git a/apps/api/src/lib/branding/transformer.ts b/apps/api/src/lib/branding/transformer.tsindex 0a0bc0db6..a79e81d6b 100644--- a/apps/api/src/lib/branding/transformer.ts+++ b/apps/api/src/lib/branding/transformer.ts@@ -2,6 +2,7 @@ import { processRawBranding } from "./processor";import { config } from "../../config";import { BrandingProfile } from "../../types/branding";import { enhanceBrandingWithLLM } from "./llm";+import { CostLimitExceededError } from "../cost-tracking";import { Meta } from "../../scraper/scrapeURL";import { Document } from "../../controllers/v2/types";import { BrandingScriptReturn, ButtonSnapshot } from "./types";@@ -210,6 +211,7 @@ export async function brandingTransformer(scrapeId: meta.id,zeroDataRetention: meta.internalOptions.zeroDataRetention,teamFlags: meta.internalOptions.teamFlags,+ costTracking: meta.costTracking,logger: meta.logger,});@@ -414,6 +416,10 @@ export async function brandingTransformer(types: inputSnapshots.map((i: any) => i.type).slice(0, 10),});} catch (error) {+ if (error instanceof CostLimitExceededError) {+ throw error;+ }+meta.logger.error("LLM branding enhancement failed, using JS analysis only",{ error },diff --git a/apps/api/src/lib/branding/types.ts b/apps/api/src/lib/branding/types.tsindex a06a10655..5410ab1f9 100644--- a/apps/api/src/lib/branding/types.ts+++ b/apps/api/src/lib/branding/types.ts@@ -1,5 +1,6 @@import { Logger } from "winston";import { BrandingProfile } from "../../types/branding";+import { CostTracking } from "../cost-tracking";export interface ButtonSnapshot {index: number;@@ -95,6 +96,7 @@ export interface BrandingLLMInput {scrapeId?: string;zeroDataRetention?: boolean;teamFlags?: { debugBranding?: boolean } | null;+ costTracking: CostTracking;logger: Logger;}diff --git a/apps/api/src/scraper/scrapeURL/transformers/calculateCost.test.ts b/apps/api/src/scraper/scrapeURL/transformers/calculateCost.test.tsnew file mode 100644index 000000000..13cf63212--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/calculateCost.test.ts@@ -0,0 +1,63 @@+import { calculateCost } from "./llmExtract";+import { logger } from "../../../lib/logger";++describe("calculateCost", () => {+ it.each([+ // [model, $ per 1M input tokens, $ per 1M output tokens]+ ["gpt-4o", 2.5, 10],+ ["openai/gpt-4o", 2.5, 10],+ ["gpt-4o-mini", 0.15, 0.6],+ ["gpt-4.1", 2, 8],+ ["gpt-4.1-mini", 0.4, 1.6],+ ["o3-mini", 1.1, 4.4],+ ["openai/o3-mini", 1.1, 4.4],+ ["gemini-2.5-flash-lite", 0.1, 0.4],+ ["google/gemini-2.5-flash-lite", 0.1, 0.4],+ ["gpt-5-mini", 0.25, 2],+ ["fireworks_ai/accounts/fireworks/models/gpt-oss-20b", 0.05, 0.2],+ ])("prices %s", (model, inputPerM, outputPerM) => {+ expect(calculateCost(model, 1_000_000, 0)).toBeCloseTo(inputPerM, 10);+ expect(calculateCost(model, 0, 1_000_000)).toBeCloseTo(outputPerM, 10);+ });++ it("falls back to the model price table for models not listed locally", () => {+ // gpt-4.1-nano is only in lib/extract/usage/model-prices.ts.+ expect(calculateCost("gpt-4.1-nano", 1_000_000, 1_000_000)).toBeCloseTo(+ 0.5,+ 10,+ );+ });++ it("keeps tiered pricing for gemini-2.5-pro", () => {+ // Up to 200k input tokens: $1.25 in / $10 out per 1M.+ expect(calculateCost("gemini-2.5-pro", 200_000, 0)).toBeCloseTo(0.25, 10);+ expect(calculateCost("gemini-2.5-pro", 200_000, 1_000_000)).toBeCloseTo(+ 10.25,+ 10,+ );+ // Above 200k input tokens: $2.50 in / $15 out per 1M.+ expect(calculateCost("gemini-2.5-pro", 200_001, 0)).toBeCloseTo(+ 0.5000025,+ 10,+ );+ expect(calculateCost("gemini-2.5-pro", 200_001, 1_000_000)).toBeCloseTo(+ 15.5000025,+ 10,+ );+ });++ it("warns once and returns 0 for an unknown model", () => {+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);++ expect(calculateCost("not-a-real-model", 1000, 1000)).toBe(0);+ expect(calculateCost("not-a-real-model", 1000, 1000)).toBe(0);++ const warnings = (warn.mock.calls as unknown[][]).filter(+ ([message]) =>+ typeof message === "string" && message.startsWith("No price for model"),+ );+ expect(warnings).toHaveLength(1);+ expect(warnings[0][1]).toMatchObject({ model: "not-a-real-model" });+ warn.mockRestore();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 87575a806..5475407d1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -273,32 +273,63 @@ export function trimToTokenLimit(}}-export function calculateCost(- model: string,- inputTokens: number,- outputTokens: number,-) {- const modelCosts = {- "openai/o3-mini": { input_cost: 1.1, output_cost: 4.4 },+// USD per million tokens. Takes precedence over modelPrices, which covers+// everything else.+const modelCosts: Record<string, { input_cost: number; output_cost: number }> =+ {"gpt-4o-mini": { input_cost: 0.15, output_cost: 0.6 },- "openai/gpt-4o-mini": { input_cost: 0.15, output_cost: 0.6 },- "openai/gpt-4o": { input_cost: 2.5, output_cost: 10 },+ "gpt-4o": { input_cost: 2.5, output_cost: 10 },+ "gpt-4.1": { input_cost: 2, output_cost: 8 },+ "gpt-4.1-mini": { input_cost: 0.4, output_cost: 1.6 },+ "o3-mini": { input_cost: 1.1, output_cost: 4.4 },"gpt-5": { input_cost: 1.25, output_cost: 10 },- "openai/gpt-5": { input_cost: 1.25, output_cost: 10 },"gpt-5-mini": { input_cost: 0.25, output_cost: 2 },- "openai/gpt-5-mini": { input_cost: 0.25, output_cost: 2 },"gpt-5-nano": { input_cost: 0.05, output_cost: 0.4 },- "openai/gpt-5-nano": { input_cost: 0.05, output_cost: 0.4 },"google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },"gemini-2.0-flash": { input_cost: 0.15, output_cost: 0.6 },+ "gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },"google/gemini-2.0-flash-thinking-exp:free": {input_cost: 0.55,output_cost: 2.19,},- "google/gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },};- let modelCost = modelCosts[model] || { input_cost: 0, output_cost: 0 };++const warnedUnpricedModels = new Set<string>();++function lookupModelCost(+ model: string,+): { input_cost: number; output_cost: number } | undefined {+ // Callers pass both bare ids ("gpt-4o") and provider-prefixed ones+ // ("openai/gpt-4o", "google/gemini-2.5-flash-lite").+ const candidates = [model];+ const slash = model.indexOf("/");+ if (slash !== -1) candidates.push(model.slice(slash + 1));++ for (const id of candidates) {+ if (modelCosts[id]) return modelCosts[id];+ }+ for (const id of candidates) {+ const price = modelPrices[id];+ if (+ typeof price?.input_cost_per_token === "number" &&+ typeof price?.output_cost_per_token === "number"+ ) {+ return {+ input_cost: price.input_cost_per_token * 1_000_000,+ output_cost: price.output_cost_per_token * 1_000_000,+ };+ }+ }+ return undefined;+}++export function calculateCost(+ model: string,+ inputTokens: number,+ outputTokens: number,+) {+ let modelCost = lookupModelCost(model);//gemini-2.5-pro-exp-03-25 pricingif (model.includes("gemini-2.5-pro")) {let inputCost = 0;@@ -312,6 +343,21 @@ export function calculateCost(}modelCost = { input_cost: inputCost, output_cost: outputCost };}++ if (!modelCost) {+ // Once per model per process: an unpriced model is a table gap, and every+ // call to it records $0.+ if (!warnedUnpricedModels.has(model)) {+ warnedUnpricedModels.add(model);+ logger.warn("No price for model, recording LLM call cost as $0", {+ module: "llmExtract",+ method: "calculateCost",+ model,+ });+ }+ return 0;+ }+const totalCost =(inputTokens * modelCost.input_cost +outputTokens * modelCost.output_cost) /@@ -494,8 +540,8 @@ export async function generateCompletions({promptTokens: result.usage?.inputTokens ?? 0,completionTokens: result.usage?.outputTokens ?? 0,totalTokens:- result.usage?.inputTokens ??- 0 + (result.usage?.outputTokens ?? 0),+ (result.usage?.inputTokens ?? 0) ++ (result.usage?.outputTokens ?? 0),},model: modelId,};@@ -601,8 +647,8 @@ export async function generateCompletions({promptTokens: result.usage?.inputTokens ?? 0,completionTokens: result.usage?.outputTokens ?? 0,totalTokens:- result.usage?.inputTokens ??- 0 + (result.usage?.outputTokens ?? 0),+ (result.usage?.inputTokens ?? 0) ++ (result.usage?.outputTokens ?? 0),},model: modelId,};diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.tsindex 7027afdea..2c211d565 100644--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts@@ -29,6 +29,9 @@ const DIRECT_QUOTE_MODEL = {provider: "fireworks" as const,// gpt-oss-20b's context window.contextTokens: 131_072,+ // The fine-tune has no per-token list price; cost tracking uses the base+ // model's serverless rate as the estimate.+ pricedAs: "fireworks_ai/accounts/fireworks/models/gpt-oss-20b",};// Room for the system prompt, the query, and the model's reasoning and answer.const DIRECT_QUOTE_RESERVED_TOKENS = 16_384;@@ -207,7 +210,11 @@ ${escapePromptTags(indexedLines)}type: "other",metadata: { feature: "query", model: modelName },model: modelName,- cost: calculateCost(modelName, inputTokens, outputTokens),+ cost: calculateCost(+ DIRECT_QUOTE_MODEL.pricedAs,+ inputTokens,+ outputTokens,+ ),tokens: { input: inputTokens, output: outputTokens },});