Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(telemetry): record X Search items fetched and price them in x-twitter cost tracking (#4824)
Abimael Martellb7bdbdc28c71
diff --git a/apps/api/src/lib/ai-usage-telemetry.test.ts b/apps/api/src/lib/ai-usage-telemetry.test.tsindex aab3a83b7..c921e9bb7 100644--- a/apps/api/src/lib/ai-usage-telemetry.test.ts+++ b/apps/api/src/lib/ai-usage-telemetry.test.ts@@ -1,3 +1,4 @@+import { createXai } from "@ai-sdk/xai";import { context, propagation, trace } from "@opentelemetry/api";import {InMemorySpanExporter,@@ -75,6 +76,60 @@ const openAIResponsesBody = {},};+// Responses API body as returned by xAI for a request that ran X Search.+// Trimmed to the fields `@ai-sdk/xai` validates plus the usage block.+function xaiResponsesBody(usage: Record<string, unknown>) {+ return {+ id: "resp_xai_test",+ object: "response",+ created_at: 1_700_000_000,+ model: "grok-4-1-fast-non-reasoning",+ status: "completed",+ output: [+ {+ type: "x_search_call",+ id: "xs_test",+ name: "x_user_search",+ arguments: '{"query":"firecrawl"}',+ status: "completed",+ },+ {+ type: "message",+ id: "msg_xai_test",+ role: "assistant",+ status: "completed",+ content: [{ type: "output_text", text: "ok", annotations: [] }],+ },+ ],+ usage: {+ input_tokens: 3000,+ input_tokens_details: { cached_tokens: 2000 },+ output_tokens: 400,+ output_tokens_details: { reasoning_tokens: 0 },+ total_tokens: 3400,+ num_server_side_tools_used: 2,+ ...usage,+ },+ };+}++async function generateWithXaiResponses(body: unknown) {+ const xai = createXai({+ apiKey: "test-key",+ fetch: async () =>+ new Response(JSON.stringify(body), {+ status: 200,+ headers: { "content-type": "application/json" },+ }),+ });+ await generateText({+ model: withUsageTelemetry(xai.responses("grok-4-1-fast-non-reasoning")),+ tools: { x_search: xai.tools.xSearch() } as any,+ prompt: "hello",+ experimental_telemetry: telemetry,+ });+}+describe("usage telemetry middleware", () => {const exporter = new InMemorySpanExporter();let provider: NodeTracerProvider;@@ -159,6 +214,48 @@ describe("usage telemetry middleware", () => {});});+ it("records X Search items fetched on the xAI Responses generateText span", async () => {+ await generateWithXaiResponses(+ xaiResponsesBody({+ server_side_tool_usage_details: {+ web_search_calls: 0,+ x_search_calls: 2,+ x_posts_fetched: 44,+ x_users_fetched: 3,+ code_interpreter_calls: 0,+ file_search_calls: 0,+ mcp_calls: 0,+ document_search_calls: 0,+ image_generation_calls: 0,+ },+ }),+ );++ const span = await spanNamed("ai.generateText.doGenerate");+ expect(span.attributes).toMatchObject({+ "ai.usage.promptTokens": 3000,+ "ai.usage.cachedInputTokens": 2000,+ "firecrawl.llm.tool.x_search_posts": 44,+ "firecrawl.llm.tool.x_search_profiles": 3,+ });+ });++ it("omits X Search attributes when xAI reports no item counts", async () => {+ await generateWithXaiResponses(+ xaiResponsesBody({+ server_side_tool_usage_details: { x_search_calls: 1 },+ }),+ );++ const span = await spanNamed("ai.generateText.doGenerate");+ expect(span.attributes["ai.usage.promptTokens"]).toBe(3000);+ expect(+ Object.keys(span.attributes).filter(key =>+ key.startsWith("firecrawl.llm.tool."),+ ),+ ).toEqual([]);+ });+it("records cache writes on the generateText span", async () => {await generateText({model: withUsageTelemetry(diff --git a/apps/api/src/lib/ai-usage-telemetry.ts b/apps/api/src/lib/ai-usage-telemetry.tsindex 509d3bbbc..fc1b0517f 100644--- a/apps/api/src/lib/ai-usage-telemetry.ts+++ b/apps/api/src/lib/ai-usage-telemetry.ts@@ -1,8 +1,10 @@import { trace, type Attributes, type Span } from "@opentelemetry/api";import { wrapLanguageModel, type LanguageModelMiddleware } from "ai";+import { xSearchUsageFromResponseBody } from "./xai-x-search";type ProviderModel = Parameters<typeof wrapLanguageModel>[0]["model"];-type ProviderUsage = Awaited<ReturnType<ProviderModel["doGenerate"]>>["usage"];+type ProviderResult = Awaited<ReturnType<ProviderModel["doGenerate"]>>;+type ProviderUsage = ProviderResult["usage"];// The AI SDK's `ai.generateObject.doGenerate` and `ai.generateText.doGenerate`// spans only record prompt and completion token totals, so cache reads and@@ -43,12 +45,28 @@ function usageTelemetryAttributes(usage: ProviderUsage): Attributes {);}+// Server-side tool fees the provider bills per item, which token counts+// cannot price. Only xAI X Search today.+function toolUsageTelemetryAttributes(result: ProviderResult): Attributes {+ const xSearch = xSearchUsageFromResponseBody(result.response?.body);+ if (!xSearch) {+ return {};+ }+ return {+ "firecrawl.llm.tool.x_search_posts": xSearch.posts,+ "firecrawl.llm.tool.x_search_profiles": xSearch.profiles,+ };+}+const usageTelemetryMiddleware: LanguageModelMiddleware = {specificationVersion: "v3",wrapGenerate: async ({ doGenerate }) => {const span = activeDoGenerateSpan();const result = await doGenerate();- span?.setAttributes(usageTelemetryAttributes(result.usage));+ span?.setAttributes({+ ...usageTelemetryAttributes(result.usage),+ ...toolUsageTelemetryAttributes(result),+ });return result;},};diff --git a/apps/api/src/lib/xai-x-search.test.ts b/apps/api/src/lib/xai-x-search.test.tsnew file mode 100644index 000000000..181386a0f--- /dev/null+++ b/apps/api/src/lib/xai-x-search.test.ts@@ -0,0 +1,72 @@+import { xSearchCost, xSearchUsageFromResponseBody } from "./xai-x-search";++describe("xSearchUsageFromResponseBody", () => {+ it("reads posts and profiles fetched from the Responses usage block", () => {+ expect(+ xSearchUsageFromResponseBody({+ usage: {+ server_side_tool_usage_details: {+ x_search_calls: 2,+ x_posts_fetched: 44,+ x_users_fetched: 3,+ },+ },+ }),+ ).toEqual({ posts: 44, profiles: 3 });+ });++ it("counts a missing field as zero when the other is reported", () => {+ expect(+ xSearchUsageFromResponseBody({+ usage: { server_side_tool_usage_details: { x_posts_fetched: 12 } },+ }),+ ).toEqual({ posts: 12, profiles: 0 });+ });++ it.each([+ ["no body", undefined],+ ["a non-object body", "not json"],+ ["no usage", { id: "resp" }],+ ["no tool usage details", { usage: { input_tokens: 10 } }],+ [+ "no item counts",+ { usage: { server_side_tool_usage_details: { x_search_calls: 1 } } },+ ],+ [+ "invalid item counts",+ {+ usage: {+ server_side_tool_usage_details: {+ x_posts_fetched: "44",+ x_users_fetched: -1,+ },+ },+ },+ ],+ [+ "one valid and one invalid item count",+ {+ usage: {+ server_side_tool_usage_details: {+ x_posts_fetched: 44,+ x_users_fetched: "3",+ },+ },+ },+ ],+ ])("returns undefined for %s", (_, body) => {+ expect(xSearchUsageFromResponseBody(body)).toBeUndefined();+ });+});++describe("xSearchCost", () => {+ it("prices posts at $5 per 1k and profiles at $10 per 1k", () => {+ expect(xSearchCost({ posts: 1000, profiles: 0 })).toBeCloseTo(5, 10);+ expect(xSearchCost({ posts: 0, profiles: 1000 })).toBeCloseTo(10, 10);+ expect(xSearchCost({ posts: 44, profiles: 3 })).toBeCloseTo(0.25, 10);+ });++ it("is zero when nothing was fetched", () => {+ expect(xSearchCost({ posts: 0, profiles: 0 })).toBe(0);+ });+});diff --git a/apps/api/src/lib/xai-x-search.ts b/apps/api/src/lib/xai-x-search.tsnew file mode 100644index 000000000..bbfc8b654--- /dev/null+++ b/apps/api/src/lib/xai-x-search.ts@@ -0,0 +1,55 @@+import { logger } from "./logger";++// xAI bills X Search per item fetched, on top of tokens+// (https://docs.x.ai/developers/pricing). USD per item.+const X_SEARCH_USD_PER_POST = 5 / 1000;+const X_SEARCH_USD_PER_PROFILE = 10 / 1000;++type XSearchUsage = { posts: number; profiles: number };++const INVALID = Symbol("invalid");++// A missing field means nothing of that kind was fetched. A field that is+// present but not a count means the provider changed shape, and reading it as+// 0 would silently drop that fee.+function count(value: unknown): number | typeof INVALID {+ if (value === undefined || value === null) {+ return 0;+ }+ return typeof value === "number" && Number.isFinite(value) && value >= 0+ ? Math.trunc(value)+ : INVALID;+}++// The Responses API reports fetched items under+// `usage.server_side_tool_usage_details`. `@ai-sdk/xai` drops that block when+// it parses usage, so read it from the raw response body.+export function xSearchUsageFromResponseBody(+ body: unknown,+): XSearchUsage | undefined {+ const details = (body as any)?.usage?.server_side_tool_usage_details;+ if (typeof details !== "object" || details === null) {+ return undefined;+ }+ if (!("x_posts_fetched" in details) && !("x_users_fetched" in details)) {+ return undefined;+ }+ const posts = count(details.x_posts_fetched);+ const profiles = count(details.x_users_fetched);+ if (posts === INVALID || profiles === INVALID) {+ logger.warn("Unreadable X Search item counts in xAI usage", {+ module: "xai-x-search",+ x_posts_fetched: details.x_posts_fetched,+ x_users_fetched: details.x_users_fetched,+ });+ return undefined;+ }+ return { posts, profiles };+}++export function xSearchCost(usage: XSearchUsage): number {+ return (+ usage.posts * X_SEARCH_USD_PER_POST ++ usage.profiles * X_SEARCH_USD_PER_PROFILE+ );+}diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.tsindex efd19f313..409f619cb 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts@@ -12,6 +12,7 @@ import {import { APICallError } from "ai";import { MockLanguageModelV3 } from "ai/test";import type { Meta } from "../..";+import { CostTracking } from "../../../../lib/cost-tracking";import { createTracerProvider } from "../../../../lib/otel-tracer";import { scrapeURLWithXTwitter } from "./index";@@ -40,11 +41,33 @@ const usage = {outputTokens: { total: 400, text: 400, reasoning: 0 },};-function grokReturns(output: unknown) {+// Raw xAI Responses body; only the usage block is read from it.+function responseBody(xSearch?: { posts?: number; users?: number }) {+ return {+ object: "response",+ usage: {+ input_tokens: 3000,+ output_tokens: 400,+ ...(xSearch && {+ server_side_tool_usage_details: {+ x_search_calls: 2,+ x_posts_fetched: xSearch.posts,+ x_users_fetched: xSearch.users,+ },+ }),+ },+ };+}++function grokReturns(+ output: unknown,+ body: unknown = responseBody({ posts: 0, users: 0 }),+) {grok.doGenerate = async () => ({content: [{ type: "text", text: JSON.stringify(output) }],finishReason: { unified: "stop", raw: "completed" },usage,+ response: { body },warnings: [],});}@@ -57,9 +80,13 @@ function makeMeta(url: string, zeroDataRetention = false): Meta {logger,abort: { asSignal: () => undefined },internalOptions: { teamId: "team-test", zeroDataRetention },+ costTracking: new CostTracking(),} as unknown as Meta;}+// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.+const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;+describe("x-twitter engine LLM telemetry", () => {const exporter = new InMemorySpanExporter();let provider: NodeTracerProvider;@@ -80,7 +107,9 @@ describe("x-twitter engine LLM telemetry", () => {propagation.disable();});- beforeEach(() => {+ beforeEach(async () => {+ // Drop spans a previous test left unflushed.+ await provider.forceFlush();exporter.reset();});@@ -117,10 +146,83 @@ describe("x-twitter engine LLM telemetry", () => {"ai.usage.promptTokens": 3000,"ai.usage.completionTokens": 400,"ai.usage.cachedInputTokens": 2000,+ "firecrawl.llm.tool.x_search_posts": 0,+ "firecrawl.llm.tool.x_search_profiles": 0,});expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");});+ it("records X Search items on the span and their fee in cost tracking for a profile lookup", async () => {+ grokReturns(+ { username: "firecrawl", latestPosts: [] },+ responseBody({ posts: 44, users: 3 }),+ );+ const meta = makeMeta("https://x.com/firecrawl");++ await scrapeURLWithXTwitter(meta);++ const span = await doGenerateSpan();+ expect(span.attributes).toMatchObject({+ "firecrawl.llm.tool.x_search_posts": 44,+ "firecrawl.llm.tool.x_search_profiles": 3,+ });+ const { calls, totalCost } = meta.costTracking.toJSON();+ expect(calls).toHaveLength(1);+ expect(calls[0]).toMatchObject({+ type: "other",+ model: "grok-4-1-fast-non-reasoning",+ tokens: { input: 3000, output: 400 },+ metadata: {+ module: "scrapeURL",+ method: "xTwitter/profile",+ xSearchPosts: 44,+ xSearchProfiles: 3,+ },+ });+ // 44 posts x $0.005 + 3 profiles x $0.01 = $0.25, plus tokens.+ expect(calls[0].cost).toBeCloseTo(0.25 + tokenCost, 10);+ expect(totalCost).toBeCloseTo(0.25 + tokenCost, 10);+ });++ it("records the X Search fee in cost tracking for a post lookup", async () => {+ grokReturns(+ { authorUsername: "firecrawl", text: "Hello from a post." },+ responseBody({ posts: 18, users: 0 }),+ );+ const meta = makeMeta("https://x.com/firecrawl/status/1234567890123");++ await scrapeURLWithXTwitter(meta);++ const { calls } = meta.costTracking.toJSON();+ expect(calls).toHaveLength(1);+ expect(calls[0].metadata).toMatchObject({+ method: "xTwitter/post",+ xSearchPosts: 18,+ xSearchProfiles: 0,+ });+ expect(calls[0].cost).toBeCloseTo(18 * 0.005 + tokenCost, 10);+ });++ it("prices tokens only when xAI reports no X Search item counts", async () => {+ grokReturns({ username: "firecrawl", latestPosts: [] }, responseBody());+ const meta = makeMeta("https://x.com/firecrawl");++ await scrapeURLWithXTwitter(meta);++ const span = await doGenerateSpan();+ expect(+ Object.keys(span.attributes).filter(key =>+ key.startsWith("firecrawl.llm.tool."),+ ),+ ).toEqual([]);+ const { calls } = meta.costTracking.toJSON();+ expect(calls[0].metadata).toMatchObject({+ xSearchPosts: 0,+ xSearchProfiles: 0,+ });+ expect(calls[0].cost).toBeCloseTo(tokenCost, 10);+ });+it("records a usage span for a post lookup", async () => {grokReturns({authorUsername: "firecrawl",@@ -154,12 +256,15 @@ describe("x-twitter engine LLM telemetry", () => {it("records no AI SDK spans for zero-data-retention scrapes", async () => {grokReturns({ username: "firecrawl", latestPosts: [] });- await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl", true));+ const meta = makeMeta("https://x.com/firecrawl", true);+ await scrapeURLWithXTwitter(meta);await provider.forceFlush();expect(exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),).toEqual([]);+ // Billing-side cost is recorded regardless of telemetry.+ expect(meta.costTracking.toJSON().calls).toHaveLength(1);});it("records the span with an error status when the Grok call is rejected", async () => {@@ -179,6 +284,11 @@ describe("x-twitter engine LLM telemetry", () => {const span = await doGenerateSpan();expect(span.status.code).toBe(SpanStatusCode.ERROR);+ expect(+ Object.keys(span.attributes).filter(key =>+ key.startsWith("firecrawl.llm.tool."),+ ),+ ).toEqual([]);expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");});});diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 10eafd98f..7d52ce212 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -1,11 +1,16 @@import { xai } from "@ai-sdk/xai";-import { generateText, jsonSchema, Output } from "ai";+import { generateText, jsonSchema, Output, type GenerateTextResult } from "ai";import { config } from "../../../../config";import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";+import {+ xSearchCost,+ xSearchUsageFromResponseBody,+} from "../../../../lib/xai-x-search";import { Meta } from "../..";import { EngineScrapeResult } from "..";import { EngineError, XTwitterConfigurationError } from "../../error";import { safeMarkdownToHtml } from "../pdf/markdownToHtml";+import { calculateCost } from "../../transformers/llmExtract";import {isXTwitterUrl,parseXTwitterUrl,@@ -29,6 +34,41 @@ function xTwitterTelemetry(functionId: string, meta: Meta) {};}+// Records the call's token cost plus the per-item X Search fee xAI bills on+// top of it.+function recordGrokCost(+ meta: Meta,+ method: string,+ {+ totalUsage,+ steps,+ }: Pick<GenerateTextResult<any, any>, "totalUsage" | "steps">,+) {+ const inputTokens = totalUsage.inputTokens ?? 0;+ const outputTokens = totalUsage.outputTokens ?? 0;+ const xSearch = { posts: 0, profiles: 0 };+ for (const step of steps) {+ const usage = xSearchUsageFromResponseBody(step.response.body);+ xSearch.posts += usage?.posts ?? 0;+ xSearch.profiles += usage?.profiles ?? 0;+ }++ meta.costTracking.addCall({+ type: "other",+ metadata: {+ module: "scrapeURL",+ method,+ xSearchPosts: xSearch.posts,+ xSearchProfiles: xSearch.profiles,+ },+ model: XAI_RESPONSES_MODEL,+ tokens: { input: inputTokens, output: outputTokens },+ cost:+ calculateCost(XAI_RESPONSES_MODEL, inputTokens, outputTokens) ++ xSearchCost(xSearch),+ });+}+type ProfilePost = {text: string;url?: string | null;@@ -408,7 +448,7 @@ async function fetchProfile(xUrl: XTwitterProfileUrl,meta: Meta,): Promise<XTwitterProfileData> {- const { output } = await generateText({+ const result = await generateText({model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),maxOutputTokens: 20000,tools: {@@ -425,8 +465,9 @@ async function fetchProfile(experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,});+ recordGrokCost(meta, "xTwitter/profile", result);- return output as XTwitterProfileData;+ return result.output as XTwitterProfileData;}async function fetchPost(@@ -445,7 +486,7 @@ async function fetchPost(enableImageUnderstanding: true,};- const { output } = await generateText({+ const result = await generateText({model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),maxOutputTokens: 20000,tools: {@@ -462,8 +503,9 @@ async function fetchPost(experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,});+ recordGrokCost(meta, "xTwitter/post", result);- return output as XTwitterPostData;+ return result.output as XTwitterPostData;}function stripAt(value: string | null | undefined): string | undefined {diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 549c82f6e..096c63fba 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -288,6 +288,7 @@ const modelCosts: Record<string, { input_cost: number; output_cost: number }> ="google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },"gemini-2.0-flash": { input_cost: 0.15, output_cost: 0.6 },"gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },+ "grok-4-1-fast-non-reasoning": { input_cost: 0.2, output_cost: 0.5 },"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },"google/gemini-2.0-flash-thinking-exp:free": {input_cost: 0.55,