Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(telemetry): record X Search items fetched and price them in x-twitter cost tracking (#4824)

Abimael Martellb7bdbdc28c71
diff --git a/apps/api/src/lib/ai-usage-telemetry.test.ts b/apps/api/src/lib/ai-usage-telemetry.test.ts
index aab3a83b7..c921e9bb7 100644
--- a/apps/api/src/lib/ai-usage-telemetry.test.ts
+++ b/apps/api/src/lib/ai-usage-telemetry.test.ts
@@ -1,3 +1,4 @@
+import { createXai } from "@ai-sdk/xai";
import { context, propagation, trace } from "@opentelemetry/api";
import {
InMemorySpanExporter,
@@ -75,6 +76,60 @@ const openAIResponsesBody = {
},
};
+// Responses API body as returned by xAI for a request that ran X Search.
+// Trimmed to the fields `@ai-sdk/xai` validates plus the usage block.
+function xaiResponsesBody(usage: Record<string, unknown>) {
+ return {
+ id: "resp_xai_test",
+ object: "response",
+ created_at: 1_700_000_000,
+ model: "grok-4-1-fast-non-reasoning",
+ status: "completed",
+ output: [
+ {
+ type: "x_search_call",
+ id: "xs_test",
+ name: "x_user_search",
+ arguments: '{"query":"firecrawl"}',
+ status: "completed",
+ },
+ {
+ type: "message",
+ id: "msg_xai_test",
+ role: "assistant",
+ status: "completed",
+ content: [{ type: "output_text", text: "ok", annotations: [] }],
+ },
+ ],
+ usage: {
+ input_tokens: 3000,
+ input_tokens_details: { cached_tokens: 2000 },
+ output_tokens: 400,
+ output_tokens_details: { reasoning_tokens: 0 },
+ total_tokens: 3400,
+ num_server_side_tools_used: 2,
+ ...usage,
+ },
+ };
+}
+
+async function generateWithXaiResponses(body: unknown) {
+ const xai = createXai({
+ apiKey: "test-key",
+ fetch: async () =>
+ new Response(JSON.stringify(body), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ });
+ await generateText({
+ model: withUsageTelemetry(xai.responses("grok-4-1-fast-non-reasoning")),
+ tools: { x_search: xai.tools.xSearch() } as any,
+ prompt: "hello",
+ experimental_telemetry: telemetry,
+ });
+}
+
describe("usage telemetry middleware", () => {
const exporter = new InMemorySpanExporter();
let provider: NodeTracerProvider;
@@ -159,6 +214,48 @@ describe("usage telemetry middleware", () => {
});
});
+ it("records X Search items fetched on the xAI Responses generateText span", async () => {
+ await generateWithXaiResponses(
+ xaiResponsesBody({
+ server_side_tool_usage_details: {
+ web_search_calls: 0,
+ x_search_calls: 2,
+ x_posts_fetched: 44,
+ x_users_fetched: 3,
+ code_interpreter_calls: 0,
+ file_search_calls: 0,
+ mcp_calls: 0,
+ document_search_calls: 0,
+ image_generation_calls: 0,
+ },
+ }),
+ );
+
+ const span = await spanNamed("ai.generateText.doGenerate");
+ expect(span.attributes).toMatchObject({
+ "ai.usage.promptTokens": 3000,
+ "ai.usage.cachedInputTokens": 2000,
+ "firecrawl.llm.tool.x_search_posts": 44,
+ "firecrawl.llm.tool.x_search_profiles": 3,
+ });
+ });
+
+ it("omits X Search attributes when xAI reports no item counts", async () => {
+ await generateWithXaiResponses(
+ xaiResponsesBody({
+ server_side_tool_usage_details: { x_search_calls: 1 },
+ }),
+ );
+
+ const span = await spanNamed("ai.generateText.doGenerate");
+ expect(span.attributes["ai.usage.promptTokens"]).toBe(3000);
+ expect(
+ Object.keys(span.attributes).filter(key =>
+ key.startsWith("firecrawl.llm.tool."),
+ ),
+ ).toEqual([]);
+ });
+
it("records cache writes on the generateText span", async () => {
await generateText({
model: withUsageTelemetry(
diff --git a/apps/api/src/lib/ai-usage-telemetry.ts b/apps/api/src/lib/ai-usage-telemetry.ts
index 509d3bbbc..fc1b0517f 100644
--- a/apps/api/src/lib/ai-usage-telemetry.ts
+++ b/apps/api/src/lib/ai-usage-telemetry.ts
@@ -1,8 +1,10 @@
import { trace, type Attributes, type Span } from "@opentelemetry/api";
import { wrapLanguageModel, type LanguageModelMiddleware } from "ai";
+import { xSearchUsageFromResponseBody } from "./xai-x-search";
type ProviderModel = Parameters<typeof wrapLanguageModel>[0]["model"];
-type ProviderUsage = Awaited<ReturnType<ProviderModel["doGenerate"]>>["usage"];
+type ProviderResult = Awaited<ReturnType<ProviderModel["doGenerate"]>>;
+type ProviderUsage = ProviderResult["usage"];
// The AI SDK's `ai.generateObject.doGenerate` and `ai.generateText.doGenerate`
// spans only record prompt and completion token totals, so cache reads and
@@ -43,12 +45,28 @@ function usageTelemetryAttributes(usage: ProviderUsage): Attributes {
);
}
+// Server-side tool fees the provider bills per item, which token counts
+// cannot price. Only xAI X Search today.
+function toolUsageTelemetryAttributes(result: ProviderResult): Attributes {
+ const xSearch = xSearchUsageFromResponseBody(result.response?.body);
+ if (!xSearch) {
+ return {};
+ }
+ return {
+ "firecrawl.llm.tool.x_search_posts": xSearch.posts,
+ "firecrawl.llm.tool.x_search_profiles": xSearch.profiles,
+ };
+}
+
const usageTelemetryMiddleware: LanguageModelMiddleware = {
specificationVersion: "v3",
wrapGenerate: async ({ doGenerate }) => {
const span = activeDoGenerateSpan();
const result = await doGenerate();
- span?.setAttributes(usageTelemetryAttributes(result.usage));
+ span?.setAttributes({
+ ...usageTelemetryAttributes(result.usage),
+ ...toolUsageTelemetryAttributes(result),
+ });
return result;
},
};
diff --git a/apps/api/src/lib/xai-x-search.test.ts b/apps/api/src/lib/xai-x-search.test.ts
new file mode 100644
index 000000000..181386a0f
--- /dev/null
+++ b/apps/api/src/lib/xai-x-search.test.ts
@@ -0,0 +1,72 @@
+import { xSearchCost, xSearchUsageFromResponseBody } from "./xai-x-search";
+
+describe("xSearchUsageFromResponseBody", () => {
+ it("reads posts and profiles fetched from the Responses usage block", () => {
+ expect(
+ xSearchUsageFromResponseBody({
+ usage: {
+ server_side_tool_usage_details: {
+ x_search_calls: 2,
+ x_posts_fetched: 44,
+ x_users_fetched: 3,
+ },
+ },
+ }),
+ ).toEqual({ posts: 44, profiles: 3 });
+ });
+
+ it("counts a missing field as zero when the other is reported", () => {
+ expect(
+ xSearchUsageFromResponseBody({
+ usage: { server_side_tool_usage_details: { x_posts_fetched: 12 } },
+ }),
+ ).toEqual({ posts: 12, profiles: 0 });
+ });
+
+ it.each([
+ ["no body", undefined],
+ ["a non-object body", "not json"],
+ ["no usage", { id: "resp" }],
+ ["no tool usage details", { usage: { input_tokens: 10 } }],
+ [
+ "no item counts",
+ { usage: { server_side_tool_usage_details: { x_search_calls: 1 } } },
+ ],
+ [
+ "invalid item counts",
+ {
+ usage: {
+ server_side_tool_usage_details: {
+ x_posts_fetched: "44",
+ x_users_fetched: -1,
+ },
+ },
+ },
+ ],
+ [
+ "one valid and one invalid item count",
+ {
+ usage: {
+ server_side_tool_usage_details: {
+ x_posts_fetched: 44,
+ x_users_fetched: "3",
+ },
+ },
+ },
+ ],
+ ])("returns undefined for %s", (_, body) => {
+ expect(xSearchUsageFromResponseBody(body)).toBeUndefined();
+ });
+});
+
+describe("xSearchCost", () => {
+ it("prices posts at $5 per 1k and profiles at $10 per 1k", () => {
+ expect(xSearchCost({ posts: 1000, profiles: 0 })).toBeCloseTo(5, 10);
+ expect(xSearchCost({ posts: 0, profiles: 1000 })).toBeCloseTo(10, 10);
+ expect(xSearchCost({ posts: 44, profiles: 3 })).toBeCloseTo(0.25, 10);
+ });
+
+ it("is zero when nothing was fetched", () => {
+ expect(xSearchCost({ posts: 0, profiles: 0 })).toBe(0);
+ });
+});
diff --git a/apps/api/src/lib/xai-x-search.ts b/apps/api/src/lib/xai-x-search.ts
new file mode 100644
index 000000000..bbfc8b654
--- /dev/null
+++ b/apps/api/src/lib/xai-x-search.ts
@@ -0,0 +1,55 @@
+import { logger } from "./logger";
+
+// xAI bills X Search per item fetched, on top of tokens
+// (https://docs.x.ai/developers/pricing). USD per item.
+const X_SEARCH_USD_PER_POST = 5 / 1000;
+const X_SEARCH_USD_PER_PROFILE = 10 / 1000;
+
+type XSearchUsage = { posts: number; profiles: number };
+
+const INVALID = Symbol("invalid");
+
+// A missing field means nothing of that kind was fetched. A field that is
+// present but not a count means the provider changed shape, and reading it as
+// 0 would silently drop that fee.
+function count(value: unknown): number | typeof INVALID {
+ if (value === undefined || value === null) {
+ return 0;
+ }
+ return typeof value === "number" && Number.isFinite(value) && value >= 0
+ ? Math.trunc(value)
+ : INVALID;
+}
+
+// The Responses API reports fetched items under
+// `usage.server_side_tool_usage_details`. `@ai-sdk/xai` drops that block when
+// it parses usage, so read it from the raw response body.
+export function xSearchUsageFromResponseBody(
+ body: unknown,
+): XSearchUsage | undefined {
+ const details = (body as any)?.usage?.server_side_tool_usage_details;
+ if (typeof details !== "object" || details === null) {
+ return undefined;
+ }
+ if (!("x_posts_fetched" in details) && !("x_users_fetched" in details)) {
+ return undefined;
+ }
+ const posts = count(details.x_posts_fetched);
+ const profiles = count(details.x_users_fetched);
+ if (posts === INVALID || profiles === INVALID) {
+ logger.warn("Unreadable X Search item counts in xAI usage", {
+ module: "xai-x-search",
+ x_posts_fetched: details.x_posts_fetched,
+ x_users_fetched: details.x_users_fetched,
+ });
+ return undefined;
+ }
+ return { posts, profiles };
+}
+
+export function xSearchCost(usage: XSearchUsage): number {
+ return (
+ usage.posts * X_SEARCH_USD_PER_POST +
+ usage.profiles * X_SEARCH_USD_PER_PROFILE
+ );
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
index efd19f313..409f619cb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
@@ -12,6 +12,7 @@ import {
import { APICallError } from "ai";
import { MockLanguageModelV3 } from "ai/test";
import type { Meta } from "../..";
+import { CostTracking } from "../../../../lib/cost-tracking";
import { createTracerProvider } from "../../../../lib/otel-tracer";
import { scrapeURLWithXTwitter } from "./index";
@@ -40,11 +41,33 @@ const usage = {
outputTokens: { total: 400, text: 400, reasoning: 0 },
};
-function grokReturns(output: unknown) {
+// Raw xAI Responses body; only the usage block is read from it.
+function responseBody(xSearch?: { posts?: number; users?: number }) {
+ return {
+ object: "response",
+ usage: {
+ input_tokens: 3000,
+ output_tokens: 400,
+ ...(xSearch && {
+ server_side_tool_usage_details: {
+ x_search_calls: 2,
+ x_posts_fetched: xSearch.posts,
+ x_users_fetched: xSearch.users,
+ },
+ }),
+ },
+ };
+}
+
+function grokReturns(
+ output: unknown,
+ body: unknown = responseBody({ posts: 0, users: 0 }),
+) {
grok.doGenerate = async () => ({
content: [{ type: "text", text: JSON.stringify(output) }],
finishReason: { unified: "stop", raw: "completed" },
usage,
+ response: { body },
warnings: [],
});
}
@@ -57,9 +80,13 @@ function makeMeta(url: string, zeroDataRetention = false): Meta {
logger,
abort: { asSignal: () => undefined },
internalOptions: { teamId: "team-test", zeroDataRetention },
+ costTracking: new CostTracking(),
} as unknown as Meta;
}
+// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.
+const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;
+
describe("x-twitter engine LLM telemetry", () => {
const exporter = new InMemorySpanExporter();
let provider: NodeTracerProvider;
@@ -80,7 +107,9 @@ describe("x-twitter engine LLM telemetry", () => {
propagation.disable();
});
- beforeEach(() => {
+ beforeEach(async () => {
+ // Drop spans a previous test left unflushed.
+ await provider.forceFlush();
exporter.reset();
});
@@ -117,10 +146,83 @@ describe("x-twitter engine LLM telemetry", () => {
"ai.usage.promptTokens": 3000,
"ai.usage.completionTokens": 400,
"ai.usage.cachedInputTokens": 2000,
+ "firecrawl.llm.tool.x_search_posts": 0,
+ "firecrawl.llm.tool.x_search_profiles": 0,
});
expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");
});
+ it("records X Search items on the span and their fee in cost tracking for a profile lookup", async () => {
+ grokReturns(
+ { username: "firecrawl", latestPosts: [] },
+ responseBody({ posts: 44, users: 3 }),
+ );
+ const meta = makeMeta("https://x.com/firecrawl");
+
+ await scrapeURLWithXTwitter(meta);
+
+ const span = await doGenerateSpan();
+ expect(span.attributes).toMatchObject({
+ "firecrawl.llm.tool.x_search_posts": 44,
+ "firecrawl.llm.tool.x_search_profiles": 3,
+ });
+ const { calls, totalCost } = meta.costTracking.toJSON();
+ expect(calls).toHaveLength(1);
+ expect(calls[0]).toMatchObject({
+ type: "other",
+ model: "grok-4-1-fast-non-reasoning",
+ tokens: { input: 3000, output: 400 },
+ metadata: {
+ module: "scrapeURL",
+ method: "xTwitter/profile",
+ xSearchPosts: 44,
+ xSearchProfiles: 3,
+ },
+ });
+ // 44 posts x $0.005 + 3 profiles x $0.01 = $0.25, plus tokens.
+ expect(calls[0].cost).toBeCloseTo(0.25 + tokenCost, 10);
+ expect(totalCost).toBeCloseTo(0.25 + tokenCost, 10);
+ });
+
+ it("records the X Search fee in cost tracking for a post lookup", async () => {
+ grokReturns(
+ { authorUsername: "firecrawl", text: "Hello from a post." },
+ responseBody({ posts: 18, users: 0 }),
+ );
+ const meta = makeMeta("https://x.com/firecrawl/status/1234567890123");
+
+ await scrapeURLWithXTwitter(meta);
+
+ const { calls } = meta.costTracking.toJSON();
+ expect(calls).toHaveLength(1);
+ expect(calls[0].metadata).toMatchObject({
+ method: "xTwitter/post",
+ xSearchPosts: 18,
+ xSearchProfiles: 0,
+ });
+ expect(calls[0].cost).toBeCloseTo(18 * 0.005 + tokenCost, 10);
+ });
+
+ it("prices tokens only when xAI reports no X Search item counts", async () => {
+ grokReturns({ username: "firecrawl", latestPosts: [] }, responseBody());
+ const meta = makeMeta("https://x.com/firecrawl");
+
+ await scrapeURLWithXTwitter(meta);
+
+ const span = await doGenerateSpan();
+ expect(
+ Object.keys(span.attributes).filter(key =>
+ key.startsWith("firecrawl.llm.tool."),
+ ),
+ ).toEqual([]);
+ const { calls } = meta.costTracking.toJSON();
+ expect(calls[0].metadata).toMatchObject({
+ xSearchPosts: 0,
+ xSearchProfiles: 0,
+ });
+ expect(calls[0].cost).toBeCloseTo(tokenCost, 10);
+ });
+
it("records a usage span for a post lookup", async () => {
grokReturns({
authorUsername: "firecrawl",
@@ -154,12 +256,15 @@ describe("x-twitter engine LLM telemetry", () => {
it("records no AI SDK spans for zero-data-retention scrapes", async () => {
grokReturns({ username: "firecrawl", latestPosts: [] });
- await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl", true));
+ const meta = makeMeta("https://x.com/firecrawl", true);
+ await scrapeURLWithXTwitter(meta);
await provider.forceFlush();
expect(
exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),
).toEqual([]);
+ // Billing-side cost is recorded regardless of telemetry.
+ expect(meta.costTracking.toJSON().calls).toHaveLength(1);
});
it("records the span with an error status when the Grok call is rejected", async () => {
@@ -179,6 +284,11 @@ describe("x-twitter engine LLM telemetry", () => {
const span = await doGenerateSpan();
expect(span.status.code).toBe(SpanStatusCode.ERROR);
+ expect(
+ Object.keys(span.attributes).filter(key =>
+ key.startsWith("firecrawl.llm.tool."),
+ ),
+ ).toEqual([]);
expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");
});
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
index 10eafd98f..7d52ce212 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -1,11 +1,16 @@
import { xai } from "@ai-sdk/xai";
-import { generateText, jsonSchema, Output } from "ai";
+import { generateText, jsonSchema, Output, type GenerateTextResult } from "ai";
import { config } from "../../../../config";
import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";
+import {
+ xSearchCost,
+ xSearchUsageFromResponseBody,
+} from "../../../../lib/xai-x-search";
import { Meta } from "../..";
import { EngineScrapeResult } from "..";
import { EngineError, XTwitterConfigurationError } from "../../error";
import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
+import { calculateCost } from "../../transformers/llmExtract";
import {
isXTwitterUrl,
parseXTwitterUrl,
@@ -29,6 +34,41 @@ function xTwitterTelemetry(functionId: string, meta: Meta) {
};
}
+// Records the call's token cost plus the per-item X Search fee xAI bills on
+// top of it.
+function recordGrokCost(
+ meta: Meta,
+ method: string,
+ {
+ totalUsage,
+ steps,
+ }: Pick<GenerateTextResult<any, any>, "totalUsage" | "steps">,
+) {
+ const inputTokens = totalUsage.inputTokens ?? 0;
+ const outputTokens = totalUsage.outputTokens ?? 0;
+ const xSearch = { posts: 0, profiles: 0 };
+ for (const step of steps) {
+ const usage = xSearchUsageFromResponseBody(step.response.body);
+ xSearch.posts += usage?.posts ?? 0;
+ xSearch.profiles += usage?.profiles ?? 0;
+ }
+
+ meta.costTracking.addCall({
+ type: "other",
+ metadata: {
+ module: "scrapeURL",
+ method,
+ xSearchPosts: xSearch.posts,
+ xSearchProfiles: xSearch.profiles,
+ },
+ model: XAI_RESPONSES_MODEL,
+ tokens: { input: inputTokens, output: outputTokens },
+ cost:
+ calculateCost(XAI_RESPONSES_MODEL, inputTokens, outputTokens) +
+ xSearchCost(xSearch),
+ });
+}
+
type ProfilePost = {
text: string;
url?: string | null;
@@ -408,7 +448,7 @@ async function fetchProfile(
xUrl: XTwitterProfileUrl,
meta: Meta,
): Promise<XTwitterProfileData> {
- const { output } = await generateText({
+ const result = await generateText({
model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
maxOutputTokens: 20000,
tools: {
@@ -425,8 +465,9 @@ async function fetchProfile(
experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),
prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
});
+ recordGrokCost(meta, "xTwitter/profile", result);
- return output as XTwitterProfileData;
+ return result.output as XTwitterProfileData;
}
async function fetchPost(
@@ -445,7 +486,7 @@ async function fetchPost(
enableImageUnderstanding: true,
};
- const { output } = await generateText({
+ const result = await generateText({
model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
maxOutputTokens: 20000,
tools: {
@@ -462,8 +503,9 @@ async function fetchPost(
experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),
prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,
});
+ recordGrokCost(meta, "xTwitter/post", result);
- return output as XTwitterPostData;
+ return result.output as XTwitterPostData;
}
function stripAt(value: string | null | undefined): string | undefined {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 549c82f6e..096c63fba 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -288,6 +288,7 @@ const modelCosts: Record<string, { input_cost: number; output_cost: number }> =
"google/gemini-2.0-flash-001": { input_cost: 0.15, output_cost: 0.6 },
"gemini-2.0-flash": { input_cost: 0.15, output_cost: 0.6 },
"gemini-2.5-flash-lite": { input_cost: 0.1, output_cost: 0.4 },
+ "grok-4-1-fast-non-reasoning": { input_cost: 0.2, output_cost: 0.5 },
"deepseek/deepseek-r1": { input_cost: 0.55, output_cost: 2.19 },
"google/gemini-2.0-flash-thinking-exp:free": {
input_cost: 0.55,