Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(telemetry): record usage spans for x-twitter Grok calls (#4821)

Abimael Martellcb388409b428
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
new file mode 100644
index 000000000..efd19f313
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
@@ -0,0 +1,184 @@
+import {
+ context,
+ propagation,
+ SpanStatusCode,
+ trace,
+} from "@opentelemetry/api";
+import {
+ InMemorySpanExporter,
+ type NodeTracerProvider,
+ type ReadableSpan,
+} from "@opentelemetry/sdk-trace-node";
+import { APICallError } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import type { Meta } from "../..";
+import { createTracerProvider } from "../../../../lib/otel-tracer";
+import { scrapeURLWithXTwitter } from "./index";
+
+const { grok } = vi.hoisted(() => {
+ process.env.XAI_API_KEY = "test-key";
+ return { grok: { doGenerate: undefined as any } };
+});
+
+vi.mock("@ai-sdk/xai", async importOriginal => {
+ const actual = await importOriginal<typeof import("@ai-sdk/xai")>();
+ return {
+ xai: {
+ responses: (modelId: string) =>
+ new MockLanguageModelV3({
+ provider: "xai.responses",
+ modelId,
+ doGenerate: (...args) => grok.doGenerate(...args),
+ }),
+ tools: actual.xai.tools,
+ },
+ };
+});
+
+const usage = {
+ inputTokens: { total: 3000, noCache: 1000, cacheRead: 2000, cacheWrite: 0 },
+ outputTokens: { total: 400, text: 400, reasoning: 0 },
+};
+
+function grokReturns(output: unknown) {
+ grok.doGenerate = async () => ({
+ content: [{ type: "text", text: JSON.stringify(output) }],
+ finishReason: { unified: "stop", raw: "completed" },
+ usage,
+ warnings: [],
+ });
+}
+
+function makeMeta(url: string, zeroDataRetention = false): Meta {
+ const logger = { info: () => {}, warn: () => {}, error: () => {} };
+ return {
+ id: "019990c0-0000-7000-8000-000000000001",
+ url,
+ logger,
+ abort: { asSignal: () => undefined },
+ internalOptions: { teamId: "team-test", zeroDataRetention },
+ } as unknown as Meta;
+}
+
+describe("x-twitter engine LLM telemetry", () => {
+ const exporter = new InMemorySpanExporter();
+ let provider: NodeTracerProvider;
+
+ beforeAll(() => {
+ provider = createTracerProvider({
+ exporter,
+ serviceName: "test-service",
+ serviceInstanceId: "pod-1",
+ });
+ provider.register();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ trace.disable();
+ context.disable();
+ propagation.disable();
+ });
+
+ beforeEach(() => {
+ exporter.reset();
+ });
+
+ async function doGenerateSpan(): Promise<ReadableSpan> {
+ await provider.forceFlush();
+ const spans = exporter
+ .getFinishedSpans()
+ .filter(s => s.name === "ai.generateText.doGenerate");
+ expect(spans).toHaveLength(1);
+ return spans[0];
+ }
+
+ it("records a usage span for a profile lookup", async () => {
+ grokReturns({
+ displayName: "Firecrawl",
+ username: "firecrawl",
+ bio: "Turn websites into LLM-ready data.",
+ followers: 1000,
+ latestPosts: [],
+ });
+
+ const result = await scrapeURLWithXTwitter(
+ makeMeta("https://x.com/firecrawl"),
+ );
+
+ expect(result.markdown).toContain("@firecrawl");
+ const span = await doGenerateSpan();
+ expect(span.attributes).toMatchObject({
+ "ai.telemetry.functionId": "xTwitter/profile",
+ "ai.telemetry.metadata.feature": "x-twitter",
+ "ai.telemetry.metadata.teamId": "team-test",
+ "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",
+ "ai.model.id": "grok-4-1-fast-non-reasoning",
+ "ai.usage.promptTokens": 3000,
+ "ai.usage.completionTokens": 400,
+ "ai.usage.cachedInputTokens": 2000,
+ });
+ expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");
+ });
+
+ it("records a usage span for a post lookup", async () => {
+ grokReturns({
+ authorUsername: "firecrawl",
+ text: "Hello from a post.",
+ likes: 10,
+ retweets: 2,
+ });
+
+ await scrapeURLWithXTwitter(
+ makeMeta("https://x.com/firecrawl/status/1234567890123"),
+ );
+
+ const span = await doGenerateSpan();
+ expect(span.attributes).toMatchObject({
+ "ai.telemetry.functionId": "xTwitter/post",
+ "ai.telemetry.metadata.feature": "x-twitter",
+ "ai.telemetry.metadata.teamId": "team-test",
+ "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",
+ "ai.model.id": "grok-4-1-fast-non-reasoning",
+ "ai.usage.promptTokens": 3000,
+ "ai.usage.completionTokens": 400,
+ "ai.usage.cachedInputTokens": 2000,
+ });
+ expect(span.attributes["ai.prompt.messages"]).toContain(
+ "post id 1234567890123",
+ );
+ });
+
+ // Covers the per-call guard on its own; ZDR scrape jobs additionally run
+ // under the tracer's ZDR context (see otel-tracer.test.ts).
+ it("records no AI SDK spans for zero-data-retention scrapes", async () => {
+ grokReturns({ username: "firecrawl", latestPosts: [] });
+
+ await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl", true));
+
+ await provider.forceFlush();
+ expect(
+ exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),
+ ).toEqual([]);
+ });
+
+ it("records the span with an error status when the Grok call is rejected", async () => {
+ grok.doGenerate = async () => {
+ throw new APICallError({
+ message: "bad request",
+ url: "https://api.x.ai/v1/responses",
+ requestBodyValues: {},
+ statusCode: 400,
+ isRetryable: false,
+ });
+ };
+
+ await expect(
+ scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl")),
+ ).rejects.toThrow("bad request");
+
+ const span = await doGenerateSpan();
+ expect(span.status.code).toBe(SpanStatusCode.ERROR);
+ expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
index 7dc4910ff..2d37404f4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -1,6 +1,7 @@
import { xai } from "@ai-sdk/xai";
import { generateText, jsonSchema, Output } from "ai";
import { config } from "../../../../config";
+import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";
import { Meta } from "../..";
import { EngineScrapeResult } from "..";
import { EngineError, XTwitterConfigurationError } from "../../error";
@@ -8,6 +9,18 @@ import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";
+function xTwitterTelemetry(functionId: string, meta: Meta) {
+ return {
+ isEnabled: !meta.internalOptions.zeroDataRetention,
+ functionId,
+ metadata: {
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId ?? "",
+ feature: "x-twitter",
+ },
+ };
+}
+
const RESERVED_PROFILE_PATHS = new Set([
"compose",
"explore",
@@ -512,7 +525,7 @@ async function fetchProfile(
meta: Meta,
): Promise<XTwitterProfileData> {
const { output } = await generateText({
- model: xai.responses(XAI_RESPONSES_MODEL),
+ model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
maxOutputTokens: 20000,
tools: {
x_search: xai.tools.xSearch(),
@@ -525,6 +538,7 @@ async function fetchProfile(
"Current public X/Twitter profile data and latest top-level posts.",
}),
abortSignal: meta.abort.asSignal(),
+ experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),
prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
});
@@ -548,7 +562,7 @@ async function fetchPost(
};
const { output } = await generateText({
- model: xai.responses(XAI_RESPONSES_MODEL),
+ model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
maxOutputTokens: 20000,
tools: {
x_search: xai.tools.xSearch(toolsOptions),
@@ -561,6 +575,7 @@ async function fetchPost(
"Current public X/Twitter post data with metrics, thread, and top comments.",
}),
abortSignal: meta.abort.asSignal(),
+ experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),
prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,
});