Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(telemetry): record usage spans for x-twitter Grok calls (#4821)
Abimael Martellcb388409b428
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.tsnew file mode 100644index 000000000..efd19f313--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts@@ -0,0 +1,184 @@+import {+ context,+ propagation,+ SpanStatusCode,+ trace,+} from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ type NodeTracerProvider,+ type ReadableSpan,+} from "@opentelemetry/sdk-trace-node";+import { APICallError } from "ai";+import { MockLanguageModelV3 } from "ai/test";+import type { Meta } from "../..";+import { createTracerProvider } from "../../../../lib/otel-tracer";+import { scrapeURLWithXTwitter } from "./index";++const { grok } = vi.hoisted(() => {+ process.env.XAI_API_KEY = "test-key";+ return { grok: { doGenerate: undefined as any } };+});++vi.mock("@ai-sdk/xai", async importOriginal => {+ const actual = await importOriginal<typeof import("@ai-sdk/xai")>();+ return {+ xai: {+ responses: (modelId: string) =>+ new MockLanguageModelV3({+ provider: "xai.responses",+ modelId,+ doGenerate: (...args) => grok.doGenerate(...args),+ }),+ tools: actual.xai.tools,+ },+ };+});++const usage = {+ inputTokens: { total: 3000, noCache: 1000, cacheRead: 2000, cacheWrite: 0 },+ outputTokens: { total: 400, text: 400, reasoning: 0 },+};++function grokReturns(output: unknown) {+ grok.doGenerate = async () => ({+ content: [{ type: "text", text: JSON.stringify(output) }],+ finishReason: { unified: "stop", raw: "completed" },+ usage,+ warnings: [],+ });+}++function makeMeta(url: string, zeroDataRetention = false): Meta {+ const logger = { info: () => {}, warn: () => {}, error: () => {} };+ return {+ id: "019990c0-0000-7000-8000-000000000001",+ url,+ logger,+ abort: { asSignal: () => undefined },+ internalOptions: { teamId: "team-test", zeroDataRetention },+ } as unknown as Meta;+}++describe("x-twitter engine LLM telemetry", () => {+ const exporter = new InMemorySpanExporter();+ let provider: NodeTracerProvider;++ beforeAll(() => {+ provider = createTracerProvider({+ exporter,+ serviceName: "test-service",+ serviceInstanceId: "pod-1",+ });+ provider.register();+ });++ afterAll(async () => {+ await provider.shutdown();+ trace.disable();+ context.disable();+ propagation.disable();+ });++ beforeEach(() => {+ exporter.reset();+ });++ async function doGenerateSpan(): Promise<ReadableSpan> {+ await provider.forceFlush();+ const spans = exporter+ .getFinishedSpans()+ .filter(s => s.name === "ai.generateText.doGenerate");+ expect(spans).toHaveLength(1);+ return spans[0];+ }++ it("records a usage span for a profile lookup", async () => {+ grokReturns({+ displayName: "Firecrawl",+ username: "firecrawl",+ bio: "Turn websites into LLM-ready data.",+ followers: 1000,+ latestPosts: [],+ });++ const result = await scrapeURLWithXTwitter(+ makeMeta("https://x.com/firecrawl"),+ );++ expect(result.markdown).toContain("@firecrawl");+ const span = await doGenerateSpan();+ expect(span.attributes).toMatchObject({+ "ai.telemetry.functionId": "xTwitter/profile",+ "ai.telemetry.metadata.feature": "x-twitter",+ "ai.telemetry.metadata.teamId": "team-test",+ "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",+ "ai.model.id": "grok-4-1-fast-non-reasoning",+ "ai.usage.promptTokens": 3000,+ "ai.usage.completionTokens": 400,+ "ai.usage.cachedInputTokens": 2000,+ });+ expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");+ });++ it("records a usage span for a post lookup", async () => {+ grokReturns({+ authorUsername: "firecrawl",+ text: "Hello from a post.",+ likes: 10,+ retweets: 2,+ });++ await scrapeURLWithXTwitter(+ makeMeta("https://x.com/firecrawl/status/1234567890123"),+ );++ const span = await doGenerateSpan();+ expect(span.attributes).toMatchObject({+ "ai.telemetry.functionId": "xTwitter/post",+ "ai.telemetry.metadata.feature": "x-twitter",+ "ai.telemetry.metadata.teamId": "team-test",+ "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",+ "ai.model.id": "grok-4-1-fast-non-reasoning",+ "ai.usage.promptTokens": 3000,+ "ai.usage.completionTokens": 400,+ "ai.usage.cachedInputTokens": 2000,+ });+ expect(span.attributes["ai.prompt.messages"]).toContain(+ "post id 1234567890123",+ );+ });++ // Covers the per-call guard on its own; ZDR scrape jobs additionally run+ // under the tracer's ZDR context (see otel-tracer.test.ts).+ it("records no AI SDK spans for zero-data-retention scrapes", async () => {+ grokReturns({ username: "firecrawl", latestPosts: [] });++ await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl", true));++ await provider.forceFlush();+ expect(+ exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),+ ).toEqual([]);+ });++ it("records the span with an error status when the Grok call is rejected", async () => {+ grok.doGenerate = async () => {+ throw new APICallError({+ message: "bad request",+ url: "https://api.x.ai/v1/responses",+ requestBodyValues: {},+ statusCode: 400,+ isRetryable: false,+ });+ };++ await expect(+ scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl")),+ ).rejects.toThrow("bad request");++ const span = await doGenerateSpan();+ expect(span.status.code).toBe(SpanStatusCode.ERROR);+ expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 7dc4910ff..2d37404f4 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -1,6 +1,7 @@import { xai } from "@ai-sdk/xai";import { generateText, jsonSchema, Output } from "ai";import { config } from "../../../../config";+import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";import { Meta } from "../..";import { EngineScrapeResult } from "..";import { EngineError, XTwitterConfigurationError } from "../../error";@@ -8,6 +9,18 @@ import { safeMarkdownToHtml } from "../pdf/markdownToHtml";const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";+function xTwitterTelemetry(functionId: string, meta: Meta) {+ return {+ isEnabled: !meta.internalOptions.zeroDataRetention,+ functionId,+ metadata: {+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId ?? "",+ feature: "x-twitter",+ },+ };+}+const RESERVED_PROFILE_PATHS = new Set(["compose","explore",@@ -512,7 +525,7 @@ async function fetchProfile(meta: Meta,): Promise<XTwitterProfileData> {const { output } = await generateText({- model: xai.responses(XAI_RESPONSES_MODEL),+ model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),maxOutputTokens: 20000,tools: {x_search: xai.tools.xSearch(),@@ -525,6 +538,7 @@ async function fetchProfile("Current public X/Twitter profile data and latest top-level posts.",}),abortSignal: meta.abort.asSignal(),+ experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,});@@ -548,7 +562,7 @@ async function fetchPost(};const { output } = await generateText({- model: xai.responses(XAI_RESPONSES_MODEL),+ model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),maxOutputTokens: 20000,tools: {x_search: xai.tools.xSearch(toolsOptions),@@ -561,6 +575,7 @@ async function fetchPost("Current public X/Twitter post data with metrics, thread, and top comments.",}),abortSignal: meta.abort.asSignal(),+ experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,});