Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(x-twitter): look up the requested handle directly on profile scrapes (#4873)
Abimael Martelle58f33e3dcd6
diff --git a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts b/apps/api/src/__tests__/snips/cost-tracking-helpers.tsnew file mode 100644index 000000000..686fbf858--- /dev/null+++ b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts@@ -0,0 +1,31 @@+import { eq } from "drizzle-orm";+import { db } from "../../db/connection";+import * as schema from "../../db/schema";++export type CostTrackingCall = {+ model: string;+ cost: number;+ metadata: Record<string, unknown>;+ tokens?: { input: number; output: number };+};++// The scrape row is written when the job finishes; give the insert a moment.+export async function getCostTrackingCalls(+ scrapeId: string,+): Promise<CostTrackingCall[]> {+ for (let attempt = 0; attempt < 10; attempt++) {+ const rows = await db+ .select({ cost_tracking: schema.scrapes.cost_tracking })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.id, scrapeId))+ .limit(1);+ if (rows.length === 1) {+ const costTracking = rows[0].cost_tracking as {+ calls?: CostTrackingCall[];+ } | null;+ return costTracking?.calls ?? [];+ }+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ throw new Error(`No scrapes row for ${scrapeId}`);+}diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 7aacb2550..e6ada1a9a 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configurationexport const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;+export const HAS_XAI = !!config.XAI_API_KEY;export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;export const HAS_PROXY = !!config.PROXY_SERVER;diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.tsindex 848d7e260..244d51971 100644--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts@@ -1,6 +1,7 @@-import { eq } from "drizzle-orm";-import { db } from "../../../db/connection";-import * as schema from "../../../db/schema";+import {+ type CostTrackingCall,+ getCostTrackingCalls,+} from "../cost-tracking-helpers";import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";import { scrape, scrapeTimeout, idmux, Identity } from "./lib";@@ -38,34 +39,6 @@ describe("Branding declared-logo fallback", () => {);});-type CostTrackingCall = {- model: string;- cost: number;- metadata: Record<string, unknown>;- tokens?: { input: number; output: number };-};--// The scrape row is written when the job finishes; give the insert a moment.-async function getCostTrackingCalls(- scrapeId: string,-): Promise<CostTrackingCall[]> {- for (let attempt = 0; attempt < 10; attempt++) {- const rows = await db- .select({ cost_tracking: schema.scrapes.cost_tracking })- .from(schema.scrapes)- .where(eq(schema.scrapes.id, scrapeId))- .limit(1);- if (rows.length === 1) {- const costTracking = rows[0].cost_tracking as {- calls?: CostTrackingCall[];- } | null;- return costTracking?.calls ?? [];- }- await new Promise(resolve => setTimeout(resolve, 1000));- }- throw new Error(`No scrapes row for ${scrapeId}`);-}-const isBrandingCall = (call: CostTrackingCall) =>call.metadata?.module === "branding" &&call.metadata?.method === "enhanceBrandingWithLLM";diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.tsnew file mode 100644index 000000000..114ecae78--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts@@ -0,0 +1,60 @@+import { getCostTrackingCalls } from "../cost-tracking-helpers";+import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "scrape-x-twitter",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++// X/Twitter scrapes go through xAI's X Search, which bills per profile and+// post fetched.+describe("X/Twitter profile scrape", () => {+ concurrentIf(TEST_PRODUCTION && HAS_XAI)(+ "returns the profile and its latest posts from a single profile lookup",+ async () => {+ const response = await scrape(+ { url: "https://x.com/NASA", timeout: scrapeTimeout },+ identity,+ );++ expect(response.markdown).toMatch(/^# .+ \(@NASA\)$/im);+ expect(response.markdown).toContain("## Latest Posts");+ expect(response.markdown).toContain("### 1. Post");++ const calls = await getCostTrackingCalls(response.metadata.scrapeId!);+ const profileCalls = calls.filter(+ call => call.metadata?.method === "xTwitter/profile",+ );+ expect(profileCalls).toHaveLength(1);+ expect(profileCalls[0].metadata.xSearchProfiles).toBeLessThanOrEqual(1);+ },+ scrapeTimeout + 15000,+ );++ concurrentIf(TEST_PRODUCTION && HAS_XAI)(+ "does not substitute another account for a nonexistent handle",+ async () => {+ // Letters and digits only, so the handle survives markdown escaping.+ const handle = `fcnx${Math.random().toString(36).slice(2, 12)}`;+ const response = await scrape(+ { url: `https://x.com/${handle}`, timeout: scrapeTimeout },+ identity,+ );++ expect(response.markdown).toMatch(+ new RegExp(`^# .+ \\(@(unknown|${handle})\\)$`, "im"),+ );+ expect(response.markdown).toContain(+ "No recent top-level posts were returned.",+ );+ expect(response.markdown).not.toContain("### 1. Post");+ },+ scrapeTimeout + 15000,+ );+});diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.tsindex 409f619cb..b5108ea3e 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts@@ -72,6 +72,22 @@ function grokReturns(});}+// Captures the call options the engine sends to the model.+function grokCapturesCall(output: unknown) {+ const calls: any[] = [];+ grokReturns(output);+ const doGenerate = grok.doGenerate;+ grok.doGenerate = async (options: any) => {+ calls.push(options);+ return doGenerate(options);+ };+ return calls;+}++function promptText(options: any): string {+ return JSON.stringify(options.prompt);+}+function makeMeta(url: string, zeroDataRetention = false): Meta {const logger = { info: () => {}, warn: () => {}, error: () => {} };return {@@ -84,6 +100,108 @@ function makeMeta(url: string, zeroDataRetention = false): Meta {} as unknown as Meta;}+describe("x-twitter engine X Search requests", () => {+ it("asks the profile lookup for the one exact handle and top-level posts only", async () => {+ const calls = grokCapturesCall({ username: "firecrawl", latestPosts: [] });++ await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl"));++ expect(calls).toHaveLength(1);+ expect(calls[0].tools).toEqual([+ expect.objectContaining({ id: "xai.x_search", args: {} }),+ ]);+ const prompt = promptText(calls[0]);+ expect(prompt).toContain(+ 'single user search for \\"firecrawl\\" that returns only 1 result',+ );+ expect(prompt).toContain("do not search for other or similarly named");+ expect(prompt).toContain("rather than another account's data");+ expect(prompt).toContain('\\"from:firecrawl -filter:replies\\"');+ });++ it("drops a profile that belongs to a different account", async () => {+ grokReturns({+ displayName: "Firecrawl Fan Club",+ username: "@firecrawlfans",+ followers: 12,+ latestPosts: [{ text: "Not from @firecrawl." }],+ });+ const meta = makeMeta("https://x.com/firecrawl");+ const warn = vi.fn();+ meta.logger.warn = warn;++ const result = await scrapeURLWithXTwitter(meta);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalledWith(expect.any(String), {+ requestedHandle: "firecrawl",+ returnedUsername: "firecrawlfans",+ });++ expect(result.markdown).toContain("# @unknown (@unknown)");+ expect(result.markdown).toContain(+ "No recent top-level posts were returned.",+ );+ expect(result.markdown).not.toContain("firecrawlfans");+ expect(result.markdown).not.toContain("Fan Club");+ });++ it("drops profile data that comes back without a username", async () => {+ grokReturns({+ displayName: "Firecrawl Fan Club",+ username: null,+ bio: "Unofficial.",+ latestPosts: [{ text: "Not from @firecrawl." }],+ });+ const meta = makeMeta("https://x.com/firecrawl");+ const warn = vi.fn();+ meta.logger.warn = warn;++ const result = await scrapeURLWithXTwitter(meta);++ expect(warn).not.toHaveBeenCalled();+ expect(result.markdown).toContain("# @unknown (@unknown)");+ expect(result.markdown).not.toContain("Fan Club");+ expect(result.markdown).not.toContain("Unofficial.");+ expect(result.markdown).not.toContain("### 1. Post");+ });++ it("keeps a profile whose username differs from the handle only in case", async () => {+ grokReturns({+ displayName: "Firecrawl",+ username: "@FireCrawl",+ latestPosts: [{ text: "Turn websites into LLM-ready data." }],+ });++ const result = await scrapeURLWithXTwitter(+ makeMeta("https://x.com/firecrawl"),+ );++ expect(result.markdown).toContain("# Firecrawl (@FireCrawl)");+ expect(result.markdown).toContain("> Turn websites into LLM-ready data.");+ });++ it("restricts the post lookup to the author's handle", async () => {+ const calls = grokCapturesCall({ authorUsername: "firecrawl", text: "Hi" });++ await scrapeURLWithXTwitter(+ makeMeta("https://x.com/firecrawl/status/1234567890123"),+ );++ expect(calls).toHaveLength(1);+ expect(calls[0].tools).toEqual([+ expect.objectContaining({+ id: "xai.x_search",+ args: {+ allowedXHandles: ["firecrawl"],+ enableVideoUnderstanding: true,+ enableImageUnderstanding: true,+ },+ }),+ ]);+ });+});+// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 7d52ce212..206b2d224 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -463,11 +463,31 @@ async function fetchProfile(}),abortSignal: meta.abort.asSignal(),experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),- prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,+ // xAI bills every profile a user search returns, and the model otherwise+ // often asks for 3 candidates. Pin the lookup to the one exact handle, and+ // filter replies in the search so they aren't fetched and then dropped.+ prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Look up this exact username directly with a single user search for "${xUrl.handle}" that returns only 1 result; do not search for other or similarly named accounts. If no account has the username ${xUrl.handle} (ignoring case), return null for every profile field and no posts rather than another account's data. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments, found with a single latest-posts search for "from:${xUrl.handle} -filter:replies" limited to 5 results. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,});recordGrokCost(meta, "xTwitter/profile", result);- return result.output as XTwitterProfileData;+ const profile = result.output as XTwitterProfileData;+ // A user search can still surface a similarly named account; never pass it+ // off as the requested one. Without a username nothing ties the data to the+ // requested account either, so that case is dropped too (it is the normal+ // result for a handle that doesn't exist, so it isn't worth a warning).+ const username = stripAt(profile.username);+ if (!username) {+ return {};+ }+ if (username.toLowerCase() !== xUrl.handle.toLowerCase()) {+ meta.logger.warn("X/Twitter profile lookup returned a different account", {+ requestedHandle: xUrl.handle,+ returnedUsername: username,+ });+ return {};+ }++ return profile;}async function fetchPost(