Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(x-twitter): look up the requested handle directly on profile scrapes (#4873)

Abimael Martelle58f33e3dcd6
diff --git a/apps/api/src/__tests__/snips/cost-tracking-helpers.ts b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts
new file mode 100644
index 000000000..686fbf858
--- /dev/null
+++ b/apps/api/src/__tests__/snips/cost-tracking-helpers.ts
@@ -0,0 +1,31 @@
+import { eq } from "drizzle-orm";
+import { db } from "../../db/connection";
+import * as schema from "../../db/schema";
+
+export type CostTrackingCall = {
+ model: string;
+ cost: number;
+ metadata: Record<string, unknown>;
+ tokens?: { input: number; output: number };
+};
+
+// The scrape row is written when the job finishes; give the insert a moment.
+export async function getCostTrackingCalls(
+ scrapeId: string,
+): Promise<CostTrackingCall[]> {
+ for (let attempt = 0; attempt < 10; attempt++) {
+ const rows = await db
+ .select({ cost_tracking: schema.scrapes.cost_tracking })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.id, scrapeId))
+ .limit(1);
+ if (rows.length === 1) {
+ const costTracking = rows[0].cost_tracking as {
+ calls?: CostTrackingCall[];
+ } | null;
+ return costTracking?.calls ?? [];
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ throw new Error(`No scrapes row for ${scrapeId}`);
+}
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 7aacb2550..e6ada1a9a 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;
// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;
+export const HAS_XAI = !!config.XAI_API_KEY;
export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;
export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
export const HAS_PROXY = !!config.PROXY_SERVER;
diff --git a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
index 848d7e260..244d51971 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-branding.test.ts
@@ -1,6 +1,7 @@
-import { eq } from "drizzle-orm";
-import { db } from "../../../db/connection";
-import * as schema from "../../../db/schema";
+import {
+ type CostTrackingCall,
+ getCostTrackingCalls,
+} from "../cost-tracking-helpers";
import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";
import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
@@ -38,34 +39,6 @@ describe("Branding declared-logo fallback", () => {
);
});
-type CostTrackingCall = {
- model: string;
- cost: number;
- metadata: Record<string, unknown>;
- tokens?: { input: number; output: number };
-};
-
-// The scrape row is written when the job finishes; give the insert a moment.
-async function getCostTrackingCalls(
- scrapeId: string,
-): Promise<CostTrackingCall[]> {
- for (let attempt = 0; attempt < 10; attempt++) {
- const rows = await db
- .select({ cost_tracking: schema.scrapes.cost_tracking })
- .from(schema.scrapes)
- .where(eq(schema.scrapes.id, scrapeId))
- .limit(1);
- if (rows.length === 1) {
- const costTracking = rows[0].cost_tracking as {
- calls?: CostTrackingCall[];
- } | null;
- return costTracking?.calls ?? [];
- }
- await new Promise(resolve => setTimeout(resolve, 1000));
- }
- throw new Error(`No scrapes row for ${scrapeId}`);
-}
-
const isBrandingCall = (call: CostTrackingCall) =>
call.metadata?.module === "branding" &&
call.metadata?.method === "enhanceBrandingWithLLM";
diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
new file mode 100644
index 000000000..114ecae78
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
@@ -0,0 +1,60 @@
+import { getCostTrackingCalls } from "../cost-tracking-helpers";
+import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";
+import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "scrape-x-twitter",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+// X/Twitter scrapes go through xAI's X Search, which bills per profile and
+// post fetched.
+describe("X/Twitter profile scrape", () => {
+ concurrentIf(TEST_PRODUCTION && HAS_XAI)(
+ "returns the profile and its latest posts from a single profile lookup",
+ async () => {
+ const response = await scrape(
+ { url: "https://x.com/NASA", timeout: scrapeTimeout },
+ identity,
+ );
+
+ expect(response.markdown).toMatch(/^# .+ \(@NASA\)$/im);
+ expect(response.markdown).toContain("## Latest Posts");
+ expect(response.markdown).toContain("### 1. Post");
+
+ const calls = await getCostTrackingCalls(response.metadata.scrapeId!);
+ const profileCalls = calls.filter(
+ call => call.metadata?.method === "xTwitter/profile",
+ );
+ expect(profileCalls).toHaveLength(1);
+ expect(profileCalls[0].metadata.xSearchProfiles).toBeLessThanOrEqual(1);
+ },
+ scrapeTimeout + 15000,
+ );
+
+ concurrentIf(TEST_PRODUCTION && HAS_XAI)(
+ "does not substitute another account for a nonexistent handle",
+ async () => {
+ // Letters and digits only, so the handle survives markdown escaping.
+ const handle = `fcnx${Math.random().toString(36).slice(2, 12)}`;
+ const response = await scrape(
+ { url: `https://x.com/${handle}`, timeout: scrapeTimeout },
+ identity,
+ );
+
+ expect(response.markdown).toMatch(
+ new RegExp(`^# .+ \\(@(unknown|${handle})\\)$`, "im"),
+ );
+ expect(response.markdown).toContain(
+ "No recent top-level posts were returned.",
+ );
+ expect(response.markdown).not.toContain("### 1. Post");
+ },
+ scrapeTimeout + 15000,
+ );
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
index 409f619cb..b5108ea3e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
@@ -72,6 +72,22 @@ function grokReturns(
});
}
+// Captures the call options the engine sends to the model.
+function grokCapturesCall(output: unknown) {
+ const calls: any[] = [];
+ grokReturns(output);
+ const doGenerate = grok.doGenerate;
+ grok.doGenerate = async (options: any) => {
+ calls.push(options);
+ return doGenerate(options);
+ };
+ return calls;
+}
+
+function promptText(options: any): string {
+ return JSON.stringify(options.prompt);
+}
+
function makeMeta(url: string, zeroDataRetention = false): Meta {
const logger = { info: () => {}, warn: () => {}, error: () => {} };
return {
@@ -84,6 +100,108 @@ function makeMeta(url: string, zeroDataRetention = false): Meta {
} as unknown as Meta;
}
+describe("x-twitter engine X Search requests", () => {
+ it("asks the profile lookup for the one exact handle and top-level posts only", async () => {
+ const calls = grokCapturesCall({ username: "firecrawl", latestPosts: [] });
+
+ await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl"));
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0].tools).toEqual([
+ expect.objectContaining({ id: "xai.x_search", args: {} }),
+ ]);
+ const prompt = promptText(calls[0]);
+ expect(prompt).toContain(
+ 'single user search for \\"firecrawl\\" that returns only 1 result',
+ );
+ expect(prompt).toContain("do not search for other or similarly named");
+ expect(prompt).toContain("rather than another account's data");
+ expect(prompt).toContain('\\"from:firecrawl -filter:replies\\"');
+ });
+
+ it("drops a profile that belongs to a different account", async () => {
+ grokReturns({
+ displayName: "Firecrawl Fan Club",
+ username: "@firecrawlfans",
+ followers: 12,
+ latestPosts: [{ text: "Not from @firecrawl." }],
+ });
+ const meta = makeMeta("https://x.com/firecrawl");
+ const warn = vi.fn();
+ meta.logger.warn = warn;
+
+ const result = await scrapeURLWithXTwitter(meta);
+
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalledWith(expect.any(String), {
+ requestedHandle: "firecrawl",
+ returnedUsername: "firecrawlfans",
+ });
+
+ expect(result.markdown).toContain("# @unknown (@unknown)");
+ expect(result.markdown).toContain(
+ "No recent top-level posts were returned.",
+ );
+ expect(result.markdown).not.toContain("firecrawlfans");
+ expect(result.markdown).not.toContain("Fan Club");
+ });
+
+ it("drops profile data that comes back without a username", async () => {
+ grokReturns({
+ displayName: "Firecrawl Fan Club",
+ username: null,
+ bio: "Unofficial.",
+ latestPosts: [{ text: "Not from @firecrawl." }],
+ });
+ const meta = makeMeta("https://x.com/firecrawl");
+ const warn = vi.fn();
+ meta.logger.warn = warn;
+
+ const result = await scrapeURLWithXTwitter(meta);
+
+ expect(warn).not.toHaveBeenCalled();
+ expect(result.markdown).toContain("# @unknown (@unknown)");
+ expect(result.markdown).not.toContain("Fan Club");
+ expect(result.markdown).not.toContain("Unofficial.");
+ expect(result.markdown).not.toContain("### 1. Post");
+ });
+
+ it("keeps a profile whose username differs from the handle only in case", async () => {
+ grokReturns({
+ displayName: "Firecrawl",
+ username: "@FireCrawl",
+ latestPosts: [{ text: "Turn websites into LLM-ready data." }],
+ });
+
+ const result = await scrapeURLWithXTwitter(
+ makeMeta("https://x.com/firecrawl"),
+ );
+
+ expect(result.markdown).toContain("# Firecrawl (@FireCrawl)");
+ expect(result.markdown).toContain("> Turn websites into LLM-ready data.");
+ });
+
+ it("restricts the post lookup to the author's handle", async () => {
+ const calls = grokCapturesCall({ authorUsername: "firecrawl", text: "Hi" });
+
+ await scrapeURLWithXTwitter(
+ makeMeta("https://x.com/firecrawl/status/1234567890123"),
+ );
+
+ expect(calls).toHaveLength(1);
+ expect(calls[0].tools).toEqual([
+ expect.objectContaining({
+ id: "xai.x_search",
+ args: {
+ allowedXHandles: ["firecrawl"],
+ enableVideoUnderstanding: true,
+ enableImageUnderstanding: true,
+ },
+ }),
+ ]);
+ });
+});
+
// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.
const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
index 7d52ce212..206b2d224 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -463,11 +463,31 @@ async function fetchProfile(
}),
abortSignal: meta.abort.asSignal(),
experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),
- prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
+ // xAI bills every profile a user search returns, and the model otherwise
+ // often asks for 3 candidates. Pin the lookup to the one exact handle, and
+ // filter replies in the search so they aren't fetched and then dropped.
+ prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Look up this exact username directly with a single user search for "${xUrl.handle}" that returns only 1 result; do not search for other or similarly named accounts. If no account has the username ${xUrl.handle} (ignoring case), return null for every profile field and no posts rather than another account's data. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments, found with a single latest-posts search for "from:${xUrl.handle} -filter:replies" limited to 5 results. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
});
recordGrokCost(meta, "xTwitter/profile", result);
- return result.output as XTwitterProfileData;
+ const profile = result.output as XTwitterProfileData;
+ // A user search can still surface a similarly named account; never pass it
+ // off as the requested one. Without a username nothing ties the data to the
+ // requested account either, so that case is dropped too (it is the normal
+ // result for a handle that doesn't exist, so it isn't worth a warning).
+ const username = stripAt(profile.username);
+ if (!username) {
+ return {};
+ }
+ if (username.toLowerCase() !== xUrl.handle.toLowerCase()) {
+ meta.logger.warn("X/Twitter profile lookup returned a different account", {
+ requestedHandle: xUrl.handle,
+ returnedUsername: username,
+ });
+ return {};
+ }
+
+ return profile;
}
async function fetchPost(