Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Route X posts and profiles through the Exchange like LinkedIn (#5656)
claude[bot]404a57968a67
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex e6ada1a9a..7aacb2550 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -23,7 +23,6 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configurationexport const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;-export const HAS_XAI = !!config.XAI_API_KEY;export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;export const HAS_PROXY = !!config.PROXY_SERVER;diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.tsindex 2dd6956f5..7c4af45f4 100644--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts@@ -293,6 +293,97 @@ describe("Exchange routing", () => {});});+ describe("X/Twitter", () => {+ const X_PROFILE_URL = "https://x.com/firecrawl";+ const X_POST_URL =+ "https://twitter.com/firecrawl/status/2037202150690844778";++ beforeEach(() => {+ exchange.setExchangeProvidersForTest([+ {+ id: "x-com",+ creditsCost: 5,+ routes: [{ domains: ["x.com", "twitter.com"], pathPrefixes: ["/"] }],+ },+ ]);+ });++ it("routes posts and profiles to the Exchange for a team outside the beta", async () => {+ for (const url of [X_PROFILE_URL, X_POST_URL]) {+ const meta = buildStubMeta(url, {});+ const fallback = await buildFallbackList(meta);++ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);+ expect(meta.exchangeProviderId).toBe("x-com");+ }+ });++ it("scrapes other X pages normally for a team outside the beta", async () => {+ const fallback = await buildFallbackList(+ buildStubMeta("https://x.com/explore", {}),+ );++ const engines = fallback.map(f => f.engine);++ expect(engines).not.toContain("exchange");+ expect(engines.length).toBeGreaterThan(0);+ });++ it("names the option the Exchange declines instead of scraping the login wall", async () => {+ const meta = buildStubMeta(X_POST_URL, {});+ meta.internalOptions.zeroDataRetention = true;++ const error = await buildFallbackList(meta).catch(e => e);++ expect(error).toBeInstanceOf(+ exchange.ThirdPartyDataUnsupportedOptionError,+ );+ expect(error.response()).toEqual(+ new exchange.ThirdPartyDataUnsupportedOptionError(+ "zero data retention",+ ).response(),+ );+ });++ it("serves a post whose request sets rendering options, which X has no page for", async () => {+ const meta = buildStubMeta(X_POST_URL, {});+ meta.options.headers = { "user-agent": "test" };+ meta.options.waitFor = 1000;++ const fallback = await buildFallbackList(meta);++ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);+ });++ it("fails closed for a post when the Exchange does not serve X", async () => {+ exchange.setExchangeProvidersForTest([]);++ await expect(+ buildFallbackList(buildStubMeta(X_POST_URL, {})),+ ).resolves.toEqual([]);+ });++ it("keeps the normal engines for a browser-profile scrape", async () => {+ const meta = buildStubMeta(X_PROFILE_URL, {});+ meta.options.profile = { name: "p" };++ const fallback = await buildFallbackList(meta);++ expect(fallback.map(f => f.engine)).not.toContain("exchange");+ });++ it("refuses a US-only team before reaching the Exchange", async () => {+ const error = await buildFallbackList(+ buildStubMeta(X_PROFILE_URL, { usOnly: true }),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.UsOnlyUnavailableError);+ expect(error.message).toBe(+ "X/Twitter scraping is not available for teams restricted to US-only processing.",+ );+ });+ });+describe("exchange engine", () => {const failure = (code: string, message: string) => ({success: false,diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.tsindex 198e6dc14..041ed5906 100644--- a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts@@ -1,7 +1,6 @@-import { getCostTrackingCalls } from "../cost-tracking-helpers";-import { HAS_JOB_LOG } from "../job-log";-import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";-import { scrape, scrapeTimeout, idmux, Identity } from "./lib";+import { config } from "../../../config";+import { concurrentIf, TEST_PRODUCTION } from "../lib";+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";let identity: Identity;@@ -13,48 +12,55 @@ beforeAll(async () => {});}, 10000 + scrapeTimeout);-// X/Twitter scrapes go through xAI's X Search, which bills per profile and-// post fetched.-describe("X/Twitter profile scrape", () => {- concurrentIf(TEST_PRODUCTION && HAS_XAI && HAS_JOB_LOG)(- "returns the profile and its latest posts from a single profile lookup",+// X/Twitter posts and profiles are served by the Exchange's X provider, the+// same route LinkedIn profiles take, for every team.+const HAS_EXCHANGE_X = TEST_PRODUCTION && !!config.FIRE_EXCHANGE_URL;++describe("X/Twitter scrape through the Exchange", () => {+ concurrentIf(HAS_EXCHANGE_X)(+ "returns the profile at the Exchange's price",async () => {const response = await scrape({ url: "https://x.com/NASA", timeout: scrapeTimeout },identity,);- expect(response.markdown).toMatch(/^# .+ \(@NASA\)$/im);- expect(response.markdown).toContain("## Latest Posts");- expect(response.markdown).toContain("### 1. Post");+ expect(response.markdown).toMatch(/nasa/i);+ expect(response.metadata.creditsUsed).toBe(5);+ },+ scrapeTimeout + 15000,+ );- const calls = await getCostTrackingCalls(response.metadata.scrapeId!);- const profileCalls = calls.filter(- call => call.metadata?.method === "xTwitter/profile",+ concurrentIf(HAS_EXCHANGE_X)(+ "returns a post at the Exchange's price",+ async () => {+ const response = await scrape(+ {+ url: "https://x.com/ericciarla/status/2037202150690844778",+ timeout: scrapeTimeout,+ },+ identity,);- expect(profileCalls).toHaveLength(1);- expect(profileCalls[0].metadata.xSearchProfiles).toBeLessThanOrEqual(1);++ expect(response.markdown).toContain("2037202150690844778");+ expect(response.markdown).toMatch(/n8n/i);+ expect(response.metadata.creditsUsed).toBe(5);},scrapeTimeout + 15000,);- concurrentIf(TEST_PRODUCTION && HAS_XAI)(- "does not substitute another account for a nonexistent handle",+ concurrentIf(HAS_EXCHANGE_X)(+ "reports a nonexistent handle as a missing record rather than another account",async () => {- // Letters and digits only, so the handle survives markdown escaping.+ // Letters and digits only, so the handle is a valid X username.const handle = `fcnx${Math.random().toString(36).slice(2, 12)}`;- const response = await scrape(+ const response = await scrapeRaw({ url: `https://x.com/${handle}`, timeout: scrapeTimeout },identity,);- expect(response.markdown).toMatch(- new RegExp(`^# .+ \\(@(unknown|${handle})\\)$`, "im"),- );- expect(response.markdown).toContain(- "No recent top-level posts were returned.",- );- expect(response.markdown).not.toContain("### 1. Post");+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");},scrapeTimeout + 15000,);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex fdfce8858..bd21dfdae 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -269,7 +269,6 @@ const configSchema = z.object({// Key for US-only teams; OPENAI_API_KEY is never used for them.OPENAI_US_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),OPENROUTER_API_KEY: z.string().optional(),- XAI_API_KEY: z.string().optional(),LLAMAPARSE_API_KEY: z.string().optional(),STRIPE_SECRET_KEY: z.string().optional(),AUTUMN_SECRET_KEY: z.string().optional(),diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex b9a2c3263..62909c41b 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -41,7 +41,6 @@ import {MediaAccessDeniedError,PromptInjectionDetectedError,JsonExtractionContentTooLargeError,- XTwitterConfigurationError,UsOnlyUnavailableError,ReservedCapacityUnavailableError,ExchangeRefusedError,@@ -97,7 +96,6 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_MEDIA_ACCESS_DENIED: MediaAccessDeniedError,SCRAPE_PROMPT_INJECTION_DETECTED: PromptInjectionDetectedError,SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,- SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,RESERVED_CAPACITY_UNAVAILABLE: ReservedCapacityUnavailableError,MAP_FAILED: MapFailedError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex c6fde4732..a77bf1539 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -42,7 +42,6 @@ export type ErrorCodes =| "SCRAPE_MEDIA_ACCESS_DENIED"| "SCRAPE_PROMPT_INJECTION_DETECTED"| "SCRAPE_JSON_CONTENT_TOO_LARGE"- | "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"| "US_ONLY_UNAVAILABLE"| "RESERVED_CAPACITY_UNAVAILABLE"| "PARSE_UNSUPPORTED_OPTIONS"diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex d30c3c785..16c8dfceb 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -6,6 +6,7 @@ import type { DocumentProvider, FormatObject } from "../controllers/v2/types";import { hasLedgerAcceptance } from "../services/alexandria/terms";import { type ErrorCodes, TransportableError } from "./error";import { logger as rootLogger } from "./logger";+import { isXTwitterUrl } from "./x-twitter-url";type OrganizationDataSourceAccessRecord = {status?: string | null;@@ -548,8 +549,14 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {}// Blocked URLs go to the Exchange for every team, since nothing else may- // serve them; elsewhere it replaces a normal scrape only for the beta.- if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {+ // serve them; elsewhere it replaces a normal scrape only for the beta. X+ // posts and profiles also go to it for every team: the Exchange's X+ // provider is what serves them, in place of a Firecrawl X engine.+ if (+ input.flags?.[EXCHANGE_BETA_FLAG] !== true &&+ input.blocked !== true &&+ !isXTwitterUrl(input.url)+ ) {return false;}diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 46b3f1648..d90b5d5a6 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -66,7 +66,7 @@ describe("calculateCreditsToBeBilled", () => {expect(credits).toBe(16);});- it("bills X/Twitter scrapes at 30 credits", async () => {+ it("bills an X/Twitter scrape at the Exchange's price, with no X surcharge", async () => {const credits = await calculateCreditsToBeBilled({formats: [{ type: "markdown" }],@@ -79,16 +79,18 @@ describe("calculateCreditsToBeBilled", () => {metadata: {statusCode: 200,proxyUsed: "basic",- postprocessorsUsed: ["x-twitter"],},} as any,{totalCost: 0,} as any,{} as any,+ undefined,+ undefined,+ { provider: { id: "x-com", creditsCost: 5, steps: [] } },);- expect(credits).toBe(30);+ expect(credits).toBe(5);});it("bills enhanced proxy scrapes the same as basic ones", async () => {diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 585c54dee..ec9670881 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -16,7 +16,6 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";const creditsPerPDFPage = 1;const unblockedDomainCostBonus = 4;-const xTwitterCostBonus = 29;const jsonCostBonus = 4;const redactPIICostBonus = 4;// Each additional PDF page also gets redacted through fire-privacy, so@@ -229,10 +228,6 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += 4;}- if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {- creditsToBeBilled += xTwitterCostBonus;- }-if (internalOptions.zeroDataRetention && !options.lockdown) {creditsToBeBilled += flags?.zdrCost ?? 1;}diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/lib/x-twitter-url.tssimilarity index 87%rename from apps/api/src/scraper/scrapeURL/engines/x-twitter/url.tsrename to apps/api/src/lib/x-twitter-url.tsindex 66ed3241a..a90a60438 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts+++ b/apps/api/src/lib/x-twitter-url.ts@@ -1,5 +1,6 @@-// Pure URL recognition for the X/Twitter engine, kept apart from the engine so-// credit estimation can use it without pulling in the scraper and AI SDK.+// The X/Twitter URLs the Exchange's X provider answers: posts and profiles.+// Scrape sends these to the Exchange for eligible teams and requests (see+// lib/exchange.ts and buildFallbackList in scraper/scrapeURL/engines).const RESERVED_PROFILE_PATHS = new Set(["compose",@@ -17,13 +18,13 @@ const RESERVED_PROFILE_PATHS = new Set(["share",]);-export type XTwitterProfileUrl = {+type XTwitterProfileUrl = {kind: "profile";handle: string;normalizedUrl: string;};-export type XTwitterPostUrl = {+type XTwitterPostUrl = {kind: "post";handle?: string;postId: string;@@ -32,7 +33,7 @@ export type XTwitterPostUrl = {type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;-export function parseXTwitterUrl(url: string): XTwitterUrl | null {+function parseXTwitterUrl(url: string): XTwitterUrl | null {let parsed: URL;try {parsed = new URL(url);diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex f3a1780e5..951d979f8 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -21,11 +21,6 @@ import {wikipediaMaxReasonableTime,isWikimediaUrl,} from "./wikipedia";-import {- scrapeURLWithXTwitter,- xTwitterMaxReasonableTime,- isXTwitterUrl,-} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";import { useFireEngine } from "./fire-engine/available";import { hasFormatOfType } from "../../../lib/format-utils";@@ -41,8 +36,11 @@ import {AgentIndexOnlyError,BrandingNotSupportedError,NoCachedDataError,+ UsOnlyUnavailableError,} from "../error";import { isUrlBlocked } from "../../WebScraper/utils/blocklist";+import { isUsOnly } from "../../../lib/us-only";+import { isXTwitterUrl } from "../../../lib/x-twitter-url";import { hasCustomRequestContext } from "../lib/request-context";import {getExchangeAccessForRequest,@@ -66,8 +64,7 @@ export type Engine =| "image"| "index"| "index;documents"- | "wikipedia"- | "x-twitter";+ | "wikipedia";const usePlaywright =config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&@@ -77,12 +74,8 @@ const useWikipedia =config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";-const useXTwitter =- (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||- config.USE_DB_AUTHENTICATION === true;const engines: Engine[] = [- ...(useXTwitter ? ["x-twitter" as const] : []),...(useWikipedia ? ["wikipedia" as const] : []),...(useIndex ? ["index" as const, "index;documents" as const] : []),...(useFireEngine@@ -215,7 +208,6 @@ const engineHandlers: {document: scrapeDocument,image: scrapeImage,wikipedia: scrapeURLWithWikipedia,- "x-twitter": scrapeURLWithXTwitter,};const engineMRTs: {@@ -242,7 +234,6 @@ const engineMRTs: {document: documentMaxReasonableTime,image: imageMaxReasonableTime,wikipedia: wikipediaMaxReasonableTime,- "x-twitter": xTwitterMaxReasonableTime,};const engineOptions: {@@ -589,28 +580,6 @@ const engineOptions: {},quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)},- "x-twitter": {- features: {- actions: false,- waitFor: false,- screenshot: false,- "screenshot@fullScreen": false,- pdf: false,- document: false,- image: false,- audio: false,- video: false,- atsv: false,- location: false,- mobile: false,- skipTlsVerification: true,- useFastMode: true,- stealthProxy: false,- branding: false,- disableAdblock: true,- },- quality: 1500,- },};export function shouldUseIndex(meta: Meta) {@@ -700,11 +669,26 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ // X posts and profiles only the Exchange serves, as blocked URLs (LinkedIn+ // profiles) are: a normal engine gets X's login wall. A profile or lockdown+ // scrape keeps the normal engines, as it did with Firecrawl's own X engine.+ const xTwitterPage =+ isXTwitterUrl(url) &&+ meta.options.profile === undefined &&+ !meta.options.lockdown;++ // They are answered by xAI, which is outside the US-only guarantee.+ if (xTwitterPage && isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {+ throw new UsOnlyUnavailableError(+ "X/Twitter scraping is not available for teams restricted to US-only processing.",+ );+ }+const exchangeAccess = await getExchangeAccessForRequest({url,teamId: meta.internalOptions.teamId ?? null,orgId: meta.internalOptions.orgId ?? null,- blocked,+ blocked: blocked || xTwitterPage,origin: meta.internalOptions.origin ?? null,formats: meta.options.formats,actions: meta.options.actions,@@ -737,9 +721,10 @@ export async function buildFallbackList(meta: Meta): Promise<// A blocked URL can only have been admitted (by scrapeBlocklistMiddleware// or batch scrape) for the Exchange; if it can no longer serve it (catalog// changed, service down), fail closed rather than letting normal engines- // scrape a blocklisted site. Unblocked URLs whose provider wants- // unaccepted terms or rejects an option fall through and scrape normally.- if (blocked) {+ // scrape a blocklisted site. X posts and profiles fail the same way.+ // Other URLs whose provider wants unaccepted terms or rejects an option+ // fall through and scrape normally.+ if (blocked || xTwitterPage) {if (exchangeAccess.termsRequired) {throw new ThirdPartyDataTermsRequiredError(exchangeAccess.terms);}@@ -957,20 +942,6 @@ export async function buildFallbackList(meta: Meta): Promise<}}- if (- !meta.options.profile &&- isXTwitterUrl(meta.url) &&- _engines.includes("x-twitter")- ) {- _engines.length = 0;- _engines.push("x-twitter");- } else if (!isXTwitterUrl(meta.url)) {- const xTwitterIndex = _engines.indexOf("x-twitter");- if (xTwitterIndex !== -1) {- _engines.splice(xTwitterIndex, 1);- }- }-const prioritySum = [...meta.featureFlags].reduce((a, x) => a + featureFlagOptions[x].priority,0,diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.tsdeleted file mode 100644index b5108ea3e..000000000--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts+++ /dev/null@@ -1,412 +0,0 @@-import {- context,- propagation,- SpanStatusCode,- trace,-} from "@opentelemetry/api";-import {- InMemorySpanExporter,- type NodeTracerProvider,- type ReadableSpan,-} from "@opentelemetry/sdk-trace-node";-import { APICallError } from "ai";-import { MockLanguageModelV3 } from "ai/test";-import type { Meta } from "../..";-import { CostTracking } from "../../../../lib/cost-tracking";-import { createTracerProvider } from "../../../../lib/otel-tracer";-import { scrapeURLWithXTwitter } from "./index";--const { grok } = vi.hoisted(() => {- process.env.XAI_API_KEY = "test-key";- return { grok: { doGenerate: undefined as any } };-});--vi.mock("@ai-sdk/xai", async importOriginal => {- const actual = await importOriginal<typeof import("@ai-sdk/xai")>();- return {- xai: {- responses: (modelId: string) =>- new MockLanguageModelV3({- provider: "xai.responses",- modelId,- doGenerate: (...args) => grok.doGenerate(...args),- }),- tools: actual.xai.tools,- },- };-});--const usage = {- inputTokens: { total: 3000, noCache: 1000, cacheRead: 2000, cacheWrite: 0 },- outputTokens: { total: 400, text: 400, reasoning: 0 },-};--// Raw xAI Responses body; only the usage block is read from it.-function responseBody(xSearch?: { posts?: number; users?: number }) {- return {- object: "response",- usage: {- input_tokens: 3000,- output_tokens: 400,- ...(xSearch && {- server_side_tool_usage_details: {- x_search_calls: 2,- x_posts_fetched: xSearch.posts,- x_users_fetched: xSearch.users,- },- }),- },- };-}--function grokReturns(- output: unknown,- body: unknown = responseBody({ posts: 0, users: 0 }),-) {- grok.doGenerate = async () => ({- content: [{ type: "text", text: JSON.stringify(output) }],- finishReason: { unified: "stop", raw: "completed" },- usage,- response: { body },- warnings: [],- });-}--// Captures the call options the engine sends to the model.-function grokCapturesCall(output: unknown) {- const calls: any[] = [];- grokReturns(output);- const doGenerate = grok.doGenerate;- grok.doGenerate = async (options: any) => {- calls.push(options);- return doGenerate(options);- };- return calls;-}--function promptText(options: any): string {- return JSON.stringify(options.prompt);-}--function makeMeta(url: string, zeroDataRetention = false): Meta {- const logger = { info: () => {}, warn: () => {}, error: () => {} };- return {- id: "019990c0-0000-7000-8000-000000000001",- url,- logger,- abort: { asSignal: () => undefined },- internalOptions: { teamId: "team-test", zeroDataRetention },- costTracking: new CostTracking(),- } as unknown as Meta;-}--describe("x-twitter engine X Search requests", () => {- it("asks the profile lookup for the one exact handle and top-level posts only", async () => {- const calls = grokCapturesCall({ username: "firecrawl", latestPosts: [] });-- await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl"));-- expect(calls).toHaveLength(1);- expect(calls[0].tools).toEqual([- expect.objectContaining({ id: "xai.x_search", args: {} }),- ]);- const prompt = promptText(calls[0]);- expect(prompt).toContain(- 'single user search for \\"firecrawl\\" that returns only 1 result',- );- expect(prompt).toContain("do not search for other or similarly named");- expect(prompt).toContain("rather than another account's data");- expect(prompt).toContain('\\"from:firecrawl -filter:replies\\"');- });-- it("drops a profile that belongs to a different account", async () => {- grokReturns({- displayName: "Firecrawl Fan Club",- username: "@firecrawlfans",- followers: 12,- latestPosts: [{ text: "Not from @firecrawl." }],- });- const meta = makeMeta("https://x.com/firecrawl");- const warn = vi.fn();- meta.logger.warn = warn;-- const result = await scrapeURLWithXTwitter(meta);-- expect(warn).toHaveBeenCalledTimes(1);- expect(warn).toHaveBeenCalledWith(expect.any(String), {- requestedHandle: "firecrawl",- returnedUsername: "firecrawlfans",- });-- expect(result.markdown).toContain("# @unknown (@unknown)");- expect(result.markdown).toContain(- "No recent top-level posts were returned.",- );- expect(result.markdown).not.toContain("firecrawlfans");- expect(result.markdown).not.toContain("Fan Club");- });-- it("drops profile data that comes back without a username", async () => {- grokReturns({- displayName: "Firecrawl Fan Club",- username: null,- bio: "Unofficial.",- latestPosts: [{ text: "Not from @firecrawl." }],- });- const meta = makeMeta("https://x.com/firecrawl");- const warn = vi.fn();- meta.logger.warn = warn;-- const result = await scrapeURLWithXTwitter(meta);-- expect(warn).not.toHaveBeenCalled();- expect(result.markdown).toContain("# @unknown (@unknown)");- expect(result.markdown).not.toContain("Fan Club");- expect(result.markdown).not.toContain("Unofficial.");- expect(result.markdown).not.toContain("### 1. Post");- });-- it("keeps a profile whose username differs from the handle only in case", async () => {- grokReturns({- displayName: "Firecrawl",- username: "@FireCrawl",- latestPosts: [{ text: "Turn websites into LLM-ready data." }],- });-- const result = await scrapeURLWithXTwitter(- makeMeta("https://x.com/firecrawl"),- );-- expect(result.markdown).toContain("# Firecrawl (@FireCrawl)");- expect(result.markdown).toContain("> Turn websites into LLM-ready data.");- });-- it("restricts the post lookup to the author's handle", async () => {- const calls = grokCapturesCall({ authorUsername: "firecrawl", text: "Hi" });-- await scrapeURLWithXTwitter(- makeMeta("https://x.com/firecrawl/status/1234567890123"),- );-- expect(calls).toHaveLength(1);- expect(calls[0].tools).toEqual([- expect.objectContaining({- id: "xai.x_search",- args: {- allowedXHandles: ["firecrawl"],- enableVideoUnderstanding: true,- enableImageUnderstanding: true,- },- }),- ]);- });-});--// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.-const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;--describe("x-twitter engine LLM telemetry", () => {- const exporter = new InMemorySpanExporter();- let provider: NodeTracerProvider;-- beforeAll(() => {- provider = createTracerProvider({- exporter,- serviceName: "test-service",- serviceInstanceId: "pod-1",- });- provider.register();- });-- afterAll(async () => {- await provider.shutdown();- trace.disable();- context.disable();- propagation.disable();- });-- beforeEach(async () => {- // Drop spans a previous test left unflushed.- await provider.forceFlush();- exporter.reset();- });-- async function doGenerateSpan(): Promise<ReadableSpan> {- await provider.forceFlush();- const spans = exporter- .getFinishedSpans()- .filter(s => s.name === "ai.generateText.doGenerate");- expect(spans).toHaveLength(1);- return spans[0];- }-- it("records a usage span for a profile lookup", async () => {- grokReturns({- displayName: "Firecrawl",- username: "firecrawl",- bio: "Turn websites into LLM-ready data.",- followers: 1000,- latestPosts: [],- });-- const result = await scrapeURLWithXTwitter(- makeMeta("https://x.com/firecrawl"),- );-- expect(result.markdown).toContain("@firecrawl");- const span = await doGenerateSpan();- expect(span.attributes).toMatchObject({- "ai.telemetry.functionId": "xTwitter/profile",- "ai.telemetry.metadata.feature": "x-twitter",- "ai.telemetry.metadata.teamId": "team-test",- "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",- "ai.model.id": "grok-4-1-fast-non-reasoning",- "ai.usage.promptTokens": 3000,- "ai.usage.completionTokens": 400,- "ai.usage.cachedInputTokens": 2000,- "firecrawl.llm.tool.x_search_posts": 0,- "firecrawl.llm.tool.x_search_profiles": 0,- });- expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");- });-- it("records X Search items on the span and their fee in cost tracking for a profile lookup", async () => {- grokReturns(- { username: "firecrawl", latestPosts: [] },- responseBody({ posts: 44, users: 3 }),- );- const meta = makeMeta("https://x.com/firecrawl");-- await scrapeURLWithXTwitter(meta);-- const span = await doGenerateSpan();- expect(span.attributes).toMatchObject({- "firecrawl.llm.tool.x_search_posts": 44,- "firecrawl.llm.tool.x_search_profiles": 3,- });- const { calls, totalCost } = meta.costTracking.toJSON();- expect(calls).toHaveLength(1);- expect(calls[0]).toMatchObject({- type: "other",- model: "grok-4-1-fast-non-reasoning",- tokens: { input: 3000, output: 400 },- metadata: {- module: "scrapeURL",- method: "xTwitter/profile",- xSearchPosts: 44,- xSearchProfiles: 3,- },- });- // 44 posts x $0.005 + 3 profiles x $0.01 = $0.25, plus tokens.- expect(calls[0].cost).toBeCloseTo(0.25 + tokenCost, 10);- expect(totalCost).toBeCloseTo(0.25 + tokenCost, 10);- });-- it("records the X Search fee in cost tracking for a post lookup", async () => {- grokReturns(- { authorUsername: "firecrawl", text: "Hello from a post." },- responseBody({ posts: 18, users: 0 }),- );- const meta = makeMeta("https://x.com/firecrawl/status/1234567890123");-- await scrapeURLWithXTwitter(meta);-- const { calls } = meta.costTracking.toJSON();- expect(calls).toHaveLength(1);- expect(calls[0].metadata).toMatchObject({- method: "xTwitter/post",- xSearchPosts: 18,- xSearchProfiles: 0,- });- expect(calls[0].cost).toBeCloseTo(18 * 0.005 + tokenCost, 10);- });-- it("prices tokens only when xAI reports no X Search item counts", async () => {- grokReturns({ username: "firecrawl", latestPosts: [] }, responseBody());- const meta = makeMeta("https://x.com/firecrawl");-- await scrapeURLWithXTwitter(meta);-- const span = await doGenerateSpan();- expect(- Object.keys(span.attributes).filter(key =>- key.startsWith("firecrawl.llm.tool."),- ),- ).toEqual([]);- const { calls } = meta.costTracking.toJSON();- expect(calls[0].metadata).toMatchObject({- xSearchPosts: 0,- xSearchProfiles: 0,- });- expect(calls[0].cost).toBeCloseTo(tokenCost, 10);- });-- it("records a usage span for a post lookup", async () => {- grokReturns({- authorUsername: "firecrawl",- text: "Hello from a post.",- likes: 10,- retweets: 2,- });-- await scrapeURLWithXTwitter(- makeMeta("https://x.com/firecrawl/status/1234567890123"),- );-- const span = await doGenerateSpan();- expect(span.attributes).toMatchObject({- "ai.telemetry.functionId": "xTwitter/post",- "ai.telemetry.metadata.feature": "x-twitter",- "ai.telemetry.metadata.teamId": "team-test",- "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",- "ai.model.id": "grok-4-1-fast-non-reasoning",- "ai.usage.promptTokens": 3000,- "ai.usage.completionTokens": 400,- "ai.usage.cachedInputTokens": 2000,- });- expect(span.attributes["ai.prompt.messages"]).toContain(- "post id 1234567890123",- );- });-- // Covers the per-call guard on its own; ZDR scrape jobs additionally run- // under the tracer's ZDR context (see otel-tracer.test.ts).- it("records no AI SDK spans for zero-data-retention scrapes", async () => {- grokReturns({ username: "firecrawl", latestPosts: [] });-- const meta = makeMeta("https://x.com/firecrawl", true);- await scrapeURLWithXTwitter(meta);-- await provider.forceFlush();- expect(- exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),- ).toEqual([]);- // Billing-side cost is recorded regardless of telemetry.- expect(meta.costTracking.toJSON().calls).toHaveLength(1);- });-- it("records the span with an error status when the Grok call is rejected", async () => {- grok.doGenerate = async () => {- throw new APICallError({- message: "bad request",- url: "https://api.x.ai/v1/responses",- requestBodyValues: {},- statusCode: 400,- isRetryable: false,- });- };-- await expect(- scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl")),- ).rejects.toThrow("bad request");-- const span = await doGenerateSpan();- expect(span.status.code).toBe(SpanStatusCode.ERROR);- expect(- Object.keys(span.attributes).filter(key =>- key.startsWith("firecrawl.llm.tool."),- ),- ).toEqual([]);- expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");- });-});diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsdeleted file mode 100644index 3a6236083..000000000--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ /dev/null@@ -1,620 +0,0 @@-import { xai } from "@ai-sdk/xai";-import { generateText, jsonSchema, Output, type GenerateTextResult } from "ai";-import { config } from "../../../../config";-import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";-import {- xSearchCost,- xSearchUsageFromResponseBody,-} from "../../../../lib/xai-x-search";-import { Meta } from "../..";-import { EngineScrapeResult } from "..";-import {- EngineError,- UsOnlyUnavailableError,- XTwitterConfigurationError,-} from "../../error";-import { isUsOnly } from "../../../../lib/us-only";-import { safeMarkdownToHtml } from "../pdf/markdownToHtml";-import { calculateCost } from "../../transformers/llmExtract";-import {- isXTwitterUrl,- parseXTwitterUrl,- type XTwitterPostUrl,- type XTwitterProfileUrl,-} from "./url";--export { isXTwitterUrl };--const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";--function xTwitterTelemetry(functionId: string, meta: Meta) {- return {- isEnabled: !meta.internalOptions.zeroDataRetention,- functionId,- metadata: {- scrapeId: meta.id,- teamId: meta.internalOptions.teamId ?? "",- feature: "x-twitter",- },- };-}--// Records the call's token cost plus the per-item X Search fee xAI bills on-// top of it.-function recordGrokCost(- meta: Meta,- method: string,- {- totalUsage,- steps,- }: Pick<GenerateTextResult<any, any>, "totalUsage" | "steps">,-) {- const inputTokens = totalUsage.inputTokens ?? 0;- const outputTokens = totalUsage.outputTokens ?? 0;- const xSearch = { posts: 0, profiles: 0 };- for (const step of steps) {- const usage = xSearchUsageFromResponseBody(step.response.body);- xSearch.posts += usage?.posts ?? 0;- xSearch.profiles += usage?.profiles ?? 0;- }-- meta.costTracking.addCall({- type: "other",- metadata: {- module: "scrapeURL",- method,- xSearchPosts: xSearch.posts,- xSearchProfiles: xSearch.profiles,- },- model: XAI_RESPONSES_MODEL,- tokens: { input: inputTokens, output: outputTokens },- cost:- calculateCost(XAI_RESPONSES_MODEL, inputTokens, outputTokens) +- xSearchCost(xSearch),- });-}--type ProfilePost = {- text: string;- url?: string | null;- createdAt?: string | null;- likes?: number | null;- retweets?: number | null;-};--type XTwitterProfileData = {- displayName?: string | null;- username?: string | null;- profilePicUrl?: string | null;- bio?: string | null;- followers?: number | null;- accountVerified?: boolean | null;- url?: string | null;- latestPosts?: ProfilePost[] | null;-};--type ThreadPost = {- authorDisplayName?: string | null;- authorUsername?: string | null;- text: string;- url?: string | null;- createdAt?: string | null;- likes?: number | null;- retweets?: number | null;-};--type PostComment = {- authorDisplayName?: string | null;- authorUsername?: string | null;- text: string;- url?: string | null;- createdAt?: string | null;- likes?: number | null;-};--type XTwitterPostData = {- authorDisplayName?: string | null;- authorUsername?: string | null;- text: string;- url?: string | null;- createdAt?: string | null;- likes?: number | null;- retweets?: number | null;- thread?: ThreadPost[] | null;- comments?: PostComment[] | null;-};--const nullableString = { type: ["string", "null"] };-const nullableNumber = { type: ["number", "null"] };-const nullableBoolean = { type: ["boolean", "null"] };--const profileSchema = {- type: "object",- properties: {- displayName: nullableString,- username: nullableString,- profilePicUrl: nullableString,- bio: nullableString,- followers: nullableNumber,- accountVerified: nullableBoolean,- url: nullableString,- latestPosts: {- type: "array",- maxItems: 5,- items: {- type: "object",- properties: {- text: { type: "string" },- url: nullableString,- createdAt: nullableString,- likes: nullableNumber,- retweets: nullableNumber,- },- required: ["text", "url", "createdAt", "likes", "retweets"],- additionalProperties: false,- },- },- },- required: [- "displayName",- "username",- "profilePicUrl",- "bio",- "followers",- "accountVerified",- "url",- "latestPosts",- ],- additionalProperties: false,-};--const postSchema = {- type: "object",- properties: {- authorDisplayName: nullableString,- authorUsername: nullableString,- text: { type: "string" },- url: nullableString,- createdAt: nullableString,- likes: nullableNumber,- retweets: nullableNumber,- thread: {- type: "array",- items: {- type: "object",- properties: {- authorDisplayName: nullableString,- authorUsername: nullableString,- text: { type: "string" },- url: nullableString,- createdAt: nullableString,- likes: nullableNumber,- retweets: nullableNumber,- },- required: [- "authorDisplayName",- "authorUsername",- "text",- "url",- "createdAt",- "likes",- "retweets",- ],- additionalProperties: false,- },- },- comments: {- type: "array",- maxItems: 5,- items: {- type: "object",- properties: {- authorDisplayName: nullableString,- authorUsername: nullableString,- text: { type: "string" },- url: nullableString,- createdAt: nullableString,- likes: nullableNumber,- },- required: [- "authorDisplayName",- "authorUsername",- "text",- "url",- "createdAt",- "likes",- ],- additionalProperties: false,- },- },- },- required: [- "authorDisplayName",- "authorUsername",- "text",- "url",- "createdAt",- "likes",- "retweets",- "thread",- "comments",- ],- additionalProperties: false,-};--export async function scrapeURLWithXTwitter(- meta: Meta,-): Promise<EngineScrapeResult> {- const urlToScrape = meta.rewrittenUrl ?? meta.url;- const xUrl = parseXTwitterUrl(urlToScrape);-- if (!xUrl) {- throw new EngineError(- `URL is not a supported X/Twitter URL: ${urlToScrape}`,- );- }-- if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {- throw new UsOnlyUnavailableError(- "X/Twitter scraping is not available for teams restricted to US-only processing.",- );- }-- if (!config.XAI_API_KEY) {- throw new XTwitterConfigurationError();- }-- meta.logger.info("Fetching X/Twitter data through Grok", {- kind: xUrl.kind,- url: xUrl.normalizedUrl,- });-- const markdown =- xUrl.kind === "profile"- ? buildProfileMarkdown(await fetchProfile(xUrl, meta))- : buildPostMarkdown(await fetchPost(xUrl, meta));-- if (markdown.trim().length === 0) {- throw new EngineError(`No X/Twitter content returned for: ${urlToScrape}`);- }-- const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);-- return {- url: xUrl.normalizedUrl,- html,- markdown,- statusCode: 200,- contentType: "text/html; charset=utf-8",- proxyUsed: "basic",- postprocessorsUsed: ["x-twitter"],- };-}--export function xTwitterMaxReasonableTime(_meta: Meta): number {- return 30000;-}--function buildProfileMarkdown(profile: XTwitterProfileData): string {- const username = stripAt(profile.username) ?? "unknown";- const displayName = profile.displayName?.trim() || `@${username}`;- const posts = (profile.latestPosts ?? []).slice(0, 5);- const lines = [- `# ${escapeMarkdownInline(displayName)} (@${escapeMarkdownInline(username)})`,- "",- ];-- if (profile.bio) {- lines.push(escapeMarkdownBlock(profile.bio), "");- }-- const facts = [- `Followers: ${- profile.followers !== null && profile.followers !== undefined- ? formatNumber(profile.followers)- : "Unknown"- }`,- `Verified: ${- profile.accountVerified !== null && profile.accountVerified !== undefined- ? profile.accountVerified- ? "yes"- : "no"- : "Unknown"- }`,- `Profile Picture: ${- profile.profilePicUrl- ? formatMarkdownImage(displayName, profile.profilePicUrl)- : "Unknown"- }`,- profile.url- ? `Source: ${formatMarkdownLink(profile.url, profile.url)}`- : undefined,- ].filter((line): line is string => line !== undefined);-- if (facts.length > 0) {- lines.push(...facts.map(fact => `- ${fact}`), "");- }-- lines.push("## Latest Posts", "");-- if (posts.length === 0) {- lines.push("No recent top-level posts were returned.");- } else {- for (const [index, post] of posts.entries()) {- lines.push(`### ${index + 1}. Post`);- if (post.createdAt) {- lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);- }- if (post.url) {- lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);- }- lines.push("");- lines.push(formatBlockquote(post.text));- const metrics = formatMetrics(post);- if (metrics.length > 0) {- lines.push("", metrics.join(" | "));- }- lines.push("");- }- }-- return trimMarkdown(lines.join("\n"));-}--function buildPostMarkdown(post: XTwitterPostData): string {- const username = stripAt(post.authorUsername);- const title = username- ? `Post by @${escapeMarkdownInline(username)}`- : "X/Twitter Post";- const lines = [`# ${title}`, ""];-- const byline = [- post.authorDisplayName?.trim()- ? escapeMarkdownInline(post.authorDisplayName)- : undefined,- username ? `@${escapeMarkdownInline(username)}` : undefined,- ].filter(Boolean);-- if (byline.length > 0) {- lines.push(`Author: ${byline.join(" ")}`);- }- if (post.createdAt) {- lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);- }- if (post.url) {- lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);- }-- const metrics = formatMetrics(post);- if (metrics.length > 0) {- lines.push(metrics.join(" | "));- }-- lines.push("", "## Post", "", escapeMarkdownBlock(post.text.trim()));-- const thread = (post.thread ?? []).filter(item => item.text?.trim());- if (thread.length > 0) {- lines.push("", "## Thread", "");- for (const [index, threadPost] of thread.entries()) {- lines.push(`### ${index + 1}. Thread Post`);- if (threadPost.authorUsername) {- lines.push(- `Author: @${escapeMarkdownInline(stripAt(threadPost.authorUsername)!)}`,- );- }- if (threadPost.createdAt) {- lines.push(`Posted: ${escapeMarkdownInline(threadPost.createdAt)}`);- }- if (threadPost.url) {- lines.push(- `URL: ${formatMarkdownLink(threadPost.url, threadPost.url)}`,- );- }- lines.push("", formatBlockquote(threadPost.text));- const threadMetrics = formatMetrics(threadPost);- if (threadMetrics.length > 0) {- lines.push("", threadMetrics.join(" | "));- }- lines.push("");- }- }-- const comments = (post.comments ?? [])- .filter(comment => comment.text?.trim())- .slice(0, 5);- lines.push("", "## Top Comments", "");- if (comments.length === 0) {- lines.push("No top comments were returned.");- } else {- for (const [index, comment] of comments.entries()) {- const author = stripAt(comment.authorUsername);- lines.push(- `### ${index + 1}. ${- author ? `@${escapeMarkdownInline(author)}` : "Comment"- }`,- );- if (comment.authorDisplayName) {- lines.push(- `Author: ${escapeMarkdownInline(comment.authorDisplayName)}`,- );- }- if (comment.createdAt) {- lines.push(`Posted: ${escapeMarkdownInline(comment.createdAt)}`);- }- if (comment.url) {- lines.push(`URL: ${formatMarkdownLink(comment.url, comment.url)}`);- }- lines.push("", formatBlockquote(comment.text));- if (comment.likes !== null && comment.likes !== undefined) {- lines.push("", `Likes: ${formatNumber(comment.likes)}`);- }- lines.push("");- }- }-- return trimMarkdown(lines.join("\n"));-}--async function fetchProfile(- xUrl: XTwitterProfileUrl,- meta: Meta,-): Promise<XTwitterProfileData> {- const result = await generateText({- model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),- maxOutputTokens: 20000,- tools: {- x_search: xai.tools.xSearch(),- } as any,- toolChoice: { type: "tool", toolName: "x_search" },- output: Output.object({- schema: jsonSchema(profileSchema as any),- name: "x_twitter_profile",- description:- "Current public X/Twitter profile data and latest top-level posts.",- }),- abortSignal: meta.abort.asSignal(),- experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),- // xAI bills every profile a user search returns, and the model otherwise- // often asks for 3 candidates. Pin the lookup to the one exact handle, and- // filter replies in the search so they aren't fetched and then dropped.- prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Look up this exact username directly with a single user search for "${xUrl.handle}" that returns only 1 result; do not search for other or similarly named accounts. If no account has the username ${xUrl.handle} (ignoring case), return null for every profile field and no posts rather than another account's data. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments, found with a single latest-posts search for "from:${xUrl.handle} -filter:replies" limited to 5 results. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,- });- recordGrokCost(meta, "xTwitter/profile", result);-- const profile = result.output as XTwitterProfileData;- // A user search can still surface a similarly named account; never pass it- // off as the requested one. Without a username nothing ties the data to the- // requested account either, so that case is dropped too (it is the normal- // result for a handle that doesn't exist, so it isn't worth a warning).- const username = stripAt(profile.username);- if (!username) {- return {};- }- if (username.toLowerCase() !== xUrl.handle.toLowerCase()) {- meta.logger.warn("X/Twitter profile lookup returned a different account", {- requestedHandle: xUrl.handle,- returnedUsername: username,- });- return {};- }-- return profile;-}--async function fetchPost(- xUrl: XTwitterPostUrl,- meta: Meta,-): Promise<XTwitterPostData> {- const handlePart = xUrl.handle ? ` by @${xUrl.handle}` : "";- const toolsOptions = xUrl.handle- ? {- allowedXHandles: [xUrl.handle],- enableVideoUnderstanding: true,- enableImageUnderstanding: true,- }- : {- enableVideoUnderstanding: true,- enableImageUnderstanding: true,- };-- const result = await generateText({- model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),- maxOutputTokens: 20000,- tools: {- x_search: xai.tools.xSearch(toolsOptions),- } as any,- toolChoice: { type: "tool", toolName: "x_search" },- output: Output.object({- schema: jsonSchema(postSchema as any),- name: "x_twitter_post",- description:- "Current public X/Twitter post data with metrics, thread, and top comments.",- }),- abortSignal: meta.abort.asSignal(),- experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),- prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,- });- recordGrokCost(meta, "xTwitter/post", result);-- return result.output as XTwitterPostData;-}--function stripAt(value: string | null | undefined): string | undefined {- const trimmed = value?.trim();- if (!trimmed) {- return undefined;- }- return trimmed.replace(/^@/, "");-}--function formatMetrics(item: {- likes?: number | null;- retweets?: number | null;-}): string[] {- return [- item.likes !== null && item.likes !== undefined- ? `Likes: ${formatNumber(item.likes)}`- : undefined,- item.retweets !== null && item.retweets !== undefined- ? `Retweets: ${formatNumber(item.retweets)}`- : undefined,- ].filter((line): line is string => line !== undefined);-}--function formatNumber(value: number): string {- return new Intl.NumberFormat("en-US").format(value);-}--function formatBlockquote(value: string): string {- const escaped = escapeMarkdownBlock(value.trim());- if (!escaped) {- return "> ";- }- return escaped- .split(/\r?\n/)- .map(line => (line.trim() ? `> ${line}` : ">"))- .join("\n");-}--function formatMarkdownLink(label: string, url: string): string {- const sanitizedUrl = sanitizeMarkdownUrl(url);- if (!sanitizedUrl) {- return escapeMarkdownInline(label);- }- return `[${escapeMarkdownInline(label)}](${sanitizedUrl})`;-}--function formatMarkdownImage(alt: string, url: string): string {- const sanitizedUrl = sanitizeMarkdownUrl(url);- if (!sanitizedUrl) {- return "Unknown";- }- return ``;-}--function sanitizeMarkdownUrl(url: string): string | undefined {- try {- const parsed = new URL(url);- if (!["http:", "https:"].includes(parsed.protocol)) {- return undefined;- }- return parsed.toString().replace(/\)/g, "%29");- } catch {- return undefined;- }-}--function escapeMarkdownInline(value: string): string {- return escapeMarkdownBlock(value).replace(/([\\`*_{}[\]()#+\-.!|])/g, "\\$1");-}--function escapeMarkdownBlock(value: string): string {- return value- .replace(/&/g, "&")- .replace(/</g, "<")- .replace(/>/g, ">");-}--function trimMarkdown(markdown: string): string {- return markdown.replace(/\n{3,}/g, "\n\n").trim() + "\n";-}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 0dd5ab7af..e44cf4c95 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -9,28 +9,6 @@ export class EngineError extends Error {}}-export class XTwitterConfigurationError extends TransportableError {- constructor() {- super(- "SCRAPE_X_TWITTER_CONFIGURATION_ERROR",- "X/Twitter scraping requires XAI_API_KEY to be configured.",- );- }-- serialize() {- return super.serialize();- }-- static deserialize(- _: ErrorCodes,- data: ReturnType<typeof this.prototype.serialize>,- ) {- const x = new XTwitterConfigurationError();- x.stack = data.stack;- return x;- }-}-/** A route the team's US-only restriction rules out, because no US-based vendor serves it. */export class UsOnlyUnavailableError extends TransportableError {constructor(message: string) {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0afd13e55..6ab8a4ee3 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -59,7 +59,6 @@ import {ProxySelectionError,ScrapeRetryLimitError,BrandingNotSupportedError,- XTwitterConfigurationError,UsOnlyUnavailableError,ReservedCapacityUnavailableError,ExchangeRefusedError,@@ -1018,12 +1017,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {continue;}}- if (error.engine === "x-twitter") {- meta.logger.warn("X/Twitter scrape failed fatally.", {- error: error.error,- });- throw error.error;- } else if (error.error instanceof EngineError) {+ if (error.error instanceof EngineError) {meta.logger.warn("Engine " + error.engine + " could not scrape the page.",{@@ -1066,7 +1060,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ProxySelectionError ||error.error instanceof NoCachedDataError ||error.error instanceof AgentIndexOnlyError ||- error.error instanceof XTwitterConfigurationError ||error.error instanceof UsOnlyUnavailableError ||error.error instanceof ReservedCapacityUnavailableError ||error.error instanceof ExchangeRefusedError ||diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 18253c613..81f09973c 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -9,7 +9,6 @@ import {} from "./store";import { judgeCreditsForJudgedCount } from "./search/billing";import type { MonitorTarget } from "./types";-import { config } from "../../config";describe("monitoring store credit helpers", () => {describe("reserving credits using previous PDF results", () => {@@ -139,108 +138,6 @@ describe("monitoring store credit helpers", () => {expect(estimateMonitorCreditsPerRun(targets, true)).toBe(12);});- // The x-twitter engine only exists with an xAI key or DB auth.- const withXTwitterEngine = (fn: () => void) => {- const saved = config.USE_DB_AUTHENTICATION;- config.USE_DB_AUTHENTICATION = true;- try {- fn();- } finally {- config.USE_DB_AUTHENTICATION = saved;- }- };-- it("includes the x-twitter surcharge, so the hold covers what is billed", () =>- withXTwitterEngine(() => {- const targets: MonitorTarget[] = [- {- id: "target-x",- type: "scrape",- urls: ["https://x.com/blknoiz06"],- scrapeOptions: {},- },- ];-- // 1 scrape + 29 x-twitter, + 1 judge.- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(30);- expect(estimateMonitorCreditsPerRun(targets, true)).toBe(31);- expect(- calculateMonitorCheckActualCreditsFromPages(- [- {- target_id: "target-x",- status: "changed",- metadata: {- creditsUsed: null,- postprocessorsUsed: ["x-twitter"],- },- judgment: { meaningful: false },- },- ],- targets,- ),- ).toBe(estimateMonitorCreditsPerRun(targets, true));- }));-- it("charges the surcharge only for URLs the x-twitter engine takes", () =>- withXTwitterEngine(() => {- const targets: MonitorTarget[] = [- {- id: "target-mixed",- type: "scrape",- urls: [- "https://twitter.com/someone/status/1234567890",- "https://x.com/home",- "https://example.com/x.com",- ],- scrapeOptions: {},- },- ];-- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(3 + 29);- }));-- it("skips the surcharge when a browser profile bypasses the x-twitter engine", () =>- withXTwitterEngine(() => {- const targets: MonitorTarget[] = [- {- id: "target-profile",- type: "scrape",- urls: ["https://x.com/someone"],- scrapeOptions: { profile: { name: "p" } },- },- ];-- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(1);- }));-- it("skips the surcharge where the x-twitter engine cannot run", () => {- const target = (- scrapeOptions: Record<string, unknown>,- ): MonitorTarget[] => [- {- id: "t",- type: "scrape",- urls: ["https://x.com/someone"],- scrapeOptions,- },- ];- withXTwitterEngine(() => {- // Lockdown serves from the index: 1 base + 4 lockdown, no surcharge.- expect(- estimateMonitorCreditsPerRun(target({ lockdown: true }), false),- ).toBe(5);- });- const saved = [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] as const;- config.USE_DB_AUTHENTICATION = false;- config.XAI_API_KEY = undefined;- try {- expect(estimateMonitorCreditsPerRun(target({}), false)).toBe(1);- } finally {- [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] = saved;- }- });-it("adds predictable lockdown costs and judge credits separately", () => {const targets: MonitorTarget[] = [{@@ -359,33 +256,6 @@ describe("monitoring store credit helpers", () => {).toBe(5);});- it("uses monitor metadata for fallback postprocessor credits and never for proxies", () => {- const targets: MonitorTarget[] = [- {- id: "target-1",- type: "scrape",- urls: ["https://x.com/firecrawl"],- scrapeOptions: {},- },- ];-- expect(- calculateMonitorCheckActualCreditsFromPages(- [- {- target_id: "target-1",- metadata: {- proxyUsed: "stealth",- postprocessorsUsed: ["x-twitter"],- },- status: "same",- },- ],- targets,- ),- ).toBe(30);- });-it("does not bill extra when enhanced proxy metadata is present", () => {const targets: MonitorTarget[] = [{diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 0b365aae7..7fd227634 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -16,13 +16,11 @@ import {import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { monitoringClaimDueMonitors } from "../../db/rpc";-import { config } from "../../config";import {getPDFMaxPages,liftCheckPromptInjection,shouldParsePDF,} from "../../controllers/v2/types";-import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";import {getNextMonitorRunAt,estimateRunsPerMonth,@@ -60,14 +58,12 @@ const DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE = 7;const SCRAPE_OPTION_CREDIT_BONUS = 4;const JUDGE_CREDITS_PER_PAGE = 1;const REMOVED_PAGE_CREDITS = 0;-const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;type MonitorCreditMetadata = {creditsUsed?: unknown;numPages?: unknown;- postprocessorsUsed?: unknown;};function formatType(format: unknown): string | null {@@ -179,37 +175,13 @@ function estimateSearchTargetCredits();}-/**- * The x-twitter engine's surcharge, which billing adds per page (see- * fallbackBaseCreditsForPage). Left out of the estimate, a judged one-URL X- * monitor reserves 2 credits and costs 31, and an account with a few credits- * left passes every hold and is never charged.- */-function xTwitterSurcharge(- url: string,- options: MonitorTarget["scrapeOptions"],-): number {- // Mirrors the engine router (scrapeURL/engines/index.ts): the engine exists- // only with an xAI key or DB auth, lockdown serves from the index alone, and- // a browser profile routes around it.- const engineEnabled =- (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||- config.USE_DB_AUTHENTICATION === true;- if (!engineEnabled || options?.lockdown || options?.profile) return 0;- return isXTwitterUrl(url) ? X_TWITTER_POSTPROCESSOR_CREDIT_BONUS : 0;-}-function estimateTargetBaseCredits(target: MonitorTarget,judgeEnabled: boolean = false,): number {const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);if (target.type === "scrape") {- return target.urls.reduce(- (sum, url) =>- sum + creditsPerPage + xTwitterSurcharge(url, target.scrapeOptions),- 0,- );+ return target.urls.length * creditsPerPage;}if (target.type === "search") {return estimateSearchTargetCredits(target, judgeEnabled);@@ -342,13 +314,6 @@ export function calculateMonitorCheckActualCreditsFromPages(credits += metadata.numPages - 1;}- if (- Array.isArray(metadata?.postprocessorsUsed) &&- metadata.postprocessorsUsed.includes("x-twitter")- ) {- credits += X_TWITTER_POSTPROCESSOR_CREDIT_BONUS;- }-return credits;}