Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Route X posts and profiles through the Exchange like LinkedIn (#5656)

claude[bot]404a57968a67
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index e6ada1a9a..7aacb2550 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -23,7 +23,6 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;
// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;
-export const HAS_XAI = !!config.XAI_API_KEY;
export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;
export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
export const HAS_PROXY = !!config.PROXY_SERVER;
diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
index 2dd6956f5..7c4af45f4 100644
--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
@@ -293,6 +293,97 @@ describe("Exchange routing", () => {
});
});
+ describe("X/Twitter", () => {
+ const X_PROFILE_URL = "https://x.com/firecrawl";
+ const X_POST_URL =
+ "https://twitter.com/firecrawl/status/2037202150690844778";
+
+ beforeEach(() => {
+ exchange.setExchangeProvidersForTest([
+ {
+ id: "x-com",
+ creditsCost: 5,
+ routes: [{ domains: ["x.com", "twitter.com"], pathPrefixes: ["/"] }],
+ },
+ ]);
+ });
+
+ it("routes posts and profiles to the Exchange for a team outside the beta", async () => {
+ for (const url of [X_PROFILE_URL, X_POST_URL]) {
+ const meta = buildStubMeta(url, {});
+ const fallback = await buildFallbackList(meta);
+
+ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);
+ expect(meta.exchangeProviderId).toBe("x-com");
+ }
+ });
+
+ it("scrapes other X pages normally for a team outside the beta", async () => {
+ const fallback = await buildFallbackList(
+ buildStubMeta("https://x.com/explore", {}),
+ );
+
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).not.toContain("exchange");
+ expect(engines.length).toBeGreaterThan(0);
+ });
+
+ it("names the option the Exchange declines instead of scraping the login wall", async () => {
+ const meta = buildStubMeta(X_POST_URL, {});
+ meta.internalOptions.zeroDataRetention = true;
+
+ const error = await buildFallbackList(meta).catch(e => e);
+
+ expect(error).toBeInstanceOf(
+ exchange.ThirdPartyDataUnsupportedOptionError,
+ );
+ expect(error.response()).toEqual(
+ new exchange.ThirdPartyDataUnsupportedOptionError(
+ "zero data retention",
+ ).response(),
+ );
+ });
+
+ it("serves a post whose request sets rendering options, which X has no page for", async () => {
+ const meta = buildStubMeta(X_POST_URL, {});
+ meta.options.headers = { "user-agent": "test" };
+ meta.options.waitFor = 1000;
+
+ const fallback = await buildFallbackList(meta);
+
+ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);
+ });
+
+ it("fails closed for a post when the Exchange does not serve X", async () => {
+ exchange.setExchangeProvidersForTest([]);
+
+ await expect(
+ buildFallbackList(buildStubMeta(X_POST_URL, {})),
+ ).resolves.toEqual([]);
+ });
+
+ it("keeps the normal engines for a browser-profile scrape", async () => {
+ const meta = buildStubMeta(X_PROFILE_URL, {});
+ meta.options.profile = { name: "p" };
+
+ const fallback = await buildFallbackList(meta);
+
+ expect(fallback.map(f => f.engine)).not.toContain("exchange");
+ });
+
+ it("refuses a US-only team before reaching the Exchange", async () => {
+ const error = await buildFallbackList(
+ buildStubMeta(X_PROFILE_URL, { usOnly: true }),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.UsOnlyUnavailableError);
+ expect(error.message).toBe(
+ "X/Twitter scraping is not available for teams restricted to US-only processing.",
+ );
+ });
+ });
+
describe("exchange engine", () => {
const failure = (code: string, message: string) => ({
success: false,
diff --git a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
index 198e6dc14..041ed5906 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-x-twitter.test.ts
@@ -1,7 +1,6 @@
-import { getCostTrackingCalls } from "../cost-tracking-helpers";
-import { HAS_JOB_LOG } from "../job-log";
-import { concurrentIf, HAS_XAI, TEST_PRODUCTION } from "../lib";
-import { scrape, scrapeTimeout, idmux, Identity } from "./lib";
+import { config } from "../../../config";
+import { concurrentIf, TEST_PRODUCTION } from "../lib";
+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";
let identity: Identity;
@@ -13,48 +12,55 @@ beforeAll(async () => {
});
}, 10000 + scrapeTimeout);
-// X/Twitter scrapes go through xAI's X Search, which bills per profile and
-// post fetched.
-describe("X/Twitter profile scrape", () => {
- concurrentIf(TEST_PRODUCTION && HAS_XAI && HAS_JOB_LOG)(
- "returns the profile and its latest posts from a single profile lookup",
+// X/Twitter posts and profiles are served by the Exchange's X provider, the
+// same route LinkedIn profiles take, for every team.
+const HAS_EXCHANGE_X = TEST_PRODUCTION && !!config.FIRE_EXCHANGE_URL;
+
+describe("X/Twitter scrape through the Exchange", () => {
+ concurrentIf(HAS_EXCHANGE_X)(
+ "returns the profile at the Exchange's price",
async () => {
const response = await scrape(
{ url: "https://x.com/NASA", timeout: scrapeTimeout },
identity,
);
- expect(response.markdown).toMatch(/^# .+ \(@NASA\)$/im);
- expect(response.markdown).toContain("## Latest Posts");
- expect(response.markdown).toContain("### 1. Post");
+ expect(response.markdown).toMatch(/nasa/i);
+ expect(response.metadata.creditsUsed).toBe(5);
+ },
+ scrapeTimeout + 15000,
+ );
- const calls = await getCostTrackingCalls(response.metadata.scrapeId!);
- const profileCalls = calls.filter(
- call => call.metadata?.method === "xTwitter/profile",
+ concurrentIf(HAS_EXCHANGE_X)(
+ "returns a post at the Exchange's price",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://x.com/ericciarla/status/2037202150690844778",
+ timeout: scrapeTimeout,
+ },
+ identity,
);
- expect(profileCalls).toHaveLength(1);
- expect(profileCalls[0].metadata.xSearchProfiles).toBeLessThanOrEqual(1);
+
+ expect(response.markdown).toContain("2037202150690844778");
+ expect(response.markdown).toMatch(/n8n/i);
+ expect(response.metadata.creditsUsed).toBe(5);
},
scrapeTimeout + 15000,
);
- concurrentIf(TEST_PRODUCTION && HAS_XAI)(
- "does not substitute another account for a nonexistent handle",
+ concurrentIf(HAS_EXCHANGE_X)(
+ "reports a nonexistent handle as a missing record rather than another account",
async () => {
- // Letters and digits only, so the handle survives markdown escaping.
+ // Letters and digits only, so the handle is a valid X username.
const handle = `fcnx${Math.random().toString(36).slice(2, 12)}`;
- const response = await scrape(
+ const response = await scrapeRaw(
{ url: `https://x.com/${handle}`, timeout: scrapeTimeout },
identity,
);
- expect(response.markdown).toMatch(
- new RegExp(`^# .+ \\(@(unknown|${handle})\\)$`, "im"),
- );
- expect(response.markdown).toContain(
- "No recent top-level posts were returned.",
- );
- expect(response.markdown).not.toContain("### 1. Post");
+ expect(response.body.success).toBe(false);
+ expect(response.body.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");
},
scrapeTimeout + 15000,
);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index fdfce8858..bd21dfdae 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -269,7 +269,6 @@ const configSchema = z.object({
// Key for US-only teams; OPENAI_API_KEY is never used for them.
OPENAI_US_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
OPENROUTER_API_KEY: z.string().optional(),
- XAI_API_KEY: z.string().optional(),
LLAMAPARSE_API_KEY: z.string().optional(),
STRIPE_SECRET_KEY: z.string().optional(),
AUTUMN_SECRET_KEY: z.string().optional(),
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index b9a2c3263..62909c41b 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -41,7 +41,6 @@ import {
MediaAccessDeniedError,
PromptInjectionDetectedError,
JsonExtractionContentTooLargeError,
- XTwitterConfigurationError,
UsOnlyUnavailableError,
ReservedCapacityUnavailableError,
ExchangeRefusedError,
@@ -97,7 +96,6 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_MEDIA_ACCESS_DENIED: MediaAccessDeniedError,
SCRAPE_PROMPT_INJECTION_DETECTED: PromptInjectionDetectedError,
SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,
- SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,
RESERVED_CAPACITY_UNAVAILABLE: ReservedCapacityUnavailableError,
MAP_FAILED: MapFailedError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index c6fde4732..a77bf1539 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -42,7 +42,6 @@ export type ErrorCodes =
| "SCRAPE_MEDIA_ACCESS_DENIED"
| "SCRAPE_PROMPT_INJECTION_DETECTED"
| "SCRAPE_JSON_CONTENT_TOO_LARGE"
- | "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"
| "US_ONLY_UNAVAILABLE"
| "RESERVED_CAPACITY_UNAVAILABLE"
| "PARSE_UNSUPPORTED_OPTIONS"
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index d30c3c785..16c8dfceb 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -6,6 +6,7 @@ import type { DocumentProvider, FormatObject } from "../controllers/v2/types";
import { hasLedgerAcceptance } from "../services/alexandria/terms";
import { type ErrorCodes, TransportableError } from "./error";
import { logger as rootLogger } from "./logger";
+import { isXTwitterUrl } from "./x-twitter-url";
type OrganizationDataSourceAccessRecord = {
status?: string | null;
@@ -548,8 +549,14 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {
}
// Blocked URLs go to the Exchange for every team, since nothing else may
- // serve them; elsewhere it replaces a normal scrape only for the beta.
- if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {
+ // serve them; elsewhere it replaces a normal scrape only for the beta. X
+ // posts and profiles also go to it for every team: the Exchange's X
+ // provider is what serves them, in place of a Firecrawl X engine.
+ if (
+ input.flags?.[EXCHANGE_BETA_FLAG] !== true &&
+ input.blocked !== true &&
+ !isXTwitterUrl(input.url)
+ ) {
return false;
}
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 46b3f1648..d90b5d5a6 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -66,7 +66,7 @@ describe("calculateCreditsToBeBilled", () => {
expect(credits).toBe(16);
});
- it("bills X/Twitter scrapes at 30 credits", async () => {
+ it("bills an X/Twitter scrape at the Exchange's price, with no X surcharge", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
@@ -79,16 +79,18 @@ describe("calculateCreditsToBeBilled", () => {
metadata: {
statusCode: 200,
proxyUsed: "basic",
- postprocessorsUsed: ["x-twitter"],
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
+ undefined,
+ undefined,
+ { provider: { id: "x-com", creditsCost: 5, steps: [] } },
);
- expect(credits).toBe(30);
+ expect(credits).toBe(5);
});
it("bills enhanced proxy scrapes the same as basic ones", async () => {
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 585c54dee..ec9670881 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -16,7 +16,6 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
const creditsPerPDFPage = 1;
const unblockedDomainCostBonus = 4;
-const xTwitterCostBonus = 29;
const jsonCostBonus = 4;
const redactPIICostBonus = 4;
// Each additional PDF page also gets redacted through fire-privacy, so
@@ -229,10 +228,6 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += 4;
}
- if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {
- creditsToBeBilled += xTwitterCostBonus;
- }
-
if (internalOptions.zeroDataRetention && !options.lockdown) {
creditsToBeBilled += flags?.zdrCost ?? 1;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/lib/x-twitter-url.ts
similarity index 87%
rename from apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
rename to apps/api/src/lib/x-twitter-url.ts
index 66ed3241a..a90a60438 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
+++ b/apps/api/src/lib/x-twitter-url.ts
@@ -1,5 +1,6 @@
-// Pure URL recognition for the X/Twitter engine, kept apart from the engine so
-// credit estimation can use it without pulling in the scraper and AI SDK.
+// The X/Twitter URLs the Exchange's X provider answers: posts and profiles.
+// Scrape sends these to the Exchange for eligible teams and requests (see
+// lib/exchange.ts and buildFallbackList in scraper/scrapeURL/engines).
const RESERVED_PROFILE_PATHS = new Set([
"compose",
@@ -17,13 +18,13 @@ const RESERVED_PROFILE_PATHS = new Set([
"share",
]);
-export type XTwitterProfileUrl = {
+type XTwitterProfileUrl = {
kind: "profile";
handle: string;
normalizedUrl: string;
};
-export type XTwitterPostUrl = {
+type XTwitterPostUrl = {
kind: "post";
handle?: string;
postId: string;
@@ -32,7 +33,7 @@ export type XTwitterPostUrl = {
type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
-export function parseXTwitterUrl(url: string): XTwitterUrl | null {
+function parseXTwitterUrl(url: string): XTwitterUrl | null {
let parsed: URL;
try {
parsed = new URL(url);
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index f3a1780e5..951d979f8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -21,11 +21,6 @@ import {
wikipediaMaxReasonableTime,
isWikimediaUrl,
} from "./wikipedia";
-import {
- scrapeURLWithXTwitter,
- xTwitterMaxReasonableTime,
- isXTwitterUrl,
-} from "./x-twitter";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { useFireEngine } from "./fire-engine/available";
import { hasFormatOfType } from "../../../lib/format-utils";
@@ -41,8 +36,11 @@ import {
AgentIndexOnlyError,
BrandingNotSupportedError,
NoCachedDataError,
+ UsOnlyUnavailableError,
} from "../error";
import { isUrlBlocked } from "../../WebScraper/utils/blocklist";
+import { isUsOnly } from "../../../lib/us-only";
+import { isXTwitterUrl } from "../../../lib/x-twitter-url";
import { hasCustomRequestContext } from "../lib/request-context";
import {
getExchangeAccessForRequest,
@@ -66,8 +64,7 @@ export type Engine =
| "image"
| "index"
| "index;documents"
- | "wikipedia"
- | "x-twitter";
+ | "wikipedia";
const usePlaywright =
config.PLAYWRIGHT_MICROSERVICE_URL !== "" &&
@@ -77,12 +74,8 @@ const useWikipedia =
config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&
config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&
config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";
-const useXTwitter =
- (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||
- config.USE_DB_AUTHENTICATION === true;
const engines: Engine[] = [
- ...(useXTwitter ? ["x-twitter" as const] : []),
...(useWikipedia ? ["wikipedia" as const] : []),
...(useIndex ? ["index" as const, "index;documents" as const] : []),
...(useFireEngine
@@ -215,7 +208,6 @@ const engineHandlers: {
document: scrapeDocument,
image: scrapeImage,
wikipedia: scrapeURLWithWikipedia,
- "x-twitter": scrapeURLWithXTwitter,
};
const engineMRTs: {
@@ -242,7 +234,6 @@ const engineMRTs: {
document: documentMaxReasonableTime,
image: imageMaxReasonableTime,
wikipedia: wikipediaMaxReasonableTime,
- "x-twitter": xTwitterMaxReasonableTime,
};
const engineOptions: {
@@ -589,28 +580,6 @@ const engineOptions: {
},
quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)
},
- "x-twitter": {
- features: {
- actions: false,
- waitFor: false,
- screenshot: false,
- "screenshot@fullScreen": false,
- pdf: false,
- document: false,
- image: false,
- audio: false,
- video: false,
- atsv: false,
- location: false,
- mobile: false,
- skipTlsVerification: true,
- useFastMode: true,
- stealthProxy: false,
- branding: false,
- disableAdblock: true,
- },
- quality: 1500,
- },
};
export function shouldUseIndex(meta: Meta) {
@@ -700,11 +669,26 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
+ // X posts and profiles only the Exchange serves, as blocked URLs (LinkedIn
+ // profiles) are: a normal engine gets X's login wall. A profile or lockdown
+ // scrape keeps the normal engines, as it did with Firecrawl's own X engine.
+ const xTwitterPage =
+ isXTwitterUrl(url) &&
+ meta.options.profile === undefined &&
+ !meta.options.lockdown;
+
+ // They are answered by xAI, which is outside the US-only guarantee.
+ if (xTwitterPage && isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
+ throw new UsOnlyUnavailableError(
+ "X/Twitter scraping is not available for teams restricted to US-only processing.",
+ );
+ }
+
const exchangeAccess = await getExchangeAccessForRequest({
url,
teamId: meta.internalOptions.teamId ?? null,
orgId: meta.internalOptions.orgId ?? null,
- blocked,
+ blocked: blocked || xTwitterPage,
origin: meta.internalOptions.origin ?? null,
formats: meta.options.formats,
actions: meta.options.actions,
@@ -737,9 +721,10 @@ export async function buildFallbackList(meta: Meta): Promise<
// A blocked URL can only have been admitted (by scrapeBlocklistMiddleware
// or batch scrape) for the Exchange; if it can no longer serve it (catalog
// changed, service down), fail closed rather than letting normal engines
- // scrape a blocklisted site. Unblocked URLs whose provider wants
- // unaccepted terms or rejects an option fall through and scrape normally.
- if (blocked) {
+ // scrape a blocklisted site. X posts and profiles fail the same way.
+ // Other URLs whose provider wants unaccepted terms or rejects an option
+ // fall through and scrape normally.
+ if (blocked || xTwitterPage) {
if (exchangeAccess.termsRequired) {
throw new ThirdPartyDataTermsRequiredError(exchangeAccess.terms);
}
@@ -957,20 +942,6 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
- if (
- !meta.options.profile &&
- isXTwitterUrl(meta.url) &&
- _engines.includes("x-twitter")
- ) {
- _engines.length = 0;
- _engines.push("x-twitter");
- } else if (!isXTwitterUrl(meta.url)) {
- const xTwitterIndex = _engines.indexOf("x-twitter");
- if (xTwitterIndex !== -1) {
- _engines.splice(xTwitterIndex, 1);
- }
- }
-
const prioritySum = [...meta.featureFlags].reduce(
(a, x) => a + featureFlagOptions[x].priority,
0,
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
deleted file mode 100644
index b5108ea3e..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.test.ts
+++ /dev/null
@@ -1,412 +0,0 @@
-import {
- context,
- propagation,
- SpanStatusCode,
- trace,
-} from "@opentelemetry/api";
-import {
- InMemorySpanExporter,
- type NodeTracerProvider,
- type ReadableSpan,
-} from "@opentelemetry/sdk-trace-node";
-import { APICallError } from "ai";
-import { MockLanguageModelV3 } from "ai/test";
-import type { Meta } from "../..";
-import { CostTracking } from "../../../../lib/cost-tracking";
-import { createTracerProvider } from "../../../../lib/otel-tracer";
-import { scrapeURLWithXTwitter } from "./index";
-
-const { grok } = vi.hoisted(() => {
- process.env.XAI_API_KEY = "test-key";
- return { grok: { doGenerate: undefined as any } };
-});
-
-vi.mock("@ai-sdk/xai", async importOriginal => {
- const actual = await importOriginal<typeof import("@ai-sdk/xai")>();
- return {
- xai: {
- responses: (modelId: string) =>
- new MockLanguageModelV3({
- provider: "xai.responses",
- modelId,
- doGenerate: (...args) => grok.doGenerate(...args),
- }),
- tools: actual.xai.tools,
- },
- };
-});
-
-const usage = {
- inputTokens: { total: 3000, noCache: 1000, cacheRead: 2000, cacheWrite: 0 },
- outputTokens: { total: 400, text: 400, reasoning: 0 },
-};
-
-// Raw xAI Responses body; only the usage block is read from it.
-function responseBody(xSearch?: { posts?: number; users?: number }) {
- return {
- object: "response",
- usage: {
- input_tokens: 3000,
- output_tokens: 400,
- ...(xSearch && {
- server_side_tool_usage_details: {
- x_search_calls: 2,
- x_posts_fetched: xSearch.posts,
- x_users_fetched: xSearch.users,
- },
- }),
- },
- };
-}
-
-function grokReturns(
- output: unknown,
- body: unknown = responseBody({ posts: 0, users: 0 }),
-) {
- grok.doGenerate = async () => ({
- content: [{ type: "text", text: JSON.stringify(output) }],
- finishReason: { unified: "stop", raw: "completed" },
- usage,
- response: { body },
- warnings: [],
- });
-}
-
-// Captures the call options the engine sends to the model.
-function grokCapturesCall(output: unknown) {
- const calls: any[] = [];
- grokReturns(output);
- const doGenerate = grok.doGenerate;
- grok.doGenerate = async (options: any) => {
- calls.push(options);
- return doGenerate(options);
- };
- return calls;
-}
-
-function promptText(options: any): string {
- return JSON.stringify(options.prompt);
-}
-
-function makeMeta(url: string, zeroDataRetention = false): Meta {
- const logger = { info: () => {}, warn: () => {}, error: () => {} };
- return {
- id: "019990c0-0000-7000-8000-000000000001",
- url,
- logger,
- abort: { asSignal: () => undefined },
- internalOptions: { teamId: "team-test", zeroDataRetention },
- costTracking: new CostTracking(),
- } as unknown as Meta;
-}
-
-describe("x-twitter engine X Search requests", () => {
- it("asks the profile lookup for the one exact handle and top-level posts only", async () => {
- const calls = grokCapturesCall({ username: "firecrawl", latestPosts: [] });
-
- await scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl"));
-
- expect(calls).toHaveLength(1);
- expect(calls[0].tools).toEqual([
- expect.objectContaining({ id: "xai.x_search", args: {} }),
- ]);
- const prompt = promptText(calls[0]);
- expect(prompt).toContain(
- 'single user search for \\"firecrawl\\" that returns only 1 result',
- );
- expect(prompt).toContain("do not search for other or similarly named");
- expect(prompt).toContain("rather than another account's data");
- expect(prompt).toContain('\\"from:firecrawl -filter:replies\\"');
- });
-
- it("drops a profile that belongs to a different account", async () => {
- grokReturns({
- displayName: "Firecrawl Fan Club",
- username: "@firecrawlfans",
- followers: 12,
- latestPosts: [{ text: "Not from @firecrawl." }],
- });
- const meta = makeMeta("https://x.com/firecrawl");
- const warn = vi.fn();
- meta.logger.warn = warn;
-
- const result = await scrapeURLWithXTwitter(meta);
-
- expect(warn).toHaveBeenCalledTimes(1);
- expect(warn).toHaveBeenCalledWith(expect.any(String), {
- requestedHandle: "firecrawl",
- returnedUsername: "firecrawlfans",
- });
-
- expect(result.markdown).toContain("# @unknown (@unknown)");
- expect(result.markdown).toContain(
- "No recent top-level posts were returned.",
- );
- expect(result.markdown).not.toContain("firecrawlfans");
- expect(result.markdown).not.toContain("Fan Club");
- });
-
- it("drops profile data that comes back without a username", async () => {
- grokReturns({
- displayName: "Firecrawl Fan Club",
- username: null,
- bio: "Unofficial.",
- latestPosts: [{ text: "Not from @firecrawl." }],
- });
- const meta = makeMeta("https://x.com/firecrawl");
- const warn = vi.fn();
- meta.logger.warn = warn;
-
- const result = await scrapeURLWithXTwitter(meta);
-
- expect(warn).not.toHaveBeenCalled();
- expect(result.markdown).toContain("# @unknown (@unknown)");
- expect(result.markdown).not.toContain("Fan Club");
- expect(result.markdown).not.toContain("Unofficial.");
- expect(result.markdown).not.toContain("### 1. Post");
- });
-
- it("keeps a profile whose username differs from the handle only in case", async () => {
- grokReturns({
- displayName: "Firecrawl",
- username: "@FireCrawl",
- latestPosts: [{ text: "Turn websites into LLM-ready data." }],
- });
-
- const result = await scrapeURLWithXTwitter(
- makeMeta("https://x.com/firecrawl"),
- );
-
- expect(result.markdown).toContain("# Firecrawl (@FireCrawl)");
- expect(result.markdown).toContain("> Turn websites into LLM-ready data.");
- });
-
- it("restricts the post lookup to the author's handle", async () => {
- const calls = grokCapturesCall({ authorUsername: "firecrawl", text: "Hi" });
-
- await scrapeURLWithXTwitter(
- makeMeta("https://x.com/firecrawl/status/1234567890123"),
- );
-
- expect(calls).toHaveLength(1);
- expect(calls[0].tools).toEqual([
- expect.objectContaining({
- id: "xai.x_search",
- args: {
- allowedXHandles: ["firecrawl"],
- enableVideoUnderstanding: true,
- enableImageUnderstanding: true,
- },
- }),
- ]);
- });
-});
-
-// grok-4-1-fast-non-reasoning at $0.20 / $0.50 per 1M input / output tokens.
-const tokenCost = (3000 * 0.2 + 400 * 0.5) / 1_000_000;
-
-describe("x-twitter engine LLM telemetry", () => {
- const exporter = new InMemorySpanExporter();
- let provider: NodeTracerProvider;
-
- beforeAll(() => {
- provider = createTracerProvider({
- exporter,
- serviceName: "test-service",
- serviceInstanceId: "pod-1",
- });
- provider.register();
- });
-
- afterAll(async () => {
- await provider.shutdown();
- trace.disable();
- context.disable();
- propagation.disable();
- });
-
- beforeEach(async () => {
- // Drop spans a previous test left unflushed.
- await provider.forceFlush();
- exporter.reset();
- });
-
- async function doGenerateSpan(): Promise<ReadableSpan> {
- await provider.forceFlush();
- const spans = exporter
- .getFinishedSpans()
- .filter(s => s.name === "ai.generateText.doGenerate");
- expect(spans).toHaveLength(1);
- return spans[0];
- }
-
- it("records a usage span for a profile lookup", async () => {
- grokReturns({
- displayName: "Firecrawl",
- username: "firecrawl",
- bio: "Turn websites into LLM-ready data.",
- followers: 1000,
- latestPosts: [],
- });
-
- const result = await scrapeURLWithXTwitter(
- makeMeta("https://x.com/firecrawl"),
- );
-
- expect(result.markdown).toContain("@firecrawl");
- const span = await doGenerateSpan();
- expect(span.attributes).toMatchObject({
- "ai.telemetry.functionId": "xTwitter/profile",
- "ai.telemetry.metadata.feature": "x-twitter",
- "ai.telemetry.metadata.teamId": "team-test",
- "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",
- "ai.model.id": "grok-4-1-fast-non-reasoning",
- "ai.usage.promptTokens": 3000,
- "ai.usage.completionTokens": 400,
- "ai.usage.cachedInputTokens": 2000,
- "firecrawl.llm.tool.x_search_posts": 0,
- "firecrawl.llm.tool.x_search_profiles": 0,
- });
- expect(span.attributes["ai.prompt.messages"]).toContain("@firecrawl");
- });
-
- it("records X Search items on the span and their fee in cost tracking for a profile lookup", async () => {
- grokReturns(
- { username: "firecrawl", latestPosts: [] },
- responseBody({ posts: 44, users: 3 }),
- );
- const meta = makeMeta("https://x.com/firecrawl");
-
- await scrapeURLWithXTwitter(meta);
-
- const span = await doGenerateSpan();
- expect(span.attributes).toMatchObject({
- "firecrawl.llm.tool.x_search_posts": 44,
- "firecrawl.llm.tool.x_search_profiles": 3,
- });
- const { calls, totalCost } = meta.costTracking.toJSON();
- expect(calls).toHaveLength(1);
- expect(calls[0]).toMatchObject({
- type: "other",
- model: "grok-4-1-fast-non-reasoning",
- tokens: { input: 3000, output: 400 },
- metadata: {
- module: "scrapeURL",
- method: "xTwitter/profile",
- xSearchPosts: 44,
- xSearchProfiles: 3,
- },
- });
- // 44 posts x $0.005 + 3 profiles x $0.01 = $0.25, plus tokens.
- expect(calls[0].cost).toBeCloseTo(0.25 + tokenCost, 10);
- expect(totalCost).toBeCloseTo(0.25 + tokenCost, 10);
- });
-
- it("records the X Search fee in cost tracking for a post lookup", async () => {
- grokReturns(
- { authorUsername: "firecrawl", text: "Hello from a post." },
- responseBody({ posts: 18, users: 0 }),
- );
- const meta = makeMeta("https://x.com/firecrawl/status/1234567890123");
-
- await scrapeURLWithXTwitter(meta);
-
- const { calls } = meta.costTracking.toJSON();
- expect(calls).toHaveLength(1);
- expect(calls[0].metadata).toMatchObject({
- method: "xTwitter/post",
- xSearchPosts: 18,
- xSearchProfiles: 0,
- });
- expect(calls[0].cost).toBeCloseTo(18 * 0.005 + tokenCost, 10);
- });
-
- it("prices tokens only when xAI reports no X Search item counts", async () => {
- grokReturns({ username: "firecrawl", latestPosts: [] }, responseBody());
- const meta = makeMeta("https://x.com/firecrawl");
-
- await scrapeURLWithXTwitter(meta);
-
- const span = await doGenerateSpan();
- expect(
- Object.keys(span.attributes).filter(key =>
- key.startsWith("firecrawl.llm.tool."),
- ),
- ).toEqual([]);
- const { calls } = meta.costTracking.toJSON();
- expect(calls[0].metadata).toMatchObject({
- xSearchPosts: 0,
- xSearchProfiles: 0,
- });
- expect(calls[0].cost).toBeCloseTo(tokenCost, 10);
- });
-
- it("records a usage span for a post lookup", async () => {
- grokReturns({
- authorUsername: "firecrawl",
- text: "Hello from a post.",
- likes: 10,
- retweets: 2,
- });
-
- await scrapeURLWithXTwitter(
- makeMeta("https://x.com/firecrawl/status/1234567890123"),
- );
-
- const span = await doGenerateSpan();
- expect(span.attributes).toMatchObject({
- "ai.telemetry.functionId": "xTwitter/post",
- "ai.telemetry.metadata.feature": "x-twitter",
- "ai.telemetry.metadata.teamId": "team-test",
- "ai.telemetry.metadata.scrapeId": "019990c0-0000-7000-8000-000000000001",
- "ai.model.id": "grok-4-1-fast-non-reasoning",
- "ai.usage.promptTokens": 3000,
- "ai.usage.completionTokens": 400,
- "ai.usage.cachedInputTokens": 2000,
- });
- expect(span.attributes["ai.prompt.messages"]).toContain(
- "post id 1234567890123",
- );
- });
-
- // Covers the per-call guard on its own; ZDR scrape jobs additionally run
- // under the tracer's ZDR context (see otel-tracer.test.ts).
- it("records no AI SDK spans for zero-data-retention scrapes", async () => {
- grokReturns({ username: "firecrawl", latestPosts: [] });
-
- const meta = makeMeta("https://x.com/firecrawl", true);
- await scrapeURLWithXTwitter(meta);
-
- await provider.forceFlush();
- expect(
- exporter.getFinishedSpans().filter(s => s.name.startsWith("ai.")),
- ).toEqual([]);
- // Billing-side cost is recorded regardless of telemetry.
- expect(meta.costTracking.toJSON().calls).toHaveLength(1);
- });
-
- it("records the span with an error status when the Grok call is rejected", async () => {
- grok.doGenerate = async () => {
- throw new APICallError({
- message: "bad request",
- url: "https://api.x.ai/v1/responses",
- requestBodyValues: {},
- statusCode: 400,
- isRetryable: false,
- });
- };
-
- await expect(
- scrapeURLWithXTwitter(makeMeta("https://x.com/firecrawl")),
- ).rejects.toThrow("bad request");
-
- const span = await doGenerateSpan();
- expect(span.status.code).toBe(SpanStatusCode.ERROR);
- expect(
- Object.keys(span.attributes).filter(key =>
- key.startsWith("firecrawl.llm.tool."),
- ),
- ).toEqual([]);
- expect(span.attributes["ai.telemetry.functionId"]).toBe("xTwitter/profile");
- });
-});
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
deleted file mode 100644
index 3a6236083..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ /dev/null
@@ -1,620 +0,0 @@
-import { xai } from "@ai-sdk/xai";
-import { generateText, jsonSchema, Output, type GenerateTextResult } from "ai";
-import { config } from "../../../../config";
-import { withUsageTelemetry } from "../../../../lib/ai-usage-telemetry";
-import {
- xSearchCost,
- xSearchUsageFromResponseBody,
-} from "../../../../lib/xai-x-search";
-import { Meta } from "../..";
-import { EngineScrapeResult } from "..";
-import {
- EngineError,
- UsOnlyUnavailableError,
- XTwitterConfigurationError,
-} from "../../error";
-import { isUsOnly } from "../../../../lib/us-only";
-import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
-import { calculateCost } from "../../transformers/llmExtract";
-import {
- isXTwitterUrl,
- parseXTwitterUrl,
- type XTwitterPostUrl,
- type XTwitterProfileUrl,
-} from "./url";
-
-export { isXTwitterUrl };
-
-const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";
-
-function xTwitterTelemetry(functionId: string, meta: Meta) {
- return {
- isEnabled: !meta.internalOptions.zeroDataRetention,
- functionId,
- metadata: {
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId ?? "",
- feature: "x-twitter",
- },
- };
-}
-
-// Records the call's token cost plus the per-item X Search fee xAI bills on
-// top of it.
-function recordGrokCost(
- meta: Meta,
- method: string,
- {
- totalUsage,
- steps,
- }: Pick<GenerateTextResult<any, any>, "totalUsage" | "steps">,
-) {
- const inputTokens = totalUsage.inputTokens ?? 0;
- const outputTokens = totalUsage.outputTokens ?? 0;
- const xSearch = { posts: 0, profiles: 0 };
- for (const step of steps) {
- const usage = xSearchUsageFromResponseBody(step.response.body);
- xSearch.posts += usage?.posts ?? 0;
- xSearch.profiles += usage?.profiles ?? 0;
- }
-
- meta.costTracking.addCall({
- type: "other",
- metadata: {
- module: "scrapeURL",
- method,
- xSearchPosts: xSearch.posts,
- xSearchProfiles: xSearch.profiles,
- },
- model: XAI_RESPONSES_MODEL,
- tokens: { input: inputTokens, output: outputTokens },
- cost:
- calculateCost(XAI_RESPONSES_MODEL, inputTokens, outputTokens) +
- xSearchCost(xSearch),
- });
-}
-
-type ProfilePost = {
- text: string;
- url?: string | null;
- createdAt?: string | null;
- likes?: number | null;
- retweets?: number | null;
-};
-
-type XTwitterProfileData = {
- displayName?: string | null;
- username?: string | null;
- profilePicUrl?: string | null;
- bio?: string | null;
- followers?: number | null;
- accountVerified?: boolean | null;
- url?: string | null;
- latestPosts?: ProfilePost[] | null;
-};
-
-type ThreadPost = {
- authorDisplayName?: string | null;
- authorUsername?: string | null;
- text: string;
- url?: string | null;
- createdAt?: string | null;
- likes?: number | null;
- retweets?: number | null;
-};
-
-type PostComment = {
- authorDisplayName?: string | null;
- authorUsername?: string | null;
- text: string;
- url?: string | null;
- createdAt?: string | null;
- likes?: number | null;
-};
-
-type XTwitterPostData = {
- authorDisplayName?: string | null;
- authorUsername?: string | null;
- text: string;
- url?: string | null;
- createdAt?: string | null;
- likes?: number | null;
- retweets?: number | null;
- thread?: ThreadPost[] | null;
- comments?: PostComment[] | null;
-};
-
-const nullableString = { type: ["string", "null"] };
-const nullableNumber = { type: ["number", "null"] };
-const nullableBoolean = { type: ["boolean", "null"] };
-
-const profileSchema = {
- type: "object",
- properties: {
- displayName: nullableString,
- username: nullableString,
- profilePicUrl: nullableString,
- bio: nullableString,
- followers: nullableNumber,
- accountVerified: nullableBoolean,
- url: nullableString,
- latestPosts: {
- type: "array",
- maxItems: 5,
- items: {
- type: "object",
- properties: {
- text: { type: "string" },
- url: nullableString,
- createdAt: nullableString,
- likes: nullableNumber,
- retweets: nullableNumber,
- },
- required: ["text", "url", "createdAt", "likes", "retweets"],
- additionalProperties: false,
- },
- },
- },
- required: [
- "displayName",
- "username",
- "profilePicUrl",
- "bio",
- "followers",
- "accountVerified",
- "url",
- "latestPosts",
- ],
- additionalProperties: false,
-};
-
-const postSchema = {
- type: "object",
- properties: {
- authorDisplayName: nullableString,
- authorUsername: nullableString,
- text: { type: "string" },
- url: nullableString,
- createdAt: nullableString,
- likes: nullableNumber,
- retweets: nullableNumber,
- thread: {
- type: "array",
- items: {
- type: "object",
- properties: {
- authorDisplayName: nullableString,
- authorUsername: nullableString,
- text: { type: "string" },
- url: nullableString,
- createdAt: nullableString,
- likes: nullableNumber,
- retweets: nullableNumber,
- },
- required: [
- "authorDisplayName",
- "authorUsername",
- "text",
- "url",
- "createdAt",
- "likes",
- "retweets",
- ],
- additionalProperties: false,
- },
- },
- comments: {
- type: "array",
- maxItems: 5,
- items: {
- type: "object",
- properties: {
- authorDisplayName: nullableString,
- authorUsername: nullableString,
- text: { type: "string" },
- url: nullableString,
- createdAt: nullableString,
- likes: nullableNumber,
- },
- required: [
- "authorDisplayName",
- "authorUsername",
- "text",
- "url",
- "createdAt",
- "likes",
- ],
- additionalProperties: false,
- },
- },
- },
- required: [
- "authorDisplayName",
- "authorUsername",
- "text",
- "url",
- "createdAt",
- "likes",
- "retweets",
- "thread",
- "comments",
- ],
- additionalProperties: false,
-};
-
-export async function scrapeURLWithXTwitter(
- meta: Meta,
-): Promise<EngineScrapeResult> {
- const urlToScrape = meta.rewrittenUrl ?? meta.url;
- const xUrl = parseXTwitterUrl(urlToScrape);
-
- if (!xUrl) {
- throw new EngineError(
- `URL is not a supported X/Twitter URL: ${urlToScrape}`,
- );
- }
-
- if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
- throw new UsOnlyUnavailableError(
- "X/Twitter scraping is not available for teams restricted to US-only processing.",
- );
- }
-
- if (!config.XAI_API_KEY) {
- throw new XTwitterConfigurationError();
- }
-
- meta.logger.info("Fetching X/Twitter data through Grok", {
- kind: xUrl.kind,
- url: xUrl.normalizedUrl,
- });
-
- const markdown =
- xUrl.kind === "profile"
- ? buildProfileMarkdown(await fetchProfile(xUrl, meta))
- : buildPostMarkdown(await fetchPost(xUrl, meta));
-
- if (markdown.trim().length === 0) {
- throw new EngineError(`No X/Twitter content returned for: ${urlToScrape}`);
- }
-
- const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);
-
- return {
- url: xUrl.normalizedUrl,
- html,
- markdown,
- statusCode: 200,
- contentType: "text/html; charset=utf-8",
- proxyUsed: "basic",
- postprocessorsUsed: ["x-twitter"],
- };
-}
-
-export function xTwitterMaxReasonableTime(_meta: Meta): number {
- return 30000;
-}
-
-function buildProfileMarkdown(profile: XTwitterProfileData): string {
- const username = stripAt(profile.username) ?? "unknown";
- const displayName = profile.displayName?.trim() || `@${username}`;
- const posts = (profile.latestPosts ?? []).slice(0, 5);
- const lines = [
- `# ${escapeMarkdownInline(displayName)} (@${escapeMarkdownInline(username)})`,
- "",
- ];
-
- if (profile.bio) {
- lines.push(escapeMarkdownBlock(profile.bio), "");
- }
-
- const facts = [
- `Followers: ${
- profile.followers !== null && profile.followers !== undefined
- ? formatNumber(profile.followers)
- : "Unknown"
- }`,
- `Verified: ${
- profile.accountVerified !== null && profile.accountVerified !== undefined
- ? profile.accountVerified
- ? "yes"
- : "no"
- : "Unknown"
- }`,
- `Profile Picture: ${
- profile.profilePicUrl
- ? formatMarkdownImage(displayName, profile.profilePicUrl)
- : "Unknown"
- }`,
- profile.url
- ? `Source: ${formatMarkdownLink(profile.url, profile.url)}`
- : undefined,
- ].filter((line): line is string => line !== undefined);
-
- if (facts.length > 0) {
- lines.push(...facts.map(fact => `- ${fact}`), "");
- }
-
- lines.push("## Latest Posts", "");
-
- if (posts.length === 0) {
- lines.push("No recent top-level posts were returned.");
- } else {
- for (const [index, post] of posts.entries()) {
- lines.push(`### ${index + 1}. Post`);
- if (post.createdAt) {
- lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);
- }
- if (post.url) {
- lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);
- }
- lines.push("");
- lines.push(formatBlockquote(post.text));
- const metrics = formatMetrics(post);
- if (metrics.length > 0) {
- lines.push("", metrics.join(" | "));
- }
- lines.push("");
- }
- }
-
- return trimMarkdown(lines.join("\n"));
-}
-
-function buildPostMarkdown(post: XTwitterPostData): string {
- const username = stripAt(post.authorUsername);
- const title = username
- ? `Post by @${escapeMarkdownInline(username)}`
- : "X/Twitter Post";
- const lines = [`# ${title}`, ""];
-
- const byline = [
- post.authorDisplayName?.trim()
- ? escapeMarkdownInline(post.authorDisplayName)
- : undefined,
- username ? `@${escapeMarkdownInline(username)}` : undefined,
- ].filter(Boolean);
-
- if (byline.length > 0) {
- lines.push(`Author: ${byline.join(" ")}`);
- }
- if (post.createdAt) {
- lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);
- }
- if (post.url) {
- lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);
- }
-
- const metrics = formatMetrics(post);
- if (metrics.length > 0) {
- lines.push(metrics.join(" | "));
- }
-
- lines.push("", "## Post", "", escapeMarkdownBlock(post.text.trim()));
-
- const thread = (post.thread ?? []).filter(item => item.text?.trim());
- if (thread.length > 0) {
- lines.push("", "## Thread", "");
- for (const [index, threadPost] of thread.entries()) {
- lines.push(`### ${index + 1}. Thread Post`);
- if (threadPost.authorUsername) {
- lines.push(
- `Author: @${escapeMarkdownInline(stripAt(threadPost.authorUsername)!)}`,
- );
- }
- if (threadPost.createdAt) {
- lines.push(`Posted: ${escapeMarkdownInline(threadPost.createdAt)}`);
- }
- if (threadPost.url) {
- lines.push(
- `URL: ${formatMarkdownLink(threadPost.url, threadPost.url)}`,
- );
- }
- lines.push("", formatBlockquote(threadPost.text));
- const threadMetrics = formatMetrics(threadPost);
- if (threadMetrics.length > 0) {
- lines.push("", threadMetrics.join(" | "));
- }
- lines.push("");
- }
- }
-
- const comments = (post.comments ?? [])
- .filter(comment => comment.text?.trim())
- .slice(0, 5);
- lines.push("", "## Top Comments", "");
- if (comments.length === 0) {
- lines.push("No top comments were returned.");
- } else {
- for (const [index, comment] of comments.entries()) {
- const author = stripAt(comment.authorUsername);
- lines.push(
- `### ${index + 1}. ${
- author ? `@${escapeMarkdownInline(author)}` : "Comment"
- }`,
- );
- if (comment.authorDisplayName) {
- lines.push(
- `Author: ${escapeMarkdownInline(comment.authorDisplayName)}`,
- );
- }
- if (comment.createdAt) {
- lines.push(`Posted: ${escapeMarkdownInline(comment.createdAt)}`);
- }
- if (comment.url) {
- lines.push(`URL: ${formatMarkdownLink(comment.url, comment.url)}`);
- }
- lines.push("", formatBlockquote(comment.text));
- if (comment.likes !== null && comment.likes !== undefined) {
- lines.push("", `Likes: ${formatNumber(comment.likes)}`);
- }
- lines.push("");
- }
- }
-
- return trimMarkdown(lines.join("\n"));
-}
-
-async function fetchProfile(
- xUrl: XTwitterProfileUrl,
- meta: Meta,
-): Promise<XTwitterProfileData> {
- const result = await generateText({
- model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
- maxOutputTokens: 20000,
- tools: {
- x_search: xai.tools.xSearch(),
- } as any,
- toolChoice: { type: "tool", toolName: "x_search" },
- output: Output.object({
- schema: jsonSchema(profileSchema as any),
- name: "x_twitter_profile",
- description:
- "Current public X/Twitter profile data and latest top-level posts.",
- }),
- abortSignal: meta.abort.asSignal(),
- experimental_telemetry: xTwitterTelemetry("xTwitter/profile", meta),
- // xAI bills every profile a user search returns, and the model otherwise
- // often asks for 3 candidates. Pin the lookup to the one exact handle, and
- // filter replies in the search so they aren't fetched and then dropped.
- prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Look up this exact username directly with a single user search for "${xUrl.handle}" that returns only 1 result; do not search for other or similarly named accounts. If no account has the username ${xUrl.handle} (ignoring case), return null for every profile field and no posts rather than another account's data. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments, found with a single latest-posts search for "from:${xUrl.handle} -filter:replies" limited to 5 results. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
- });
- recordGrokCost(meta, "xTwitter/profile", result);
-
- const profile = result.output as XTwitterProfileData;
- // A user search can still surface a similarly named account; never pass it
- // off as the requested one. Without a username nothing ties the data to the
- // requested account either, so that case is dropped too (it is the normal
- // result for a handle that doesn't exist, so it isn't worth a warning).
- const username = stripAt(profile.username);
- if (!username) {
- return {};
- }
- if (username.toLowerCase() !== xUrl.handle.toLowerCase()) {
- meta.logger.warn("X/Twitter profile lookup returned a different account", {
- requestedHandle: xUrl.handle,
- returnedUsername: username,
- });
- return {};
- }
-
- return profile;
-}
-
-async function fetchPost(
- xUrl: XTwitterPostUrl,
- meta: Meta,
-): Promise<XTwitterPostData> {
- const handlePart = xUrl.handle ? ` by @${xUrl.handle}` : "";
- const toolsOptions = xUrl.handle
- ? {
- allowedXHandles: [xUrl.handle],
- enableVideoUnderstanding: true,
- enableImageUnderstanding: true,
- }
- : {
- enableVideoUnderstanding: true,
- enableImageUnderstanding: true,
- };
-
- const result = await generateText({
- model: withUsageTelemetry(xai.responses(XAI_RESPONSES_MODEL)),
- maxOutputTokens: 20000,
- tools: {
- x_search: xai.tools.xSearch(toolsOptions),
- } as any,
- toolChoice: { type: "tool", toolName: "x_search" },
- output: Output.object({
- schema: jsonSchema(postSchema as any),
- name: "x_twitter_post",
- description:
- "Current public X/Twitter post data with metrics, thread, and top comments.",
- }),
- abortSignal: meta.abort.asSignal(),
- experimental_telemetry: xTwitterTelemetry("xTwitter/post", meta),
- prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,
- });
- recordGrokCost(meta, "xTwitter/post", result);
-
- return result.output as XTwitterPostData;
-}
-
-function stripAt(value: string | null | undefined): string | undefined {
- const trimmed = value?.trim();
- if (!trimmed) {
- return undefined;
- }
- return trimmed.replace(/^@/, "");
-}
-
-function formatMetrics(item: {
- likes?: number | null;
- retweets?: number | null;
-}): string[] {
- return [
- item.likes !== null && item.likes !== undefined
- ? `Likes: ${formatNumber(item.likes)}`
- : undefined,
- item.retweets !== null && item.retweets !== undefined
- ? `Retweets: ${formatNumber(item.retweets)}`
- : undefined,
- ].filter((line): line is string => line !== undefined);
-}
-
-function formatNumber(value: number): string {
- return new Intl.NumberFormat("en-US").format(value);
-}
-
-function formatBlockquote(value: string): string {
- const escaped = escapeMarkdownBlock(value.trim());
- if (!escaped) {
- return "> ";
- }
- return escaped
- .split(/\r?\n/)
- .map(line => (line.trim() ? `> ${line}` : ">"))
- .join("\n");
-}
-
-function formatMarkdownLink(label: string, url: string): string {
- const sanitizedUrl = sanitizeMarkdownUrl(url);
- if (!sanitizedUrl) {
- return escapeMarkdownInline(label);
- }
- return `[${escapeMarkdownInline(label)}](${sanitizedUrl})`;
-}
-
-function formatMarkdownImage(alt: string, url: string): string {
- const sanitizedUrl = sanitizeMarkdownUrl(url);
- if (!sanitizedUrl) {
- return "Unknown";
- }
- return `![${escapeMarkdownInline(alt)}](${sanitizedUrl})`;
-}
-
-function sanitizeMarkdownUrl(url: string): string | undefined {
- try {
- const parsed = new URL(url);
- if (!["http:", "https:"].includes(parsed.protocol)) {
- return undefined;
- }
- return parsed.toString().replace(/\)/g, "%29");
- } catch {
- return undefined;
- }
-}
-
-function escapeMarkdownInline(value: string): string {
- return escapeMarkdownBlock(value).replace(/([\\`*_{}[\]()#+\-.!|])/g, "\\$1");
-}
-
-function escapeMarkdownBlock(value: string): string {
- return value
- .replace(/&/g, "&amp;")
- .replace(/</g, "&lt;")
- .replace(/>/g, "&gt;");
-}
-
-function trimMarkdown(markdown: string): string {
- return markdown.replace(/\n{3,}/g, "\n\n").trim() + "\n";
-}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 0dd5ab7af..e44cf4c95 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -9,28 +9,6 @@ export class EngineError extends Error {
}
}
-export class XTwitterConfigurationError extends TransportableError {
- constructor() {
- super(
- "SCRAPE_X_TWITTER_CONFIGURATION_ERROR",
- "X/Twitter scraping requires XAI_API_KEY to be configured.",
- );
- }
-
- serialize() {
- return super.serialize();
- }
-
- static deserialize(
- _: ErrorCodes,
- data: ReturnType<typeof this.prototype.serialize>,
- ) {
- const x = new XTwitterConfigurationError();
- x.stack = data.stack;
- return x;
- }
-}
-
/** A route the team's US-only restriction rules out, because no US-based vendor serves it. */
export class UsOnlyUnavailableError extends TransportableError {
constructor(message: string) {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0afd13e55..6ab8a4ee3 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -59,7 +59,6 @@ import {
ProxySelectionError,
ScrapeRetryLimitError,
BrandingNotSupportedError,
- XTwitterConfigurationError,
UsOnlyUnavailableError,
ReservedCapacityUnavailableError,
ExchangeRefusedError,
@@ -1018,12 +1017,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
continue;
}
}
- if (error.engine === "x-twitter") {
- meta.logger.warn("X/Twitter scrape failed fatally.", {
- error: error.error,
- });
- throw error.error;
- } else if (error.error instanceof EngineError) {
+ if (error.error instanceof EngineError) {
meta.logger.warn(
"Engine " + error.engine + " could not scrape the page.",
{
@@ -1066,7 +1060,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ProxySelectionError ||
error.error instanceof NoCachedDataError ||
error.error instanceof AgentIndexOnlyError ||
- error.error instanceof XTwitterConfigurationError ||
error.error instanceof UsOnlyUnavailableError ||
error.error instanceof ReservedCapacityUnavailableError ||
error.error instanceof ExchangeRefusedError ||
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 18253c613..81f09973c 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -9,7 +9,6 @@ import {
} from "./store";
import { judgeCreditsForJudgedCount } from "./search/billing";
import type { MonitorTarget } from "./types";
-import { config } from "../../config";
describe("monitoring store credit helpers", () => {
describe("reserving credits using previous PDF results", () => {
@@ -139,108 +138,6 @@ describe("monitoring store credit helpers", () => {
expect(estimateMonitorCreditsPerRun(targets, true)).toBe(12);
});
- // The x-twitter engine only exists with an xAI key or DB auth.
- const withXTwitterEngine = (fn: () => void) => {
- const saved = config.USE_DB_AUTHENTICATION;
- config.USE_DB_AUTHENTICATION = true;
- try {
- fn();
- } finally {
- config.USE_DB_AUTHENTICATION = saved;
- }
- };
-
- it("includes the x-twitter surcharge, so the hold covers what is billed", () =>
- withXTwitterEngine(() => {
- const targets: MonitorTarget[] = [
- {
- id: "target-x",
- type: "scrape",
- urls: ["https://x.com/blknoiz06"],
- scrapeOptions: {},
- },
- ];
-
- // 1 scrape + 29 x-twitter, + 1 judge.
- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(30);
- expect(estimateMonitorCreditsPerRun(targets, true)).toBe(31);
- expect(
- calculateMonitorCheckActualCreditsFromPages(
- [
- {
- target_id: "target-x",
- status: "changed",
- metadata: {
- creditsUsed: null,
- postprocessorsUsed: ["x-twitter"],
- },
- judgment: { meaningful: false },
- },
- ],
- targets,
- ),
- ).toBe(estimateMonitorCreditsPerRun(targets, true));
- }));
-
- it("charges the surcharge only for URLs the x-twitter engine takes", () =>
- withXTwitterEngine(() => {
- const targets: MonitorTarget[] = [
- {
- id: "target-mixed",
- type: "scrape",
- urls: [
- "https://twitter.com/someone/status/1234567890",
- "https://x.com/home",
- "https://example.com/x.com",
- ],
- scrapeOptions: {},
- },
- ];
-
- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(3 + 29);
- }));
-
- it("skips the surcharge when a browser profile bypasses the x-twitter engine", () =>
- withXTwitterEngine(() => {
- const targets: MonitorTarget[] = [
- {
- id: "target-profile",
- type: "scrape",
- urls: ["https://x.com/someone"],
- scrapeOptions: { profile: { name: "p" } },
- },
- ];
-
- expect(estimateMonitorCreditsPerRun(targets, false)).toBe(1);
- }));
-
- it("skips the surcharge where the x-twitter engine cannot run", () => {
- const target = (
- scrapeOptions: Record<string, unknown>,
- ): MonitorTarget[] => [
- {
- id: "t",
- type: "scrape",
- urls: ["https://x.com/someone"],
- scrapeOptions,
- },
- ];
- withXTwitterEngine(() => {
- // Lockdown serves from the index: 1 base + 4 lockdown, no surcharge.
- expect(
- estimateMonitorCreditsPerRun(target({ lockdown: true }), false),
- ).toBe(5);
- });
- const saved = [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] as const;
- config.USE_DB_AUTHENTICATION = false;
- config.XAI_API_KEY = undefined;
- try {
- expect(estimateMonitorCreditsPerRun(target({}), false)).toBe(1);
- } finally {
- [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] = saved;
- }
- });
-
it("adds predictable lockdown costs and judge credits separately", () => {
const targets: MonitorTarget[] = [
{
@@ -359,33 +256,6 @@ describe("monitoring store credit helpers", () => {
).toBe(5);
});
- it("uses monitor metadata for fallback postprocessor credits and never for proxies", () => {
- const targets: MonitorTarget[] = [
- {
- id: "target-1",
- type: "scrape",
- urls: ["https://x.com/firecrawl"],
- scrapeOptions: {},
- },
- ];
-
- expect(
- calculateMonitorCheckActualCreditsFromPages(
- [
- {
- target_id: "target-1",
- metadata: {
- proxyUsed: "stealth",
- postprocessorsUsed: ["x-twitter"],
- },
- status: "same",
- },
- ],
- targets,
- ),
- ).toBe(30);
- });
-
it("does not bill extra when enhanced proxy metadata is present", () => {
const targets: MonitorTarget[] = [
{
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 0b365aae7..7fd227634 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -16,13 +16,11 @@ import {
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { monitoringClaimDueMonitors } from "../../db/rpc";
-import { config } from "../../config";
import {
getPDFMaxPages,
liftCheckPromptInjection,
shouldParsePDF,
} from "../../controllers/v2/types";
-import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";
import {
getNextMonitorRunAt,
estimateRunsPerMonth,
@@ -60,14 +58,12 @@ const DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE = 7;
const SCRAPE_OPTION_CREDIT_BONUS = 4;
const JUDGE_CREDITS_PER_PAGE = 1;
const REMOVED_PAGE_CREDITS = 0;
-const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;
const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;
const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;
type MonitorCreditMetadata = {
creditsUsed?: unknown;
numPages?: unknown;
- postprocessorsUsed?: unknown;
};
function formatType(format: unknown): string | null {
@@ -179,37 +175,13 @@ function estimateSearchTargetCredits(
);
}
-/**
- * The x-twitter engine's surcharge, which billing adds per page (see
- * fallbackBaseCreditsForPage). Left out of the estimate, a judged one-URL X
- * monitor reserves 2 credits and costs 31, and an account with a few credits
- * left passes every hold and is never charged.
- */
-function xTwitterSurcharge(
- url: string,
- options: MonitorTarget["scrapeOptions"],
-): number {
- // Mirrors the engine router (scrapeURL/engines/index.ts): the engine exists
- // only with an xAI key or DB auth, lockdown serves from the index alone, and
- // a browser profile routes around it.
- const engineEnabled =
- (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||
- config.USE_DB_AUTHENTICATION === true;
- if (!engineEnabled || options?.lockdown || options?.profile) return 0;
- return isXTwitterUrl(url) ? X_TWITTER_POSTPROCESSOR_CREDIT_BONUS : 0;
-}
-
function estimateTargetBaseCredits(
target: MonitorTarget,
judgeEnabled: boolean = false,
): number {
const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);
if (target.type === "scrape") {
- return target.urls.reduce(
- (sum, url) =>
- sum + creditsPerPage + xTwitterSurcharge(url, target.scrapeOptions),
- 0,
- );
+ return target.urls.length * creditsPerPage;
}
if (target.type === "search") {
return estimateSearchTargetCredits(target, judgeEnabled);
@@ -342,13 +314,6 @@ export function calculateMonitorCheckActualCreditsFromPages(
credits += metadata.numPages - 1;
}
- if (
- Array.isArray(metadata?.postprocessorsUsed) &&
- metadata.postprocessorsUsed.includes("x-twitter")
- ) {
- credits += X_TWITTER_POSTPROCESSOR_CREDIT_BONUS;
- }
-
return credits;
}