Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): X via Grok

mogery476318bd0ea4
diff --git a/apps/api/package.json b/apps/api/package.json
index ab7bed49e..857984dc0 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -81,6 +81,7 @@
"@ai-sdk/google-vertex": "^3.0.86",
"@ai-sdk/groq": "^2.0.28",
"@ai-sdk/openai": "2.0.64",
+ "@ai-sdk/xai": "^3.0.83",
"@apidevtools/json-schema-ref-parser": "^15.1.2",
"@bull-board/api": "^6.14.0",
"@bull-board/express": "^6.14.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 60e9f996c..a1ee1f588 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -51,6 +51,9 @@ importers:
'@ai-sdk/openai':
specifier: 2.0.64
version: 2.0.64([email protected])
+ '@ai-sdk/xai':
+ specifier: ^3.0.83
+ version: 3.0.83([email protected])
'@apidevtools/json-schema-ref-parser':
specifier: ^15.1.2
version: 15.1.2(@types/[email protected])
@@ -448,6 +451,12 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-kNAGINk71AlOXx10Dq/PXw4t/9XjdK8uxfpVElRwtSFMdeSiLVt58p9TPx4/FJD+hxZuVhvxYj9r42osxWq79g==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ zod: ^3.25.76 || ^4.1.8
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-+1mqxn42uB32DPZ6kurSyGAmL3MgCaDpkYU7zNDWI4NLy3Zg97RxTsI1jBCGIqkEVvRZKJlIMYtb89OvMnq3AQ==}
engines: {node: '>=18'}
@@ -496,6 +505,12 @@ packages:
peerDependencies:
zod: ^3.25.76 || ^4.1.8
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-z8GlDaCmRSDlqkMF2f4/RFgWxdarvIbyuk+m6WXT1LYgsnGiXRJGTD2Z1+SDl3LqtFuRtGX1aghYvQLoHL/9pg==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ zod: ^3.25.76 || ^4.1.8
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==}
engines: {node: '>=18'}
@@ -512,6 +527,12 @@ packages:
resolution: {integrity: sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==}
engines: {node: '>=18'}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-SuQz68BZGeuZjrSUJAzku97IlhdiNJJBsvG/Tvm3K2tuxkBS7TJq0fH4/AzAM7w2H2jxVUgboP7kRR6IfpRxcg==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ zod: ^3.25.76 || ^4.1.8
+
'@apidevtools/[email protected]':
resolution: {integrity: sha512-54vnqDyGsDAVM0LOIdGMnfVyCN1NlqoGnHEGpMfaDBLMvClfT4j2XbJgvfuF0Ca0kxT6Gb7xUgS5W1I14QqjiQ==}
engines: {node: '>=20'}
@@ -5994,6 +6015,12 @@ snapshots:
'@ai-sdk/provider-utils': 3.0.18([email protected])
zod: 4.1.12
+ dependencies:
+ '@ai-sdk/provider': 3.0.8
+ '@ai-sdk/provider-utils': 4.0.23([email protected])
+ zod: 4.1.12
+
dependencies:
'@ai-sdk/provider': 2.0.0
@@ -6046,6 +6073,13 @@ snapshots:
eventsource-parser: 3.0.6
zod: 4.1.12
+ dependencies:
+ '@ai-sdk/provider': 3.0.8
+ '@standard-schema/spec': 1.1.0
+ eventsource-parser: 3.0.6
+ zod: 4.1.12
+
'@ai-sdk/[email protected]':
dependencies:
json-schema: 0.4.0
@@ -6062,6 +6096,13 @@ snapshots:
dependencies:
json-schema: 0.4.0
+ dependencies:
+ '@ai-sdk/openai-compatible': 2.0.41([email protected])
+ '@ai-sdk/provider': 3.0.8
+ '@ai-sdk/provider-utils': 4.0.23([email protected])
+ zod: 4.1.12
+
'@apidevtools/[email protected](@types/[email protected])':
dependencies:
'@types/json-schema': 7.0.15
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index adf824efd..7d328d3d2 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -45,6 +45,7 @@ const configSchema = z.object({
OPENAI_API_KEY: z.string().optional(),
OPENAI_BASE_URL: z.string().optional(),
OPENROUTER_API_KEY: z.string().optional(),
+ XAI_API_KEY: z.string().optional(),
LLAMAPARSE_API_KEY: z.string().optional(),
STRIPE_SECRET_KEY: z.string().optional(),
AUTUMN_SECRET_KEY: z.string().optional(),
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index bad342c2c..1a6c31afb 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -32,6 +32,7 @@ import {
ScrapeRetryLimitError,
BrandingNotSupportedError,
AudioUnsupportedUrlError,
+ XTwitterConfigurationError,
} from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -64,6 +65,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_SITEMAP_ERROR: SitemapError,
CRAWL_DENIAL: CrawlDenialError,
SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,
+ SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
MAP_FAILED: MapFailedError,
// Zod errors
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index d80a33d91..838eb3167 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -26,6 +26,7 @@ export type ErrorCodes =
| "SCRAPE_BRANDING_NOT_SUPPORTED"
| "AGENT_INDEX_ONLY"
| "SCRAPE_AUDIO_UNSUPPORTED_URL"
+ | "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"
| "PARSE_UNSUPPORTED_OPTIONS"
| "CRAWL_DENIAL"
| "MAP_FAILED"
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 3520616fd..b14e2d469 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -14,6 +14,7 @@ import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
const unblockedDomainCostBonus = 4;
+const xTwitterCostBonus = 4;
export async function calculateCreditsToBeBilled(
options: ScrapeOptions,
@@ -82,6 +83,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += 4;
}
+ if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {
+ creditsToBeBilled += xTwitterCostBonus;
+ }
+
if (internalOptions.zeroDataRetention && !options.lockdown) {
creditsToBeBilled += flags?.zdrCost ?? 1;
}
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 4f4676b41..6809f3a9f 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -11,6 +11,32 @@ type BlocklistBlob = {
allowedKeywords: string[];
};
+function allowedKeywordMatches(url: string, allowedKeyword: string): boolean {
+ const keyword = allowedKeyword.trim();
+ if (!keyword) {
+ return false;
+ }
+
+ if (keyword.startsWith("regex:")) {
+ try {
+ return new RegExp(keyword.slice("regex:".length), "i").test(url);
+ } catch {
+ return false;
+ }
+ }
+
+ const regexMatch = keyword.match(/^\/(.+)\/([dgimsuvy]*)$/);
+ if (regexMatch) {
+ try {
+ return new RegExp(regexMatch[1], regexMatch[2]).test(url);
+ } catch {
+ return false;
+ }
+ }
+
+ return url.toLowerCase().includes(keyword.toLowerCase());
+}
+
let blob: BlocklistBlob | null = null;
export async function initializeBlocklist() {
@@ -73,9 +99,7 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {
// Check if URL contains any allowed keyword
if (
- blob.allowedKeywords.some(keyword =>
- lowerCaseUrl.includes(keyword.toLowerCase()),
- )
+ blob.allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))
) {
return false;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 5c77720c4..509a46fac 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -19,6 +19,11 @@ import {
wikipediaMaxReasonableTime,
isWikimediaUrl,
} from "./wikipedia";
+import {
+ scrapeURLWithXTwitter,
+ xTwitterMaxReasonableTime,
+ isXTwitterUrl,
+} from "./x-twitter";
import { queryEngpickerVerdict, useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
import { getPDFMaxPages } from "../../../controllers/v2/types";
@@ -39,7 +44,8 @@ export type Engine =
| "document"
| "index"
| "index;documents"
- | "wikipedia";
+ | "wikipedia"
+ | "x-twitter";
const useFireEngine =
config.FIRE_ENGINE_BETA_URL !== "" &&
@@ -52,8 +58,12 @@ const useWikipedia =
config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&
config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&
config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";
+const useXTwitter =
+ (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||
+ config.USE_DB_AUTHENTICATION === true;
const engines: Engine[] = [
+ ...(useXTwitter ? ["x-twitter" as const] : []),
...(useWikipedia ? ["wikipedia" as const] : []),
...(useIndex ? ["index" as const, "index;documents" as const] : []),
...(useFireEngine
@@ -166,6 +176,7 @@ const engineHandlers: {
pdf: scrapePDF,
document: scrapeDocument,
wikipedia: scrapeURLWithWikipedia,
+ "x-twitter": scrapeURLWithXTwitter,
};
const engineMRTs: {
@@ -190,6 +201,7 @@ const engineMRTs: {
pdf: pdfMaxReasonableTime,
document: documentMaxReasonableTime,
wikipedia: wikipediaMaxReasonableTime,
+ "x-twitter": xTwitterMaxReasonableTime,
};
const engineOptions: {
@@ -462,6 +474,26 @@ const engineOptions: {
},
quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)
},
+ "x-twitter": {
+ features: {
+ actions: false,
+ waitFor: false,
+ screenshot: false,
+ "screenshot@fullScreen": false,
+ pdf: false,
+ document: false,
+ audio: false,
+ atsv: false,
+ location: false,
+ mobile: false,
+ skipTlsVerification: true,
+ useFastMode: true,
+ stealthProxy: false,
+ branding: false,
+ disableAdblock: true,
+ },
+ quality: 1500,
+ },
};
export function shouldUseIndex(meta: Meta) {
@@ -547,6 +579,13 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
+ if (!isXTwitterUrl(meta.url)) {
+ const xTwitterIndex = _engines.indexOf("x-twitter");
+ if (xTwitterIndex !== -1) {
+ _engines.splice(xTwitterIndex, 1);
+ }
+ }
+
const prioritySum = [...meta.featureFlags].reduce(
(a, x) => a + featureFlagOptions[x].priority,
0,
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
new file mode 100644
index 000000000..adc57d9f7
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -0,0 +1,656 @@
+import { xai } from "@ai-sdk/xai";
+import { generateText, jsonSchema, Output } from "ai";
+import { config } from "../../../../config";
+import { Meta } from "../..";
+import { EngineScrapeResult } from "..";
+import { EngineError, XTwitterConfigurationError } from "../../error";
+import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
+
+const XAI_RESPONSES_MODEL = "grok-4.20-0309-non-reasoning";
+
+const RESERVED_PROFILE_PATHS = new Set([
+ "compose",
+ "explore",
+ "hashtag",
+ "home",
+ "i",
+ "intent",
+ "login",
+ "logout",
+ "messages",
+ "notifications",
+ "search",
+ "settings",
+ "share",
+]);
+
+type XTwitterProfileUrl = {
+ kind: "profile";
+ handle: string;
+ normalizedUrl: string;
+};
+
+type XTwitterPostUrl = {
+ kind: "post";
+ handle?: string;
+ postId: string;
+ normalizedUrl: string;
+};
+
+type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
+
+type ProfilePost = {
+ text: string;
+ url?: string | null;
+ createdAt?: string | null;
+ likes?: number | null;
+ retweets?: number | null;
+};
+
+type XTwitterProfileData = {
+ displayName?: string | null;
+ username?: string | null;
+ profilePicUrl?: string | null;
+ bio?: string | null;
+ followers?: number | null;
+ accountVerified?: boolean | null;
+ url?: string | null;
+ latestPosts?: ProfilePost[] | null;
+};
+
+type ThreadPost = {
+ authorDisplayName?: string | null;
+ authorUsername?: string | null;
+ text: string;
+ url?: string | null;
+ createdAt?: string | null;
+ likes?: number | null;
+ retweets?: number | null;
+};
+
+type PostComment = {
+ authorDisplayName?: string | null;
+ authorUsername?: string | null;
+ text: string;
+ url?: string | null;
+ createdAt?: string | null;
+ likes?: number | null;
+};
+
+type XTwitterPostData = {
+ authorDisplayName?: string | null;
+ authorUsername?: string | null;
+ text: string;
+ url?: string | null;
+ createdAt?: string | null;
+ likes?: number | null;
+ retweets?: number | null;
+ thread?: ThreadPost[] | null;
+ comments?: PostComment[] | null;
+};
+
+const nullableString = { type: ["string", "null"] };
+const nullableNumber = { type: ["number", "null"] };
+const nullableBoolean = { type: ["boolean", "null"] };
+
+const profileSchema = {
+ type: "object",
+ properties: {
+ displayName: nullableString,
+ username: nullableString,
+ profilePicUrl: nullableString,
+ bio: nullableString,
+ followers: nullableNumber,
+ accountVerified: nullableBoolean,
+ url: nullableString,
+ latestPosts: {
+ type: "array",
+ maxItems: 5,
+ items: {
+ type: "object",
+ properties: {
+ text: { type: "string" },
+ url: nullableString,
+ createdAt: nullableString,
+ likes: nullableNumber,
+ retweets: nullableNumber,
+ },
+ required: ["text", "url", "createdAt", "likes", "retweets"],
+ additionalProperties: false,
+ },
+ },
+ },
+ required: [
+ "displayName",
+ "username",
+ "profilePicUrl",
+ "bio",
+ "followers",
+ "accountVerified",
+ "url",
+ "latestPosts",
+ ],
+ additionalProperties: false,
+};
+
+const postSchema = {
+ type: "object",
+ properties: {
+ authorDisplayName: nullableString,
+ authorUsername: nullableString,
+ text: { type: "string" },
+ url: nullableString,
+ createdAt: nullableString,
+ likes: nullableNumber,
+ retweets: nullableNumber,
+ thread: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ authorDisplayName: nullableString,
+ authorUsername: nullableString,
+ text: { type: "string" },
+ url: nullableString,
+ createdAt: nullableString,
+ likes: nullableNumber,
+ retweets: nullableNumber,
+ },
+ required: [
+ "authorDisplayName",
+ "authorUsername",
+ "text",
+ "url",
+ "createdAt",
+ "likes",
+ "retweets",
+ ],
+ additionalProperties: false,
+ },
+ },
+ comments: {
+ type: "array",
+ maxItems: 5,
+ items: {
+ type: "object",
+ properties: {
+ authorDisplayName: nullableString,
+ authorUsername: nullableString,
+ text: { type: "string" },
+ url: nullableString,
+ createdAt: nullableString,
+ likes: nullableNumber,
+ },
+ required: [
+ "authorDisplayName",
+ "authorUsername",
+ "text",
+ "url",
+ "createdAt",
+ "likes",
+ ],
+ additionalProperties: false,
+ },
+ },
+ },
+ required: [
+ "authorDisplayName",
+ "authorUsername",
+ "text",
+ "url",
+ "createdAt",
+ "likes",
+ "retweets",
+ "thread",
+ "comments",
+ ],
+ additionalProperties: false,
+};
+
+function parseXTwitterUrl(url: string): XTwitterUrl | null {
+ let parsed: URL;
+ try {
+ parsed = new URL(url);
+ } catch {
+ return null;
+ }
+
+ if (!["http:", "https:"].includes(parsed.protocol)) {
+ return null;
+ }
+
+ const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
+ if (
+ hostname !== "x.com" &&
+ hostname !== "twitter.com" &&
+ hostname !== "mobile.twitter.com"
+ ) {
+ return null;
+ }
+
+ const segments = parsed.pathname
+ .split("/")
+ .map(segment => segment.trim())
+ .filter(Boolean);
+
+ if (segments.length === 0) {
+ return null;
+ }
+
+ if (
+ segments.length >= 4 &&
+ segments[0] === "i" &&
+ segments[1] === "web" &&
+ segments[2] === "status" &&
+ isPostId(segments[3])
+ ) {
+ return {
+ kind: "post",
+ postId: segments[3],
+ normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,
+ };
+ }
+
+ if (
+ segments.length >= 3 &&
+ segments[0] === "i" &&
+ segments[1] === "status" &&
+ isPostId(segments[2])
+ ) {
+ return {
+ kind: "post",
+ postId: segments[2],
+ normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,
+ };
+ }
+
+ if (
+ segments.length >= 3 &&
+ isHandle(segments[0]) &&
+ ["status", "statuses"].includes(segments[1]) &&
+ isPostId(segments[2])
+ ) {
+ const handle = segments[0];
+ const postId = segments[2];
+ return {
+ kind: "post",
+ handle,
+ postId,
+ normalizedUrl: `https://x.com/${handle}/status/${postId}`,
+ };
+ }
+
+ if (
+ segments.length === 1 &&
+ isHandle(segments[0]) &&
+ !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())
+ ) {
+ const handle = segments[0];
+ return {
+ kind: "profile",
+ handle,
+ normalizedUrl: `https://x.com/${handle}`,
+ };
+ }
+
+ return null;
+}
+
+export function isXTwitterUrl(url: string): boolean {
+ return parseXTwitterUrl(url) !== null;
+}
+
+export async function scrapeURLWithXTwitter(
+ meta: Meta,
+): Promise<EngineScrapeResult> {
+ const urlToScrape = meta.rewrittenUrl ?? meta.url;
+ const xUrl = parseXTwitterUrl(urlToScrape);
+
+ if (!xUrl) {
+ throw new EngineError(
+ `URL is not a supported X/Twitter URL: ${urlToScrape}`,
+ );
+ }
+
+ if (!config.XAI_API_KEY) {
+ throw new XTwitterConfigurationError();
+ }
+
+ meta.logger.info("Fetching X/Twitter data through Grok", {
+ kind: xUrl.kind,
+ url: xUrl.normalizedUrl,
+ });
+
+ const markdown =
+ xUrl.kind === "profile"
+ ? buildProfileMarkdown(await fetchProfile(xUrl, meta))
+ : buildPostMarkdown(await fetchPost(xUrl, meta));
+
+ if (markdown.trim().length === 0) {
+ throw new EngineError(`No X/Twitter content returned for: ${urlToScrape}`);
+ }
+
+ const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);
+
+ return {
+ url: xUrl.normalizedUrl,
+ html,
+ markdown,
+ statusCode: 200,
+ contentType: "text/html; charset=utf-8",
+ proxyUsed: "basic",
+ postprocessorsUsed: ["x-twitter"],
+ };
+}
+
+export function xTwitterMaxReasonableTime(_meta: Meta): number {
+ return 30000;
+}
+
+function buildProfileMarkdown(profile: XTwitterProfileData): string {
+ const username = stripAt(profile.username) ?? "unknown";
+ const displayName = profile.displayName?.trim() || `@${username}`;
+ const posts = (profile.latestPosts ?? []).slice(0, 5);
+ const lines = [
+ `# ${escapeMarkdownInline(displayName)} (@${escapeMarkdownInline(username)})`,
+ "",
+ ];
+
+ if (profile.bio) {
+ lines.push(escapeMarkdownBlock(profile.bio), "");
+ }
+
+ const facts = [
+ `Followers: ${
+ profile.followers !== null && profile.followers !== undefined
+ ? formatNumber(profile.followers)
+ : "Unknown"
+ }`,
+ `Verified: ${
+ profile.accountVerified !== null && profile.accountVerified !== undefined
+ ? profile.accountVerified
+ ? "yes"
+ : "no"
+ : "Unknown"
+ }`,
+ `Profile Picture: ${
+ profile.profilePicUrl
+ ? formatMarkdownImage(displayName, profile.profilePicUrl)
+ : "Unknown"
+ }`,
+ profile.url
+ ? `Source: ${formatMarkdownLink(profile.url, profile.url)}`
+ : undefined,
+ ].filter((line): line is string => line !== undefined);
+
+ if (facts.length > 0) {
+ lines.push(...facts.map(fact => `- ${fact}`), "");
+ }
+
+ lines.push("## Latest Posts", "");
+
+ if (posts.length === 0) {
+ lines.push("No recent top-level posts were returned.");
+ } else {
+ for (const [index, post] of posts.entries()) {
+ lines.push(`### ${index + 1}. Post`);
+ if (post.createdAt) {
+ lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);
+ }
+ if (post.url) {
+ lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);
+ }
+ lines.push("");
+ lines.push(formatBlockquote(post.text));
+ const metrics = formatMetrics(post);
+ if (metrics.length > 0) {
+ lines.push("", metrics.join(" | "));
+ }
+ lines.push("");
+ }
+ }
+
+ return trimMarkdown(lines.join("\n"));
+}
+
+function buildPostMarkdown(post: XTwitterPostData): string {
+ const username = stripAt(post.authorUsername);
+ const title = username
+ ? `Post by @${escapeMarkdownInline(username)}`
+ : "X/Twitter Post";
+ const lines = [`# ${title}`, ""];
+
+ const byline = [
+ post.authorDisplayName?.trim()
+ ? escapeMarkdownInline(post.authorDisplayName)
+ : undefined,
+ username ? `@${escapeMarkdownInline(username)}` : undefined,
+ ].filter(Boolean);
+
+ if (byline.length > 0) {
+ lines.push(`Author: ${byline.join(" ")}`);
+ }
+ if (post.createdAt) {
+ lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);
+ }
+ if (post.url) {
+ lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);
+ }
+
+ const metrics = formatMetrics(post);
+ if (metrics.length > 0) {
+ lines.push(metrics.join(" | "));
+ }
+
+ lines.push("", "## Post", "", formatBlockquote(post.text));
+
+ const thread = (post.thread ?? []).filter(item => item.text?.trim());
+ if (thread.length > 0) {
+ lines.push("", "## Thread", "");
+ for (const [index, threadPost] of thread.entries()) {
+ lines.push(`### ${index + 1}. Thread Post`);
+ if (threadPost.authorUsername) {
+ lines.push(
+ `Author: @${escapeMarkdownInline(stripAt(threadPost.authorUsername)!)}`,
+ );
+ }
+ if (threadPost.createdAt) {
+ lines.push(`Posted: ${escapeMarkdownInline(threadPost.createdAt)}`);
+ }
+ if (threadPost.url) {
+ lines.push(
+ `URL: ${formatMarkdownLink(threadPost.url, threadPost.url)}`,
+ );
+ }
+ lines.push("", formatBlockquote(threadPost.text));
+ const threadMetrics = formatMetrics(threadPost);
+ if (threadMetrics.length > 0) {
+ lines.push("", threadMetrics.join(" | "));
+ }
+ lines.push("");
+ }
+ }
+
+ const comments = (post.comments ?? [])
+ .filter(comment => comment.text?.trim())
+ .slice(0, 5);
+ lines.push("", "## Top Comments", "");
+ if (comments.length === 0) {
+ lines.push("No top comments were returned.");
+ } else {
+ for (const [index, comment] of comments.entries()) {
+ const author = stripAt(comment.authorUsername);
+ lines.push(
+ `### ${index + 1}. ${
+ author ? `@${escapeMarkdownInline(author)}` : "Comment"
+ }`,
+ );
+ if (comment.authorDisplayName) {
+ lines.push(
+ `Author: ${escapeMarkdownInline(comment.authorDisplayName)}`,
+ );
+ }
+ if (comment.createdAt) {
+ lines.push(`Posted: ${escapeMarkdownInline(comment.createdAt)}`);
+ }
+ if (comment.url) {
+ lines.push(`URL: ${formatMarkdownLink(comment.url, comment.url)}`);
+ }
+ lines.push("", formatBlockquote(comment.text));
+ if (comment.likes !== null && comment.likes !== undefined) {
+ lines.push("", `Likes: ${formatNumber(comment.likes)}`);
+ }
+ lines.push("");
+ }
+ }
+
+ return trimMarkdown(lines.join("\n"));
+}
+
+async function fetchProfile(
+ xUrl: XTwitterProfileUrl,
+ meta: Meta,
+): Promise<XTwitterProfileData> {
+ const { output } = await generateText({
+ model: xai.responses(XAI_RESPONSES_MODEL),
+ maxOutputTokens: 20000,
+ tools: {
+ x_search: xai.tools.xSearch(),
+ } as any,
+ toolChoice: { type: "tool", toolName: "x_search" },
+ output: Output.object({
+ schema: jsonSchema(profileSchema as any),
+ name: "x_twitter_profile",
+ description:
+ "Current public X/Twitter profile data and latest top-level posts.",
+ }),
+ abortSignal: meta.abort.asSignal(),
+ prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,
+ });
+
+ return output as XTwitterProfileData;
+}
+
+async function fetchPost(
+ xUrl: XTwitterPostUrl,
+ meta: Meta,
+): Promise<XTwitterPostData> {
+ const handlePart = xUrl.handle ? ` by @${xUrl.handle}` : "";
+ const toolsOptions = xUrl.handle
+ ? {
+ allowedXHandles: [xUrl.handle],
+ enableVideoUnderstanding: true,
+ enableImageUnderstanding: true,
+ }
+ : {
+ enableVideoUnderstanding: true,
+ enableImageUnderstanding: true,
+ };
+
+ const { output } = await generateText({
+ model: xai.responses(XAI_RESPONSES_MODEL),
+ maxOutputTokens: 20000,
+ tools: {
+ x_search: xai.tools.xSearch(toolsOptions),
+ } as any,
+ toolChoice: { type: "tool", toolName: "x_search" },
+ output: Output.object({
+ schema: jsonSchema(postSchema as any),
+ name: "x_twitter_post",
+ description:
+ "Current public X/Twitter post data with metrics, thread, and top comments.",
+ }),
+ abortSignal: meta.abort.asSignal(),
+ prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post text, author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,
+ });
+
+ return output as XTwitterPostData;
+}
+
+function isHandle(value: string): boolean {
+ return /^[A-Za-z0-9_]{1,15}$/.test(value);
+}
+
+function isPostId(value: string): boolean {
+ return /^\d{5,}$/.test(value);
+}
+
+function stripAt(value: string | null | undefined): string | undefined {
+ const trimmed = value?.trim();
+ if (!trimmed) {
+ return undefined;
+ }
+ return trimmed.replace(/^@/, "");
+}
+
+function formatMetrics(item: {
+ likes?: number | null;
+ retweets?: number | null;
+}): string[] {
+ return [
+ item.likes !== null && item.likes !== undefined
+ ? `Likes: ${formatNumber(item.likes)}`
+ : undefined,
+ item.retweets !== null && item.retweets !== undefined
+ ? `Retweets: ${formatNumber(item.retweets)}`
+ : undefined,
+ ].filter((line): line is string => line !== undefined);
+}
+
+function formatNumber(value: number): string {
+ return new Intl.NumberFormat("en-US").format(value);
+}
+
+function formatBlockquote(value: string): string {
+ const escaped = escapeMarkdownBlock(value.trim());
+ if (!escaped) {
+ return "> ";
+ }
+ return escaped
+ .split(/\r?\n/)
+ .map(line => (line.trim() ? `> ${line}` : ">"))
+ .join("\n");
+}
+
+function formatMarkdownLink(label: string, url: string): string {
+ const sanitizedUrl = sanitizeMarkdownUrl(url);
+ if (!sanitizedUrl) {
+ return escapeMarkdownInline(label);
+ }
+ return `[${escapeMarkdownInline(label)}](${sanitizedUrl})`;
+}
+
+function formatMarkdownImage(alt: string, url: string): string {
+ const sanitizedUrl = sanitizeMarkdownUrl(url);
+ if (!sanitizedUrl) {
+ return "Unknown";
+ }
+ return `![${escapeMarkdownInline(alt)}](${sanitizedUrl})`;
+}
+
+function sanitizeMarkdownUrl(url: string): string | undefined {
+ try {
+ const parsed = new URL(url);
+ if (!["http:", "https:"].includes(parsed.protocol)) {
+ return undefined;
+ }
+ return parsed.toString().replace(/\)/g, "%29");
+ } catch {
+ return undefined;
+ }
+}
+
+function escapeMarkdownInline(value: string): string {
+ return escapeMarkdownBlock(value).replace(/([\\`*_{}[\]()#+\-.!|])/g, "\\$1");
+}
+
+function escapeMarkdownBlock(value: string): string {
+ return value
+ .replace(/&/g, "&amp;")
+ .replace(/</g, "&lt;")
+ .replace(/>/g, "&gt;");
+}
+
+function trimMarkdown(markdown: string): string {
+ return markdown.replace(/\n{3,}/g, "\n\n").trim() + "\n";
+}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 5b7782f98..acfcebc2f 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -9,6 +9,28 @@ export class EngineError extends Error {
}
}
+export class XTwitterConfigurationError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_X_TWITTER_CONFIGURATION_ERROR",
+ "X/Twitter scraping requires XAI_API_KEY to be configured.",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new XTwitterConfigurationError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class NoEnginesLeftError extends TransportableError {
public fallbackList: Engine[];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 4c48fc99e..a19213d5f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -52,6 +52,7 @@ import {
ProxySelectionError,
ScrapeRetryLimitError,
BrandingNotSupportedError,
+ XTwitterConfigurationError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
import { executeTransformers } from "./transformers";
@@ -806,7 +807,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof PDFInsufficientTimeError ||
error.error instanceof ProxySelectionError ||
error.error instanceof NoCachedDataError ||
- error.error instanceof AgentIndexOnlyError
+ error.error instanceof AgentIndexOnlyError ||
+ error.error instanceof XTwitterConfigurationError
) {
throw error.error;
} else if (error.error instanceof LLMRefusalError) {