Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): X via Grok
mogery476318bd0ea4
diff --git a/apps/api/package.json b/apps/api/package.jsonindex ab7bed49e..857984dc0 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -81,6 +81,7 @@"@ai-sdk/google-vertex": "^3.0.86","@ai-sdk/groq": "^2.0.28","@ai-sdk/openai": "2.0.64",+ "@ai-sdk/xai": "^3.0.83","@apidevtools/json-schema-ref-parser": "^15.1.2","@bull-board/api": "^6.14.0","@bull-board/express": "^6.14.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 60e9f996c..a1ee1f588 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -51,6 +51,9 @@ importers:'@ai-sdk/openai':specifier: 2.0.64version: 2.0.64([email protected])+ '@ai-sdk/xai':+ specifier: ^3.0.83+ version: 3.0.83([email protected])'@apidevtools/json-schema-ref-parser':specifier: ^15.1.2version: 15.1.2(@types/[email protected])@@ -448,6 +451,12 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-kNAGINk71AlOXx10Dq/PXw4t/9XjdK8uxfpVElRwtSFMdeSiLVt58p9TPx4/FJD+hxZuVhvxYj9r42osxWq79g==}+ engines: {node: '>=18'}+ peerDependencies:+ zod: ^3.25.76 || ^4.1.8+'@ai-sdk/[email protected]':resolution: {integrity: sha512-+1mqxn42uB32DPZ6kurSyGAmL3MgCaDpkYU7zNDWI4NLy3Zg97RxTsI1jBCGIqkEVvRZKJlIMYtb89OvMnq3AQ==}engines: {node: '>=18'}@@ -496,6 +505,12 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-z8GlDaCmRSDlqkMF2f4/RFgWxdarvIbyuk+m6WXT1LYgsnGiXRJGTD2Z1+SDl3LqtFuRtGX1aghYvQLoHL/9pg==}+ engines: {node: '>=18'}+ peerDependencies:+ zod: ^3.25.76 || ^4.1.8+'@ai-sdk/[email protected]':resolution: {integrity: sha512-f9jSYwKMdXvm44Dmab1vUBnfCDSFfI5rOtvV1W9oKB7WYHR5dGvCC6x68Mk3NUfrdmNoMVHGoh6JT9HCVMlMow==}engines: {node: '>=18'}@@ -512,6 +527,12 @@ packages:resolution: {integrity: sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==}engines: {node: '>=18'}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-SuQz68BZGeuZjrSUJAzku97IlhdiNJJBsvG/Tvm3K2tuxkBS7TJq0fH4/AzAM7w2H2jxVUgboP7kRR6IfpRxcg==}+ engines: {node: '>=18'}+ peerDependencies:+ zod: ^3.25.76 || ^4.1.8+'@apidevtools/[email protected]':resolution: {integrity: sha512-54vnqDyGsDAVM0LOIdGMnfVyCN1NlqoGnHEGpMfaDBLMvClfT4j2XbJgvfuF0Ca0kxT6Gb7xUgS5W1I14QqjiQ==}engines: {node: '>=20'}@@ -5994,6 +6015,12 @@ snapshots:'@ai-sdk/provider-utils': 3.0.18([email protected])zod: 4.1.12+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 3.0.8+ '@ai-sdk/provider-utils': 4.0.23([email protected])+ zod: 4.1.12+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 2.0.0@@ -6046,6 +6073,13 @@ snapshots:eventsource-parser: 3.0.6zod: 4.1.12+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 3.0.8+ '@standard-schema/spec': 1.1.0+ eventsource-parser: 3.0.6+ zod: 4.1.12+'@ai-sdk/[email protected]':dependencies:json-schema: 0.4.0@@ -6062,6 +6096,13 @@ snapshots:dependencies:json-schema: 0.4.0+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/openai-compatible': 2.0.41([email protected])+ '@ai-sdk/provider': 3.0.8+ '@ai-sdk/provider-utils': 4.0.23([email protected])+ zod: 4.1.12+'@apidevtools/[email protected](@types/[email protected])':dependencies:'@types/json-schema': 7.0.15diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex adf824efd..7d328d3d2 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -45,6 +45,7 @@ const configSchema = z.object({OPENAI_API_KEY: z.string().optional(),OPENAI_BASE_URL: z.string().optional(),OPENROUTER_API_KEY: z.string().optional(),+ XAI_API_KEY: z.string().optional(),LLAMAPARSE_API_KEY: z.string().optional(),STRIPE_SECRET_KEY: z.string().optional(),AUTUMN_SECRET_KEY: z.string().optional(),diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex bad342c2c..1a6c31afb 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -32,6 +32,7 @@ import {ScrapeRetryLimitError,BrandingNotSupportedError,AudioUnsupportedUrlError,+ XTwitterConfigurationError,} from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -64,6 +65,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,+ SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,MAP_FAILED: MapFailedError,// Zod errorsdiff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex d80a33d91..838eb3167 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -26,6 +26,7 @@ export type ErrorCodes =| "SCRAPE_BRANDING_NOT_SUPPORTED"| "AGENT_INDEX_ONLY"| "SCRAPE_AUDIO_UNSUPPORTED_URL"+ | "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"| "PARSE_UNSUPPORTED_OPTIONS"| "CRAWL_DENIAL"| "MAP_FAILED"diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 3520616fd..b14e2d469 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -14,6 +14,7 @@ import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;const unblockedDomainCostBonus = 4;+const xTwitterCostBonus = 4;export async function calculateCreditsToBeBilled(options: ScrapeOptions,@@ -82,6 +83,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += 4;}+ if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {+ creditsToBeBilled += xTwitterCostBonus;+ }+if (internalOptions.zeroDataRetention && !options.lockdown) {creditsToBeBilled += flags?.zdrCost ?? 1;}diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 4f4676b41..6809f3a9f 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -11,6 +11,32 @@ type BlocklistBlob = {allowedKeywords: string[];};+function allowedKeywordMatches(url: string, allowedKeyword: string): boolean {+ const keyword = allowedKeyword.trim();+ if (!keyword) {+ return false;+ }++ if (keyword.startsWith("regex:")) {+ try {+ return new RegExp(keyword.slice("regex:".length), "i").test(url);+ } catch {+ return false;+ }+ }++ const regexMatch = keyword.match(/^\/(.+)\/([dgimsuvy]*)$/);+ if (regexMatch) {+ try {+ return new RegExp(regexMatch[1], regexMatch[2]).test(url);+ } catch {+ return false;+ }+ }++ return url.toLowerCase().includes(keyword.toLowerCase());+}+let blob: BlocklistBlob | null = null;export async function initializeBlocklist() {@@ -73,9 +99,7 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {// Check if URL contains any allowed keywordif (- blob.allowedKeywords.some(keyword =>- lowerCaseUrl.includes(keyword.toLowerCase()),- )+ blob.allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))) {return false;}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 5c77720c4..509a46fac 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -19,6 +19,11 @@ import {wikipediaMaxReasonableTime,isWikimediaUrl,} from "./wikipedia";+import {+ scrapeURLWithXTwitter,+ xTwitterMaxReasonableTime,+ isXTwitterUrl,+} from "./x-twitter";import { queryEngpickerVerdict, useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";import { getPDFMaxPages } from "../../../controllers/v2/types";@@ -39,7 +44,8 @@ export type Engine =| "document"| "index"| "index;documents"- | "wikipedia";+ | "wikipedia"+ | "x-twitter";const useFireEngine =config.FIRE_ENGINE_BETA_URL !== "" &&@@ -52,8 +58,12 @@ const useWikipedia =config.WIKIPEDIA_ENTERPRISE_USERNAME !== "" &&config.WIKIPEDIA_ENTERPRISE_PASSWORD !== undefined &&config.WIKIPEDIA_ENTERPRISE_PASSWORD !== "";+const useXTwitter =+ (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||+ config.USE_DB_AUTHENTICATION === true;const engines: Engine[] = [+ ...(useXTwitter ? ["x-twitter" as const] : []),...(useWikipedia ? ["wikipedia" as const] : []),...(useIndex ? ["index" as const, "index;documents" as const] : []),...(useFireEngine@@ -166,6 +176,7 @@ const engineHandlers: {pdf: scrapePDF,document: scrapeDocument,wikipedia: scrapeURLWithWikipedia,+ "x-twitter": scrapeURLWithXTwitter,};const engineMRTs: {@@ -190,6 +201,7 @@ const engineMRTs: {pdf: pdfMaxReasonableTime,document: documentMaxReasonableTime,wikipedia: wikipediaMaxReasonableTime,+ "x-twitter": xTwitterMaxReasonableTime,};const engineOptions: {@@ -462,6 +474,26 @@ const engineOptions: {},quality: 500, // below index (1000) so cache is tried first, above fire-engine (50)},+ "x-twitter": {+ features: {+ actions: false,+ waitFor: false,+ screenshot: false,+ "screenshot@fullScreen": false,+ pdf: false,+ document: false,+ audio: false,+ atsv: false,+ location: false,+ mobile: false,+ skipTlsVerification: true,+ useFastMode: true,+ stealthProxy: false,+ branding: false,+ disableAdblock: true,+ },+ quality: 1500,+ },};export function shouldUseIndex(meta: Meta) {@@ -547,6 +579,13 @@ export async function buildFallbackList(meta: Meta): Promise<}}+ if (!isXTwitterUrl(meta.url)) {+ const xTwitterIndex = _engines.indexOf("x-twitter");+ if (xTwitterIndex !== -1) {+ _engines.splice(xTwitterIndex, 1);+ }+ }+const prioritySum = [...meta.featureFlags].reduce((a, x) => a + featureFlagOptions[x].priority,0,diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsnew file mode 100644index 000000000..adc57d9f7--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -0,0 +1,656 @@+import { xai } from "@ai-sdk/xai";+import { generateText, jsonSchema, Output } from "ai";+import { config } from "../../../../config";+import { Meta } from "../..";+import { EngineScrapeResult } from "..";+import { EngineError, XTwitterConfigurationError } from "../../error";+import { safeMarkdownToHtml } from "../pdf/markdownToHtml";++const XAI_RESPONSES_MODEL = "grok-4.20-0309-non-reasoning";++const RESERVED_PROFILE_PATHS = new Set([+ "compose",+ "explore",+ "hashtag",+ "home",+ "i",+ "intent",+ "login",+ "logout",+ "messages",+ "notifications",+ "search",+ "settings",+ "share",+]);++type XTwitterProfileUrl = {+ kind: "profile";+ handle: string;+ normalizedUrl: string;+};++type XTwitterPostUrl = {+ kind: "post";+ handle?: string;+ postId: string;+ normalizedUrl: string;+};++type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;++type ProfilePost = {+ text: string;+ url?: string | null;+ createdAt?: string | null;+ likes?: number | null;+ retweets?: number | null;+};++type XTwitterProfileData = {+ displayName?: string | null;+ username?: string | null;+ profilePicUrl?: string | null;+ bio?: string | null;+ followers?: number | null;+ accountVerified?: boolean | null;+ url?: string | null;+ latestPosts?: ProfilePost[] | null;+};++type ThreadPost = {+ authorDisplayName?: string | null;+ authorUsername?: string | null;+ text: string;+ url?: string | null;+ createdAt?: string | null;+ likes?: number | null;+ retweets?: number | null;+};++type PostComment = {+ authorDisplayName?: string | null;+ authorUsername?: string | null;+ text: string;+ url?: string | null;+ createdAt?: string | null;+ likes?: number | null;+};++type XTwitterPostData = {+ authorDisplayName?: string | null;+ authorUsername?: string | null;+ text: string;+ url?: string | null;+ createdAt?: string | null;+ likes?: number | null;+ retweets?: number | null;+ thread?: ThreadPost[] | null;+ comments?: PostComment[] | null;+};++const nullableString = { type: ["string", "null"] };+const nullableNumber = { type: ["number", "null"] };+const nullableBoolean = { type: ["boolean", "null"] };++const profileSchema = {+ type: "object",+ properties: {+ displayName: nullableString,+ username: nullableString,+ profilePicUrl: nullableString,+ bio: nullableString,+ followers: nullableNumber,+ accountVerified: nullableBoolean,+ url: nullableString,+ latestPosts: {+ type: "array",+ maxItems: 5,+ items: {+ type: "object",+ properties: {+ text: { type: "string" },+ url: nullableString,+ createdAt: nullableString,+ likes: nullableNumber,+ retweets: nullableNumber,+ },+ required: ["text", "url", "createdAt", "likes", "retweets"],+ additionalProperties: false,+ },+ },+ },+ required: [+ "displayName",+ "username",+ "profilePicUrl",+ "bio",+ "followers",+ "accountVerified",+ "url",+ "latestPosts",+ ],+ additionalProperties: false,+};++const postSchema = {+ type: "object",+ properties: {+ authorDisplayName: nullableString,+ authorUsername: nullableString,+ text: { type: "string" },+ url: nullableString,+ createdAt: nullableString,+ likes: nullableNumber,+ retweets: nullableNumber,+ thread: {+ type: "array",+ items: {+ type: "object",+ properties: {+ authorDisplayName: nullableString,+ authorUsername: nullableString,+ text: { type: "string" },+ url: nullableString,+ createdAt: nullableString,+ likes: nullableNumber,+ retweets: nullableNumber,+ },+ required: [+ "authorDisplayName",+ "authorUsername",+ "text",+ "url",+ "createdAt",+ "likes",+ "retweets",+ ],+ additionalProperties: false,+ },+ },+ comments: {+ type: "array",+ maxItems: 5,+ items: {+ type: "object",+ properties: {+ authorDisplayName: nullableString,+ authorUsername: nullableString,+ text: { type: "string" },+ url: nullableString,+ createdAt: nullableString,+ likes: nullableNumber,+ },+ required: [+ "authorDisplayName",+ "authorUsername",+ "text",+ "url",+ "createdAt",+ "likes",+ ],+ additionalProperties: false,+ },+ },+ },+ required: [+ "authorDisplayName",+ "authorUsername",+ "text",+ "url",+ "createdAt",+ "likes",+ "retweets",+ "thread",+ "comments",+ ],+ additionalProperties: false,+};++function parseXTwitterUrl(url: string): XTwitterUrl | null {+ let parsed: URL;+ try {+ parsed = new URL(url);+ } catch {+ return null;+ }++ if (!["http:", "https:"].includes(parsed.protocol)) {+ return null;+ }++ const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");+ if (+ hostname !== "x.com" &&+ hostname !== "twitter.com" &&+ hostname !== "mobile.twitter.com"+ ) {+ return null;+ }++ const segments = parsed.pathname+ .split("/")+ .map(segment => segment.trim())+ .filter(Boolean);++ if (segments.length === 0) {+ return null;+ }++ if (+ segments.length >= 4 &&+ segments[0] === "i" &&+ segments[1] === "web" &&+ segments[2] === "status" &&+ isPostId(segments[3])+ ) {+ return {+ kind: "post",+ postId: segments[3],+ normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,+ };+ }++ if (+ segments.length >= 3 &&+ segments[0] === "i" &&+ segments[1] === "status" &&+ isPostId(segments[2])+ ) {+ return {+ kind: "post",+ postId: segments[2],+ normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,+ };+ }++ if (+ segments.length >= 3 &&+ isHandle(segments[0]) &&+ ["status", "statuses"].includes(segments[1]) &&+ isPostId(segments[2])+ ) {+ const handle = segments[0];+ const postId = segments[2];+ return {+ kind: "post",+ handle,+ postId,+ normalizedUrl: `https://x.com/${handle}/status/${postId}`,+ };+ }++ if (+ segments.length === 1 &&+ isHandle(segments[0]) &&+ !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())+ ) {+ const handle = segments[0];+ return {+ kind: "profile",+ handle,+ normalizedUrl: `https://x.com/${handle}`,+ };+ }++ return null;+}++export function isXTwitterUrl(url: string): boolean {+ return parseXTwitterUrl(url) !== null;+}++export async function scrapeURLWithXTwitter(+ meta: Meta,+): Promise<EngineScrapeResult> {+ const urlToScrape = meta.rewrittenUrl ?? meta.url;+ const xUrl = parseXTwitterUrl(urlToScrape);++ if (!xUrl) {+ throw new EngineError(+ `URL is not a supported X/Twitter URL: ${urlToScrape}`,+ );+ }++ if (!config.XAI_API_KEY) {+ throw new XTwitterConfigurationError();+ }++ meta.logger.info("Fetching X/Twitter data through Grok", {+ kind: xUrl.kind,+ url: xUrl.normalizedUrl,+ });++ const markdown =+ xUrl.kind === "profile"+ ? buildProfileMarkdown(await fetchProfile(xUrl, meta))+ : buildPostMarkdown(await fetchPost(xUrl, meta));++ if (markdown.trim().length === 0) {+ throw new EngineError(`No X/Twitter content returned for: ${urlToScrape}`);+ }++ const html = await safeMarkdownToHtml(markdown, meta.logger, meta.id);++ return {+ url: xUrl.normalizedUrl,+ html,+ markdown,+ statusCode: 200,+ contentType: "text/html; charset=utf-8",+ proxyUsed: "basic",+ postprocessorsUsed: ["x-twitter"],+ };+}++export function xTwitterMaxReasonableTime(_meta: Meta): number {+ return 30000;+}++function buildProfileMarkdown(profile: XTwitterProfileData): string {+ const username = stripAt(profile.username) ?? "unknown";+ const displayName = profile.displayName?.trim() || `@${username}`;+ const posts = (profile.latestPosts ?? []).slice(0, 5);+ const lines = [+ `# ${escapeMarkdownInline(displayName)} (@${escapeMarkdownInline(username)})`,+ "",+ ];++ if (profile.bio) {+ lines.push(escapeMarkdownBlock(profile.bio), "");+ }++ const facts = [+ `Followers: ${+ profile.followers !== null && profile.followers !== undefined+ ? formatNumber(profile.followers)+ : "Unknown"+ }`,+ `Verified: ${+ profile.accountVerified !== null && profile.accountVerified !== undefined+ ? profile.accountVerified+ ? "yes"+ : "no"+ : "Unknown"+ }`,+ `Profile Picture: ${+ profile.profilePicUrl+ ? formatMarkdownImage(displayName, profile.profilePicUrl)+ : "Unknown"+ }`,+ profile.url+ ? `Source: ${formatMarkdownLink(profile.url, profile.url)}`+ : undefined,+ ].filter((line): line is string => line !== undefined);++ if (facts.length > 0) {+ lines.push(...facts.map(fact => `- ${fact}`), "");+ }++ lines.push("## Latest Posts", "");++ if (posts.length === 0) {+ lines.push("No recent top-level posts were returned.");+ } else {+ for (const [index, post] of posts.entries()) {+ lines.push(`### ${index + 1}. Post`);+ if (post.createdAt) {+ lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);+ }+ if (post.url) {+ lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);+ }+ lines.push("");+ lines.push(formatBlockquote(post.text));+ const metrics = formatMetrics(post);+ if (metrics.length > 0) {+ lines.push("", metrics.join(" | "));+ }+ lines.push("");+ }+ }++ return trimMarkdown(lines.join("\n"));+}++function buildPostMarkdown(post: XTwitterPostData): string {+ const username = stripAt(post.authorUsername);+ const title = username+ ? `Post by @${escapeMarkdownInline(username)}`+ : "X/Twitter Post";+ const lines = [`# ${title}`, ""];++ const byline = [+ post.authorDisplayName?.trim()+ ? escapeMarkdownInline(post.authorDisplayName)+ : undefined,+ username ? `@${escapeMarkdownInline(username)}` : undefined,+ ].filter(Boolean);++ if (byline.length > 0) {+ lines.push(`Author: ${byline.join(" ")}`);+ }+ if (post.createdAt) {+ lines.push(`Posted: ${escapeMarkdownInline(post.createdAt)}`);+ }+ if (post.url) {+ lines.push(`URL: ${formatMarkdownLink(post.url, post.url)}`);+ }++ const metrics = formatMetrics(post);+ if (metrics.length > 0) {+ lines.push(metrics.join(" | "));+ }++ lines.push("", "## Post", "", formatBlockquote(post.text));++ const thread = (post.thread ?? []).filter(item => item.text?.trim());+ if (thread.length > 0) {+ lines.push("", "## Thread", "");+ for (const [index, threadPost] of thread.entries()) {+ lines.push(`### ${index + 1}. Thread Post`);+ if (threadPost.authorUsername) {+ lines.push(+ `Author: @${escapeMarkdownInline(stripAt(threadPost.authorUsername)!)}`,+ );+ }+ if (threadPost.createdAt) {+ lines.push(`Posted: ${escapeMarkdownInline(threadPost.createdAt)}`);+ }+ if (threadPost.url) {+ lines.push(+ `URL: ${formatMarkdownLink(threadPost.url, threadPost.url)}`,+ );+ }+ lines.push("", formatBlockquote(threadPost.text));+ const threadMetrics = formatMetrics(threadPost);+ if (threadMetrics.length > 0) {+ lines.push("", threadMetrics.join(" | "));+ }+ lines.push("");+ }+ }++ const comments = (post.comments ?? [])+ .filter(comment => comment.text?.trim())+ .slice(0, 5);+ lines.push("", "## Top Comments", "");+ if (comments.length === 0) {+ lines.push("No top comments were returned.");+ } else {+ for (const [index, comment] of comments.entries()) {+ const author = stripAt(comment.authorUsername);+ lines.push(+ `### ${index + 1}. ${+ author ? `@${escapeMarkdownInline(author)}` : "Comment"+ }`,+ );+ if (comment.authorDisplayName) {+ lines.push(+ `Author: ${escapeMarkdownInline(comment.authorDisplayName)}`,+ );+ }+ if (comment.createdAt) {+ lines.push(`Posted: ${escapeMarkdownInline(comment.createdAt)}`);+ }+ if (comment.url) {+ lines.push(`URL: ${formatMarkdownLink(comment.url, comment.url)}`);+ }+ lines.push("", formatBlockquote(comment.text));+ if (comment.likes !== null && comment.likes !== undefined) {+ lines.push("", `Likes: ${formatNumber(comment.likes)}`);+ }+ lines.push("");+ }+ }++ return trimMarkdown(lines.join("\n"));+}++async function fetchProfile(+ xUrl: XTwitterProfileUrl,+ meta: Meta,+): Promise<XTwitterProfileData> {+ const { output } = await generateText({+ model: xai.responses(XAI_RESPONSES_MODEL),+ maxOutputTokens: 20000,+ tools: {+ x_search: xai.tools.xSearch(),+ } as any,+ toolChoice: { type: "tool", toolName: "x_search" },+ output: Output.object({+ schema: jsonSchema(profileSchema as any),+ name: "x_twitter_profile",+ description:+ "Current public X/Twitter profile data and latest top-level posts.",+ }),+ abortSignal: meta.abort.asSignal(),+ prompt: `Give me current public X/Twitter profile details for @${xUrl.handle}: display name, username, profile picture URL, bio, follower count, verification status, and profile URL. Also return exactly the 5 latest posts authored by @${xUrl.handle} that are top-level posts, not replies or comments. Include fewer posts only if fewer public non-reply posts are available. Use the current public X data available to x_search.`,+ });++ return output as XTwitterProfileData;+}++async function fetchPost(+ xUrl: XTwitterPostUrl,+ meta: Meta,+): Promise<XTwitterPostData> {+ const handlePart = xUrl.handle ? ` by @${xUrl.handle}` : "";+ const toolsOptions = xUrl.handle+ ? {+ allowedXHandles: [xUrl.handle],+ enableVideoUnderstanding: true,+ enableImageUnderstanding: true,+ }+ : {+ enableVideoUnderstanding: true,+ enableImageUnderstanding: true,+ };++ const { output } = await generateText({+ model: xai.responses(XAI_RESPONSES_MODEL),+ maxOutputTokens: 20000,+ tools: {+ x_search: xai.tools.xSearch(toolsOptions),+ } as any,+ toolChoice: { type: "tool", toolName: "x_search" },+ output: Output.object({+ schema: jsonSchema(postSchema as any),+ name: "x_twitter_post",+ description:+ "Current public X/Twitter post data with metrics, thread, and top comments.",+ }),+ abortSignal: meta.abort.asSignal(),+ prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post text, author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,+ });++ return output as XTwitterPostData;+}++function isHandle(value: string): boolean {+ return /^[A-Za-z0-9_]{1,15}$/.test(value);+}++function isPostId(value: string): boolean {+ return /^\d{5,}$/.test(value);+}++function stripAt(value: string | null | undefined): string | undefined {+ const trimmed = value?.trim();+ if (!trimmed) {+ return undefined;+ }+ return trimmed.replace(/^@/, "");+}++function formatMetrics(item: {+ likes?: number | null;+ retweets?: number | null;+}): string[] {+ return [+ item.likes !== null && item.likes !== undefined+ ? `Likes: ${formatNumber(item.likes)}`+ : undefined,+ item.retweets !== null && item.retweets !== undefined+ ? `Retweets: ${formatNumber(item.retweets)}`+ : undefined,+ ].filter((line): line is string => line !== undefined);+}++function formatNumber(value: number): string {+ return new Intl.NumberFormat("en-US").format(value);+}++function formatBlockquote(value: string): string {+ const escaped = escapeMarkdownBlock(value.trim());+ if (!escaped) {+ return "> ";+ }+ return escaped+ .split(/\r?\n/)+ .map(line => (line.trim() ? `> ${line}` : ">"))+ .join("\n");+}++function formatMarkdownLink(label: string, url: string): string {+ const sanitizedUrl = sanitizeMarkdownUrl(url);+ if (!sanitizedUrl) {+ return escapeMarkdownInline(label);+ }+ return `[${escapeMarkdownInline(label)}](${sanitizedUrl})`;+}++function formatMarkdownImage(alt: string, url: string): string {+ const sanitizedUrl = sanitizeMarkdownUrl(url);+ if (!sanitizedUrl) {+ return "Unknown";+ }+ return ``;+}++function sanitizeMarkdownUrl(url: string): string | undefined {+ try {+ const parsed = new URL(url);+ if (!["http:", "https:"].includes(parsed.protocol)) {+ return undefined;+ }+ return parsed.toString().replace(/\)/g, "%29");+ } catch {+ return undefined;+ }+}++function escapeMarkdownInline(value: string): string {+ return escapeMarkdownBlock(value).replace(/([\\`*_{}[\]()#+\-.!|])/g, "\\$1");+}++function escapeMarkdownBlock(value: string): string {+ return value+ .replace(/&/g, "&")+ .replace(/</g, "<")+ .replace(/>/g, ">");+}++function trimMarkdown(markdown: string): string {+ return markdown.replace(/\n{3,}/g, "\n\n").trim() + "\n";+}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 5b7782f98..acfcebc2f 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -9,6 +9,28 @@ export class EngineError extends Error {}}+export class XTwitterConfigurationError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_X_TWITTER_CONFIGURATION_ERROR",+ "X/Twitter scraping requires XAI_API_KEY to be configured.",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new XTwitterConfigurationError();+ x.stack = data.stack;+ return x;+ }+}+export class NoEnginesLeftError extends TransportableError {public fallbackList: Engine[];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 4c48fc99e..a19213d5f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -52,6 +52,7 @@ import {ProxySelectionError,ScrapeRetryLimitError,BrandingNotSupportedError,+ XTwitterConfigurationError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";@@ -806,7 +807,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof PDFInsufficientTimeError ||error.error instanceof ProxySelectionError ||error.error instanceof NoCachedDataError ||- error.error instanceof AgentIndexOnlyError+ error.error instanceof AgentIndexOnlyError ||+ error.error instanceof XTwitterConfigurationError) {throw error.error;} else if (error.error instanceof LLMRefusalError) {