Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: extract post content as markdown (#3722)
tomsideguide222606e53db9
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 57c812cd7..7dc4910ff 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -442,7 +442,7 @@ function buildPostMarkdown(post: XTwitterPostData): string {lines.push(metrics.join(" | "));}- lines.push("", "## Post", "", formatBlockquote(post.text));+ lines.push("", "## Post", "", escapeMarkdownBlock(post.text.trim()));const thread = (post.thread ?? []).filter(item => item.text?.trim());if (thread.length > 0) {@@ -561,7 +561,7 @@ async function fetchPost("Current public X/Twitter post data with metrics, thread, and top comments.",}),abortSignal: meta.abort.asSignal(),- prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post text, author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,+ prompt: `Fetch the public X/Twitter post${handlePart} with post id ${xUrl.postId} at ${xUrl.normalizedUrl}. Return the post body in text as GitHub-flavored Markdown, preserving its original structure like headings and lists. Also return author, URL, created date, likes, and retweets. If this post is part of a thread, return the unrolled thread in chronological order under thread. Return the top 5 public comments or replies to the post under comments. Use the current public X data available to x_search.`,});return output as XTwitterPostData;