Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): stop escaping underscores in text/plain markdown output
firecrawl-spring[bot]0b4806840de7
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 2cc13bc0f..4ae10eca4 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -415,6 +415,25 @@ describe("Scrape tests", () => {);});+ describeIf(ALLOW_TEST_SUITE_WEBSITE)("text/plain scrape support", () => {+ it.concurrent(+ "does not escape underscores in text/plain markdown",+ async () => {+ const response = await scrape(+ {+ url: `${base}/llms-underscore.txt`,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.markdown).toContain("access_policies");+ expect(response.markdown).not.toContain("access\\_policies");+ },+ scrapeTimeout,+ );+ });+describeIf(TEST_PRODUCTION)("Fire-Engine scraping", () => {it.concurrent("scrape status works",diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 5d11c12ea..6e80fb74b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -138,6 +138,21 @@ async function deriveMarkdownFromHTML(return document;}+ // text/plain responses (e.g. llms.txt) are already plain text/markdown.+ // Running them through the HTML-to-markdown converter escapes markdown+ // punctuation like "_", which corrupts underscores inside link URLs. Pass+ // the raw body through untouched instead.+ if (document.metadata.contentType?.includes("text/plain")) {+ if (document.rawHtml === undefined) {+ throw new Error(+ "rawHtml is undefined -- this transformer is being called out of order",+ );+ }++ document.markdown = document.rawHtml;+ return document;+ }+// Use scrape ID or crawl ID as request_id for tracingconst requestId = meta.id || meta.internalOptions.crawlId;document.markdown = await parseMarkdown(document.html, {diff --git a/apps/test-site/public/llms-underscore.txt b/apps/test-site/public/llms-underscore.txtnew file mode 100644index 000000000..a27b004a1--- /dev/null+++ b/apps/test-site/public/llms-underscore.txt@@ -0,0 +1,4 @@+# Test llms.txt++- [Create an access policy](https://docs.langchain.com/langsmith/smith-api/access_policies/create-an-access-policy.md)+- [List access policies](https://docs.langchain.com/langsmith/smith-api/access_policies/list-access-policies.md)