Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): stop escaping underscores in text/plain markdown output

firecrawl-spring[bot]0b4806840de7
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 2cc13bc0f..4ae10eca4 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -415,6 +415,25 @@ describe("Scrape tests", () => {
);
});
+ describeIf(ALLOW_TEST_SUITE_WEBSITE)("text/plain scrape support", () => {
+ it.concurrent(
+ "does not escape underscores in text/plain markdown",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${base}/llms-underscore.txt`,
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toContain("access_policies");
+ expect(response.markdown).not.toContain("access\\_policies");
+ },
+ scrapeTimeout,
+ );
+ });
+
describeIf(TEST_PRODUCTION)("Fire-Engine scraping", () => {
it.concurrent(
"scrape status works",
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 5d11c12ea..6e80fb74b 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -138,6 +138,21 @@ async function deriveMarkdownFromHTML(
return document;
}
+ // text/plain responses (e.g. llms.txt) are already plain text/markdown.
+ // Running them through the HTML-to-markdown converter escapes markdown
+ // punctuation like "_", which corrupts underscores inside link URLs. Pass
+ // the raw body through untouched instead.
+ if (document.metadata.contentType?.includes("text/plain")) {
+ if (document.rawHtml === undefined) {
+ throw new Error(
+ "rawHtml is undefined -- this transformer is being called out of order",
+ );
+ }
+
+ document.markdown = document.rawHtml;
+ return document;
+ }
+
// Use scrape ID or crawl ID as request_id for tracing
const requestId = meta.id || meta.internalOptions.crawlId;
document.markdown = await parseMarkdown(document.html, {
diff --git a/apps/test-site/public/llms-underscore.txt b/apps/test-site/public/llms-underscore.txt
new file mode 100644
index 000000000..a27b004a1
--- /dev/null
+++ b/apps/test-site/public/llms-underscore.txt
@@ -0,0 +1,4 @@
+# Test llms.txt
+
+- [Create an access policy](https://docs.langchain.com/langsmith/smith-api/access_policies/create-an-access-policy.md)
+- [List access policies](https://docs.langchain.com/langsmith/smith-api/access_policies/list-access-policies.md)