Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Disable markdown conversion for sitemap scrapes (#2461)

Abimael Martell090861122b6d
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 5be946ad0..d9fe3294c 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1431,9 +1431,12 @@ export function toLegacyDocument(
return { url: document.metadata.sourceURL! };
}
+ // backwards compatibility to v0 API
+ const markdown = document.markdown ?? "";
+
return {
- content: document.markdown!,
- markdown: document.markdown!,
+ content: markdown,
+ markdown: markdown,
html: document.html,
rawHtml: document.rawHtml,
linksOnPage: document.links,
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index 707e23fff..e8336d999 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -7,6 +7,17 @@ import { scrapeOptions } from "../../controllers/v2/types";
import { Engine } from "./engines";
import { CostTracking } from "../../lib/cost-tracking";
+// Mock parseMarkdown but delegate to real implementation for other tests
+jest.mock("../../lib/html-to-markdown", () => {
+ const actual = jest.requireActual("../../lib/html-to-markdown");
+ return {
+ ...actual,
+ parseMarkdown: jest.fn(actual.parseMarkdown),
+ };
+});
+
+import { parseMarkdown } from "../../lib/html-to-markdown";
+
const testEngines: (Engine | undefined)[] = [
undefined,
"fire-engine;chrome-cdp",
@@ -529,4 +540,35 @@ describe("Standalone scrapeURL tests", () => {
},
30000,
);
+
+ it("Sitemap scrape should not convert to markdown", async () => {
+ const mockParseMarkdown = parseMarkdown as jest.MockedFunction<
+ typeof parseMarkdown
+ >;
+ mockParseMarkdown.mockClear();
+
+ const out = await scrapeURL(
+ "test:sitemap-no-markdown",
+ "https://www.scrapethissite.com/sitemap.xml",
+ scrapeOptions.parse({
+ formats: ["rawHtml"],
+ }),
+ { teamId: "sitemap" },
+ new CostTracking(),
+ );
+
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ // Verify markdown conversion was never called
+ expect(mockParseMarkdown).not.toHaveBeenCalled();
+ // Sitemap scrapes should not have markdown field
+ expect(out.document).not.toHaveProperty("markdown");
+ // But should have rawHtml
+ expect(out.document).toHaveProperty("rawHtml");
+ expect(out.document.rawHtml).toBeTruthy();
+ expect(out.document).toHaveProperty("metadata");
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 30000);
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 956f46490..65a5ca9cc 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -71,6 +71,22 @@ async function deriveMarkdownFromHTML(
);
}
+ // Only derive markdown if markdown format is requested or if formats that require markdown are requested:
+ // - changeTracking requires markdown
+ // - json format requires markdown (for LLM extraction)
+ // - summary format requires markdown (for summarization)
+ const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
+ const hasChangeTracking = hasFormatOfType(
+ meta.options.formats,
+ "changeTracking",
+ );
+ const hasJson = hasFormatOfType(meta.options.formats, "json");
+ const hasSummary = hasFormatOfType(meta.options.formats, "summary");
+
+ if (!hasMarkdown && !hasChangeTracking && !hasJson && !hasSummary) {
+ return document;
+ }
+
if (document.metadata.contentType?.includes("application/json")) {
if (document.rawHtml === undefined) {
throw new Error(