Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Disable markdown conversion for sitemap scrapes (#2461)
Abimael Martell090861122b6d
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 5be946ad0..d9fe3294c 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1431,9 +1431,12 @@ export function toLegacyDocument(return { url: document.metadata.sourceURL! };}+ // backwards compatibility to v0 API+ const markdown = document.markdown ?? "";+return {- content: document.markdown!,- markdown: document.markdown!,+ content: markdown,+ markdown: markdown,html: document.html,rawHtml: document.rawHtml,linksOnPage: document.links,diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex 707e23fff..e8336d999 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -7,6 +7,17 @@ import { scrapeOptions } from "../../controllers/v2/types";import { Engine } from "./engines";import { CostTracking } from "../../lib/cost-tracking";+// Mock parseMarkdown but delegate to real implementation for other tests+jest.mock("../../lib/html-to-markdown", () => {+ const actual = jest.requireActual("../../lib/html-to-markdown");+ return {+ ...actual,+ parseMarkdown: jest.fn(actual.parseMarkdown),+ };+});++import { parseMarkdown } from "../../lib/html-to-markdown";+const testEngines: (Engine | undefined)[] = [undefined,"fire-engine;chrome-cdp",@@ -529,4 +540,35 @@ describe("Standalone scrapeURL tests", () => {},30000,);++ it("Sitemap scrape should not convert to markdown", async () => {+ const mockParseMarkdown = parseMarkdown as jest.MockedFunction<+ typeof parseMarkdown+ >;+ mockParseMarkdown.mockClear();++ const out = await scrapeURL(+ "test:sitemap-no-markdown",+ "https://www.scrapethissite.com/sitemap.xml",+ scrapeOptions.parse({+ formats: ["rawHtml"],+ }),+ { teamId: "sitemap" },+ new CostTracking(),+ );++ expect(out.success).toBe(true);+ if (out.success) {+ expect(out.document.warning).toBeUndefined();+ // Verify markdown conversion was never called+ expect(mockParseMarkdown).not.toHaveBeenCalled();+ // Sitemap scrapes should not have markdown field+ expect(out.document).not.toHaveProperty("markdown");+ // But should have rawHtml+ expect(out.document).toHaveProperty("rawHtml");+ expect(out.document.rawHtml).toBeTruthy();+ expect(out.document).toHaveProperty("metadata");+ expect(out.document.metadata.error).toBeUndefined();+ }+ }, 30000);});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 956f46490..65a5ca9cc 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -71,6 +71,22 @@ async function deriveMarkdownFromHTML();}+ // Only derive markdown if markdown format is requested or if formats that require markdown are requested:+ // - changeTracking requires markdown+ // - json format requires markdown (for LLM extraction)+ // - summary format requires markdown (for summarization)+ const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");+ const hasChangeTracking = hasFormatOfType(+ meta.options.formats,+ "changeTracking",+ );+ const hasJson = hasFormatOfType(meta.options.formats, "json");+ const hasSummary = hasFormatOfType(meta.options.formats, "summary");++ if (!hasMarkdown && !hasChangeTracking && !hasJson && !hasSummary) {+ return document;+ }+if (document.metadata.contentType?.includes("application/json")) {if (document.rawHtml === undefined) {throw new Error(