Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scraper): preserve postprocessor markdown in transformer (#2694)
firecrawl-spring[bot]fce1f4a78c92
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex f2a4a6ede..b5317556b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -87,6 +87,15 @@ async function deriveMarkdownFromHTML(return document;}+ // Skip markdown derivation if a postprocessor already set it+ if (document.metadata.postprocessorsUsed?.length && document.markdown) {+ meta.logger.debug(+ "Skipping markdown derivation - postprocessor already set markdown",+ { postprocessorsUsed: document.metadata.postprocessorsUsed },+ );+ return document;+ }+if (document.metadata.contentType?.includes("application/json")) {if (document.rawHtml === undefined) {throw new Error(