Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): skip HTML transformations for sitemap XML (#4615)
tomsideguide30a9697084c8
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 3f797fcd2..102f81368 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1200,7 +1200,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {: warning;}- // NOTE: for sitemap, we don't need all the transformers, need to skip unused onesdocument = await executeTransformers(meta, document);// Set final span attributesdiff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 3695ae78f..e447aab2d 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -670,6 +670,12 @@ export async function executeTransformers(meta: Meta,document: Document,): Promise<Document> {+ if (meta.internalOptions.teamId === "sitemap") {+ document.metadata.scrapeId = meta.id;+ if (useIndex) document = await sendDocumentToIndex(meta, document);+ return coerceFieldsToFormats(meta, document);+ }+if (hasFormatOfType(meta.options.formats, "rawBase64")) {return coerceFieldsToFormats(meta, document);}