Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): skip HTML transformations for sitemap XML (#4615)

tomsideguide30a9697084c8
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 3f797fcd2..102f81368 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1200,7 +1200,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
: warning;
}
- // NOTE: for sitemap, we don't need all the transformers, need to skip unused ones
document = await executeTransformers(meta, document);
// Set final span attributes
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 3695ae78f..e447aab2d 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -670,6 +670,12 @@ export async function executeTransformers(
meta: Meta,
document: Document,
): Promise<Document> {
+ if (meta.internalOptions.teamId === "sitemap") {
+ document.metadata.scrapeId = meta.id;
+ if (useIndex) document = await sendDocumentToIndex(meta, document);
+ return coerceFieldsToFormats(meta, document);
+ }
+
if (hasFormatOfType(meta.options.formats, "rawBase64")) {
return coerceFieldsToFormats(meta, document);
}