Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(removeUnwantedElements): post-includeTags excludeTags

Móricz Gergőfaf58dfca74c
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index 92df68f47..57015557e 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -54,7 +54,7 @@ export const htmlTransform = (
url: string,
scrapeOptions: ScrapeOptions,
) => {
- const soup = load(html);
+ let soup = load(html);
// remove unwanted elements
if (
@@ -68,7 +68,8 @@ export const htmlTransform = (
newRoot.append(soup(element).clone());
});
});
- return newRoot.html() ?? "";
+
+ soup = load(newRoot.html() ?? "");
}
soup("script, style, noscript, meta, head").remove();