Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(removeUnwantedElements): post-includeTags excludeTags
Móricz Gergőfaf58dfca74c
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex 92df68f47..57015557e 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -54,7 +54,7 @@ export const htmlTransform = (url: string,scrapeOptions: ScrapeOptions,) => {- const soup = load(html);+ let soup = load(html);// remove unwanted elementsif (@@ -68,7 +68,8 @@ export const htmlTransform = (newRoot.append(soup(element).clone());});});- return newRoot.html() ?? "";++ soup = load(newRoot.html() ?? "");}soup("script, style, noscript, meta, head").remove();