Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/removeUnwantedElements): try to fix onlyMainContent for poorly structured sites
Gergő Móricz6b1f30e0fbf2
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex e06ee3378..9458ed0f9 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -50,6 +50,10 @@ const excludeNonMainTags = ["#cookie"];+const forceIncludeMainTags = [+ "#main"+];+export const removeUnwantedElements = (html: string,scrapeOptions: ScrapeOptions@@ -101,7 +105,9 @@ export const removeUnwantedElements = (if (scrapeOptions.onlyMainContent) {excludeNonMainTags.forEach((tag) => {- const elementsToRemove = soup(tag);+ const elementsToRemove = soup(tag)+ .filter(forceIncludeMainTags.map(x => ":not(:has(" + x + "))").join(""));+elementsToRemove.remove();});}