Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update excludeTags.ts
Nicolas2c1a98f01976
diff --git a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts b/apps/api/src/scraper/WebScraper/utils/excludeTags.tsindex 400ef84fe..b813c813d 100644--- a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts+++ b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts@@ -6,7 +6,6 @@ export const excludeNonMainTags = [".header",".top",".navbar",- "#header",".footer",".bottom","#footer",@@ -39,8 +38,6 @@ export const excludeNonMainTags = ["#search",".share","#share",- ".widget",- "#widget",".cookie","#cookie"];