Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update excludeTags.ts

Nicolas2c1a98f01976
diff --git a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
index 400ef84fe..b813c813d 100644
--- a/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
+++ b/apps/api/src/scraper/WebScraper/utils/excludeTags.ts
@@ -6,7 +6,6 @@ export const excludeNonMainTags = [
".header",
".top",
".navbar",
- "#header",
".footer",
".bottom",
"#footer",
@@ -39,8 +38,6 @@ export const excludeNonMainTags = [
"#search",
".share",
"#share",
- ".widget",
- "#widget",
".cookie",
"#cookie"
];