Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update blocklist.ts
Nicolasd734df3dfc03
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex f375f384c..5c14cec46 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -92,8 +92,10 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {// Block different TLDs of the same base domainconst baseDomain = domain.split(".")[0]; // Extract the base domain (e.g., "facebook" from "facebook.com")+ // Only apply TLD-variant blocking for base domains longer than 2 charactersif (publicSuffix &&+ baseDomain.length > 2 &&blockedlist.some(blocked => blocked.startsWith(baseDomain + ".") && blocked !== domain,)