Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick:

Nicolas9fc9f66e3c8e
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 5c14cec46..4f4676b41 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -92,7 +92,7 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {
// Block different TLDs of the same base domain
const baseDomain = domain.split(".")[0]; // Extract the base domain (e.g., "facebook" from "facebook.com")
- // Only apply TLD-variant blocking for base domains longer than 2 characters
+
if (
publicSuffix &&
baseDomain.length > 2 &&