Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update blocklist.ts

Nicolasae010a76c19e
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 02aab89f2..21889795c 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -67,6 +67,7 @@ const urlBlocklist = [
"vMdzZ33BXoyWVZnAPOBcrg==",
"l8GDVI8w/ueHnNzdN1ODuQ==",
"+yz9bnYYMnC0trJZGJwf6Q==",
+ "oTdhIjEjqdT2pEvyxD1Ssg==",
]
const allowedKeywords = [
@@ -155,4 +156,4 @@ export function isUrlBlocked(url: string): boolean {
}
return false;
-}
+}
\ No newline at end of file