Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(blocklist): behavioural fixes

Gergő Móriczbc787de62a36
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 71d1f454f..02aab89f2 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -94,7 +94,7 @@ const allowedKeywords = [
"://www.facebook.com/ads/library",
];
-function decryptedBlocklist(list: string[]): string[] {
+export function decryptedBlocklist(list: string[]): string[] {
return hashKey.length > 0
? list.map((ciphertext) => decryptAES(ciphertext, hashKey))
: [];
@@ -148,7 +148,7 @@ export function isUrlBlocked(url: string): boolean {
if (
publicSuffix &&
blockedlist.some(
- (blocked) => blocked.startsWith(baseDomain) && blocked !== domain,
+ (blocked) => blocked.startsWith(baseDomain + ".") && blocked !== domain,
)
) {
return true;