Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(blocklist): behavioural fixes
Gergő Móriczbc787de62a36
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 71d1f454f..02aab89f2 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -94,7 +94,7 @@ const allowedKeywords = ["://www.facebook.com/ads/library",];-function decryptedBlocklist(list: string[]): string[] {+export function decryptedBlocklist(list: string[]): string[] {return hashKey.length > 0? list.map((ciphertext) => decryptAES(ciphertext, hashKey)): [];@@ -148,7 +148,7 @@ export function isUrlBlocked(url: string): boolean {if (publicSuffix &&blockedlist.some(- (blocked) => blocked.startsWith(baseDomain) && blocked !== domain,+ (blocked) => blocked.startsWith(baseDomain + ".") && blocked !== domain,)) {return true;