Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update blocklist.ts
Nicolasae010a76c19e
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 02aab89f2..21889795c 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -67,6 +67,7 @@ const urlBlocklist = ["vMdzZ33BXoyWVZnAPOBcrg==","l8GDVI8w/ueHnNzdN1ODuQ==","+yz9bnYYMnC0trJZGJwf6Q==",+ "oTdhIjEjqdT2pEvyxD1Ssg==",]const allowedKeywords = [@@ -155,4 +156,4 @@ export function isUrlBlocked(url: string): boolean {}return false;-}+}\ No newline at end of file