Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add another blocked URL exception
Micah Stairse8531b99456c
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 4f6bf822d..e574929fa 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -96,6 +96,7 @@ const allowedKeywords = ["://facebook.com/ads/library","://www.facebook.com/ads/library","://meta.com/experiences",+ "://www.meta.com/experiences",];export function decryptedBlocklist(list: string[]): string[] {