Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add blocked domain exception
Micah Stairs8e125b794a64
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex d6b32538c..14fc6144e 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -98,6 +98,8 @@ const allowedKeywords = ["://meta.com/experiences","://www.meta.com/experiences","://creditcards.aa.com",+ "://aa.org",+ "://www.aa.org",];export function decryptedBlocklist(list: string[]): string[] {