Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add a couple exceptions to our blocked list (#1816)
Micah Stairse8d4e516b0e7
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 6c44682af..4f6bf822d 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -92,8 +92,10 @@ const allowedKeywords = ["://ads.tiktok.com","://tiktok.com/business","://developers.facebook.com",+ "://developers.meta.com","://facebook.com/ads/library","://www.facebook.com/ads/library",+ "://meta.com/experiences",];export function decryptedBlocklist(list: string[]): string[] {@@ -104,7 +106,7 @@ export function decryptedBlocklist(list: string[]): string[] {export function isUrlBlocked(url: string, flags: TeamFlags): boolean {const lowerCaseUrl = url.trim().toLowerCase();-+let blockedlist = decryptedBlocklist(urlBlocklist);if (flags?.unblockedDomains) {