Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(blocklist): unblock
Gergő Móriczec039dcb8f8e
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 16e9e45fd..21993cf1b 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -94,6 +94,8 @@ const allowedKeywords = ["://ads.tiktok.com","://tiktok.com/business","://developers.facebook.com",+ "://facebook.com/ads/library",+ "://www.facebook.com/ads/library",];export function isUrlBlocked(url: string): boolean {