Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add a couple exceptions to our blocked list (#1816)

Micah Stairse8d4e516b0e7
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 6c44682af..4f6bf822d 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -92,8 +92,10 @@ const allowedKeywords = [
"://ads.tiktok.com",
"://tiktok.com/business",
"://developers.facebook.com",
+ "://developers.meta.com",
"://facebook.com/ads/library",
"://www.facebook.com/ads/library",
+ "://meta.com/experiences",
];
export function decryptedBlocklist(list: string[]): string[] {
@@ -104,7 +106,7 @@ export function decryptedBlocklist(list: string[]): string[] {
export function isUrlBlocked(url: string, flags: TeamFlags): boolean {
const lowerCaseUrl = url.trim().toLowerCase();
-
+
let blockedlist = decryptedBlocklist(urlBlocklist);
if (flags?.unblockedDomains) {