Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): move blocklist to DB (#2186)
Gergő Móricz4d460eacdf88
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 1c95201af..9eb6822a3 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -39,6 +39,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";+import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -138,7 +139,14 @@ app.use(domainFrequencyRouter);const DEFAULT_PORT = process.env.PORT ?? 3002;const HOST = process.env.HOST ?? "localhost";-function startServer(port = DEFAULT_PORT) {+async function startServer(port = DEFAULT_PORT) {+ try {+ await initializeBlocklist();+ } catch (error) {+ logger.error("Failed to initialize blocklist", { error });+ throw error;+ }+// Attach WebSocket proxy to the Express appattachWsProxy(app);@@ -175,7 +183,10 @@ function startServer(port = DEFAULT_PORT) {}if (require.main === module) {- startServer();+ startServer().catch(error => {+ logger.error("Failed to start server", { error });+ process.exit(1);+ });}app.get("/is-production", (req, res) => {diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.tsdeleted file mode 100644index 96b82a2e3..000000000--- a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts+++ /dev/null@@ -1,177 +0,0 @@-import { decryptAES, isUrlBlocked } from "../blocklist";--const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");--describe("isUrlBlocked function", () => {- beforeAll(() => {- // Mock the decryptedBlocklist function to return known values- jest- .spyOn(require("../blocklist"), "decryptedBlocklist")- .mockReturnValue([- "h8ngAFXUNLO3ZqQufJjGVA==",- "fEGiDm/TWDBkXUXejFVICg==",- "l6Mei7IGbEmTTFoSudUnqQ==",- "4OjallJzXRiZUAWDiC2Xww==",- "ReSvkSfx34TNEdecmmSDdQ==",- "X1E4WtdmXAv3SAX9xN925Q==",- "VTzBQfMtXZzM05mnNkWkjA==",- "m/q4Lb2Z8cxwU7/CoztOFg==",- "UbVnmRaeG+gKcyVDLAm0vg==",- "xNQhczYG22tTVc6lYE3qwg==",- "CQfGDydbg4l1swRCru6O6Q==",- "l86LQxm2NonTWMauXwEsPw==",- "6v4QDUcwjnID80G+uU+tgw==",- "pCF/6nrKZAxaYntzEGluZQ==",- "r0CRhAmQqSe7V2s3073T00sAh4WcS5779jwuGJ26ows==",- "aBOVqRFBM4UVg33usY10NdiF0HCnFH/ImtD0n+zIpc8==",- "QV436UZuQ6D0Dqrx9MwaGw==",- "OYVvrwILYbzA2mSSqOPPpw==",- "xW2i4C0Dzcnp+qu12u0SAw==",- "OLHba209l0dfl0MI4EnQonBITK9z8Qwgd/NsuaTkXmA=",- "X0VynmNjpL3PrYxpUIG7sFMBt8OlrmQWtxj8oXVu2QM=",- "ObdlM5NEkvBJ/sojRW5K/Q==",- "C8Th38X0SjsE1vL/OsD8bA==",- "PTbGg8PK/h0Seyw4HEpK4Q==",- "lZdQMknjHb7+4+sjF3qNTw==",- "LsgSq54q5oDysbva29JxnQ==",- "KZfBtpwjOpdSoqacRbz7og==",- "Indtl4yxJMHCKBGF4KABCQ==",- "e3HFXLVgxhaVoadYpwb2BA==",- "b+asgLayXQ5Jq+se+q56jA==",- "86ZDUI7vmp4MvNq3fvZrGQ==",- "sEGFoYZ6GEg4Zocd+TiyfQ==",- "6OOL72eXthgnJ1Hj4PfOQQ==",- "g/ME+Sh1CAFboKrwkVb+5Q==",- "Pw+xawUoX8xBYbX2yqqGWQ==",- "k6vBalxYFhAvkPsF19t9gQ==",- "b+asgLayXQ5Jq+se+q56jA==",- "KKttwRz4w+AMJrZcB828WQ==",- "vMdzZ33BXoyWVZnAPOBcrg==",- "l8GDVI8w/ueHnNzdN1ODuQ==",- "+yz9bnYYMnC0trJZGJwf6Q==",- ]);- });-- test("Blocks exact domain with and without protocol", () => {- expect(- isUrlBlocked(decryptAES("KZfBtpwjOpdSoqacRbz7og==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("TemsdmaA9kBK9cVJTaAmZksAh4WcS5779jwuGJ26ows=", hashKey),- null,- ),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("0pCVMPgc7+IMrLjIA5lFV0ttO4rKIA14yZBb+2FDG7I=", hashKey),- null,- ),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("m+PjIWE9E4GF3lA/B9cUMDj3smbHhZYOGxP74UTmd3M=", hashKey),- null,- ),- ).toBe(true);- });-- test("Blocks subdomains of a blocked domain", () => {- expect(- isUrlBlocked(- decryptAES("m+PjIWE9E4GF3lA/B9cUMDj3smbHhZYOGxP74UTmd3M=", hashKey),- null,- ),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("o/ClKrW6Qo0uidbD2X8cVjj3smbHhZYOGxP74UTmd3M=", hashKey),- null,- ),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("Z53Ny7rvn7cBX/2bYpOZrRDosKfU7BiSM0OClb4bdWY=", hashKey),- null,- ),- ).toBe(true);- });-- test("Blocks different TLDs (BLOCKED-DOMAIN.pt, BLOCKED-DOMAIN.io)", () => {- expect(- isUrlBlocked(decryptAES("vUMeqQdqk7ajwczYBr6prA==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(decryptAES("WOjW9VwGwrPu846jDo6VQg==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(decryptAES("Ti3vVa6sRew3wyTZ7a/Yag==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(- decryptAES("0pCVMPgc7+IMrLjIA5lFV5cYWcOWC5LGWwvlbCW2GH4=", hashKey),- null,- ),- ).toBe(true);- });-- test("Allows unrelated domains like whateverfacebook.com", () => {- expect(isUrlBlocked("whateverfacebook.com", null)).toBe(false);- expect(isUrlBlocked("https://whateverfacebook.com", null)).toBe(false);- });-- test("Blocks other domains from the blocklist", () => {- expect(- isUrlBlocked(decryptAES("e3HFXLVgxhaVoadYpwb2BA==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(decryptAES("XS61fAjZb5JfAWsyzzOoCQ==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(decryptAES("Indtl4yxJMHCKBGF4KABCQ==", hashKey), null),- ).toBe(true);- expect(- isUrlBlocked(decryptAES("86ZDUI7vmp4MvNq3fvZrGQ==", hashKey), null),- ).toBe(true);- });-- test("Allows allowed keywords URLs [developers.*, library.*, ads.*]", () => {- expect(- isUrlBlocked(- decryptAES(- "4H7Uyz6sSCwE3mne1SsGU+6gs7VssjM3e5C6qsyUPUnhsthhQp2bAQwZ9xSCJsjB",- hashKey,- ),- null,- ),- ).toBe(false);- expect(- isUrlBlocked(- decryptAES("rNA7JWR/voEnzAqpC4QJAYgZUratpaNBCBVujdFqDb0=", hashKey),- null,- ),- ).toBe(false);- expect(- isUrlBlocked(- decryptAES("ipHiDz83ep6vbIMee94+4XtxxVy1YMYWlaGnWKcG9gQ=", hashKey),- null,- ),- ).toBe(false);- });-- test("Should return false if the URL is invalid", () => {- expect(isUrlBlocked("randomstring", null)).toBe(false);- expect(isUrlBlocked("htp://bad.url", null)).toBe(false);- expect(isUrlBlocked("", null)).toBe(false);- });-- test("Should respect flags", () => {- const decryptedDomain = decryptAES("e3HFXLVgxhaVoadYpwb2BA==", hashKey);-- expect(- isUrlBlocked(decryptedDomain, {- unblockedDomains: [decryptedDomain],- }),- ).toBe(false);- });-});diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 90896d746..d70dfe51f 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -1,109 +1,44 @@import { configDotenv } from "dotenv";-import crypto from "crypto";import { parse } from "tldts";import { TeamFlags } from "../../../controllers/v1/types";+import { supabase_rr_service } from "../../../services/supabase";configDotenv();-const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");-const algorithm = "aes-256-ecb";+type BlocklistBlob = {+ blocklist: string[];+ allowedKeywords: string[];+};-export function decryptAES(ciphertext: string, key: Buffer): string {- const decipher = crypto.createDecipheriv(algorithm, key, null);- const decrypted = Buffer.concat([- decipher.update(Buffer.from(ciphertext, "base64")),- decipher.final(),- ]);- return decrypted.toString("utf-8");-}+let blob: BlocklistBlob | null = null;++export async function initializeBlocklist() {+ if (process.env.USE_DB_AUTHENTICATION !== "true") {+ blob = {+ blocklist: [],+ allowedKeywords: [],+ };+ return;+ }-const urlBlocklist = [- "h8ngAFXUNLO3ZqQufJjGVA==",- "fEGiDm/TWDBkXUXejFVICg==",- "l6Mei7IGbEmTTFoSudUnqQ==",- "4OjallJzXRiZUAWDiC2Xww==",- "ReSvkSfx34TNEdecmmSDdQ==",- "X1E4WtdmXAv3SAX9xN925Q==",- "VTzBQfMtXZzM05mnNkWkjA==",- "m/q4Lb2Z8cxwU7/CoztOFg==",- "UbVnmRaeG+gKcyVDLAm0vg==",- "xNQhczYG22tTVc6lYE3qwg==",- "CQfGDydbg4l1swRCru6O6Q==",- "l86LQxm2NonTWMauXwEsPw==",- "6v4QDUcwjnID80G+uU+tgw==",- "pCF/6nrKZAxaYntzEGluZQ==",- "r0CRhAmQqSe7V2s3073T00sAh4WcS5779jwuGJ26ows==",- "aBOVqRFBM4UVg33usY10NdiF0HCnFH/ImtD0n+zIpc8==",- "QV436UZuQ6D0Dqrx9MwaGw==",- "OYVvrwILYbzA2mSSqOPPpw==",- "xW2i4C0Dzcnp+qu12u0SAw==",- "OLHba209l0dfl0MI4EnQonBITK9z8Qwgd/NsuaTkXmA=",- "X0VynmNjpL3PrYxpUIG7sFMBt8OlrmQWtxj8oXVu2QM=",- "ObdlM5NEkvBJ/sojRW5K/Q==",- "C8Th38X0SjsE1vL/OsD8bA==",- "PTbGg8PK/h0Seyw4HEpK4Q==",- "lZdQMknjHb7+4+sjF3qNTw==",- "LsgSq54q5oDysbva29JxnQ==",- "KZfBtpwjOpdSoqacRbz7og==",- "Indtl4yxJMHCKBGF4KABCQ==",- "e3HFXLVgxhaVoadYpwb2BA==",- "b+asgLayXQ5Jq+se+q56jA==",- "sEGFoYZ6GEg4Zocd+TiyfQ==",- "6OOL72eXthgnJ1Hj4PfOQQ==",- "g/ME+Sh1CAFboKrwkVb+5Q==",- "Pw+xawUoX8xBYbX2yqqGWQ==",- "k6vBalxYFhAvkPsF19t9gQ==",- "b+asgLayXQ5Jq+se+q56jA==",- "KKttwRz4w+AMJrZcB828WQ==",- "vMdzZ33BXoyWVZnAPOBcrg==",- "l8GDVI8w/ueHnNzdN1ODuQ==",- "+yz9bnYYMnC0trJZGJwf6Q==",- "oTdhIjEjqdT2pEvyxD1Ssg==",-];--const allowedKeywords = [- "pulse",- "privacy",- "terms",- "policy",- "user-agreement",- "legal",- "help",- "policies",- "support",- "contact",- "about",- "careers",- "blog",- "press",- "conditions",- "tos",- "://library.tiktok.com",- "://ads.tiktok.com",- "://tiktok.com/business",- "://developers.facebook.com",- "://developers.meta.com",- "://facebook.com/ads/library",- "://www.facebook.com/ads/library",- "://meta.com/experiences",- "://www.meta.com/experiences",- "://creditcards.aa.com",- "://aa.org",- "://www.aa.org",- "://www.reddit.com/dev/api",- "://ads-api.reddit.com",-];--function decryptedBlocklist(list: string[]): string[] {- return hashKey.length > 0- ? list.map(ciphertext => decryptAES(ciphertext, hashKey))- : [];+ const { data, error } = await supabase_rr_service+ .from("blocklist")+ .select("*")+ .single();+ if (error || !data) {+ throw new Error("Error getting blocklist");+ }+ blob = data.data;}export function isUrlBlocked(url: string, flags: TeamFlags): boolean {+ if (blob === null) {+ throw new Error("Blocklist not initialized");+ }+const lowerCaseUrl = url.trim().toLowerCase();- let blockedlist = decryptedBlocklist(urlBlocklist);+ let blockedlist = [...blob.blocklist];if (flags?.unblockedDomains) {blockedlist = blockedlist.filter(@@ -132,7 +67,7 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {// Check if URL contains any allowed keywordif (- allowedKeywords.some(keyword =>+ blob.allowedKeywords.some(keyword =>lowerCaseUrl.includes(keyword.toLowerCase()),)) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex af87f1775..82327ade3 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -38,6 +38,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { BullMQOtel } from "bullmq-otel";import { pathToFileURL } from "url";import { getErrorContactMessage } from "../lib/deployment";+import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";configDotenv();@@ -496,6 +497,8 @@ app.listen(workerPort, () => {});(async () => {+ await initializeBlocklist().catch(e => { _logger.error("Failed to initialize blocklist", { error: e }); process.exit(1); });+await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex e5fc91d01..ccb6a4e28 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -4,8 +4,16 @@ import { processJobInternal } from "./scrape-worker";import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";import Express from "express";import { _ } from "ajv";+import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";(async () => {+ try {+ await initializeBlocklist();+ } catch (error) {+ _logger.error("Failed to initialize blocklist", { error });+ process.exit(1);+ }+let isShuttingDown = false;const myLock = crypto.randomUUID();