Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): move blocklist to DB (#2186)

Gergő Móricz4d460eacdf88
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 1c95201af..9eb6822a3 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -39,6 +39,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
+import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -138,7 +139,14 @@ app.use(domainFrequencyRouter);
const DEFAULT_PORT = process.env.PORT ?? 3002;
const HOST = process.env.HOST ?? "localhost";
-function startServer(port = DEFAULT_PORT) {
+async function startServer(port = DEFAULT_PORT) {
+ try {
+ await initializeBlocklist();
+ } catch (error) {
+ logger.error("Failed to initialize blocklist", { error });
+ throw error;
+ }
+
// Attach WebSocket proxy to the Express app
attachWsProxy(app);
@@ -175,7 +183,10 @@ function startServer(port = DEFAULT_PORT) {
}
if (require.main === module) {
- startServer();
+ startServer().catch(error => {
+ logger.error("Failed to start server", { error });
+ process.exit(1);
+ });
}
app.get("/is-production", (req, res) => {
diff --git a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts
deleted file mode 100644
index 96b82a2e3..000000000
--- a/apps/api/src/scraper/WebScraper/utils/__tests__/blocklist.test.ts
+++ /dev/null
@@ -1,177 +0,0 @@
-import { decryptAES, isUrlBlocked } from "../blocklist";
-
-const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");
-
-describe("isUrlBlocked function", () => {
- beforeAll(() => {
- // Mock the decryptedBlocklist function to return known values
- jest
- .spyOn(require("../blocklist"), "decryptedBlocklist")
- .mockReturnValue([
- "h8ngAFXUNLO3ZqQufJjGVA==",
- "fEGiDm/TWDBkXUXejFVICg==",
- "l6Mei7IGbEmTTFoSudUnqQ==",
- "4OjallJzXRiZUAWDiC2Xww==",
- "ReSvkSfx34TNEdecmmSDdQ==",
- "X1E4WtdmXAv3SAX9xN925Q==",
- "VTzBQfMtXZzM05mnNkWkjA==",
- "m/q4Lb2Z8cxwU7/CoztOFg==",
- "UbVnmRaeG+gKcyVDLAm0vg==",
- "xNQhczYG22tTVc6lYE3qwg==",
- "CQfGDydbg4l1swRCru6O6Q==",
- "l86LQxm2NonTWMauXwEsPw==",
- "6v4QDUcwjnID80G+uU+tgw==",
- "pCF/6nrKZAxaYntzEGluZQ==",
- "r0CRhAmQqSe7V2s3073T00sAh4WcS5779jwuGJ26ows==",
- "aBOVqRFBM4UVg33usY10NdiF0HCnFH/ImtD0n+zIpc8==",
- "QV436UZuQ6D0Dqrx9MwaGw==",
- "OYVvrwILYbzA2mSSqOPPpw==",
- "xW2i4C0Dzcnp+qu12u0SAw==",
- "OLHba209l0dfl0MI4EnQonBITK9z8Qwgd/NsuaTkXmA=",
- "X0VynmNjpL3PrYxpUIG7sFMBt8OlrmQWtxj8oXVu2QM=",
- "ObdlM5NEkvBJ/sojRW5K/Q==",
- "C8Th38X0SjsE1vL/OsD8bA==",
- "PTbGg8PK/h0Seyw4HEpK4Q==",
- "lZdQMknjHb7+4+sjF3qNTw==",
- "LsgSq54q5oDysbva29JxnQ==",
- "KZfBtpwjOpdSoqacRbz7og==",
- "Indtl4yxJMHCKBGF4KABCQ==",
- "e3HFXLVgxhaVoadYpwb2BA==",
- "b+asgLayXQ5Jq+se+q56jA==",
- "86ZDUI7vmp4MvNq3fvZrGQ==",
- "sEGFoYZ6GEg4Zocd+TiyfQ==",
- "6OOL72eXthgnJ1Hj4PfOQQ==",
- "g/ME+Sh1CAFboKrwkVb+5Q==",
- "Pw+xawUoX8xBYbX2yqqGWQ==",
- "k6vBalxYFhAvkPsF19t9gQ==",
- "b+asgLayXQ5Jq+se+q56jA==",
- "KKttwRz4w+AMJrZcB828WQ==",
- "vMdzZ33BXoyWVZnAPOBcrg==",
- "l8GDVI8w/ueHnNzdN1ODuQ==",
- "+yz9bnYYMnC0trJZGJwf6Q==",
- ]);
- });
-
- test("Blocks exact domain with and without protocol", () => {
- expect(
- isUrlBlocked(decryptAES("KZfBtpwjOpdSoqacRbz7og==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("TemsdmaA9kBK9cVJTaAmZksAh4WcS5779jwuGJ26ows=", hashKey),
- null,
- ),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("0pCVMPgc7+IMrLjIA5lFV0ttO4rKIA14yZBb+2FDG7I=", hashKey),
- null,
- ),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("m+PjIWE9E4GF3lA/B9cUMDj3smbHhZYOGxP74UTmd3M=", hashKey),
- null,
- ),
- ).toBe(true);
- });
-
- test("Blocks subdomains of a blocked domain", () => {
- expect(
- isUrlBlocked(
- decryptAES("m+PjIWE9E4GF3lA/B9cUMDj3smbHhZYOGxP74UTmd3M=", hashKey),
- null,
- ),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("o/ClKrW6Qo0uidbD2X8cVjj3smbHhZYOGxP74UTmd3M=", hashKey),
- null,
- ),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("Z53Ny7rvn7cBX/2bYpOZrRDosKfU7BiSM0OClb4bdWY=", hashKey),
- null,
- ),
- ).toBe(true);
- });
-
- test("Blocks different TLDs (BLOCKED-DOMAIN.pt, BLOCKED-DOMAIN.io)", () => {
- expect(
- isUrlBlocked(decryptAES("vUMeqQdqk7ajwczYBr6prA==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(decryptAES("WOjW9VwGwrPu846jDo6VQg==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(decryptAES("Ti3vVa6sRew3wyTZ7a/Yag==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(
- decryptAES("0pCVMPgc7+IMrLjIA5lFV5cYWcOWC5LGWwvlbCW2GH4=", hashKey),
- null,
- ),
- ).toBe(true);
- });
-
- test("Allows unrelated domains like whateverfacebook.com", () => {
- expect(isUrlBlocked("whateverfacebook.com", null)).toBe(false);
- expect(isUrlBlocked("https://whateverfacebook.com", null)).toBe(false);
- });
-
- test("Blocks other domains from the blocklist", () => {
- expect(
- isUrlBlocked(decryptAES("e3HFXLVgxhaVoadYpwb2BA==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(decryptAES("XS61fAjZb5JfAWsyzzOoCQ==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(decryptAES("Indtl4yxJMHCKBGF4KABCQ==", hashKey), null),
- ).toBe(true);
- expect(
- isUrlBlocked(decryptAES("86ZDUI7vmp4MvNq3fvZrGQ==", hashKey), null),
- ).toBe(true);
- });
-
- test("Allows allowed keywords URLs [developers.*, library.*, ads.*]", () => {
- expect(
- isUrlBlocked(
- decryptAES(
- "4H7Uyz6sSCwE3mne1SsGU+6gs7VssjM3e5C6qsyUPUnhsthhQp2bAQwZ9xSCJsjB",
- hashKey,
- ),
- null,
- ),
- ).toBe(false);
- expect(
- isUrlBlocked(
- decryptAES("rNA7JWR/voEnzAqpC4QJAYgZUratpaNBCBVujdFqDb0=", hashKey),
- null,
- ),
- ).toBe(false);
- expect(
- isUrlBlocked(
- decryptAES("ipHiDz83ep6vbIMee94+4XtxxVy1YMYWlaGnWKcG9gQ=", hashKey),
- null,
- ),
- ).toBe(false);
- });
-
- test("Should return false if the URL is invalid", () => {
- expect(isUrlBlocked("randomstring", null)).toBe(false);
- expect(isUrlBlocked("htp://bad.url", null)).toBe(false);
- expect(isUrlBlocked("", null)).toBe(false);
- });
-
- test("Should respect flags", () => {
- const decryptedDomain = decryptAES("e3HFXLVgxhaVoadYpwb2BA==", hashKey);
-
- expect(
- isUrlBlocked(decryptedDomain, {
- unblockedDomains: [decryptedDomain],
- }),
- ).toBe(false);
- });
-});
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 90896d746..d70dfe51f 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -1,109 +1,44 @@
import { configDotenv } from "dotenv";
-import crypto from "crypto";
import { parse } from "tldts";
import { TeamFlags } from "../../../controllers/v1/types";
+import { supabase_rr_service } from "../../../services/supabase";
configDotenv();
-const hashKey = Buffer.from(process.env.HASH_KEY || "", "utf-8");
-const algorithm = "aes-256-ecb";
+type BlocklistBlob = {
+ blocklist: string[];
+ allowedKeywords: string[];
+};
-export function decryptAES(ciphertext: string, key: Buffer): string {
- const decipher = crypto.createDecipheriv(algorithm, key, null);
- const decrypted = Buffer.concat([
- decipher.update(Buffer.from(ciphertext, "base64")),
- decipher.final(),
- ]);
- return decrypted.toString("utf-8");
-}
+let blob: BlocklistBlob | null = null;
+
+export async function initializeBlocklist() {
+ if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ blob = {
+ blocklist: [],
+ allowedKeywords: [],
+ };
+ return;
+ }
-const urlBlocklist = [
- "h8ngAFXUNLO3ZqQufJjGVA==",
- "fEGiDm/TWDBkXUXejFVICg==",
- "l6Mei7IGbEmTTFoSudUnqQ==",
- "4OjallJzXRiZUAWDiC2Xww==",
- "ReSvkSfx34TNEdecmmSDdQ==",
- "X1E4WtdmXAv3SAX9xN925Q==",
- "VTzBQfMtXZzM05mnNkWkjA==",
- "m/q4Lb2Z8cxwU7/CoztOFg==",
- "UbVnmRaeG+gKcyVDLAm0vg==",
- "xNQhczYG22tTVc6lYE3qwg==",
- "CQfGDydbg4l1swRCru6O6Q==",
- "l86LQxm2NonTWMauXwEsPw==",
- "6v4QDUcwjnID80G+uU+tgw==",
- "pCF/6nrKZAxaYntzEGluZQ==",
- "r0CRhAmQqSe7V2s3073T00sAh4WcS5779jwuGJ26ows==",
- "aBOVqRFBM4UVg33usY10NdiF0HCnFH/ImtD0n+zIpc8==",
- "QV436UZuQ6D0Dqrx9MwaGw==",
- "OYVvrwILYbzA2mSSqOPPpw==",
- "xW2i4C0Dzcnp+qu12u0SAw==",
- "OLHba209l0dfl0MI4EnQonBITK9z8Qwgd/NsuaTkXmA=",
- "X0VynmNjpL3PrYxpUIG7sFMBt8OlrmQWtxj8oXVu2QM=",
- "ObdlM5NEkvBJ/sojRW5K/Q==",
- "C8Th38X0SjsE1vL/OsD8bA==",
- "PTbGg8PK/h0Seyw4HEpK4Q==",
- "lZdQMknjHb7+4+sjF3qNTw==",
- "LsgSq54q5oDysbva29JxnQ==",
- "KZfBtpwjOpdSoqacRbz7og==",
- "Indtl4yxJMHCKBGF4KABCQ==",
- "e3HFXLVgxhaVoadYpwb2BA==",
- "b+asgLayXQ5Jq+se+q56jA==",
- "sEGFoYZ6GEg4Zocd+TiyfQ==",
- "6OOL72eXthgnJ1Hj4PfOQQ==",
- "g/ME+Sh1CAFboKrwkVb+5Q==",
- "Pw+xawUoX8xBYbX2yqqGWQ==",
- "k6vBalxYFhAvkPsF19t9gQ==",
- "b+asgLayXQ5Jq+se+q56jA==",
- "KKttwRz4w+AMJrZcB828WQ==",
- "vMdzZ33BXoyWVZnAPOBcrg==",
- "l8GDVI8w/ueHnNzdN1ODuQ==",
- "+yz9bnYYMnC0trJZGJwf6Q==",
- "oTdhIjEjqdT2pEvyxD1Ssg==",
-];
-
-const allowedKeywords = [
- "pulse",
- "privacy",
- "terms",
- "policy",
- "user-agreement",
- "legal",
- "help",
- "policies",
- "support",
- "contact",
- "about",
- "careers",
- "blog",
- "press",
- "conditions",
- "tos",
- "://library.tiktok.com",
- "://ads.tiktok.com",
- "://tiktok.com/business",
- "://developers.facebook.com",
- "://developers.meta.com",
- "://facebook.com/ads/library",
- "://www.facebook.com/ads/library",
- "://meta.com/experiences",
- "://www.meta.com/experiences",
- "://creditcards.aa.com",
- "://aa.org",
- "://www.aa.org",
- "://www.reddit.com/dev/api",
- "://ads-api.reddit.com",
-];
-
-function decryptedBlocklist(list: string[]): string[] {
- return hashKey.length > 0
- ? list.map(ciphertext => decryptAES(ciphertext, hashKey))
- : [];
+ const { data, error } = await supabase_rr_service
+ .from("blocklist")
+ .select("*")
+ .single();
+ if (error || !data) {
+ throw new Error("Error getting blocklist");
+ }
+ blob = data.data;
}
export function isUrlBlocked(url: string, flags: TeamFlags): boolean {
+ if (blob === null) {
+ throw new Error("Blocklist not initialized");
+ }
+
const lowerCaseUrl = url.trim().toLowerCase();
- let blockedlist = decryptedBlocklist(urlBlocklist);
+ let blockedlist = [...blob.blocklist];
if (flags?.unblockedDomains) {
blockedlist = blockedlist.filter(
@@ -132,7 +67,7 @@ export function isUrlBlocked(url: string, flags: TeamFlags): boolean {
// Check if URL contains any allowed keyword
if (
- allowedKeywords.some(keyword =>
+ blob.allowedKeywords.some(keyword =>
lowerCaseUrl.includes(keyword.toLowerCase()),
)
) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index af87f1775..82327ade3 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -38,6 +38,7 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { BullMQOtel } from "bullmq-otel";
import { pathToFileURL } from "url";
import { getErrorContactMessage } from "../lib/deployment";
+import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
configDotenv();
@@ -496,6 +497,8 @@ app.listen(workerPort, () => {
});
(async () => {
+ await initializeBlocklist().catch(e => { _logger.error("Failed to initialize blocklist", { error: e }); process.exit(1); });
+
await Promise.all([
workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index e5fc91d01..ccb6a4e28 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -4,8 +4,16 @@ import { processJobInternal } from "./scrape-worker";
import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";
import Express from "express";
import { _ } from "ajv";
+import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
(async () => {
+ try {
+ await initializeBlocklist();
+ } catch (error) {
+ _logger.error("Failed to initialize blocklist", { error });
+ process.exit(1);
+ }
+
let isShuttingDown = false;
const myLock = crypto.randomUUID();