Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): keep blocklist keyword exemptions off URLs the Exchange claims (#5011)

Gergő Móriczf9f2e3dd5406
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index 703a2de30..a9bc26dc3 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -281,6 +281,15 @@ function providerMatchesUrl(
});
}
+/** Whether a catalog provider claims the URL, read from the cached catalog: false until it first loads. */
+export function exchangeClaimsUrl(inputUrl: string): boolean {
+ return (
+ cachedProviders?.value?.some(provider =>
+ providerMatchesUrl(provider, inputUrl),
+ ) ?? false
+ );
+}
+
// Organizations kept on the direct FullEnrich path for LinkedIn, which predates
// enrichment preferences: they skip the enrichment provider and resolve to the
// next claimant in the catalog.
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts
index c290111ac..50fa1143d 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts
@@ -1,4 +1,8 @@
import type { TeamFlags } from "../../../controllers/v1/types";
+import {
+ clearExchangeProvidersForTest,
+ setExchangeProvidersForTest,
+} from "../../../lib/exchange";
import {
hasOrgScopedBlocklist,
initializeBlocklist,
@@ -168,6 +172,47 @@ describe("isUrlBlocked", () => {
});
});
+ describe("allowed keywords on URLs the Exchange claims", () => {
+ beforeAll(async () => {
+ dbState.rows = [
+ {
+ org_id: null,
+ data: {
+ blocklist: ["example.com"],
+ allowedKeywords: ["about", "press"],
+ },
+ },
+ ];
+ await initializeBlocklist();
+ setExchangeProvidersForTest([
+ {
+ id: "acme",
+ routes: [
+ { domains: ["example.com"], pathPrefixes: ["/company/", "/in/"] },
+ ],
+ },
+ ]);
+ });
+
+ afterAll(() => {
+ clearExchangeProvidersForTest();
+ });
+
+ it("keeps blocking a claimed URL that contains an allowed keyword", () => {
+ expect(
+ isUrlBlocked("https://example.com/company/example-co/about/", null),
+ ).toBe(true);
+ expect(isUrlBlocked("https://example.com/in/example-presser", null)).toBe(
+ true,
+ );
+ });
+
+ it("still exempts unclaimed URLs that contain an allowed keyword", () => {
+ expect(isUrlBlocked("https://example.com/about", null)).toBe(false);
+ expect(isUrlBlocked("https://example.com/press/2026", null)).toBe(false);
+ });
+ });
+
describe("row loading", () => {
it("merges multiple rows for the same org", async () => {
dbState.rows = [
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index b426fe959..a8b7f8ed7 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -5,6 +5,7 @@ import { parse } from "tldts";
import { TeamFlags } from "../../../controllers/v1/types";
import { db, dbRr } from "../../../db/connection";
import * as schema from "../../../db/schema";
+import { exchangeClaimsUrl } from "../../../lib/exchange";
configDotenv();
@@ -223,8 +224,13 @@ function findBlockedMatch(
return null;
}
- // Check if URL contains any allowed keyword
- if (allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))) {
+ // Check if URL contains any allowed keyword. A keyword in a URL the
+ // Exchange claims (a profile slug, or a sub-page like /about) doesn't
+ // exempt it: only the Exchange may serve those.
+ if (
+ allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword)) &&
+ !exchangeClaimsUrl(url)
+ ) {
return null;
}