Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep blocklist keyword exemptions off URLs the Exchange claims (#5011)
Gergő Móriczf9f2e3dd5406
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex 703a2de30..a9bc26dc3 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -281,6 +281,15 @@ function providerMatchesUrl(});}+/** Whether a catalog provider claims the URL, read from the cached catalog: false until it first loads. */+export function exchangeClaimsUrl(inputUrl: string): boolean {+ return (+ cachedProviders?.value?.some(provider =>+ providerMatchesUrl(provider, inputUrl),+ ) ?? false+ );+}+// Organizations kept on the direct FullEnrich path for LinkedIn, which predates// enrichment preferences: they skip the enrichment provider and resolve to the// next claimant in the catalog.diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.test.tsindex c290111ac..50fa1143d 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts@@ -1,4 +1,8 @@import type { TeamFlags } from "../../../controllers/v1/types";+import {+ clearExchangeProvidersForTest,+ setExchangeProvidersForTest,+} from "../../../lib/exchange";import {hasOrgScopedBlocklist,initializeBlocklist,@@ -168,6 +172,47 @@ describe("isUrlBlocked", () => {});});+ describe("allowed keywords on URLs the Exchange claims", () => {+ beforeAll(async () => {+ dbState.rows = [+ {+ org_id: null,+ data: {+ blocklist: ["example.com"],+ allowedKeywords: ["about", "press"],+ },+ },+ ];+ await initializeBlocklist();+ setExchangeProvidersForTest([+ {+ id: "acme",+ routes: [+ { domains: ["example.com"], pathPrefixes: ["/company/", "/in/"] },+ ],+ },+ ]);+ });++ afterAll(() => {+ clearExchangeProvidersForTest();+ });++ it("keeps blocking a claimed URL that contains an allowed keyword", () => {+ expect(+ isUrlBlocked("https://example.com/company/example-co/about/", null),+ ).toBe(true);+ expect(isUrlBlocked("https://example.com/in/example-presser", null)).toBe(+ true,+ );+ });++ it("still exempts unclaimed URLs that contain an allowed keyword", () => {+ expect(isUrlBlocked("https://example.com/about", null)).toBe(false);+ expect(isUrlBlocked("https://example.com/press/2026", null)).toBe(false);+ });+ });+describe("row loading", () => {it("merges multiple rows for the same org", async () => {dbState.rows = [diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex b426fe959..a8b7f8ed7 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -5,6 +5,7 @@ import { parse } from "tldts";import { TeamFlags } from "../../../controllers/v1/types";import { db, dbRr } from "../../../db/connection";import * as schema from "../../../db/schema";+import { exchangeClaimsUrl } from "../../../lib/exchange";configDotenv();@@ -223,8 +224,13 @@ function findBlockedMatch(return null;}- // Check if URL contains any allowed keyword- if (allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))) {+ // Check if URL contains any allowed keyword. A keyword in a URL the+ // Exchange claims (a profile slug, or a sub-page like /about) doesn't+ // exempt it: only the Exchange may serve those.+ if (+ allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword)) &&+ !exchangeClaimsUrl(url)+ ) {return null;}