Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): threat protection β€” enterprise domain risk blocking (#3949)

GergΕ‘ MΓ³ricz6db599637f2c
diff --git a/apps/api/knip.config.ts b/apps/api/knip.config.ts
index 6c8f8de9c..e73777ef8 100644
--- a/apps/api/knip.config.ts
+++ b/apps/api/knip.config.ts
@@ -15,6 +15,9 @@ const config: KnipConfig = {
ignore: [
"native/**",
"src/scraper/scrapeURL/engines/fire-engine/branding-script/**",
+ // Shared type contract co-owned by concurrent threat-protection branches;
+ // the provider/verdict types are consumed by the core-lib branch.
+ "src/lib/threat-protection/types.ts",
],
ignoreDependencies: ["undici-types", "stripe"],
};
diff --git a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts
new file mode 100644
index 000000000..d34552388
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts
@@ -0,0 +1,167 @@
+import request from "supertest";
+import {
+ describeIf,
+ idmux,
+ Identity,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+} from "../lib";
+import { THREAT_PROTECTION_POLICY_DEFAULTS } from "../../../lib/threat-protection/types";
+
+async function getConfigRaw(identity: Identity) {
+ return await request(TEST_API_URL)
+ .get("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+async function putConfigRaw(body: unknown, identity: Identity) {
+ return await request(TEST_API_URL)
+ .put("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body as object);
+}
+
+// Requires DB authentication + idmux-provisioned team flags, which only exist
+// in the production test configuration.
+describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {
+ describe("without the team flag", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "team-threat-protection/no-flag",
+ });
+ });
+
+ it("GET returns 403", async () => {
+ const res = await getConfigRaw(identity);
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("enterprise feature");
+ });
+
+ it("PUT returns 403", async () => {
+ const res = await putConfigRaw({ mode: "normal" }, identity);
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("enterprise feature");
+ });
+ });
+
+ describe("with the team flag", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "team-threat-protection/flagged",
+ flags: {
+ threatProtection: "allowed",
+ },
+ });
+ });
+
+ it("GET returns the default (unconfigured) effective config", async () => {
+ const res = await getConfigRaw(identity);
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ expect(res.body.data).toMatchObject({
+ mode: "off",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ allowRequestOverrides: true,
+ configured: false,
+ });
+ expect(res.body.data).not.toHaveProperty("siem");
+ });
+
+ it("PUT round-trips a full config document", async () => {
+ const doc = {
+ mode: "normal",
+ riskScoreThreshold: 60,
+ blacklist: ["*.bad.example"],
+ whitelist: ["firecrawl.dev"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+ allowRequestOverrides: false,
+ };
+
+ const put = await putConfigRaw(doc, identity);
+ expect(put.statusCode).toBe(200);
+ expect(put.body.success).toBe(true);
+ expect(put.body.data).toMatchObject({
+ mode: "normal",
+ riskScoreThreshold: 60,
+ blacklist: ["*.bad.example"],
+ whitelist: ["firecrawl.dev"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+ allowRequestOverrides: false,
+ configured: true,
+ });
+ // Retired policy fields must not appear in the served document.
+ expect(put.body.data).not.toHaveProperty("siem");
+ expect(put.body.data).not.toHaveProperty("deniedCategories");
+ expect(put.body.data).not.toHaveProperty("maxDomainAgeDays");
+ expect(put.body.data).not.toHaveProperty("blockedCountries");
+
+ const get = await getConfigRaw(identity);
+ expect(get.statusCode).toBe(200);
+ expect(get.body.data).toMatchObject({
+ mode: "normal",
+ riskScoreThreshold: 60,
+ configured: true,
+ });
+ expect(get.body.data).not.toHaveProperty("siem");
+ expect(get.body.data).not.toHaveProperty("deniedCategories");
+ expect(get.body.data).not.toHaveProperty("maxDomainAgeDays");
+ expect(get.body.data).not.toHaveProperty("blockedCountries");
+ });
+
+ it("PUT is a full-document update (unspecified fields reset to defaults)", async () => {
+ const put = await putConfigRaw({ mode: "normal" }, identity);
+ expect(put.statusCode).toBe(200);
+ expect(put.body.data).toMatchObject({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ allowRequestOverrides: true,
+ configured: true,
+ });
+ expect(put.body.data).not.toHaveProperty("siem");
+ });
+
+ it("PUT rejects an invalid document with 400", async () => {
+ const res = await putConfigRaw(
+ {
+ mode: "normal",
+ riskScoreThreshold: 500,
+ blacklist: ["https://not-a-domain"],
+ },
+ identity,
+ );
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ });
+
+ it('PUT rejects the retired "enhanced" mode with 400', async () => {
+ const res = await putConfigRaw({ mode: "enhanced" }, identity);
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ });
+
+ it("PUT rejects retired policy fields with 400", async () => {
+ for (const retired of [
+ { siem: { url: "https://siem.example.com/ingest" } },
+ { deniedCategories: ["Malicious"] },
+ { maxDomainAgeDays: 30 },
+ { blockedCountries: ["KP"] },
+ ]) {
+ const res = await putConfigRaw(
+ { mode: "normal", ...retired },
+ identity,
+ );
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ }
+ });
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.ts
new file mode 100644
index 000000000..006b690e6
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.ts
@@ -0,0 +1,402 @@
+import http from "http";
+import {
+ describeIf,
+ idmux,
+ Identity,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+ scrapeTimeout,
+} from "../lib";
+import {
+ batchScrape,
+ crawl,
+ creditUsage,
+ scrape,
+ scrapeRaw,
+ searchRaw,
+} from "./lib";
+import {
+ createWebRiskMockCounters,
+ createWebRiskMockHandler,
+ WebRiskMockDatabase,
+} from "../../../lib/threat-protection/providers/web-risk/testing";
+
+// =========================================
+// Threat protection billing (ENG-4985, ZDR rework ENG-5004)
+//
+// Billing rules under test:
+// - +2 credits per domain scanned in "normal" mode (Google Web Risk)
+// - a "scan" = a ThreatDecision with providerConsulted β€” +2 per consulted
+// verdict. There is no verdict cache anymore (ZDR): every request scans
+// afresh, and dedup only applies within one request/job.
+// - blocked requests still bill the scan fee (no base scrape cost, matching
+// how other failed scrapes bill)
+// - local-only decisions (whitelist/blacklist/blocked-tld) never bill
+//
+// These tests need the mock provider below. Start the harness with the
+// base-URL override pointing at it, e.g.:
+//
+// GOOGLE_WEB_RISK_API_KEY=test \
+// GOOGLE_WEB_RISK_API_URL=http://localhost:4519 \
+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-billing.test.ts
+//
+// Tests that need the provider self-skip when the override is not set to a
+// local address. NOTE: this file binds the mock port itself β€” do not run it
+// in the same pass as another file using the same port (e.g.
+// threat-protection-enforcement.test.ts if given the same Web Risk port).
+// =========================================
+
+// Fixture domains. *.example.com is reserved (RFC 2606) β€” these domains are
+// never actually fetched: blocked scrapes fail before any outbound request.
+const BLACKLISTED_DOMAIN = "threat-billing-blacklisted.example.com";
+const RISKY_DOMAIN = "threat-billing-risky.example.com";
+
+const CLEAN_URL = TEST_SUITE_WEBSITE;
+
+// A stable cross-hostname redirect: google.com 301s to www.google.com. The
+// mock flags only the redirect target, so the scrape consults the provider
+// twice (initial domain + redirect re-check) before being blocked.
+// NOTE the two hostnames are different domains for the request-scoped dedup,
+// so the redirect re-check is a second scan (two consulted decisions).
+const REDIRECT_SOURCE_URL = "https://google.com/";
+const REDIRECT_TARGET_DOMAIN = "www.google.com";
+
+const MOCK_RISKY_DOMAINS = [RISKY_DOMAIN, REDIRECT_TARGET_DOMAIN];
+
+const isLocalMock = (url: string) =>
+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(url);
+
+const webRiskMockUrl = process.env.GOOGLE_WEB_RISK_API_URL ?? "";
+const HAS_WEB_RISK_MOCK =
+ isLocalMock(webRiskMockUrl) && !!process.env.GOOGLE_WEB_RISK_API_KEY;
+
+const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms));
+// Credit deductions land through the batched billing queue; see billing.test.ts.
+const sleepForBatchBilling = () => sleep(40000);
+
+// Update API mock (ZDR rework): the API syncs the mock's threat lists via
+// threatLists:computeDiff and confirms local prefix hits via hashes:search β€”
+// clean domains resolve locally and never reach the mock at all.
+const webRiskDb = new WebRiskMockDatabase();
+for (const domain of MOCK_RISKY_DOMAINS) {
+ webRiskDb.addRiskyDomain(domain, "MALWARE");
+}
+const webRiskCounters = createWebRiskMockCounters();
+const webRiskHandler = createWebRiskMockHandler(webRiskDb, webRiskCounters);
+
+let webRiskServer: http.Server | null = null;
+
+/** hashes:search confirmations whose prefix belongs to `domain`. */
+const webRiskHitsFor = (domain: string) =>
+ webRiskCounters.hashesSearchRequestsForDomain(domain);
+
+function startWebRiskMock(): Promise<void> {
+ const port = Number(new URL(webRiskMockUrl).port);
+ webRiskServer = http.createServer((req, res) => {
+ if (!webRiskHandler(req, res)) {
+ res.statusCode = 404;
+ res.end("{}");
+ }
+ });
+ return new Promise((resolve, reject) => {
+ webRiskServer!.once("error", reject);
+ webRiskServer!.listen(port, () => resolve());
+ });
+}
+
+describeIf(TEST_PRODUCTION)("Threat protection billing", () => {
+ beforeAll(async () => {
+ if (HAS_WEB_RISK_MOCK) {
+ await startWebRiskMock();
+ }
+ });
+
+ afterAll(async () => {
+ if (webRiskServer) {
+ await new Promise(resolve => webRiskServer!.close(resolve));
+ webRiskServer = null;
+ }
+ });
+
+ describe("scrape", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-billing/scrape",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "normal-mode clean scrape bills base + 2",
+ async () => {
+ const doc = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(doc.metadata.creditsUsed).toBe(3);
+ },
+ scrapeTimeout,
+ );
+
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "a rescrape of the same domain is a fresh scan and bills + 2 again",
+ async () => {
+ // ZDR: there is no cross-request verdict cache β€” every request scans
+ // afresh and bills per consulted verdict. A clean domain's scan is
+ // entirely local (hash-prefix lookup against the synced list), so
+ // Google is never contacted for it on either scrape.
+ const cleanDomain = new URL(CLEAN_URL).hostname;
+
+ const first = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(first.metadata.creditsUsed).toBe(3);
+
+ const second = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(second.metadata.creditsUsed).toBe(3);
+
+ // Clean domains never trigger a hashes:search confirmation.
+ expect(webRiskHitsFor(cleanDomain)).toBe(0);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it(
+ "scan costs are visible in the document's creditsUsed metadata",
+ async () => {
+ // Local-only allow (whitelist): no provider consulted, so a plain
+ // 1-credit scrape β€” proves the fee only appears when a scan happened.
+ const cleanDomain = new URL(CLEAN_URL).hostname;
+ const doc = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: {
+ mode: "normal",
+ whitelist: [cleanDomain],
+ failurePolicy: "closed",
+ },
+ } as any,
+ identity,
+ );
+ expect(doc.metadata.creditsUsed).toBe(1);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("blocked scrapes (end-to-end credit deduction)", () => {
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "bills scan fees for blocked scrapes, nothing for local-only blocks",
+ async () => {
+ // Dedicated team so the credit delta is exactly attributable.
+ const identity = await idmux({
+ name: "threat-protection-billing/blocked",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+
+ // A fresh team's credit grant can land a moment after idmux returns
+ // (and a small default allotment can show up before the 1M grant);
+ // poll until the baseline read reflects the full grant so the delta
+ // is exact.
+ let before = 0;
+ for (let i = 0; i < 30 && before < 100_000; i++) {
+ before = (await creditUsage(identity)).remainingCredits;
+ if (before < 100_000) await sleep(1000);
+ }
+ expect(before).toBeGreaterThanOrEqual(100_000);
+
+ // 1. Blocked by provider verdict: 0 base + 2 scan.
+ const blocked = await scrapeRaw(
+ {
+ url: `https://${RISKY_DOMAIN}/`,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(blocked.statusCode).toBe(403);
+ expect(blocked.body.code).toBe("unsafe_domain_blocked");
+
+ // 2. Blocked by blacklist (local-only): no scan fee, no base cost.
+ const blacklisted = await scrapeRaw(
+ {
+ url: `https://${BLACKLISTED_DOMAIN}/page`,
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ } as any,
+ identity,
+ );
+ expect(blacklisted.statusCode).toBe(403);
+ expect(blacklisted.body.code).toBe("unsafe_domain_blocked");
+
+ // 3. Clean domain redirecting to a blocked domain: two consulted
+ // decisions (initial + redirect re-check) β†’ two scan fees.
+ const redirected = await scrapeRaw(
+ {
+ url: REDIRECT_SOURCE_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(redirected.statusCode).toBe(403);
+ expect(redirected.body.code).toBe("unsafe_domain_blocked");
+ expect(redirected.body.error).toContain(REDIRECT_TARGET_DOMAIN);
+
+ await sleepForBatchBilling();
+
+ const after = (await creditUsage(identity)).remainingCredits;
+ // 2 (blocked scan) + 0 (blacklist) + 4 (redirect: 2 scans) = 6
+ expect(before - after).toBe(6);
+ },
+ scrapeTimeout * 3 + 60000,
+ );
+ });
+
+ describe("crawl and batch scrape (per-document scan fees)", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-billing/crawl-batch",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "batch scrape bills base + 2 per document (each job scans afresh)",
+ async () => {
+ const res = await batchScrape(
+ {
+ urls: [CLEAN_URL, `${CLEAN_URL}/blog`],
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(res.data.length).toBe(2);
+ for (const doc of res.data) {
+ expect(doc.metadata.creditsUsed).toBe(3);
+ }
+ },
+ scrapeTimeout * 2,
+ );
+
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "crawl bills base + 2 per scraped document",
+ async () => {
+ const res = await crawl(
+ {
+ url: CLEAN_URL,
+ limit: 2,
+ scrapeOptions: {
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ },
+ } as any,
+ identity,
+ );
+ expect(res.status).toBe("completed");
+ expect(res.data.length).toBeGreaterThan(0);
+ for (const doc of res.data) {
+ expect(doc.metadata.creditsUsed).toBe(3);
+ }
+ },
+ scrapeTimeout * 5,
+ );
+ });
+
+ describe("search", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-billing/search",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ (HAS_WEB_RISK_MOCK ? it : it.skip)(
+ "bills a scan fee per result domain on top of normal search billing",
+ async () => {
+ const limit = 20;
+ const res = await searchRaw(
+ {
+ query: "firecrawl",
+ includeDomains: ["firecrawl.dev"],
+ limit,
+ timeout: 120000,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+
+ const web: { url: string }[] = res.body.data.web ?? [];
+ expect(web.length).toBeGreaterThan(0);
+
+ const uniqueDomains = new Set(
+ web.map(x => new URL(x.url).hostname.toLowerCase()),
+ );
+ const searchCredits = Math.ceil(web.length / 10) * 2;
+
+ if (web.length < limit) {
+ // Nothing was sliced off, so the returned set IS the scanned set:
+ // exactly one +2 scan fee per unique result domain.
+ expect(res.body.creditsUsed).toBe(
+ searchCredits + 2 * uniqueDomains.size,
+ );
+ } else {
+ // The provider over-fetched (limit * 2 buffer) and results were
+ // sliced; scanned domains βŠ‡ returned domains.
+ expect(res.body.creditsUsed).toBeGreaterThanOrEqual(
+ searchCredits + 2 * uniqueDomains.size,
+ );
+ }
+ },
+ scrapeTimeout * 2,
+ );
+
+ it(
+ "search without provider scans bills plain search credits",
+ async () => {
+ // Local-only policy (everything blacklist-missed resolves via
+ // failurePolicy "open" without a provider configured => no consulted
+ // decisions => no scan fees). With the mock provider configured this
+ // still holds a stronger property: whitelisted domains short-circuit.
+ const res = await searchRaw(
+ {
+ query: "firecrawl",
+ includeDomains: ["firecrawl.dev"],
+ limit: 5,
+ timeout: 120000,
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ const web: { url: string }[] = res.body.data.web ?? [];
+ expect(res.body.creditsUsed).toBe(Math.ceil(web.length / 10) * 2);
+ },
+ scrapeTimeout * 2,
+ );
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
new file mode 100644
index 000000000..796d5e613
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
@@ -0,0 +1,613 @@
+import http from "http";
+import request from "supertest";
+import {
+ describeIf,
+ idmux,
+ Identity,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+ scrapeTimeout,
+} from "../lib";
+import { crawl, crawlStart, map, scrape, scrapeRaw, search } from "./lib";
+import {
+ createWebRiskMockCounters,
+ createWebRiskMockHandler,
+ WebRiskMockDatabase,
+} from "../../../lib/threat-protection/providers/web-risk/testing";
+
+// =========================================
+// Threat protection enforcement (ENG-4982/4983/4984)
+//
+// Most tests here use per-request threatProtection overrides with local-only
+// rules (blacklist/whitelist/risk threshold), which never hit the provider β€”
+// they only need a team with the threatProtection flag (via idmux) and are
+// gated on TEST_PRODUCTION.
+//
+// Provider-verdict tests (risky domain, redirect re-check) need the mock
+// Google Web Risk server below. The API must be started with the base-URL
+// override pointing at it, e.g.:
+//
+// GOOGLE_WEB_RISK_API_KEY=test \
+// GOOGLE_WEB_RISK_API_URL=http://localhost:4517 \
+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-enforcement.test.ts
+//
+// Those tests are skipped when the override is not set to a local address.
+//
+// Org-config-based tests (PUT /v2/team/threat-protection) additionally need
+// the threat_protection_config table (DDL ships with the config PR); they
+// self-skip with a warning when the config API is unavailable.
+// =========================================
+
+// Fixture domains. *.example.com is reserved (RFC 2606) β€” these domains are
+// never actually fetched: blocked scrapes fail before any outbound request.
+const BLACKLISTED_DOMAIN = "threat-blacklisted.example.com";
+const RISKY_DOMAIN = "threat-risky.example.com";
+
+// The scrape target for happy-path tests.
+const CLEAN_URL = TEST_SUITE_WEBSITE;
+const CLEAN_DOMAIN = new URL(TEST_SUITE_WEBSITE).hostname;
+
+// A stable cross-hostname redirect: google.com 301s to www.google.com.
+const REDIRECT_SOURCE_URL = "https://google.com/";
+const REDIRECT_TARGET_DOMAIN = "www.google.com";
+
+// Domains the mock provider's threat lists flag as MALWARE (risk score 100).
+const MOCK_RISKY_DOMAINS = [RISKY_DOMAIN, REDIRECT_TARGET_DOMAIN];
+
+const mockProviderUrl = process.env.GOOGLE_WEB_RISK_API_URL ?? "";
+const HAS_MOCK_PROVIDER =
+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(mockProviderUrl) &&
+ !!process.env.GOOGLE_WEB_RISK_API_KEY;
+
+// Update API mock (ZDR rework): the API syncs the mock's threat lists via
+// threatLists:computeDiff and confirms local prefix hits via hashes:search β€”
+// there is no uris:search anymore, and clean domains never reach the mock.
+const webRiskDb = new WebRiskMockDatabase();
+for (const domain of MOCK_RISKY_DOMAINS) {
+ webRiskDb.addRiskyDomain(domain, "MALWARE");
+}
+const webRiskCounters = createWebRiskMockCounters();
+const webRiskHandler = createWebRiskMockHandler(webRiskDb, webRiskCounters);
+
+let mockServer: http.Server | null = null;
+
+/** hashes:search confirmations whose prefix belongs to `domain`. */
+const providerHitsFor = (domain: string) =>
+ webRiskCounters.hashesSearchRequestsForDomain(domain);
+
+function startMockProvider(): Promise<void> {
+ const port = Number(new URL(mockProviderUrl).port);
+ mockServer = http.createServer((req, res) => {
+ if (!webRiskHandler(req, res)) {
+ res.statusCode = 404;
+ res.end("{}");
+ }
+ });
+ return new Promise((resolve, reject) => {
+ mockServer!.once("error", reject);
+ mockServer!.listen(port, () => resolve());
+ });
+}
+
+async function putThreatProtectionConfig(body: unknown, identity: Identity) {
+ return await request(TEST_API_URL)
+ .put("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body as object);
+}
+
+describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {
+ beforeAll(async () => {
+ if (HAS_MOCK_PROVIDER) {
+ await startMockProvider();
+ }
+ });
+
+ afterAll(async () => {
+ if (mockServer) {
+ await new Promise(resolve => mockServer!.close(resolve));
+ mockServer = null;
+ }
+ });
+
+ describe("without the team flag", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-enforcement/no-flag",
+ });
+ });
+
+ it(
+ "rejects any per-request threatProtection option with 403",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "normal" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("enterprise feature");
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("with the team flag (per-request overrides)", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-enforcement/allowed",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ it(
+ "scrape: clean domain passes with an active policy",
+ async () => {
+ // failurePolicy "open" makes this deterministic whether or not a
+ // provider is configured: clean verdict β†’ allow; no provider β†’ allow.
+ const doc = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(doc.metadata.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: blacklisted domain is blocked with zero provider calls",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: `https://${BLACKLISTED_DOMAIN}/some/page`,
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ expect(res.body.error).toContain(BLACKLISTED_DOMAIN);
+ if (HAS_MOCK_PROVIDER) {
+ expect(providerHitsFor(BLACKLISTED_DOMAIN)).toBe(0);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: whitelisted domain passes without a provider call",
+ async () => {
+ // failurePolicy "closed" + no whitelist would block when the provider
+ // is unavailable β€” so a success here proves the whitelist
+ // short-circuited the check in environments without a provider, and
+ // the hit counter proves it where the mock is running.
+ const before = providerHitsFor(CLEAN_DOMAIN);
+ const doc = await scrape(
+ {
+ url: CLEAN_URL,
+ threatProtection: {
+ mode: "normal",
+ whitelist: [CLEAN_DOMAIN],
+ failurePolicy: "closed",
+ },
+ } as any,
+ identity,
+ );
+ expect(doc.metadata.statusCode).toBe(200);
+ if (HAS_MOCK_PROVIDER) {
+ expect(providerHitsFor(CLEAN_DOMAIN)).toBe(before);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: risk threshold blocks (verdict or fail-closed, deterministic)",
+ async () => {
+ // riskScoreThreshold 0 blocks any domain with a verdict (score >= 0),
+ // and failurePolicy "closed" blocks when no provider is configured β€”
+ // so this asserts the block path in every environment.
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: {
+ mode: "normal",
+ riskScoreThreshold: 0,
+ failurePolicy: "closed",
+ },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "batch scrape: blocked URLs are rejected at enqueue, rest succeed",
+ async () => {
+ const start = await request(TEST_API_URL)
+ .post("/v2/batch/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ urls: [CLEAN_URL, `https://${BLACKLISTED_DOMAIN}/page`],
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ failurePolicy: "open",
+ },
+ });
+
+ expect(start.statusCode).toBe(200);
+ expect(start.body.success).toBe(true);
+ expect(start.body.invalidURLs).toContain(
+ `https://${BLACKLISTED_DOMAIN}/page`,
+ );
+
+ let status: any;
+ do {
+ await new Promise(resolve => setTimeout(resolve, 500));
+ const res = await request(TEST_API_URL)
+ .get(`/v2/batch/scrape/${start.body.id}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .send();
+ expect(res.statusCode).toBe(200);
+ status = res.body;
+ } while (status.status === "scraping");
+
+ expect(status.status).toBe("completed");
+ expect(status.data.length).toBe(1);
+ expect(status.data[0].metadata.sourceURL).toBe(CLEAN_URL);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it(
+ "crawl: blocked seed URL is rejected at kickoff",
+ async () => {
+ const res = await crawlStart(
+ {
+ url: `https://${BLACKLISTED_DOMAIN}/`,
+ limit: 2,
+ scrapeOptions: {
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "crawl: blocked discoveries are skipped and the crawl completes",
+ async () => {
+ const blockedDiscoveryDomain = "github.com";
+ const res = await crawl(
+ {
+ url: CLEAN_URL,
+ limit: 3,
+ maxDiscoveryDepth: 1,
+ scrapeOptions: {
+ threatProtection: {
+ mode: "normal",
+ blacklist: [blockedDiscoveryDomain],
+ failurePolicy: "open",
+ },
+ },
+ } as any,
+ identity,
+ );
+ expect(res.status).toBe("completed");
+ expect(res.data.length).toBeGreaterThan(0);
+ for (const doc of res.data) {
+ const url = doc.metadata.url ?? doc.metadata.sourceURL;
+ if (!url) continue;
+ const hostname = new URL(url).hostname;
+ expect(
+ hostname === blockedDiscoveryDomain ||
+ hostname.endsWith(`.${blockedDiscoveryDomain}`),
+ ).toBe(false);
+ }
+ },
+ scrapeTimeout * 5,
+ );
+
+ it(
+ "map: blocked domains are filtered out of the returned URL list",
+ async () => {
+ const blockedSubdomain = `docs.${CLEAN_DOMAIN}`;
+ const res = await map(
+ {
+ url: CLEAN_URL,
+ limit: 100,
+ includeSubdomains: true,
+ threatProtection: {
+ mode: "normal",
+ blacklist: [blockedSubdomain],
+ failurePolicy: "open",
+ },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ for (const link of res.body.links) {
+ const hostname = new URL(link.url).hostname;
+ expect(
+ hostname === blockedSubdomain ||
+ hostname.endsWith(`.${blockedSubdomain}`),
+ ).toBe(false);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "search: blocked domains are removed from results entirely",
+ async () => {
+ const results = await search(
+ {
+ query: "firecrawl",
+ limit: 10,
+ threatProtection: {
+ mode: "normal",
+ blacklist: ["firecrawl.dev"],
+ failurePolicy: "open",
+ },
+ } as any,
+ identity,
+ );
+ for (const result of results.web ?? []) {
+ const hostname = new URL(result.url).hostname;
+ expect(
+ hostname === "firecrawl.dev" || hostname.endsWith(".firecrawl.dev"),
+ ).toBe(false);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "extract: blocked target URLs are rejected before fetching",
+ async () => {
+ const res = await request(TEST_API_URL)
+ .post("/v2/extract")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ urls: [`https://${BLACKLISTED_DOMAIN}/page`],
+ prompt: "Extract the page title.",
+ ignoreInvalidURLs: false,
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ });
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain(BLACKLISTED_DOMAIN);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("provider verdicts (mock Google Web Risk)", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ if (!HAS_MOCK_PROVIDER) return;
+ identity = await idmux({
+ name: "threat-protection-enforcement/provider",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ (HAS_MOCK_PROVIDER ? it : it.skip)(
+ "scrape: risky domain is blocked by the provider verdict",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: `https://${RISKY_DOMAIN}/`,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ // The domain's hash prefix is in the synced local list; the block
+ // required a hashes:search confirmation against the mock.
+ expect(res.body.error).toContain(RISKY_DOMAIN);
+ expect(providerHitsFor(RISKY_DOMAIN)).toBeGreaterThanOrEqual(1);
+ },
+ scrapeTimeout,
+ );
+
+ (HAS_MOCK_PROVIDER ? it : it.skip)(
+ "scrape: clean domain redirecting to a blocked domain is blocked",
+ async () => {
+ // google.com 301s to www.google.com; the mock flags only the
+ // redirect target as risky, so the initial check passes and the
+ // redirect re-check must catch it.
+ const res = await scrapeRaw(
+ {
+ url: REDIRECT_SOURCE_URL,
+ threatProtection: { mode: "normal", failurePolicy: "open" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ expect(res.body.error).toContain(REDIRECT_TARGET_DOMAIN);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("with the team flag forced", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-enforcement/forced",
+ flags: { threatProtection: "forced" },
+ credits: 100_000,
+ });
+ });
+
+ it(
+ "rejects a per-request override that disables threat protection",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "off" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("cannot be disabled");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "still allows narrowing overrides (mode stays on)",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: `https://${BLACKLISTED_DOMAIN}/`,
+ threatProtection: {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describe("org config enforcement + override lockout", () => {
+ let identity: Identity;
+ let configApiAvailable = false;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-enforcement/org-config",
+ flags: { threatProtection: "allowed" },
+ credits: 100_000,
+ });
+
+ // The config API needs the threat_protection_config table (DDL ships
+ // with the config PR). Skip these tests gracefully where it is not
+ // provisioned yet.
+ const res = await putThreatProtectionConfig(
+ {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ failurePolicy: "open",
+ allowRequestOverrides: true,
+ },
+ identity,
+ );
+ configApiAvailable = res.statusCode === 200;
+ if (!configApiAvailable) {
+ console.warn(
+ "threat protection config API unavailable (missing DDL?); skipping org config tests",
+ res.statusCode,
+ res.body,
+ );
+ }
+ });
+
+ afterAll(async () => {
+ if (configApiAvailable) {
+ // Reset so reruns against the same idmux team start clean.
+ await putThreatProtectionConfig(
+ { mode: "off", allowRequestOverrides: true },
+ identity,
+ );
+ }
+ });
+
+ it(
+ "org policy applies without any per-request option",
+ async () => {
+ if (!configApiAvailable) return;
+ const res = await scrapeRaw(
+ { url: `https://${BLACKLISTED_DOMAIN}/` },
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "request override is rejected when the org locks overrides down",
+ async () => {
+ if (!configApiAvailable) return;
+ const lock = await putThreatProtectionConfig(
+ {
+ mode: "normal",
+ blacklist: [BLACKLISTED_DOMAIN],
+ failurePolicy: "open",
+ allowRequestOverrides: false,
+ },
+ identity,
+ );
+ expect(lock.statusCode).toBe(200);
+
+ // The org config read is cached for ~60s; the PUT invalidates it, so
+ // the lockout applies immediately.
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: { whitelist: [BLACKLISTED_DOMAIN] },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("overrides are disabled");
+ },
+ scrapeTimeout,
+ );
+ });
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 0ce129200..35c082082 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -49,6 +49,33 @@ const configSchema = z.object({
// TOR). Unset disables the check entirely (keyless behaves as before).
SPUR_API_KEY: z.string().optional(),
+ // Threat protection (enterprise domain risk blocking). "normal" mode uses
+ // Google Web Risk. An unset key disables the provider (lookups then fail
+ // per the org's failurePolicy).
+ GOOGLE_WEB_RISK_API_KEY: z.string().optional(),
+ GOOGLE_WEB_RISK_API_URL: z
+ .string()
+ .url()
+ .default("https://webrisk.googleapis.com"),
+ // Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run
+ // against a locally synced hash-prefix database (threatLists:computeDiff)
+ // instead of sending URLs to Google, and verdicts are never persisted.
+ //
+ // Floor for how often threatLists:computeDiff may run per list. Google's
+ // recommendedNextDiff is respected when it is later than this floor.
+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(60),
+ // A synced threat list older than this is treated as unavailable
+ // (provider-failure semantics β†’ the org's failurePolicy decides).
+ THREAT_LIST_STALENESS_SECONDS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(6 * 60 * 60),
+
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
OPENAI_API_KEY: z.string().optional(),
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 2a30a540e..a45b7efc1 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -26,6 +26,13 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -49,7 +56,22 @@ export async function batchScrapeController(
req.body = batchScrapeRequestSchema.parse(req.body);
}
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
+ const permissions = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
if (permissions.error) {
return res.status(403).json({
success: false,
@@ -129,6 +151,68 @@ export async function batchScrapeController(
}
}
+ // Threat protection: reject/report blocked URLs at enqueue time so they
+ // never consume scrape slots (mirrors the isUrlBlocked handling above).
+ if (threatProtection.policy) {
+ const { blocked } = await checkUrlsAgainstThreatPolicy(
+ urls,
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ if (blocked.length > 0) {
+ // Blocked domains whose decision consulted the classifier (fresh or
+ // cached verdict) bill the scan fee (+2 per scanned domain) even
+ // though they will never be scraped β€” the scan already happened.
+ // Allowed URLs are not billed here: their scrape jobs re-check the
+ // cached verdict and bill there.
+ const blockedDecisionsByDomain = new Map(
+ blocked.map(x => [x.domain, x.decision]),
+ );
+ const threatScanCredits = calculateThreatScanCredits(
+ blockedDecisionsByDomain.values(),
+ );
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "batch_scrape", jobId: id },
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ if (req.body.ignoreInvalidURLs) {
+ const blockedSet = new Set(blocked.map(x => x.url));
+ const keptUnnormalized: string[] = [];
+ const keptUrls: string[] = [];
+ urls.forEach((u, i) => {
+ if (blockedSet.has(u)) {
+ invalidURLs!.push(unnormalizedURLs[i] ?? u);
+ } else {
+ keptUrls.push(u);
+ keptUnnormalized.push(unnormalizedURLs[i]);
+ }
+ });
+ urls = keptUrls;
+ unnormalizedURLs = keptUnnormalized;
+ } else {
+ const first = blocked[0];
+ const error = new UnsafeDomainBlockedError(
+ first.domain,
+ first.decision,
+ );
+ return res.status(403).json({
+ success: false,
+ code: error.code,
+ error: error.message,
+ });
+ }
+ }
+ }
+
if (urls.length === 0) {
return res.status(400).json({
success: false,
@@ -176,6 +260,7 @@ export async function batchScrapeController(
: false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 57674d8f4..9c2c8702d 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -29,6 +29,12 @@ import {
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import { checkDomain } from "../../lib/threat-protection";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { normalizeDomain } from "../../lib/threat-protection/verdict";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -37,9 +43,23 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.scrapeOptions?.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
const permissions = checkPermissions(
{ ...req.body, crawlerOptions: req.body },
req.acuc?.flags,
+ { threatProtectionOrgConfig: threatProtection.orgConfig },
);
if (permissions.error) {
return res.status(403).json({
@@ -48,6 +68,39 @@ export async function crawlController(
});
}
+ // Threat protection: check the seed URL before kicking off the crawl.
+ if (threatProtection.policy) {
+ const seedDomain = normalizeDomain(req.body.url);
+ const decision = await checkDomain(seedDomain, threatProtection.policy, {
+ teamId: req.auth.team_id,
+ });
+ if (!decision.allowed) {
+ // A blocked seed still bills the scan fee when the classifier was
+ // consulted β€” the scan already happened. (An allowed seed is not billed
+ // here: its scrape job re-checks the cached verdict and bills there.)
+ const threatScanCredits = calculateThreatScanCredits([decision]);
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "crawl" },
+ ).catch(error => {
+ _logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ const error = new UnsafeDomainBlockedError(seedDomain, decision);
+ return res.status(403).json({
+ success: false,
+ code: error.code,
+ error: error.message,
+ });
+ }
+ }
+
const keyRestriction = await checkKeyFormatRestriction(
formatTypesOf(req.body.scrapeOptions?.formats),
actionTypesOf(req.body.scrapeOptions?.actions),
@@ -151,6 +204,7 @@ export async function crawlController(
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 5c498aa8c..6662f67ba 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -23,6 +23,13 @@ import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { config } from "../../config";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>,
@@ -133,6 +140,64 @@ export async function extractController(
}
}
+ // Threat protection: check target URLs before fetching. Blocked URLs are
+ // reported via invalidURLs (with ignoreInvalidURLs) or reject the request.
+ // Discovered URLs are enforced in the scrape pipeline via the policy
+ // threaded through the extract job.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {
+ const { blocked, decisionsByDomain } = await checkUrlsAgainstThreatPolicy(
+ req.body.urls ?? [],
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ // Every consulted decision (fresh or cached provider verdict) is a
+ // billable scan (+2 per scanned domain) β€” including when the request is
+ // rejected below: the scans already happened.
+ const threatScanCredits = calculateThreatScanCredits(
+ decisionsByDomain.values(),
+ );
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "extract" },
+ ).catch(error => {
+ _logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ if (blocked.length > 0) {
+ if (req.body.ignoreInvalidURLs) {
+ invalidURLs.push(...blocked.map(x => x.url));
+ } else {
+ const first = blocked[0];
+ const error = new UnsafeDomainBlockedError(
+ first.domain,
+ first.decision,
+ );
+ return res.status(403).json({
+ success: false,
+ error: error.message,
+ });
+ }
+ }
+ }
+
const extractId = uuidv7();
_logger.info("Extract starting...", {
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index b9751c055..93aa6895e 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -31,6 +31,12 @@ import {
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { normalizeDomain } from "../../lib/threat-protection/verdict";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
configDotenv();
const redis = new Redis(config.REDIS_URL!);
@@ -379,7 +385,22 @@ export async function mapController(
});
}
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
+ const permissions = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
if (permissions.error) {
return res.status(403).json({
success: false,
@@ -464,16 +485,34 @@ export async function mapController(
}
}
+ // Threat protection: remove links on blocked domains from the returned
+ // URL list entirely. Every consulted decision (fresh or cached provider
+ // verdict) is a billable scan (+2 per scanned domain).
+ let threatScanCredits = 0;
+ if (threatProtection.policy && result.links.length > 0) {
+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(
+ result.links,
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ threatScanCredits = calculateThreatScanCredits(decisionsByDomain.values());
+ result.links = result.links.filter(x => {
+ const decision = decisionsByDomain.get(normalizeDomain(x));
+ return decision === undefined || decision.allowed;
+ });
+ }
+
// Bill the team
+ const creditsToBill = 1 + threatScanCredits;
billTeam(
req.auth.team_id,
req.acuc?.sub_id,
- 1,
+ creditsToBill,
req.acuc?.api_key_id ?? null,
{ endpoint: "map", jobId: mapId },
).catch(error => {
logger.error(
- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,
+ `Failed to bill team ${req.auth.team_id} for ${creditsToBill} credit(s): ${error}`,
);
});
@@ -494,7 +533,7 @@ export async function mapController(
location: req.body.location,
},
results: result.links,
- credits_cost: 1,
+ credits_cost: creditsToBill,
zeroDataRetention: false, // not supported
});
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index f17ef0337..7fd083171 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -36,6 +36,7 @@ import {
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -50,7 +51,22 @@ export async function scrapeController(
const preNormalizedBody = { ...req.body };
req.body = scrapeRequestSchema.parse(req.body);
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
+ const permissions = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
if (permissions.error) {
return res.status(403).json({
success: false,
@@ -212,6 +228,7 @@ export async function scrapeController(
zeroDataRetention,
teamFlags: req.acuc?.flags ?? null,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
},
skipNuq: true,
origin,
@@ -275,6 +292,14 @@ export async function scrapeController(
});
}
+ if (e.code === "unsafe_domain_blocked") {
+ return res.status(403).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({
success: false,
code: e.code,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 018698bb9..bbf4a09a4 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -22,6 +22,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
import { z } from "zod";
import { executeSearch } from "../../search/execute";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
import {
DocumentWithCostTracking,
scrapeSearchResults,
@@ -151,6 +152,22 @@ export async function searchController(
origin: req.body.origin,
});
+ // Threat protection: resolve the effective policy. Blocked domains are
+ // removed from search results entirely.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override:
+ req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
await logRequest({
id: jobId,
kind: "search",
@@ -228,6 +245,7 @@ export async function searchController(
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
keylessReserved: reservedKeylessCredits > 0,
+ threatProtectionPolicy: threatProtection.policy,
},
logger,
);
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index eac175261..7e0db4d88 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -19,6 +19,7 @@ import { webhookSchema } from "../../services/webhook/schema";
import { BrandingProfile } from "../../types/branding";
import { ProductProfile } from "../../types/product";
import { MenuProfile } from "../../types/menu";
+import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";
type Format =
| "markdown"
@@ -528,6 +529,9 @@ const baseScrapeOptions = z.strictObject({
.gte(0)
.prefault(1 * 24 * 60 * 60 * 1000),
storeInCache: z.boolean().prefault(true),
+ // Enterprise: per-request field-level override of the org's threat
+ // protection policy. Gated on the team flag + org config (checkPermissions).
+ threatProtection: threatProtectionOverrideSchema.optional(),
// @deprecated
__experimental_cache: z.boolean().prefault(false).optional(),
__searchPreviewToken: z.string().optional(),
@@ -744,6 +748,7 @@ const extractV1Options = z
__experimental_showCostTracking: z.boolean().prefault(false),
ignoreInvalidURLs: z.boolean().prefault(false),
webhook: webhookSchema.optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
})
.refine(obj => obj.urls || obj.prompt, {
error: "Either 'urls' or 'prompt' must be provided.",
@@ -972,6 +977,7 @@ const mapRequestSchemaBase = crawlerOptions
ignoreCache: z.boolean().prefault(false),
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
});
export const mapRequestSchema = mapRequestSchemaBase.strict();
@@ -1302,6 +1308,7 @@ export type AuthCreditUsageChunk = {
export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
+ threatProtection?: "disabled" | "allowed" | "forced";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
@@ -1559,6 +1566,7 @@ export const searchRequestSchema = z
timeout: z.int().positive().finite().prefault(60000),
ignoreInvalidURLs: z.boolean().optional().prefault(false),
__searchPreviewToken: z.string().optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
scrapeOptions: baseScrapeOptions
.extend({
formats: z
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index e478f4128..24b2572f6 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -11,6 +11,13 @@ import { logRequest } from "../../services/logging/log_job";
import { config } from "../../config";
import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
export async function agentController(
req: RequestWithAuth<{}, AgentResponse, AgentRequest>,
@@ -46,6 +53,63 @@ export async function agentController(
zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
});
+ // Threat protection: check the agent's starting URLs before handing off to
+ // the agent service. Content the agent fetches through the API
+ // (agent-interop scrapes) is additionally enforced by the scrape pipeline's
+ // org-policy resolution; in-page navigations performed by the remote
+ // browser cannot be intercepted here.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {
+ const { blocked } = await checkUrlsAgainstThreatPolicy(
+ req.body.urls ?? [],
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ if (blocked.length > 0) {
+ // Blocked domains whose decision consulted the classifier (fresh or
+ // cached verdict) bill the scan fee (+2 per scanned domain) even
+ // though the request is rejected β€” the scan already happened. Allowed
+ // start URLs are not billed here: the agent's API-driven scrapes
+ // re-check the policy in the scrape pipeline and bill there.
+ const blockedDecisionsByDomain = new Map(
+ blocked.map(x => [x.domain, x.decision]),
+ );
+ const threatScanCredits = calculateThreatScanCredits(
+ blockedDecisionsByDomain.values(),
+ );
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "agent", jobId: agentId },
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ const first = blocked[0];
+ const error = new UnsafeDomainBlockedError(first.domain, first.decision);
+ return res.status(403).json({
+ success: false,
+ error: error.message,
+ });
+ }
+ }
+
if (!config.EXTRACT_V3_BETA_URL) {
throw new Error("Agent beta is not enabled.");
}
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index ad9df8c80..5d74d8894 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -38,6 +38,13 @@ import {
import { logRequest } from "../../services/logging/log_job";
import type { BillingMetadata } from "../../services/billing/types";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -50,7 +57,22 @@ export async function batchScrapeController(
req.body = batchScrapeRequestSchema.parse(req.body);
}
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
+ const permissions = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
if (permissions.error) {
return res.status(403).json({
success: false,
@@ -150,6 +172,75 @@ export async function batchScrapeController(
}
}
+ // Threat protection: reject/report blocked URLs at enqueue time so they
+ // never consume scrape slots. Mirrors the isUrlBlocked handling above:
+ // with ignoreInvalidURLs they are reported in invalidURLs; without it the
+ // whole request is rejected. Any URL that slips through (e.g. via a
+ // redirect) is still blocked in the scrape pipeline and its job document
+ // gets the unsafe_domain_blocked error.
+ if (threatProtection.policy) {
+ const { blocked } = await checkUrlsAgainstThreatPolicy(
+ urls,
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ if (blocked.length > 0) {
+ // Blocked domains whose decision consulted the classifier (fresh or
+ // cached verdict) bill the scan fee (+2 per scanned domain) even
+ // though they will never be scraped β€” the scan already happened.
+ // Allowed URLs are not billed here: their scrape jobs re-check the
+ // cached verdict and bill there.
+ const blockedDecisionsByDomain = new Map(
+ blocked.map(x => [x.domain, x.decision]),
+ );
+ const threatScanCredits = calculateThreatScanCredits(
+ blockedDecisionsByDomain.values(),
+ );
+ if (
+ threatScanCredits > 0 &&
+ (req.body.__agentInterop?.shouldBill ?? true)
+ ) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ billing,
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ if (req.body.ignoreInvalidURLs) {
+ const blockedSet = new Set(blocked.map(x => x.url));
+ const keptUnnormalized: string[] = [];
+ const keptUrls: string[] = [];
+ urls.forEach((u, i) => {
+ if (blockedSet.has(u)) {
+ invalidURLs!.push(unnormalizedURLs[i] ?? u);
+ } else {
+ keptUrls.push(u);
+ keptUnnormalized.push(unnormalizedURLs[i]);
+ }
+ });
+ urls = keptUrls;
+ unnormalizedURLs = keptUnnormalized;
+ } else {
+ const first = blocked[0];
+ const error = new UnsafeDomainBlockedError(
+ first.domain,
+ first.decision,
+ );
+ return res.status(403).json({
+ success: false,
+ code: error.code,
+ error: error.message,
+ });
+ }
+ }
+ }
+
if (urls.length === 0) {
return res.status(400).json({
success: false,
@@ -191,6 +282,7 @@ export async function batchScrapeController(
zeroDataRetention,
bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 0568e9705..cb07fb6ca 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -31,6 +31,12 @@ import {
} from "../../services/worker/nuq-router";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import { checkDomain } from "../../lib/threat-protection";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { normalizeDomain } from "../../lib/threat-protection/verdict";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -39,9 +45,23 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.scrapeOptions?.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
const permissions = checkPermissions(
{ ...req.body, crawlerOptions: req.body },
req.acuc?.flags,
+ { threatProtectionOrgConfig: threatProtection.orgConfig },
);
if (permissions.error) {
return res.status(403).json({
@@ -50,6 +70,41 @@ export async function crawlController(
});
}
+ // Threat protection: check the seed URL before kicking off the crawl.
+ // Blocked seed => request-level error. Discovered links are checked during
+ // link discovery in the workers (blocked ones are silently skipped).
+ if (threatProtection.policy) {
+ const seedDomain = normalizeDomain(req.body.url);
+ const decision = await checkDomain(seedDomain, threatProtection.policy, {
+ teamId: req.auth.team_id,
+ });
+ if (!decision.allowed) {
+ // A blocked seed still bills the scan fee when the classifier was
+ // consulted β€” the scan already happened. (An allowed seed is not billed
+ // here: its scrape job re-checks the cached verdict and bills there.)
+ const threatScanCredits = calculateThreatScanCredits([decision]);
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "crawl" },
+ ).catch(error => {
+ _logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ const error = new UnsafeDomainBlockedError(seedDomain, decision);
+ return res.status(403).json({
+ success: false,
+ code: error.code,
+ error: error.message,
+ });
+ }
+ }
+
const keyRestriction = await checkKeyFormatRestriction(
formatTypesOf(req.body.scrapeOptions?.formats),
actionTypesOf(req.body.scrapeOptions?.actions),
@@ -210,6 +265,7 @@ export async function crawlController(
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
},
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index 5842dafd2..6bdbf8090 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -14,6 +14,13 @@ import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../../services/logging/log_job";
import { config } from "../../config";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
+import { billTeam } from "../../services/billing/credit_billing";
/**
* Extracts data from the provided URLs based on the request parameters.
@@ -74,6 +81,64 @@ export async function extractController(
}
}
+ // Threat protection: check target URLs before fetching. Blocked URLs are
+ // reported via invalidURLs (with ignoreInvalidURLs) or reject the request.
+ // Discovered URLs are enforced in the scrape pipeline via the policy
+ // threaded through the extract job.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {
+ const { blocked, decisionsByDomain } = await checkUrlsAgainstThreatPolicy(
+ req.body.urls ?? [],
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ // Every consulted decision (fresh or cached provider verdict) is a
+ // billable scan (+2 per scanned domain) β€” including when the request is
+ // rejected below: the scans already happened.
+ const threatScanCredits = calculateThreatScanCredits(
+ decisionsByDomain.values(),
+ );
+ if (threatScanCredits > 0) {
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ threatScanCredits,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "extract", jobId: extractId },
+ ).catch(error => {
+ _logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,
+ );
+ });
+ }
+ if (blocked.length > 0) {
+ if (req.body.ignoreInvalidURLs) {
+ invalidURLs.push(...blocked.map(x => x.url));
+ } else {
+ const first = blocked[0];
+ const error = new UnsafeDomainBlockedError(
+ first.domain,
+ first.decision,
+ );
+ return res.status(403).json({
+ success: false,
+ error: error.message,
+ });
+ }
+ }
+ }
+
await logRequest({
id: extractId,
kind: "extract",
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 0728b662f..cc0203bfe 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -16,6 +16,12 @@ import { v7 as uuidv7 } from "uuid";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { resolveViaAvgrab } from "../../lib/avgrab-resolve";
+import {
+ checkUrlsAgainstThreatPolicy,
+ resolveThreatProtection,
+} from "../../lib/threat-protection/request";
+import { normalizeDomain } from "../../lib/threat-protection/verdict";
+import { calculateThreatScanCredits } from "../../lib/scrape-billing";
configDotenv();
@@ -38,7 +44,22 @@ export async function mapController(
const originalRequest = req.body;
req.body = mapRequestSchema.parse(req.body);
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
+ const permissions = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
if (permissions.error) {
return res.status(403).json({
success: false,
@@ -189,17 +210,37 @@ export async function mapController(
}
}
+ // Threat protection: remove links on blocked domains from the returned
+ // URL list entirely. Every consulted decision (fresh or cached provider
+ // verdict) is a billable scan (+2 per scanned domain).
+ let threatScanCredits = 0;
+ if (threatProtection.policy && result.mapResults.length > 0) {
+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(
+ result.mapResults.map(x => x.url),
+ threatProtection.policy,
+ { teamId: req.auth.team_id },
+ );
+ threatScanCredits = calculateThreatScanCredits(decisionsByDomain.values());
+ result.mapResults = result.mapResults.filter(x => {
+ const decision = decisionsByDomain.get(normalizeDomain(x.url));
+ return decision === undefined || decision.allowed;
+ });
+ }
+
// Bill the team
+ const creditsToBill = 1 + threatScanCredits;
billTeam(
req.auth.team_id,
req.acuc?.sub_id ?? undefined,
- 1,
+ creditsToBill,
req.acuc?.api_key_id ?? null,
{ endpoint: "map", jobId: mapId },
).catch(error => {
- logger.error(
- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,
- );
+ logger.error("Failed to bill team for map credits", {
+ teamId: req.auth.team_id,
+ creditsToBill,
+ error,
+ });
});
logMap({
@@ -217,7 +258,7 @@ export async function mapController(
location: req.body.location,
},
results: result.mapResults,
- credits_cost: 1,
+ credits_cost: creditsToBill,
zeroDataRetention: false, // not supported
}).catch(error => {
logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 44823a2f7..4de9b9108 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -37,6 +37,7 @@ import {
} from "../../lib/keyless";
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
@@ -72,11 +73,33 @@ export async function scrapeController(
});
});
+ // Threat protection: resolve the effective policy (org config +
+ // per-request override). No-ops (null policy, zero I/O) for teams
+ // without the flag.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override: req.body.threatProtection,
+ });
+ if (threatProtection.error) {
+ setSpanAttributes(span, {
+ "scrape.error": threatProtection.error,
+ "scrape.status_code": 403,
+ });
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
// Permission check span
const permissions = await withSpan(
"api.scrape.check_permissions",
async permSpan => {
- const perms = checkPermissions(req.body, req.acuc?.flags);
+ const perms = checkPermissions(req.body, req.acuc?.flags, {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ });
setSpanAttributes(permSpan, {
"permissions.success": !perms.error,
"permissions.error": perms.error,
@@ -306,6 +329,7 @@ export async function scrapeController(
zeroDataRetention,
teamFlags: req.acuc?.flags ?? null,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
+ threatProtection: threatProtection.policy ?? undefined,
},
skipNuq: true,
origin,
@@ -416,6 +440,17 @@ export async function scrapeController(
});
}
+ if (e.code === "unsafe_domain_blocked") {
+ setSpanAttributes(span, {
+ "scrape.status_code": 403,
+ });
+ return res.status(403).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;
setSpanAttributes(span, {
"scrape.status_code": statusCode,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 7d2cdb43b..ffe8dd4b7 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -28,6 +28,7 @@ import type { BillingMetadata } from "../../services/billing/types";
import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";
import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -100,6 +101,23 @@ export async function searchController(
});
}
+ // Threat protection: resolve the effective policy. Blocked domains are
+ // removed from search results entirely, and scraped results inherit the
+ // policy through the scrape pipeline.
+ const threatProtection = await resolveThreatProtection({
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ override:
+ req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,
+ });
+ if (threatProtection.error) {
+ return res.status(403).json({
+ success: false,
+ error: threatProtection.error,
+ });
+ }
+
const shouldBill = req.body.__agentInterop?.shouldBill ?? true;
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
const billing: BillingMetadata = req.body.__agentInterop
@@ -211,6 +229,7 @@ export async function searchController(
billing,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
keylessReserved: reservedKeylessCredits > 0,
+ threatProtectionPolicy: threatProtection.policy,
},
logger,
);
diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.ts
new file mode 100644
index 000000000..ea2528203
--- /dev/null
+++ b/apps/api/src/controllers/v2/team-threat-protection.ts
@@ -0,0 +1,125 @@
+import { Response } from "express";
+import { z } from "zod";
+import { logger as _logger } from "../../lib/logger";
+import { RequestWithAuth } from "./types";
+import { getThreatProtection } from "../../lib/zdr-helpers";
+import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";
+import {
+ getOrgIdForTeam,
+ getOrgThreatProtectionConfig,
+ resolveEffectivePolicy,
+ upsertOrgThreatProtectionConfig,
+ type OrgThreatProtectionConfig,
+} from "../../lib/threat-protection/store";
+
+const logger = _logger.child({ module: "team-threat-protection" });
+
+const SUPPORT_EMAIL = "[email protected]";
+
+function rejectWithoutFlag(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+): boolean {
+ const mode = getThreatProtection(req.acuc?.flags);
+ if (mode !== "allowed" && mode !== "forced") {
+ res.status(403).json({
+ success: false,
+ error: `Threat protection is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
+ });
+ return true;
+ }
+ return false;
+}
+
+async function resolveOrgId(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+): Promise<string | null> {
+ const orgId = req.auth.org_id ?? (await getOrgIdForTeam(req.auth.team_id));
+ if (!orgId) {
+ logger.error("Failed to resolve org for team", {
+ teamId: req.auth.team_id,
+ });
+ res.status(500).json({
+ success: false,
+ error: "Failed to resolve the organization for this team.",
+ });
+ return null;
+ }
+ return orgId;
+}
+
+/**
+ * Effective config document served by GET and PUT. Always includes every
+ * policy field (defaults applied), so the dashboard can render the form
+ * without knowing the defaults.
+ */
+function serializeConfig(orgConfig: OrgThreatProtectionConfig | null) {
+ const policy = resolveEffectivePolicy(orgConfig);
+ return {
+ ...policy,
+ allowRequestOverrides: orgConfig?.allowRequestOverrides ?? true,
+ configured: orgConfig !== null,
+ updatedAt: orgConfig?.updatedAt ?? null,
+ };
+}
+
+function changedFields(
+ previous: OrgThreatProtectionConfig | null,
+ next: OrgThreatProtectionConfig,
+): string[] {
+ const before = serializeConfig(previous) as Record<string, unknown>;
+ const after = serializeConfig(next) as Record<string, unknown>;
+ return Object.keys(after).filter(
+ key =>
+ key !== "updatedAt" &&
+ key !== "configured" &&
+ JSON.stringify(before[key]) !== JSON.stringify(after[key]),
+ );
+}
+
+export async function getTeamThreatProtectionController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ const orgConfig = await getOrgThreatProtectionConfig(orgId);
+
+ res.status(200).json({
+ success: true,
+ data: serializeConfig(orgConfig),
+ });
+}
+
+export async function putTeamThreatProtectionController(
+ req: RequestWithAuth<{}, any, unknown>,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+
+ const input = threatProtectionConfigSchema.parse(req.body);
+
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ const previous = await getOrgThreatProtectionConfig(orgId);
+
+ const updated = await upsertOrgThreatProtectionConfig(orgId, input);
+
+ // Audit log β€” org-level security configuration change.
+ logger.info("Threat protection config updated", {
+ teamId: req.auth.team_id,
+ orgId,
+ mode: updated.policy.mode,
+ changedFields: changedFields(previous, updated),
+ });
+
+ res.status(200).json({
+ success: true,
+ data: serializeConfig(updated),
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 61ee4ad23..37b95e62a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -28,6 +28,7 @@ import {
import { BrandingProfile } from "../../types/branding";
import { ProductProfile } from "../../types/product";
import { MenuProfile } from "../../types/menu";
+import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";
// Base URL schema with common validation logic
export const URL = z.preprocess(
@@ -692,6 +693,9 @@ const baseScrapeOptions = z.strictObject({
storeInCache: z.boolean().prefault(true),
lockdown: z.boolean().prefault(false),
redactPII: redactPIISchema,
+ // Enterprise: per-request field-level override of the org's threat
+ // protection policy. Gated on the team flag + org config (checkPermissions).
+ threatProtection: threatProtectionOverrideSchema.optional(),
profile: z
.object({
@@ -880,6 +884,7 @@ const extractOptions = z
__experimental_showCostTracking: z.boolean().prefault(false),
ignoreInvalidURLs: z.boolean().prefault(true),
webhook: webhookSchema.optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
})
.refine(obj => obj.urls || obj.prompt, {
error: "Either 'urls' or 'prompt' must be provided.",
@@ -940,6 +945,7 @@ export const agentRequestSchema = z.strictObject({
overrideWhitelist: z.string().optional(),
model: z.enum(["spark-1-pro", "spark-1-mini"]).default("spark-1-pro"),
+ threatProtection: threatProtectionOverrideSchema.optional(),
});
export type AgentRequest = z.infer<typeof agentRequestSchema>;
@@ -1191,6 +1197,7 @@ const mapRequestSchemaBase = crawlerOptions
ignoreCache: z.boolean().prefault(false),
location: locationSchema,
headers: z.record(z.string(), z.string()).optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
});
export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);
@@ -1534,6 +1541,7 @@ type Account = {
export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
+ threatProtection?: "disabled" | "allowed" | "forced";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
@@ -1943,6 +1951,7 @@ export const searchRequestSchema = z
// scrapeURL. Falls back to the provider snippet when the URL isn't indexed.
highlights: z.boolean().optional().prefault(false),
__searchPreviewToken: z.string().optional(),
+ threatProtection: threatProtectionOverrideSchema.optional(),
scrapeOptions: baseScrapeOptions
.extend({
formats: z
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 8fbdc3c4c..fa29eb7ae 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -654,6 +654,19 @@ export const teams = pgTable("teams", {
org_id: uuid("org_id").notNull(),
});
+// Org-scoped threat protection configuration (enterprise feature, gated by the
+// `threatProtection` team flag). One row per org; DDL is applied out-of-band.
+// The policy document lives in the `config` jsonb column (parsed tolerantly in
+// lib/threat-protection/store.ts); only `mode` is a dedicated column.
+export const threat_protection_config = pgTable("threat_protection_config", {
+ id: uuid("id").notNull().defaultRandom(),
+ org_id: uuid("org_id").notNull().unique(),
+ mode: varchar("mode").notNull().default("off"),
+ config: jsonb("config").notNull().default({}),
+ created_at: ts("created_at").notNull().defaultNow(),
+ updated_at: ts("updated_at").notNull().defaultNow(),
+});
+
export const user_notifications = pgTable("user_notifications", {
id: uuid("id").notNull().defaultRandom(),
team_id: uuid("team_id"),
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index f82c9feee..697fb774c 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -73,6 +73,28 @@ export async function recordRobotsBlocked(crawlId: string, url: string) {
);
}
+/**
+ * Records a URL that was silently skipped during crawl link discovery because
+ * its domain was blocked by the team's threat protection policy. The crawl
+ * continues without it; the record (url -> full ThreatDecision JSON) is kept
+ * for the security-logging layer to read.
+ *
+ * ZDR posture: this hash follows the same rules as the rest of the transient
+ * crawl bookkeeping in this file (crawl doc, visited sets, robots_blocked) β€”
+ * it necessarily holds URLs while the crawl runs, lives in the evict Redis,
+ * and expires after 24h like every other crawl key. Additionally, for
+ * zero-data-retention crawls it is deleted eagerly in {@link finishCrawl}.
+ */
+export async function recordThreatBlocked(
+ crawlId: string,
+ url: string,
+ decision: unknown,
+) {
+ const key = "crawl:" + crawlId + ":threat_blocked";
+ await redisEvictConnection.hset(key, url, JSON.stringify(decision));
+ await redisEvictConnection.expire(key, 24 * 60 * 60);
+}
+
export async function markCrawlActive(id: string) {
await redisEvictConnection.sadd("active_crawls", id);
}
@@ -331,6 +353,13 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {
// Clear visited sets to save memory
await redisEvictConnection.del("crawl:" + id + ":visited");
await redisEvictConnection.del("crawl:" + id + ":visited_unique");
+
+ // Eagerly drop the threat-protection bookkeeping (URL -> decision records
+ // of silently skipped discoveries) instead of letting it ride out its 24h
+ // TTL. Nothing reads it after the crawl finishes, and deleting it
+ // unconditionally keeps the ZDR guarantee even when the crawl document is
+ // missing or unreadable at finish time.
+ await redisEvictConnection.del("crawl:" + id + ":threat_blocked");
}
export async function getCrawlJobs(id: string): Promise<string[]> {
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index cdf4ddf17..935ef612b 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -35,6 +35,7 @@ import {
VideoUnsupportedUrlError,
XTwitterConfigurationError,
} from "../scraper/scrapeURL/error";
+import { UnsafeDomainBlockedError } from "./threat-protection/error";
// TODO: figure out correct typing for this
const errorMap: Record<ErrorCodes, any> = {
@@ -69,6 +70,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_VIDEO_UNSUPPORTED_URL: VideoUnsupportedUrlError,
SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
MAP_FAILED: MapFailedError,
+ unsafe_domain_blocked: UnsafeDomainBlockedError,
// Zod errors
BAD_REQUEST: null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index d45ffdd5d..62df2572b 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -32,7 +32,10 @@ export type ErrorCodes =
| "CRAWL_DENIAL"
| "MAP_FAILED"
| "BAD_REQUEST_INVALID_JSON"
- | "BAD_REQUEST";
+ | "BAD_REQUEST"
+ // Threat protection (enterprise domain risk blocking). Lowercase by design:
+ // this is the documented, user-facing error code for the feature.
+ | "unsafe_domain_blocked";
export class TransportableError extends Error {
public readonly code: ErrorCodes;
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 0b884f62a..08a0b0b77 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -12,6 +12,7 @@ import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
import { scrapeQueue } from "../../../services/worker/nuq-router";
+import type { ThreatProtectionPolicy } from "../../threat-protection/types";
interface ScrapeDocumentOptions {
url: string;
@@ -22,6 +23,8 @@ interface ScrapeDocumentOptions {
flags: TeamFlags | null;
apiKeyId: number | null;
requestId?: string;
+ /** Effective threat protection policy; enforced in the scrape pipeline. */
+ threatProtectionPolicy?: ThreatProtectionPolicy | null;
}
export async function scrapeDocument_F0(
@@ -66,6 +69,7 @@ export async function scrapeDocument_F0(
internalOptions: {
teamId: options.teamId,
bypassBilling: true,
+ threatProtection: options.threatProtectionPolicy ?? undefined,
},
origin: options.origin,
is_scrape: true,
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index fecfa4b27..aeb3224a1 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -39,6 +39,7 @@ import {
} from "./usage/llm-cost-f0";
import { SourceTracker_F0 } from "./helpers/source-tracker-f0";
import { getACUCTeam } from "../../../controllers/auth";
+import { resolveThreatProtection } from "../../threat-protection/request";
interface ExtractServiceOptions {
request: ExtractRequest;
@@ -92,6 +93,18 @@ export async function performExtraction_F0(
const acuc = await getACUCTeam(teamId);
+ // Threat protection: resolve the effective policy once for this extract
+ // job (org config + the request's threatProtection override, which the
+ // controller already validated). Threaded into every document scrape so
+ // both user-provided and discovered URLs are enforced in the pipeline.
+ const threatProtectionPolicy = (
+ await resolveThreatProtection({
+ teamId,
+ flags: acuc?.flags ?? null,
+ override: request.threatProtection,
+ })
+ ).policy;
+
const logger = _logger.child({
module: "extract",
method: "performExtraction",
@@ -341,6 +354,7 @@ export async function performExtraction_F0(
flags: acuc?.flags ?? null,
apiKeyId,
requestId: extractId,
+ threatProtectionPolicy,
},
urlTraces,
logger.child({
@@ -636,6 +650,7 @@ export async function performExtraction_F0(
flags: acuc?.flags ?? null,
apiKeyId,
requestId: extractId,
+ threatProtectionPolicy,
},
urlTraces,
logger.child({
diff --git a/apps/api/src/lib/permissions.test.ts b/apps/api/src/lib/permissions.test.ts
new file mode 100644
index 000000000..d97818301
--- /dev/null
+++ b/apps/api/src/lib/permissions.test.ts
@@ -0,0 +1,55 @@
+import { checkPermissions } from "./permissions";
+
+describe("checkPermissions β€” threat protection", () => {
+ const requestWithOption = { threatProtection: { mode: "normal" } };
+
+ it("allows requests without a threatProtection option regardless of flags", () => {
+ expect(checkPermissions({}, null)).toEqual({});
+ expect(checkPermissions({}, { threatProtection: "disabled" })).toEqual({});
+ });
+
+ it("rejects a per-request option when the flag is missing or disabled", () => {
+ expect(checkPermissions(requestWithOption, null).error).toMatch(
+ /enterprise feature/,
+ );
+ expect(
+ checkPermissions(requestWithOption, { threatProtection: "disabled" })
+ .error,
+ ).toMatch(/enterprise feature/);
+ });
+
+ it.each(["allowed", "forced"] as const)(
+ "allows a per-request option when the flag is %s",
+ mode => {
+ expect(
+ checkPermissions(requestWithOption, { threatProtection: mode }),
+ ).toEqual({});
+ },
+ );
+
+ it("rejects a per-request option when the org disables overrides", () => {
+ const result = checkPermissions(
+ requestWithOption,
+ { threatProtection: "allowed" },
+ { threatProtectionOrgConfig: { allowRequestOverrides: false } },
+ );
+ expect(result.error).toMatch(/overrides are disabled/);
+ });
+
+ it("allows a per-request option when the org config allows overrides", () => {
+ expect(
+ checkPermissions(
+ requestWithOption,
+ { threatProtection: "allowed" },
+ { threatProtectionOrgConfig: { allowRequestOverrides: true } },
+ ),
+ ).toEqual({});
+ expect(
+ checkPermissions(
+ requestWithOption,
+ { threatProtection: "allowed" },
+ { threatProtectionOrgConfig: null },
+ ),
+ ).toEqual({});
+ });
+});
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index 4f7478072..bb116f47a 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -3,20 +3,40 @@ import {
getScrapeZDR,
getIgnoreRobots,
getCustomRobotsAgent,
+ getThreatProtection,
} from "./zdr-helpers";
+import {
+ THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,
+ THREAT_PROTECTION_NOT_ENABLED_MESSAGE,
+ THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,
+} from "./threat-protection/request";
type LocationOptions = { country?: string };
+type ThreatProtectionOption = { mode?: string };
interface APIRequest {
zeroDataRetention?: boolean;
location?: LocationOptions;
scrapeOptions?: {
location?: LocationOptions;
+ threatProtection?: ThreatProtectionOption;
};
crawlerOptions?: {
ignoreRobotsTxt?: boolean;
robotsUserAgent?: string;
};
+ // Per-request threat protection policy override (field-level override of
+ // the org config). Presence of any value gates on the team flag.
+ threatProtection?: ThreatProtectionOption;
+}
+
+interface PermissionOptions {
+ /**
+ * Org-level threat protection config (or the relevant slice of it), if
+ * already loaded. When the org disables request overrides, any per-request
+ * threatProtection option is rejected.
+ */
+ threatProtectionOrgConfig?: { allowRequestOverrides: boolean } | null;
}
const SUPPORT_EMAIL = "[email protected]";
@@ -24,6 +44,7 @@ const SUPPORT_EMAIL = "[email protected]";
export function checkPermissions(
request: APIRequest,
flags?: TeamFlags,
+ options?: PermissionOptions,
): { error?: string } {
// zdr perms β€” scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDR
const scrapeMode = getScrapeZDR(flags);
@@ -59,6 +80,25 @@ export function checkPermissions(
};
}
+ // threat protection perms β€” the flag must be 'allowed' or 'forced' for any
+ // per-request threatProtection option, the org must not have locked down
+ // request-level overrides, and a 'forced' team may never disable the
+ // feature per-request.
+ const threatProtectionOption =
+ request.threatProtection ?? request.scrapeOptions?.threatProtection;
+ if (threatProtectionOption !== undefined) {
+ const threatMode = getThreatProtection(flags);
+ if (threatMode !== "allowed" && threatMode !== "forced") {
+ return { error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE };
+ }
+ if (options?.threatProtectionOrgConfig?.allowRequestOverrides === false) {
+ return { error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE };
+ }
+ if (threatMode === "forced" && threatProtectionOption.mode === "off") {
+ return { error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE };
+ }
+ }
+
// ip whitelist perms
const needsWhitelist =
request.location?.country === "us-whitelist" ||
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index cdf49cb52..68d80fb5d 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -1,4 +1,9 @@
-import { calculateCreditsToBeBilled } from "./scrape-billing";
+import {
+ calculateCreditsToBeBilled,
+ calculateThreatScanCredits,
+} from "./scrape-billing";
+import { UnsafeDomainBlockedError } from "./threat-protection/error";
+import type { ThreatDecision } from "./threat-protection/types";
describe("calculateCreditsToBeBilled", () => {
it("bills handled data layer successes at 15 credits", async () => {
@@ -114,3 +119,190 @@ describe("calculateCreditsToBeBilled", () => {
expect(credits).toBe(3);
});
});
+
+// =========================================
+// Threat protection scan fees (ENG-4985)
+// =========================================
+
+const consulted = (allowed = true): ThreatDecision => ({
+ allowed,
+ rule: allowed ? "default-allow" : "risk-score",
+ providerConsulted: true,
+ verdict: {
+ provider: "google-web-risk",
+ riskScore: allowed ? 0 : 100,
+ categories: [],
+ fromCache: false,
+ raw: {},
+ },
+ mode: "normal",
+});
+
+const localOnly = (
+ rule: ThreatDecision["rule"],
+ allowed: boolean,
+): ThreatDecision => ({
+ allowed,
+ rule,
+ providerConsulted: false,
+ verdict: null,
+ mode: "normal",
+});
+
+const billWithDecisions = (args: {
+ document: { metadata: Record<string, unknown> } | null;
+ error?: Error | null;
+ threatDecisions?: ThreatDecision[];
+}) =>
+ calculateCreditsToBeBilled(
+ { formats: [{ type: "markdown" }] } as any,
+ { teamId: "team-id" },
+ args.document as any,
+ { totalCost: 0 } as any,
+ {} as any,
+ args.error,
+ undefined,
+ undefined,
+ args.threatDecisions,
+ );
+
+const successDocument = {
+ metadata: { statusCode: 200, proxyUsed: "basic" },
+};
+
+describe("calculateThreatScanCredits", () => {
+ it("bills nothing for no decisions", () => {
+ expect(calculateThreatScanCredits([])).toBe(0);
+ });
+
+ it("bills +2 per consulted decision", () => {
+ expect(calculateThreatScanCredits([consulted()])).toBe(2);
+ expect(calculateThreatScanCredits([consulted(), consulted()])).toBe(4);
+ });
+
+ it("bills consulted decisions regardless of the allow/block outcome", () => {
+ expect(calculateThreatScanCredits([consulted(false)])).toBe(2);
+ });
+
+ it("bills cached verdicts the same as fresh ones", () => {
+ const cached = consulted();
+ cached.verdict = { ...cached.verdict!, fromCache: true };
+ expect(calculateThreatScanCredits([cached])).toBe(2);
+ });
+
+ it("never bills local-only decisions", () => {
+ expect(
+ calculateThreatScanCredits([
+ localOnly("whitelist", true),
+ localOnly("blacklist", false),
+ localOnly("blocked-tld", false),
+ localOnly("provider-failure", false),
+ ]),
+ ).toBe(0);
+ });
+
+ it("sums mixed decisions", () => {
+ expect(
+ calculateThreatScanCredits([
+ consulted(),
+ consulted(false),
+ localOnly("blacklist", false),
+ ]),
+ ).toBe(4);
+ });
+});
+
+describe("calculateCreditsToBeBilled β€” threat protection scan fees", () => {
+ it("adds +2 to a successful scrape with a consulted decision", async () => {
+ expect(
+ await billWithDecisions({
+ document: successDocument,
+ threatDecisions: [consulted()],
+ }),
+ ).toBe(3);
+ });
+
+ it("bills each consulted decision on a redirect (two domains scanned)", async () => {
+ expect(
+ await billWithDecisions({
+ document: successDocument,
+ threatDecisions: [consulted(), consulted()],
+ }),
+ ).toBe(5);
+ });
+
+ it("adds nothing for local-only decisions on success", async () => {
+ expect(
+ await billWithDecisions({
+ document: successDocument,
+ threatDecisions: [localOnly("whitelist", true)],
+ }),
+ ).toBe(1);
+ });
+
+ it("bills the scan fee (and no base cost) for a blocked scrape", async () => {
+ const decision = consulted(false);
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),
+ threatDecisions: [decision],
+ }),
+ ).toBe(2);
+ });
+
+ it("bills a blocked scrape from the error decision when the decisions array is missing", async () => {
+ const decision = consulted(false);
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),
+ }),
+ ).toBe(2);
+ });
+
+ it("does not double-bill when the error decision is also in the decisions array", async () => {
+ const initial = consulted();
+ const redirectBlock = consulted(false);
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new UnsafeDomainBlockedError(
+ "blocked.example.com",
+ redirectBlock,
+ ),
+ threatDecisions: [initial, redirectBlock],
+ }),
+ ).toBe(4);
+ });
+
+ it("bills nothing for a blocked scrape decided by local-only rules", async () => {
+ const decision = localOnly("blacklist", false);
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),
+ threatDecisions: [decision],
+ }),
+ ).toBe(0);
+ });
+
+ it("bills nothing for unrelated failures without decisions", async () => {
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new Error("engine failure"),
+ }),
+ ).toBe(0);
+ });
+
+ it("adds the scan fee on top of other failure billing (scan happened before the failure)", async () => {
+ expect(
+ await billWithDecisions({
+ document: null,
+ error: new Error("engine failure"),
+ threatDecisions: [consulted()],
+ }),
+ ).toBe(2);
+ });
+});
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 23b334c84..bef1ebdc9 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -14,6 +14,8 @@ import {
DataLayerScrapeMetadata,
getDataLayerSuccessCredits,
} from "./data-layer";
+import type { ThreatDecision } from "./threat-protection/types";
+import { UnsafeDomainBlockedError } from "./threat-protection/error";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
@@ -23,6 +25,29 @@ const redactPIICostBonus = 4;
// Each additional PDF page also gets redacted through fire-privacy, so
// the per-page surcharge mirrors the +4 base β€” same tier as lockdown.
const redactPIIPdfPageCostBonus = 4;
+// Threat protection domain scans: +2 per scanned domain in "normal" mode
+// (Google Web Risk). A "scan" is any ThreatDecision with providerConsulted
+// set β€” +2 per consulted verdict. Checks deduplicated within one request/job
+// (e.g. a scrape and its same-domain redirect re-check) share a single
+// decision and therefore bill once; verdicts are never reused across
+// requests (no verdict cache β€” ZDR). Local-only decisions (whitelist/
+// blacklist/blocked-tld, mode off, provider failure) never bill.
+const threatScanCost = 2;
+
+/**
+ * Sums the scan fees for a set of threat protection decisions. Only decisions
+ * that consulted the provider bill; the fee is +2 per scanned domain.
+ */
+export function calculateThreatScanCredits(
+ decisions: Iterable<ThreatDecision>,
+): number {
+ let credits = 0;
+ for (const decision of decisions) {
+ if (!decision.providerConsulted) continue;
+ credits += threatScanCost;
+ }
+ return credits;
+}
export async function calculateCreditsToBeBilled(
options: ScrapeOptions,
@@ -33,10 +58,28 @@ export async function calculateCreditsToBeBilled(
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
dataLayer?: DataLayerScrapeMetadata,
+ // Threat protection decisions for this scrape (initial + redirect checks,
+ // in order). Each decision with `providerConsulted` bills a scan fee (+2
+ // per scanned domain) on top of the scrape's own cost β€” on both success
+ // and failure (a scrape blocked by threat protection still consulted the
+ // classifier). For scrapes blocked by threat protection, the
+ // UnsafeDomainBlockedError in `error` also carries its decision, which is
+ // used as a fallback when the decisions array did not make it here.
+ threatDecisions?: ThreatDecision[],
) {
const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =
costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;
+ const effectiveThreatDecisions: ThreatDecision[] =
+ threatDecisions && threatDecisions.length > 0
+ ? threatDecisions
+ : error instanceof UnsafeDomainBlockedError
+ ? [error.decision]
+ : [];
+ const threatScanCredits = calculateThreatScanCredits(
+ effectiveThreatDecisions,
+ );
+
if (document === null) {
// Failure -- check cost tracking if FIRE-1
let creditsToBeBilled = 0;
@@ -57,7 +100,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = 1;
}
- return creditsToBeBilled;
+ // Failed scrapes bill no base cost (except the cases above), but threat
+ // protection scans that already happened still bill β€” including scrapes
+ // blocked by the policy itself.
+ return creditsToBeBilled + threatScanCredits;
}
const dataLayerCredits = getDataLayerSuccessCredits({
@@ -65,7 +111,7 @@ export async function calculateCreditsToBeBilled(
statusCode: document.metadata?.statusCode,
});
if (dataLayerCredits !== null) {
- return dataLayerCredits;
+ return dataLayerCredits + threatScanCredits;
}
let creditsToBeBilled = 1; // Assuming 1 credit per document
@@ -167,5 +213,7 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += unblockedDomainCostBonus;
}
+ creditsToBeBilled += threatScanCredits;
+
return creditsToBeBilled;
}
diff --git a/apps/api/src/lib/threat-protection/check-domain.test.ts b/apps/api/src/lib/threat-protection/check-domain.test.ts
new file mode 100644
index 000000000..e0c0a2872
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/check-domain.test.ts
@@ -0,0 +1,279 @@
+import http from "http";
+import { AddressInfo } from "net";
+
+// checkDomain is enforcement-only: it emits/exports no security events, so
+// there are no event sinks to stub here.
+// The Web Risk threat-list store lives on the durable Redis connection β€”
+// swap in an in-memory fake. (fake-redis.ts has no runtime imports, so the
+// factory cannot re-enter the module being mocked.)
+vi.mock("../../services/queue-service", async () => {
+ const { createFakeWebRiskRedis } = await import(
+ "./providers/web-risk/fake-redis.js"
+ );
+ const client = createFakeWebRiskRedis();
+ return { getRedisConnection: () => client };
+});
+
+import { config } from "../../config";
+import {
+ checkDomain,
+ THREAT_PROTECTION_POLICY_DEFAULTS,
+ ThreatCheckDedup,
+ ThreatProtectionPolicy,
+ UnsafeDomainBlockedError,
+} from "./index";
+import {
+ createWebRiskMockCounters,
+ createWebRiskMockHandler,
+ WebRiskMockDatabase,
+} from "./providers/web-risk/testing";
+
+function policy(
+ overrides: Partial<ThreatProtectionPolicy> = {},
+): ThreatProtectionPolicy {
+ return {
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ ...overrides,
+ };
+}
+
+// The mock provider: a flagged fixture domain in the MALWARE list, served
+// through the Update API endpoints (computeDiff for the local list sync,
+// hashes:search for prefix-hit confirmation).
+const RISKY_DOMAIN = "threat-risky.example";
+
+const db = new WebRiskMockDatabase();
+db.addRiskyDomain(RISKY_DOMAIN, "MALWARE");
+
+const counters = createWebRiskMockCounters();
+const webRiskHandler = createWebRiskMockHandler(db, counters);
+
+// While > 0, hashes:search requests fail with 503 (decremented per request);
+// Infinity = permanently down (provider-failure paths).
+let failHashesSearches = 0;
+
+let server: http.Server;
+
+const originalConfig = {
+ webRiskUrl: config.GOOGLE_WEB_RISK_API_URL,
+ webRiskKey: config.GOOGLE_WEB_RISK_API_KEY,
+};
+
+beforeAll(async () => {
+ await new Promise<void>(resolve => {
+ server = http.createServer((req, res) => {
+ if (
+ failHashesSearches > 0 &&
+ (req.url ?? "").startsWith("/v1/hashes:search")
+ ) {
+ failHashesSearches--;
+ res.statusCode = 503;
+ res.end("{}");
+ return;
+ }
+ if (!webRiskHandler(req, res)) {
+ res.statusCode = 404;
+ res.end("{}");
+ }
+ });
+ server.listen(0, "127.0.0.1", () => {
+ const addr = server.address() as AddressInfo;
+ config.GOOGLE_WEB_RISK_API_URL = `http://127.0.0.1:${addr.port}`;
+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";
+ resolve();
+ });
+ });
+});
+
+afterAll(async () => {
+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.webRiskUrl;
+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.webRiskKey;
+ await new Promise<void>(resolve => server.close(() => resolve()));
+});
+
+beforeEach(() => {
+ failHashesSearches = 0;
+});
+
+const riskyHits = () => counters.hashesSearchRequestsForDomain(RISKY_DOMAIN);
+
+describe("checkDomain", () => {
+ it("allows immediately when mode is off, with no provider call", async () => {
+ const before = counters.hashesSearchRequests;
+ const decision = await checkDomain("example.com", policy({ mode: "off" }), {
+ teamId: "team-1",
+ });
+
+ expect(decision).toEqual({
+ allowed: true,
+ rule: "default-allow",
+ providerConsulted: false,
+ verdict: null,
+ mode: "off",
+ });
+ expect(counters.hashesSearchRequests).toBe(before);
+ });
+
+ it("skips the provider scan when a local rule is decisive", async () => {
+ const before = counters.hashesSearchRequests;
+ const decision = await checkDomain(
+ "cdn.blocked.com",
+ policy({ blacklist: ["blocked.com"] }),
+ { teamId: "team-1" },
+ );
+
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "blacklist",
+ providerConsulted: false,
+ verdict: null,
+ });
+ expect(counters.hashesSearchRequests).toBe(before);
+ });
+
+ it("consults the provider and blocks a flagged domain (fresh scan)", async () => {
+ const before = riskyHits();
+ const decision = await checkDomain(RISKY_DOMAIN.toUpperCase(), policy(), {
+ teamId: "team-1",
+ });
+
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "risk-score",
+ providerConsulted: true,
+ mode: "normal",
+ });
+ expect(decision.verdict).toMatchObject({
+ provider: "google-web-risk",
+ riskScore: 100,
+ categories: ["MALWARE"],
+ fromCache: false,
+ });
+ // The threat list synced locally; the hit was confirmed by exactly one
+ // hashes:search call carrying only the anonymized hash prefix.
+ expect(counters.computeDiffRequests).toBeGreaterThanOrEqual(3);
+ expect(riskyHits()).toBe(before + 1);
+ });
+
+ it("resolves clean domains locally with zero Google calls", async () => {
+ const before = counters.hashesSearchRequests;
+ const decision = await checkDomain("safe.example", policy(), {});
+
+ expect(decision).toMatchObject({
+ allowed: true,
+ rule: "default-allow",
+ providerConsulted: true,
+ });
+ expect(decision.verdict?.fromCache).toBe(false);
+ expect(counters.hashesSearchRequests).toBe(before);
+ });
+
+ it("re-scans on a second lookup without a dedup handle (no verdict cache)", async () => {
+ const before = riskyHits();
+
+ const first = await checkDomain(RISKY_DOMAIN, policy(), {});
+ const second = await checkDomain(RISKY_DOMAIN, policy(), {});
+
+ expect(first).toMatchObject({ allowed: false, providerConsulted: true });
+ expect(second).toMatchObject({ allowed: false, providerConsulted: true });
+ expect(second).not.toBe(first);
+ // Two independent scans, two confirmation calls β€” nothing was cached.
+ expect(riskyHits()).toBe(before + 2);
+ });
+
+ it("shares one in-flight decision within a request via ctx.dedup", async () => {
+ const before = riskyHits();
+ const dedup: ThreatCheckDedup = new Map();
+
+ // Concurrent + sequential re-checks of the same domain in one request.
+ const [first, second] = await Promise.all([
+ checkDomain(RISKY_DOMAIN, policy(), { dedup }),
+ checkDomain(RISKY_DOMAIN, policy(), { dedup }),
+ ]);
+ const third = await checkDomain(`${RISKY_DOMAIN}.`, policy(), { dedup });
+
+ // Same decision object β†’ a single scan β†’ a single billable decision.
+ expect(second).toBe(first);
+ expect(third).toBe(first); // normalization applies before dedup
+ expect(riskyHits()).toBe(before + 1);
+ });
+
+ it("scans each distinct domain in a dedup scope separately", async () => {
+ const dedup: ThreatCheckDedup = new Map();
+ const before = riskyHits();
+
+ const clean = await checkDomain("safe.example", policy(), { dedup });
+ const risky = await checkDomain(RISKY_DOMAIN, policy(), { dedup });
+
+ expect(clean.allowed).toBe(true);
+ expect(risky.allowed).toBe(false);
+ expect(riskyHits()).toBe(before + 1);
+ });
+
+ it("retries once and succeeds when the first confirmation attempt fails", async () => {
+ failHashesSearches = 1; // exactly the first hashes:search attempt 503s
+
+ const decision = await checkDomain(RISKY_DOMAIN, policy(), {});
+
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "risk-score",
+ providerConsulted: true,
+ });
+ expect(failHashesSearches).toBe(0);
+ });
+
+ it("fails closed when confirmation is down and failurePolicy is closed", async () => {
+ failHashesSearches = Infinity;
+
+ const decision = await checkDomain(
+ RISKY_DOMAIN,
+ policy({ failurePolicy: "closed" }),
+ { teamId: "team-1" },
+ );
+
+ expect(decision).toEqual({
+ allowed: false,
+ rule: "provider-failure",
+ providerConsulted: false,
+ verdict: null,
+ mode: "normal",
+ });
+ });
+
+ it("fails open when confirmation is down and failurePolicy is open", async () => {
+ failHashesSearches = Infinity;
+
+ const decision = await checkDomain(
+ RISKY_DOMAIN,
+ policy({ failurePolicy: "open" }),
+ {},
+ );
+
+ expect(decision).toMatchObject({
+ allowed: true,
+ rule: "provider-failure",
+ providerConsulted: false,
+ verdict: null,
+ });
+ });
+});
+
+describe("UnsafeDomainBlockedError", () => {
+ it("carries the decision and the unsafe_domain_blocked code", async () => {
+ const decision = await checkDomain(
+ "bad.example",
+ policy({ blacklist: ["bad.example"] }),
+ {},
+ );
+ const error = new UnsafeDomainBlockedError("bad.example", decision);
+
+ expect(error.code).toBe("unsafe_domain_blocked");
+ expect(error.name).toBe("UnsafeDomainBlockedError");
+ expect(error.domain).toBe("bad.example");
+ expect(error.decision).toBe(decision);
+ expect(error.message).toContain("threat protection policy");
+ expect(error.message).toContain("blacklist");
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/config.test.ts b/apps/api/src/lib/threat-protection/config.test.ts
new file mode 100644
index 000000000..d43ddd820
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/config.test.ts
@@ -0,0 +1,152 @@
+import {
+ threatProtectionConfigSchema,
+ threatProtectionOverrideSchema,
+ threatProtectionPolicySchema,
+} from "./config";
+import { THREAT_PROTECTION_POLICY_DEFAULTS } from "./types";
+
+describe("threatProtectionPolicySchema", () => {
+ it("applies defaults for a minimal document", () => {
+ const policy = threatProtectionPolicySchema.parse({ mode: "off" });
+ expect(policy).toEqual({
+ mode: "off",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ });
+ });
+
+ it("accepts a full valid document", () => {
+ const policy = threatProtectionPolicySchema.parse({
+ mode: "normal",
+ riskScoreThreshold: 50,
+ blacklist: ["bad.example.com", "*.malware.example"],
+ whitelist: ["example.com", "*.example.org"],
+ blockedTlds: ["zip", "mov"],
+ failurePolicy: "open",
+ });
+ expect(policy.mode).toBe("normal");
+ expect(policy.riskScoreThreshold).toBe(50);
+ expect(policy.failurePolicy).toBe("open");
+ });
+
+ it("rejects an invalid mode", () => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({ mode: "paranoid" }),
+ ).toThrow();
+ });
+
+ it('rejects the retired "enhanced" mode', () => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({ mode: "enhanced" }),
+ ).toThrow();
+ expect(() =>
+ threatProtectionOverrideSchema.parse({ mode: "enhanced" }),
+ ).toThrow();
+ expect(() =>
+ threatProtectionConfigSchema.parse({ mode: "enhanced" }),
+ ).toThrow();
+ });
+
+ it("rejects unknown keys", () => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({ mode: "off", nope: true }),
+ ).toThrow();
+ });
+
+ it("rejects retired policy fields as unknown keys", () => {
+ for (const retired of [
+ { deniedCategories: ["Malicious"] },
+ { maxDomainAgeDays: 30 },
+ { blockedCountries: ["KP"] },
+ ]) {
+ expect(() =>
+ threatProtectionPolicySchema.parse({ mode: "normal", ...retired }),
+ ).toThrow();
+ expect(() => threatProtectionOverrideSchema.parse(retired)).toThrow();
+ }
+ });
+
+ describe("riskScoreThreshold", () => {
+ it.each([0, 100, 75])("accepts %d", value => {
+ expect(
+ threatProtectionPolicySchema.parse({
+ mode: "normal",
+ riskScoreThreshold: value,
+ }).riskScoreThreshold,
+ ).toBe(value);
+ });
+
+ it.each([-1, 101, 50.5])("rejects %d", value => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({
+ mode: "normal",
+ riskScoreThreshold: value,
+ }),
+ ).toThrow();
+ });
+ });
+
+ describe("blacklist / whitelist globs", () => {
+ it("normalizes case and whitespace", () => {
+ const policy = threatProtectionPolicySchema.parse({
+ mode: "normal",
+ blacklist: [" Bad.Example.COM ", "*.Evil.Example"],
+ });
+ expect(policy.blacklist).toEqual(["bad.example.com", "*.evil.example"]);
+ });
+
+ it.each([
+ "https://example.com",
+ "example.com/path",
+ "example.com:8080",
+ "*.",
+ "*",
+ "*.*.example.com",
+ "foo",
+ "ex ample.com",
+ "-bad-.example.com",
+ "exa_mple.com",
+ ])("rejects garbage entry %j with a clear message", entry => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({
+ mode: "normal",
+ blacklist: [entry],
+ }),
+ ).toThrow(/Invalid domain entry/);
+ });
+ });
+
+ describe("blockedTlds", () => {
+ it("accepts and normalizes TLDs", () => {
+ const policy = threatProtectionPolicySchema.parse({
+ mode: "normal",
+ blockedTlds: ["ZIP", " mov "],
+ });
+ expect(policy.blockedTlds).toEqual(["zip", "mov"]);
+ });
+
+ it.each([".zip", "z!p", "co.uk", ""])("rejects %j", entry => {
+ expect(() =>
+ threatProtectionPolicySchema.parse({
+ mode: "normal",
+ blockedTlds: [entry],
+ }),
+ ).toThrow(/Invalid TLD/);
+ });
+ });
+});
+
+describe("threatProtectionConfigSchema", () => {
+ it("applies defaults for allowRequestOverrides", () => {
+ const config = threatProtectionConfigSchema.parse({ mode: "normal" });
+ expect(config.allowRequestOverrides).toBe(true);
+ });
+
+ it("rejects the retired siem field", () => {
+ expect(() =>
+ threatProtectionConfigSchema.parse({
+ mode: "normal",
+ siem: { url: "https://siem.example.com/ingest" },
+ }),
+ ).toThrow();
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.ts
new file mode 100644
index 000000000..34d439546
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/config.ts
@@ -0,0 +1,113 @@
+import { z } from "zod";
+import {
+ THREAT_PROTECTION_POLICY_DEFAULTS,
+ type ThreatProtectionPolicy,
+} from "./types";
+
+// =========================================
+// Field schemas
+// =========================================
+
+// One DNS label: alphanumeric, optionally with inner hyphens, max 63 chars.
+const DOMAIN_LABEL = "[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?";
+// Plain domain ("example.com", "sub.example.co.uk") or a single leading
+// wildcard label ("*.example.com"). No protocol, path, port, or inner "*".
+const DOMAIN_GLOB_REGEX = new RegExp(
+ `^(\\*\\.)?(?:${DOMAIN_LABEL}\\.)+${DOMAIN_LABEL}$`,
+);
+
+const domainGlobSchema = z
+ .string()
+ .trim()
+ .toLowerCase()
+ .refine(value => value.length <= 253 && DOMAIN_GLOB_REGEX.test(value), {
+ error: iss =>
+ `Invalid domain entry ${JSON.stringify(iss.input)}: must be a plain domain like "example.com" or a wildcard glob like "*.example.com" (no protocol, path, or port)`,
+ });
+
+const tldSchema = z
+ .string()
+ .trim()
+ .toLowerCase()
+ .refine(value => /^[a-z0-9]{1,63}$/.test(value), {
+ error: iss =>
+ `Invalid TLD ${JSON.stringify(iss.input)}: must be a lowercase alphanumeric TLD without the leading dot, e.g. "zip"`,
+ });
+
+// =========================================
+// Policy + org config schemas
+// =========================================
+
+/**
+ * Field-for-field zod schema for {@link ThreatProtectionPolicy}. Every field
+ * except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.
+ */
+export const threatProtectionPolicySchema = z.strictObject({
+ mode: z.enum(["off", "normal"]),
+ riskScoreThreshold: z
+ .number()
+ .int()
+ .min(0)
+ .max(100)
+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.riskScoreThreshold),
+ blacklist: z
+ .array(domainGlobSchema)
+ .max(1000)
+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.blacklist),
+ whitelist: z
+ .array(domainGlobSchema)
+ .max(1000)
+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.whitelist),
+ blockedTlds: z
+ .array(tldSchema)
+ .max(1000)
+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.blockedTlds),
+ failurePolicy: z
+ .enum(["open", "closed"])
+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),
+});
+
+// Compile-time assertion that the schema output matches the shared contract
+// in ./types.ts β€” fails to typecheck if the two drift apart.
+const _policyContractCheck = (
+ x: z.infer<typeof threatProtectionPolicySchema>,
+): ThreatProtectionPolicy => x;
+void _policyContractCheck;
+
+/**
+ * Per-request `threatProtection` option: a field-level override of the org
+ * config. Mirrors {@link threatProtectionPolicySchema} but every field is
+ * optional and NO defaults are injected β€” only fields the caller explicitly
+ * provides replace the org policy's values (see `resolveEffectivePolicy`).
+ */
+export const threatProtectionOverrideSchema = z.strictObject({
+ mode: z.enum(["off", "normal"]).optional(),
+ riskScoreThreshold: z.number().int().min(0).max(100).optional(),
+ blacklist: z.array(domainGlobSchema).max(1000).optional(),
+ whitelist: z.array(domainGlobSchema).max(1000).optional(),
+ blockedTlds: z.array(tldSchema).max(1000).optional(),
+ failurePolicy: z.enum(["open", "closed"]).optional(),
+});
+
+type ThreatProtectionOverride = z.infer<typeof threatProtectionOverrideSchema>;
+
+// Compile-time assertion that the override shape stays assignable to
+// Partial<ThreatProtectionPolicy> (what `resolveEffectivePolicy` consumes).
+const _overrideContractCheck = (
+ x: ThreatProtectionOverride,
+): Partial<ThreatProtectionPolicy> => x;
+void _overrideContractCheck;
+
+/**
+ * Full org-level configuration document, as accepted by
+ * `PUT /v2/team/threat-protection`.
+ */
+export const threatProtectionConfigSchema = threatProtectionPolicySchema.extend(
+ {
+ allowRequestOverrides: z.boolean().prefault(true),
+ },
+);
+
+export type ThreatProtectionConfigInput = z.infer<
+ typeof threatProtectionConfigSchema
+>;
diff --git a/apps/api/src/lib/threat-protection/error.ts b/apps/api/src/lib/threat-protection/error.ts
new file mode 100644
index 000000000..bc12f70c6
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/error.ts
@@ -0,0 +1,40 @@
+import { ErrorCodes, TransportableError } from "../error";
+import type { ThreatDecision } from "./types";
+
+// Error surfaced when threat protection blocks a domain. Wired into the shared
+// TransportableError machinery (src/lib/error.ts + src/lib/error-serde.ts) so
+// blocked scrapes surface as a clean, documented API error β€” both on sync
+// responses (`code: "unsafe_domain_blocked"`) and on crawl/batch job document
+// errors. The full ThreatDecision rides along in serialize() so the billing
+// and security-logging layers can read it even across the job queue.
+
+export class UnsafeDomainBlockedError extends TransportableError {
+ constructor(
+ public readonly domain: string,
+ public readonly decision: ThreatDecision,
+ ) {
+ super(
+ "unsafe_domain_blocked",
+ `This domain (${domain}) is blocked by your organization's threat protection policy (rule: ${decision.rule}). ` +
+ `If you believe this is a mistake, contact your organization administrator to adjust the policy (e.g. whitelist the domain).`,
+ );
+ this.name = "UnsafeDomainBlockedError";
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ domain: this.domain,
+ decision: this.decision,
+ };
+ }
+
+ static deserialize(
+ _code: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new UnsafeDomainBlockedError(data.domain, data.decision);
+ x.stack = data.stack;
+ return x;
+ }
+}
diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.ts
new file mode 100644
index 000000000..b5f5e739a
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/index.ts
@@ -0,0 +1,184 @@
+import { logger } from "../logger";
+import { fetchGoogleWebRiskVerdict } from "./providers/google-web-risk";
+import type {
+ RawVerdict,
+ ThreatCheckDedup,
+ ThreatDecision,
+ ThreatProtectionMode,
+ ThreatProtectionPolicy,
+} from "./types";
+import { evaluatePolicy, localOnlyDecision, normalizeDomain } from "./verdict";
+
+/**
+ * Request context threaded into {@link checkDomain}. Enforcement-only: the
+ * feature no longer emits or exports security events (both the ClickHouse
+ * security log and the SIEM push were removed), so this carries just the
+ * request/job-scoped dedup handle plus the team id used for server-side
+ * logging. Everything is optional β€” call sites pass whatever is cheaply
+ * available.
+ */
+export interface ThreatCheckContext {
+ teamId?: string;
+ /**
+ * Request/job-scoped dedup handle (see {@link ThreatCheckDedup}). When set,
+ * checkDomain() reuses the in-flight decision for a domain already checked
+ * within this request instead of re-scanning it. Strictly in-memory and
+ * request-scoped β€” never persisted, never shared across requests (ZDR).
+ */
+ dedup?: ThreatCheckDedup;
+}
+
+// Public entry point for the threat protection core library (enterprise
+// domain risk blocking). Flow per domain:
+// 1. mode "off" β†’ allow, no provider, no billing
+// 2. request-scoped dedup (ctx.dedup) β†’ a domain already checked within
+// this request/job reuses the same in-flight decision
+// 3. local-only rules (whitelist/blacklist/blocked-tld) β†’ decide without a
+// provider call (no billing)
+// 4. provider ("normal" = Google Web Risk local hash-prefix database),
+// with a per-attempt timeout and one retry
+// 5. evaluate the policy against the verdict; provider failure β†’ the org's
+// failurePolicy decides (fail-open allows, fail-closed blocks)
+// Any decision backed by a verdict sets providerConsulted, which drives
+// billing (+2 credits per scanned domain) in the enforcement layer. This
+// module performs no billing or pipeline integration itself.
+//
+// ZDR boundary: verdicts are NEVER persisted β€” there is no cross-request
+// verdict cache (scrape-target domains must not be stored at rest). The only
+// reuse of a decision is via the caller-provided, request-scoped in-memory
+// dedup map. The org-config cache in ./store.ts is unaffected: it caches the
+// org's own settings, not scrape-derived data.
+//
+// This feature is enforcement-only: it emits and exports NO security events.
+// (There is no built-in audit trail β€” both the ClickHouse security log and
+// the SIEM push that once consumed decisions have been removed.)
+
+// Policy evaluation helpers (evaluatePolicy, localOnlyDecision) are exported
+// from ./verdict, and the shared contract types from ./types β€” import those
+// directly; index.ts only re-exports what checkDomain callers need.
+export { UnsafeDomainBlockedError } from "./error";
+export * from "./types";
+
+const PROVIDER_TIMEOUT_MS = 5000;
+const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry
+
+/**
+ * Single mode→provider dispatch point. Every provider is a separate module
+ * under ./providers exporting `fetch<X>Verdict(domain) β†’ RawVerdict`; this
+ * switch is the only place that knows which mode maps to which classifier.
+ * Deliberately kept as a dispatch even with one live branch β€” future partner
+ * classifiers add a mode + a case here and nothing else changes.
+ */
+async function fetchProviderVerdict(
+ domain: string,
+ mode: Exclude<ThreatProtectionMode, "off">,
+): Promise<RawVerdict> {
+ let lastError: unknown;
+ for (let attempt = 1; attempt <= PROVIDER_ATTEMPTS; attempt++) {
+ try {
+ const signal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS);
+ switch (mode) {
+ case "normal":
+ return await fetchGoogleWebRiskVerdict(domain, { signal });
+ }
+ } catch (error) {
+ lastError = error;
+ logger.warn("Threat protection provider lookup failed", {
+ canonicalLog: "threat-protection/provider",
+ domain,
+ mode,
+ attempt,
+ error,
+ });
+ }
+ }
+ throw lastError;
+}
+
+async function checkDomainFresh(
+ normalized: string,
+ policy: ThreatProtectionPolicy,
+ ctx: ThreatCheckContext,
+): Promise<ThreatDecision> {
+ // Local rules first: when whitelist/blacklist/blocked-tld are decisive we
+ // skip the paid provider scan entirely.
+ const local = localOnlyDecision(normalized, policy);
+ if (local !== null) {
+ return local;
+ }
+
+ let verdict: RawVerdict | null = null;
+ try {
+ verdict = await fetchProviderVerdict(
+ normalized,
+ policy.mode as Exclude<ThreatProtectionMode, "off">,
+ );
+ } catch {
+ // Already logged per-attempt; a null verdict routes the decision
+ // through the org's failurePolicy below.
+ verdict = null;
+ }
+
+ const decision = evaluatePolicy(normalized, verdict, policy);
+ if (!decision.allowed || decision.rule === "provider-failure") {
+ logger.info("Threat protection decision", {
+ canonicalLog: "threat-protection/check",
+ teamId: ctx.teamId,
+ domain: normalized,
+ mode: policy.mode,
+ allowed: decision.allowed,
+ rule: decision.rule,
+ providerConsulted: decision.providerConsulted,
+ riskScore: verdict?.riskScore ?? null,
+ categories: verdict?.categories ?? [],
+ rawVerdict: verdict?.raw,
+ });
+ }
+ return decision;
+}
+
+/**
+ * Classify a domain against an org's threat protection policy. Never throws:
+ * provider failures resolve through the policy's failurePolicy
+ * ("provider-failure" rule). Callers bill +2 credits per scanned domain when
+ * the returned decision has `providerConsulted` set.
+ *
+ * When `ctx.dedup` is provided (a request/job-scoped map created by the call
+ * site), repeated checks of the same domain within that scope share one
+ * in-flight decision: one scan, one billable consulted verdict. There is
+ * deliberately no cross-request or persisted verdict reuse (ZDR:
+ * scrape-target domains are never stored at rest).
+ *
+ * Enforcement-only: no security event is emitted or exported for a scan
+ * (neither to a ClickHouse log nor to a SIEM destination β€” both were
+ * removed). There is no built-in audit trail.
+ */
+export async function checkDomain(
+ domain: string,
+ policy: ThreatProtectionPolicy,
+ ctx: ThreatCheckContext,
+): Promise<ThreatDecision> {
+ const normalized = normalizeDomain(domain);
+
+ if (policy.mode === "off") {
+ return {
+ allowed: true,
+ rule: "default-allow",
+ providerConsulted: false,
+ verdict: null,
+ mode: "off",
+ };
+ }
+
+ if (ctx.dedup) {
+ const existing = ctx.dedup.get(normalized);
+ if (existing !== undefined) {
+ return existing;
+ }
+ const pending = checkDomainFresh(normalized, policy, ctx);
+ ctx.dedup.set(normalized, pending);
+ return pending;
+ }
+
+ return checkDomainFresh(normalized, policy, ctx);
+}
diff --git a/apps/api/src/lib/threat-protection/providers/google-web-risk.ts b/apps/api/src/lib/threat-protection/providers/google-web-risk.ts
new file mode 100644
index 000000000..cd4867cfd
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/google-web-risk.ts
@@ -0,0 +1,200 @@
+import { createHash } from "crypto";
+import { config } from "../../../config";
+import type { RawVerdict, ThreatProvider } from "../types";
+import { generateHostExpressions } from "./web-risk/canonicalize";
+import {
+ getWebRiskListStore,
+ WEB_RISK_THREAT_TYPES,
+ type WebRiskLookupResult,
+ type WebRiskThreatType,
+} from "./web-risk/store";
+import {
+ ensureThreatListBootSync,
+ ensureThreatListSyncLoop,
+} from "./web-risk/sync";
+
+// Google Web Risk provider ("normal" threat protection mode), built on the
+// Update API for ZDR compliance:
+//
+// 1. The domain is canonicalized into host-suffix expressions per the Safe
+// Browsing spec, each expression is SHA-256 hashed, and the 4-byte hash
+// prefixes are checked against the locally synced threat lists
+// (./web-risk/store.ts + ./web-risk/sync.ts). No network I/O, and the
+// target domain is never transmitted anywhere.
+// 2. Only when a local prefix matches (rare) is Google consulted:
+// GET /v1/hashes:search with the matched (anonymized, non-reversible)
+// hash prefix. The returned full hashes are compared locally β€” a full
+// match confirms the threat, a mismatch was a prefix collision (clean).
+// The response's ttl/negativeExpireTime metadata is deliberately ignored:
+// verdicts are never persisted (ZDR).
+//
+// Cost note: hashes:search is ~100x the price of the old uris:search per
+// call, so it must ONLY ever run to confirm a local prefix hit β€” never per
+// check. computeDiff (the sync path) is free.
+//
+// Web Risk gives a boolean-ish signal (flagged for a threat type or not), not
+// a granular score β€” so we normalize any confirmed threat to riskScore 100 and
+// a clean lookup to 0, and surface the threat types as category strings.
+
+const PROVIDER: ThreatProvider = "google-web-risk";
+
+interface HashesSearchThreat {
+ hash?: string;
+ threatTypes?: string[];
+ expireTime?: string;
+}
+
+type HashesSearchResponse = {
+ threats?: HashesSearchThreat[];
+ negativeExpireTime?: string;
+};
+
+function isGoogleWebRiskConfigured(): boolean {
+ return (
+ typeof config.GOOGLE_WEB_RISK_API_KEY === "string" &&
+ config.GOOGLE_WEB_RISK_API_KEY.trim().length > 0
+ );
+}
+
+async function searchHashPrefix(
+ prefix: Buffer,
+ signal: AbortSignal | undefined,
+): Promise<HashesSearchResponse> {
+ const params = new URLSearchParams();
+ params.append("hashPrefix", prefix.toString("base64"));
+ for (const threatType of WEB_RISK_THREAT_TYPES) {
+ params.append("threatTypes", threatType);
+ }
+ params.append("key", config.GOOGLE_WEB_RISK_API_KEY!);
+
+ const response = await fetch(
+ `${config.GOOGLE_WEB_RISK_API_URL}/v1/hashes:search?${params.toString()}`,
+ { method: "GET", signal },
+ );
+ if (!response.ok) {
+ throw new Error(
+ `Google Web Risk hash search failed with status ${response.status}`,
+ );
+ }
+ return (await response.json()) as HashesSearchResponse;
+}
+
+/**
+ * Look up a domain against Google Web Risk. Throws on any transport/API
+ * error, and when the local threat lists are unavailable or stale, so the
+ * caller can apply the org's failurePolicy.
+ */
+export async function fetchGoogleWebRiskVerdict(
+ domain: string,
+ options?: { signal?: AbortSignal },
+): Promise<RawVerdict> {
+ if (!isGoogleWebRiskConfigured()) {
+ throw new Error("Google Web Risk is not configured");
+ }
+
+ ensureThreatListSyncLoop();
+
+ const expressions = generateHostExpressions(domain);
+ if (expressions.length === 0) {
+ throw new Error(`Cannot canonicalize domain for Web Risk lookup`);
+ }
+ const fullHashes = expressions.map(expression =>
+ createHash("sha256").update(expression).digest(),
+ );
+
+ const store = getWebRiskListStore();
+
+ // Wait for the once-per-process boot sync (bounded by the caller's timeout
+ // signal) so a fresh deployment doesn't fail every check while the lists
+ // download; afterwards this resolves instantly.
+ await abortable(ensureThreatListBootSync(), options?.signal);
+
+ const lookup: WebRiskLookupResult = await store.lookup(fullHashes);
+ if (lookup.status !== "ok") {
+ throw new Error(`Web Risk threat lists are ${lookup.status}`);
+ }
+
+ if (lookup.hits.length === 0) {
+ // The overwhelmingly common case: clean domain, zero Google calls,
+ // nothing transmitted anywhere.
+ return {
+ provider: PROVIDER,
+ riskScore: 0,
+ categories: [],
+ fromCache: false,
+ raw: { localPrefixMatch: false },
+ };
+ }
+
+ // Local prefix hit: confirm with hashes:search (only the anonymized hash
+ // prefix leaves our infrastructure), then compare full hashes locally.
+ const uniquePrefixes = new Map<string, Buffer>();
+ for (const hit of lookup.hits) {
+ uniquePrefixes.set(hit.prefix.toString("base64"), hit.prefix);
+ }
+ const responses = await Promise.all(
+ [...uniquePrefixes.values()].map(prefix =>
+ searchHashPrefix(prefix, options?.signal),
+ ),
+ );
+
+ const fullHashSet = new Set(fullHashes.map(hash => hash.toString("base64")));
+ const confirmedThreatTypes = new Set<string>();
+ for (const response of responses) {
+ for (const threat of response.threats ?? []) {
+ if (threat.hash && fullHashSet.has(threat.hash)) {
+ for (const threatType of threat.threatTypes ?? []) {
+ if (
+ (WEB_RISK_THREAT_TYPES as readonly string[]).includes(threatType)
+ ) {
+ confirmedThreatTypes.add(threatType);
+ }
+ }
+ }
+ }
+ }
+
+ const categories = [...confirmedThreatTypes] as WebRiskThreatType[];
+ return {
+ provider: PROVIDER,
+ // No confirmed full-hash match = prefix collision = clean.
+ riskScore: categories.length > 0 ? 100 : 0,
+ categories,
+ fromCache: false,
+ raw: {
+ localPrefixMatch: true,
+ prefixesSearched: uniquePrefixes.size,
+ threats: responses.flatMap(response =>
+ (response.threats ?? []).map(threat => ({
+ hash: threat.hash,
+ threatTypes: threat.threatTypes,
+ })),
+ ),
+ },
+ };
+}
+
+/** Await a promise, rejecting early if the signal aborts. */
+function abortable<T>(
+ promise: Promise<T>,
+ signal: AbortSignal | undefined,
+): Promise<T> {
+ if (!signal) return promise;
+ if (signal.aborted) {
+ return Promise.reject(new Error("Web Risk lookup aborted"));
+ }
+ return new Promise<T>((resolve, reject) => {
+ const onAbort = () => reject(new Error("Web Risk lookup aborted"));
+ signal.addEventListener("abort", onAbort, { once: true });
+ promise.then(
+ value => {
+ signal.removeEventListener("abort", onAbort);
+ resolve(value);
+ },
+ error => {
+ signal.removeEventListener("abort", onAbort);
+ reject(error);
+ },
+ );
+ });
+}
diff --git a/apps/api/src/lib/threat-protection/providers/providers.test.ts b/apps/api/src/lib/threat-protection/providers/providers.test.ts
new file mode 100644
index 000000000..a2d5ae811
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/providers.test.ts
@@ -0,0 +1,225 @@
+import http from "http";
+import { AddressInfo } from "net";
+
+// The Web Risk threat-list store lives on the durable Redis connection β€”
+// swap in an in-memory fake. (fake-redis.ts has no runtime imports, so the
+// factory cannot re-enter the module being mocked.)
+vi.mock("../../../services/queue-service", async () => {
+ const { createFakeWebRiskRedis } = await import("./web-risk/fake-redis.js");
+ const client = createFakeWebRiskRedis();
+ return { getRedisConnection: () => client };
+});
+
+import { config } from "../../../config";
+import { fetchGoogleWebRiskVerdict } from "./google-web-risk";
+import { domainExpressionHash, WebRiskMockDatabase } from "./web-risk/testing";
+
+// Mocked-HTTP provider tests: a local http server stands in for the real
+// provider API via the config URL override (same pattern as
+// src/lib/fire-privacy-client.test.ts). For Google Web Risk the mock serves
+// the Update API endpoints (threatLists:computeDiff + hashes:search); the
+// old uris:search endpoint intentionally no longer exists.
+
+type SeenRequest = { url: string; method: string; body: unknown };
+
+let server: http.Server;
+let baseUrl: string;
+let seenRequests: SeenRequest[] = [];
+
+// Fixture threat lists, fixed for the whole file (the local list is synced
+// once per process by the provider's boot sync).
+const CONFIRMED_DOMAIN = "malware.example";
+const COLLISION_DOMAIN = "collision.example";
+
+const webRiskDb = new WebRiskMockDatabase();
+webRiskDb.addRiskyDomain(CONFIRMED_DOMAIN, "MALWARE");
+webRiskDb.addRiskyDomain(CONFIRMED_DOMAIN, "SOCIAL_ENGINEERING");
+// A list entry that shares the 4-byte prefix of COLLISION_DOMAIN's expression
+// hash but is a different full hash β†’ local hit, unconfirmed by hashes:search.
+webRiskDb.addCollidingFullHash(
+ Buffer.concat([
+ domainExpressionHash(COLLISION_DOMAIN).subarray(0, 4),
+ Buffer.alloc(28, 0xab),
+ ]),
+ "UNWANTED_SOFTWARE",
+);
+
+// While > 0, hashes:search requests fail with 503 (decremented per request).
+let failHashesSearches = 0;
+
+const originalConfig = {
+ webRiskUrl: config.GOOGLE_WEB_RISK_API_URL,
+ webRiskKey: config.GOOGLE_WEB_RISK_API_KEY,
+};
+
+beforeAll(async () => {
+ await new Promise<void>(resolve => {
+ server = http.createServer((req, res) => {
+ const chunks: Buffer[] = [];
+ req.on("data", chunk => chunks.push(chunk));
+ req.on("end", () => {
+ const rawBody = Buffer.concat(chunks).toString("utf8");
+ let body: unknown = null;
+ try {
+ body = rawBody ? JSON.parse(rawBody) : null;
+ } catch {}
+ seenRequests.push({
+ url: req.url ?? "",
+ method: req.method ?? "",
+ body,
+ });
+
+ const url = new URL(req.url ?? "/", "http://localhost");
+ const path = url.pathname;
+
+ // Web Risk Update API endpoints.
+ if (path === "/v1/threatLists:computeDiff") {
+ res.setHeader("content-type", "application/json");
+ res.end(
+ JSON.stringify(
+ webRiskDb.computeDiffResponse(
+ url.searchParams.get("threatType") ?? "",
+ ),
+ ),
+ );
+ return;
+ }
+ if (path === "/v1/hashes:search") {
+ if (failHashesSearches > 0) {
+ failHashesSearches--;
+ res.statusCode = 503;
+ res.end("{}");
+ return;
+ }
+ res.setHeader("content-type", "application/json");
+ res.end(
+ JSON.stringify(
+ webRiskDb.hashesSearchResponse(
+ url.searchParams.get("hashPrefix") ?? "",
+ ),
+ ),
+ );
+ return;
+ }
+
+ res.statusCode = 404;
+ res.end("{}");
+ });
+ });
+ server.listen(0, "127.0.0.1", () => {
+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+ config.GOOGLE_WEB_RISK_API_URL = baseUrl;
+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";
+ resolve();
+ });
+ });
+});
+
+afterAll(async () => {
+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.webRiskUrl;
+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.webRiskKey;
+ await new Promise<void>(resolve => server.close(() => resolve()));
+});
+
+beforeEach(() => {
+ seenRequests = [];
+ failHashesSearches = 0;
+});
+
+const hashesSearchRequests = () =>
+ seenRequests.filter(r => r.url.startsWith("/v1/hashes:search"));
+const urisSearchRequests = () =>
+ seenRequests.filter(r => r.url.startsWith("/v1/uris:search"));
+
+describe("fetchGoogleWebRiskVerdict", () => {
+ it("confirms a local prefix hit via hashes:search β†’ riskScore 100 with categories", async () => {
+ const verdict = await fetchGoogleWebRiskVerdict(CONFIRMED_DOMAIN);
+
+ expect(verdict).toMatchObject({
+ provider: "google-web-risk",
+ riskScore: 100,
+ fromCache: false,
+ });
+ expect([...verdict.categories].sort()).toEqual([
+ "MALWARE",
+ "SOCIAL_ENGINEERING",
+ ]);
+
+ // Exactly one confirmation call, carrying ONLY the anonymized 4-byte
+ // hash prefix (never the domain or URL), plus the API key.
+ const confirmations = hashesSearchRequests();
+ expect(confirmations).toHaveLength(1);
+ const url = new URL(baseUrl + confirmations[0].url);
+ expect(confirmations[0].method).toBe("GET");
+ expect(url.searchParams.get("hashPrefix")).toBe(
+ domainExpressionHash(CONFIRMED_DOMAIN).subarray(0, 4).toString("base64"),
+ );
+ expect(url.searchParams.getAll("threatTypes")).toEqual([
+ "MALWARE",
+ "SOCIAL_ENGINEERING",
+ "UNWANTED_SOFTWARE",
+ ]);
+ expect(url.searchParams.get("key")).toBe("test-web-risk-key");
+ expect(confirmations[0].url).not.toContain(CONFIRMED_DOMAIN);
+ // The legacy full-URL lookup endpoint is never used.
+ expect(urisSearchRequests()).toHaveLength(0);
+ });
+
+ it("flags subdomains of a listed domain through host-suffix expressions", async () => {
+ const verdict = await fetchGoogleWebRiskVerdict(
+ `cdn.assets.${CONFIRMED_DOMAIN}`,
+ );
+
+ expect(verdict.riskScore).toBe(100);
+ expect(verdict.categories).toContain("MALWARE");
+ });
+
+ it("resolves clean domains locally with zero Google calls", async () => {
+ const verdict = await fetchGoogleWebRiskVerdict("safe.example");
+
+ expect(verdict).toMatchObject({
+ provider: "google-web-risk",
+ riskScore: 0,
+ categories: [],
+ fromCache: false,
+ raw: { localPrefixMatch: false },
+ });
+ // The common case transmits nothing: no hashes:search, no uris:search.
+ expect(hashesSearchRequests()).toHaveLength(0);
+ expect(urisSearchRequests()).toHaveLength(0);
+ });
+
+ it("treats an unconfirmed prefix hit (collision) as clean", async () => {
+ const verdict = await fetchGoogleWebRiskVerdict(COLLISION_DOMAIN);
+
+ expect(verdict).toMatchObject({
+ provider: "google-web-risk",
+ riskScore: 0,
+ categories: [],
+ });
+ // The collision DID require a confirmation round trip…
+ expect(hashesSearchRequests()).toHaveLength(1);
+ // …whose returned full hash didn't match any of our expression hashes.
+ expect(verdict.raw).toMatchObject({ localPrefixMatch: true });
+ });
+
+ it("throws on hashes:search errors so failurePolicy can apply", async () => {
+ failHashesSearches = Infinity;
+
+ await expect(fetchGoogleWebRiskVerdict(CONFIRMED_DOMAIN)).rejects.toThrow(
+ /status 503/,
+ );
+ });
+
+ it("throws when the API key is not configured", async () => {
+ config.GOOGLE_WEB_RISK_API_KEY = undefined;
+ try {
+ await expect(fetchGoogleWebRiskVerdict("safe.example")).rejects.toThrow(
+ /not configured/,
+ );
+ } finally {
+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";
+ }
+ expect(seenRequests).toHaveLength(0);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.ts
new file mode 100644
index 000000000..54234e5fe
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.ts
@@ -0,0 +1,119 @@
+import {
+ canonicalizeHost,
+ canonicalizeUrl,
+ generateHostExpressions,
+} from "./canonicalize";
+
+// Google's published canonicalization test vectors, verbatim:
+// https://cloud.google.com/web-risk/docs/urls-hashing#canonicalization
+// (identical to the Safe Browsing v4 vectors).
+const CANONICALIZATION_VECTORS: [string, string][] = [
+ ["http://host/%25%32%35", "http://host/%25"],
+ ["http://host/%25%32%35%25%32%35", "http://host/%25%25"],
+ ["http://host/%2525252525252525", "http://host/%25"],
+ ["http://host/asdf%25%32%35asd", "http://host/asdf%25asd"],
+ ["http://host/%%%25%32%35asd%%", "http://host/%25%25%25asd%25%25"],
+ ["http://www.google.com/", "http://www.google.com/"],
+ [
+ "http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
+ "http://168.188.99.26/.secure/www.ebay.com/",
+ ],
+ [
+ "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
+ "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
+ ],
+ [
+ "http://host%23.com/%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_55%252B",
+ "http://host%23.com/~a!b@c%23d$e%25f^00&11*22(33)44_55+",
+ ],
+ ["http://3279880203/blah", "http://195.127.0.11/blah"],
+ ["http://www.google.com/blah/..", "http://www.google.com/"],
+ ["www.google.com/", "http://www.google.com/"],
+ ["www.google.com", "http://www.google.com/"],
+ ["http://www.evil.com/blah#frag", "http://www.evil.com/blah"],
+ ["http://www.GOOgle.com/", "http://www.google.com/"],
+ ["http://www.google.com.../", "http://www.google.com/"],
+ [
+ "http://www.google.com/foo\tbar\rbaz\n2",
+ "http://www.google.com/foobarbaz2",
+ ],
+ ["http://www.google.com/q?", "http://www.google.com/q?"],
+ ["http://www.google.com/q?r?", "http://www.google.com/q?r?"],
+ ["http://www.google.com/q?r?s", "http://www.google.com/q?r?s"],
+ ["http://evil.com/foo#bar#baz", "http://evil.com/foo"],
+ ["http://evil.com/foo;", "http://evil.com/foo;"],
+ ["http://evil.com/foo?bar;", "http://evil.com/foo?bar;"],
+ ["http://\x01\x80.com/", "http://%01%80.com/"],
+ ["http://notrailingslash.com", "http://notrailingslash.com/"],
+ ["http://www.gotaport.com:1234/", "http://www.gotaport.com/"],
+ [" http://www.google.com/ ", "http://www.google.com/"],
+ ["http:// leadingspace.com/", "http://%20leadingspace.com/"],
+ ["http://%20leadingspace.com/", "http://%20leadingspace.com/"],
+ ["%20leadingspace.com/", "http://%20leadingspace.com/"],
+ ["https://www.securesite.com/", "https://www.securesite.com/"],
+ ["http://host.com/ab%23cd", "http://host.com/ab%23cd"],
+ [
+ "http://host.com//twoslashes?more//slashes",
+ "http://host.com/twoslashes?more//slashes",
+ ],
+];
+
+describe("canonicalizeUrl", () => {
+ it.each(CANONICALIZATION_VECTORS)("canonicalizes %j", (input, expected) => {
+ expect(canonicalizeUrl(input)).toBe(expected);
+ });
+});
+
+describe("canonicalizeHost", () => {
+ it("normalizes dots and case", () => {
+ expect(canonicalizeHost(".Example..COM.")).toBe("example.com");
+ });
+
+ it("normalizes integer, octal and hex IP forms", () => {
+ expect(canonicalizeHost("3279880203")).toBe("195.127.0.11");
+ expect(canonicalizeHost("0303.0177.0.013")).toBe("195.127.0.11");
+ expect(canonicalizeHost("0xc3.0x7f.0x0.0xb")).toBe("195.127.0.11");
+ expect(canonicalizeHost("195.127.11")).toBe("195.127.0.11");
+ });
+
+ it("leaves regular hostnames that merely look numeric-ish alone", () => {
+ expect(canonicalizeHost("1234x.com")).toBe("1234x.com");
+ expect(canonicalizeHost("256.256.256.256")).toBe("256.256.256.256");
+ });
+});
+
+describe("generateHostExpressions", () => {
+ // Spec example (host component of "http://a.b.c.d.e.f.g/1.html"): the
+ // exact host plus the four suffixes built from the last five components.
+ it("walks at most the last five host components", () => {
+ expect(generateHostExpressions("a.b.c.d.e.f.g")).toEqual([
+ "a.b.c.d.e.f.g/",
+ "c.d.e.f.g/",
+ "d.e.f.g/",
+ "e.f.g/",
+ "f.g/",
+ ]);
+ });
+
+ it("generates exact host + registrable suffixes for short hosts", () => {
+ expect(generateHostExpressions("a.b.c")).toEqual(["a.b.c/", "b.c/"]);
+ expect(generateHostExpressions("example.com")).toEqual(["example.com/"]);
+ expect(generateHostExpressions("www.phishing.example.com")).toEqual([
+ "www.phishing.example.com/",
+ "phishing.example.com/",
+ "example.com/",
+ ]);
+ });
+
+ it("generates a single expression for IP hosts (no suffix walk)", () => {
+ expect(generateHostExpressions("195.127.0.11")).toEqual(["195.127.0.11/"]);
+ expect(generateHostExpressions("3279880203")).toEqual(["195.127.0.11/"]);
+ });
+
+ it("canonicalizes the host before expanding", () => {
+ expect(generateHostExpressions("WWW.Example.COM.")).toEqual([
+ "www.example.com/",
+ "example.com/",
+ ]);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.ts b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.ts
new file mode 100644
index 000000000..f150a68a2
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.ts
@@ -0,0 +1,281 @@
+// URL canonicalization and lookup-expression generation for the Google Web
+// Risk / Safe Browsing hash-prefix protocol.
+// https://cloud.google.com/web-risk/docs/urls-hashing
+//
+// The threat lists store SHA-256 hash prefixes of *canonicalized expressions*
+// (host[/path] strings). To check a target locally we must produce byte-exact
+// expressions per the published spec β€” the canonicalization rules here are
+// verified against Google's published test vectors (see canonicalize.test.ts).
+//
+// Everything in this file is pure string/byte manipulation: nothing is
+// transmitted anywhere (ZDR β€” the check path only ever sends anonymized hash
+// prefixes to Google, and only on a local prefix hit).
+
+/**
+ * Treats a JS string as a byte sequence: code units <= 0xFF are bytes,
+ * anything above is UTF-8 encoded. Matches how reference implementations
+ * operate on raw URL bytes.
+ */
+function toBytes(input: string): number[] {
+ const bytes: number[] = [];
+ for (const char of input) {
+ const code = char.codePointAt(0)!;
+ if (code <= 0xff) {
+ bytes.push(code);
+ } else {
+ bytes.push(...Buffer.from(char, "utf8"));
+ }
+ }
+ return bytes;
+}
+
+function fromBytes(bytes: number[]): string {
+ return bytes.map(b => String.fromCharCode(b)).join("");
+}
+
+const HEX = "0123456789ABCDEF";
+
+function isHexDigit(char: string): boolean {
+ return /^[0-9a-fA-F]$/.test(char);
+}
+
+/** One pass of percent-unescaping (only valid %XX sequences). */
+function unescapeOnce(input: string): string {
+ let out = "";
+ for (let i = 0; i < input.length; i++) {
+ if (
+ input[i] === "%" &&
+ i + 2 < input.length &&
+ isHexDigit(input[i + 1]) &&
+ isHexDigit(input[i + 2])
+ ) {
+ out += String.fromCharCode(parseInt(input.slice(i + 1, i + 3), 16));
+ i += 2;
+ } else {
+ out += input[i];
+ }
+ }
+ return out;
+}
+
+/** Repeatedly percent-unescape until the string stops changing (spec step). */
+function fullyUnescape(input: string): string {
+ let prev = input;
+ // Bounded to defend against adversarial inputs; real URLs converge fast.
+ for (let i = 0; i < 1024; i++) {
+ const next = unescapeOnce(prev);
+ if (next === prev) return next;
+ prev = next;
+ }
+ return prev;
+}
+
+/**
+ * Percent-escape bytes <= 0x20, >= 0x7F, '#' and '%', uppercase hex β€” the
+ * spec's escape character class.
+ */
+function escapeBytes(input: string): string {
+ const bytes = toBytes(input);
+ let out = "";
+ for (const byte of bytes) {
+ if (
+ byte <= 0x20 ||
+ byte >= 0x7f ||
+ byte === 0x23 /* # */ ||
+ byte === 0x25 /* % */
+ ) {
+ out += "%" + HEX[byte >> 4] + HEX[byte & 0xf];
+ } else {
+ out += String.fromCharCode(byte);
+ }
+ }
+ return out;
+}
+
+/**
+ * inet_aton-style IP parsing: 1-4 dot-separated components, each decimal,
+ * octal (leading 0) or hex (0x); the last component fills the remaining
+ * bytes. Returns the normalized dotted-quad, or null if not an IP.
+ */
+function canonicalizeIp(host: string): string | null {
+ if (
+ !/^(?:0x[0-9a-f]+|[0-9]+)(?:\.(?:0x[0-9a-f]+|[0-9]+)){0,3}$/i.test(host)
+ ) {
+ return null;
+ }
+ const parts = host.split(".");
+ const values: number[] = [];
+ for (const part of parts) {
+ let value: number;
+ if (/^0x/i.test(part)) {
+ value = parseInt(part.slice(2), 16);
+ } else if (/^0[0-7]*$/.test(part)) {
+ value = parseInt(part, 8);
+ } else if (/^[0-9]+$/.test(part)) {
+ value = parseInt(part, 10);
+ } else {
+ return null;
+ }
+ if (!Number.isFinite(value) || value < 0) return null;
+ values.push(value);
+ }
+
+ // All but the last component must fit in one byte; the last fills the rest.
+ const tailBytes = 4 - (values.length - 1);
+ const last = values[values.length - 1];
+ if (last >= 2 ** (8 * tailBytes)) return null;
+ for (let i = 0; i < values.length - 1; i++) {
+ if (values[i] > 0xff) return null;
+ }
+
+ const bytes: number[] = values.slice(0, -1);
+ for (let i = tailBytes - 1; i >= 0; i--) {
+ bytes.push((last >> (8 * i)) & 0xff);
+ }
+ return bytes.join(".");
+}
+
+/**
+ * Canonicalize a hostname per the spec: fully unescape, remove leading /
+ * trailing dots, collapse consecutive dots, normalize IP forms to
+ * dotted-quad, lowercase, then re-escape.
+ */
+export function canonicalizeHost(host: string): string {
+ let out = fullyUnescape(host);
+ out = out
+ .replace(/^\.+/, "")
+ .replace(/\.+$/, "")
+ .replace(/\.{2,}/g, ".");
+ out = out.toLowerCase();
+ const ip = canonicalizeIp(out);
+ if (ip !== null) out = ip;
+ return escapeBytes(out);
+}
+
+/** Resolve "/./" and "/../" segments and collapse runs of slashes. */
+function canonicalizePath(path: string): string {
+ const segments = path.split("/");
+ const out: string[] = [];
+ for (let i = 0; i < segments.length; i++) {
+ const segment = segments[i];
+ if (segment === "" || segment === ".") {
+ // Empty segments are runs of slashes (or the leading slash) β€” skip.
+ continue;
+ }
+ if (segment === "..") {
+ out.pop();
+ continue;
+ }
+ out.push(segment);
+ }
+ const endsWithSlash =
+ path.endsWith("/") ||
+ path.endsWith("/.") ||
+ path.endsWith("/..") ||
+ out.length === 0;
+ return "/" + out.join("/") + (endsWithSlash && out.length > 0 ? "/" : "");
+}
+
+interface ParsedUrl {
+ scheme: string;
+ host: string;
+ path: string;
+ query: string | null;
+}
+
+/**
+ * Lenient URL splitter (real URL parsers reject hosts the spec requires us to
+ * handle, e.g. embedded spaces or raw control bytes).
+ */
+function splitUrl(input: string): ParsedUrl {
+ let rest = input;
+ let scheme = "http";
+ const schemeMatch = rest.match(/^([a-zA-Z][a-zA-Z0-9+.-]*):\/\//);
+ if (schemeMatch) {
+ scheme = schemeMatch[1].toLowerCase();
+ rest = rest.slice(schemeMatch[0].length);
+ }
+
+ let hostEnd = rest.length;
+ for (let i = 0; i < rest.length; i++) {
+ if (rest[i] === "/" || rest[i] === "?") {
+ hostEnd = i;
+ break;
+ }
+ }
+ let authority = rest.slice(0, hostEnd);
+ const afterHost = rest.slice(hostEnd);
+
+ // Strip userinfo and port from the authority.
+ const at = authority.lastIndexOf("@");
+ if (at !== -1) authority = authority.slice(at + 1);
+ const portMatch = authority.match(/^(.*):(\d*)$/);
+ if (portMatch) authority = portMatch[1];
+
+ let path = "/";
+ let query: string | null = null;
+ if (afterHost.length > 0) {
+ const q = afterHost.indexOf("?");
+ if (q !== -1) {
+ path = q === 0 ? "/" : afterHost.slice(0, q);
+ query = afterHost.slice(q + 1);
+ } else {
+ path = afterHost;
+ }
+ }
+
+ return { scheme, host: authority, path, query };
+}
+
+/**
+ * Full URL canonicalization per the Web Risk / Safe Browsing spec. Verified
+ * against Google's published test vectors.
+ */
+export function canonicalizeUrl(input: string): string {
+ // Leading/trailing whitespace, then embedded tab/CR/LF, then the fragment.
+ let url = input.trim().replace(/[\t\r\n]/g, "");
+ const hash = url.indexOf("#");
+ if (hash !== -1) url = url.slice(0, hash);
+
+ const { scheme, host, path, query } = splitUrl(url);
+
+ const canonicalHost = canonicalizeHost(host);
+ const canonicalPath = escapeBytes(canonicalizePath(fullyUnescape(path)));
+ const canonicalQuery = query === null ? null : escapeBytes(query);
+
+ return (
+ `${scheme}://${canonicalHost}${canonicalPath}` +
+ (canonicalQuery !== null ? `?${canonicalQuery}` : "")
+ );
+}
+
+/** Whether a canonicalized host is a (dotted-quad) IP address. */
+function isIpHost(host: string): boolean {
+ return /^\d+\.\d+\.\d+\.\d+$/.test(host);
+}
+
+/**
+ * Host-suffix lookup expressions for a domain-level check, per the spec: the
+ * exact hostname plus up to four suffixes formed from the last five host
+ * components, successively dropping the leading component (never the bare
+ * TLD), each with the "/" root path. IP hosts get a single expression.
+ *
+ * Firecrawl's threat protection is domain-level (we classify the target
+ * domain, not individual URLs), so path variants are intentionally not
+ * generated β€” this matches the previous uris:search behavior of looking up
+ * `http://<domain>/`.
+ */
+export function generateHostExpressions(domain: string): string[] {
+ const host = canonicalizeHost(domain.trim());
+ if (host.length === 0) return [];
+ if (isIpHost(host)) return [host + "/"];
+
+ const expressions = [host + "/"];
+ const parts = host.split(".");
+ const maxSuffix = Math.min(5, parts.length - 1);
+ for (let suffixLen = maxSuffix; suffixLen >= 2; suffixLen--) {
+ const suffix = parts.slice(parts.length - suffixLen).join(".");
+ if (suffix !== host) expressions.push(suffix + "/");
+ }
+ return expressions;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.ts b/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.ts
new file mode 100644
index 000000000..262b1c506
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.ts
@@ -0,0 +1,73 @@
+import type { WebRiskRedis } from "./store";
+
+// In-memory stand-in for the durable Redis connection used by the threat-list
+// store and sync lock (supports EX/NX set semantics and the buffer read
+// variants the store uses). Test helper only.
+//
+// Lives in its own module with a type-only import so that vi.mock factories
+// for services/queue-service can import it without re-entering the module
+// being mocked (testing.ts β†’ store.ts β†’ queue-service would deadlock).
+
+export function createFakeWebRiskRedis(): WebRiskRedis & {
+ dump(): Map<string, Buffer>;
+} {
+ const store = new Map<string, { value: Buffer; expiresAt: number | null }>();
+
+ const live = (key: string): Buffer | null => {
+ const entry = store.get(key);
+ if (!entry) return null;
+ if (entry.expiresAt !== null && entry.expiresAt <= Date.now()) {
+ store.delete(key);
+ return null;
+ }
+ return entry.value;
+ };
+
+ return {
+ async get(key) {
+ const value = live(key);
+ return value === null ? null : value.toString("utf8");
+ },
+ async mget(...keys) {
+ return keys.map(key => {
+ const value = live(key);
+ return value === null ? null : value.toString("utf8");
+ });
+ },
+ async mgetBuffer(...keys) {
+ return keys.map(key => live(key));
+ },
+ async set(key, value, ...args) {
+ const upper = args.map(a => String(a).toUpperCase());
+ if (upper.includes("NX") && live(key) !== null) return null;
+ const exIndex = upper.indexOf("EX");
+ const expiresAt =
+ exIndex !== -1 ? Date.now() + Number(args[exIndex + 1]) * 1000 : null;
+ store.set(key, {
+ value: Buffer.isBuffer(value) ? value : Buffer.from(String(value)),
+ expiresAt,
+ });
+ return "OK";
+ },
+ async del(...keys) {
+ let deleted = 0;
+ for (const key of keys) {
+ if (store.delete(key)) deleted++;
+ }
+ return deleted;
+ },
+ async expire(key, seconds) {
+ const entry = store.get(key);
+ if (!entry || live(key) === null) return 0;
+ entry.expiresAt = Date.now() + seconds * 1000;
+ return 1;
+ },
+ dump() {
+ return new Map(
+ [...store.entries()]
+ .filter(([key]) => live(key) !== null)
+ .map(([key, entry]) => [key, entry.value]),
+ );
+ },
+ };
+}
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/store.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/store.test.ts
new file mode 100644
index 000000000..d58380dab
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/store.test.ts
@@ -0,0 +1,148 @@
+import { config } from "../../../../config";
+import {
+ WEB_RISK_THREAT_TYPES,
+ WebRiskListStore,
+ type ThreatListMeta,
+} from "./store";
+import { createFakeWebRiskRedis, sha256 } from "./testing";
+
+function meta(overrides: Partial<ThreatListMeta> = {}): ThreatListMeta {
+ return {
+ versionToken: "dG9rZW4=",
+ checksum: "",
+ count: 0,
+ syncedAt: new Date().toISOString(),
+ nextDiffAt: new Date(Date.now() + 60_000).toISOString(),
+ ...overrides,
+ };
+}
+
+function sorted(entries: Buffer[]): Buffer[] {
+ return [...entries].sort(Buffer.compare);
+}
+
+describe("WebRiskListStore", () => {
+ it("round-trips entries through the bucketed layout", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+
+ // Hashes spread across many buckets, plus one long (8-byte) entry.
+ const fullHashes = Array.from({ length: 500 }, (_, i) =>
+ sha256(`domain-${i}.example/`),
+ );
+ const entries = sorted([
+ ...fullHashes.map(hash => hash.subarray(0, 4)),
+ sha256("long-entry.example/").subarray(0, 8),
+ ]);
+
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ await store.writeVersion(type, entries, meta({ count: entries.length }));
+ }
+
+ const roundTripped = await store.loadEntries(
+ "MALWARE",
+ (await store.getPointer("MALWARE"))!.version,
+ );
+ expect(roundTripped.map(e => e.toString("hex"))).toEqual(
+ entries.map(e => e.toString("hex")),
+ );
+
+ // Membership: a listed 4-byte prefix hits, an unlisted hash misses.
+ const hit = await store.lookup([fullHashes[123]]);
+ expect(hit).toMatchObject({ status: "ok" });
+ if (hit.status === "ok") {
+ expect(hit.hits.length).toBe(3); // present in every list
+ expect(hit.hits[0].prefix.equals(fullHashes[123].subarray(0, 4))).toBe(
+ true,
+ );
+ }
+
+ const miss = await store.lookup([sha256("clean.example/")]);
+ expect(miss).toMatchObject({ status: "ok", hits: [] });
+
+ // Long entries match by prefix comparison against the full hash.
+ const longHit = await store.lookup([sha256("long-entry.example/")]);
+ expect(longHit).toMatchObject({ status: "ok" });
+ if (longHit.status === "ok") {
+ expect(longHit.hits[0].prefix.length).toBe(8);
+ }
+ });
+
+ it("reports unavailable until every list has a version", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+
+ expect(await store.lookup([sha256("x/")])).toEqual({
+ status: "unavailable",
+ });
+
+ await store.writeVersion("MALWARE", [], meta());
+ expect(await store.lookup([sha256("x/")])).toEqual({
+ status: "unavailable",
+ });
+
+ await store.writeVersion("SOCIAL_ENGINEERING", [], meta());
+ await store.writeVersion("UNWANTED_SOFTWARE", [], meta());
+ expect(await store.lookup([sha256("x/")])).toMatchObject({ status: "ok" });
+ });
+
+ it("reports stale when a list exceeds the staleness threshold", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ const staleSyncedAt = new Date(
+ Date.now() - (config.THREAT_LIST_STALENESS_SECONDS + 60) * 1000,
+ ).toISOString();
+
+ await store.writeVersion("MALWARE", [], meta({ syncedAt: staleSyncedAt }));
+ await store.writeVersion("SOCIAL_ENGINEERING", [], meta());
+ await store.writeVersion("UNWANTED_SOFTWARE", [], meta());
+
+ const result = await store.lookup([sha256("x/")]);
+ expect(result).toMatchObject({ status: "stale" });
+ if (result.status === "stale") {
+ expect(result.ageSeconds).toBeGreaterThan(
+ config.THREAT_LIST_STALENESS_SECONDS,
+ );
+ }
+ });
+
+ it("swaps the version pointer atomically and serves the new content", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+
+ const v1Hash = sha256("v1-only.example/");
+ const v2Hash = sha256("v2-only.example/");
+
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ await store.writeVersion(type, [v1Hash.subarray(0, 4)], meta());
+ }
+ const v1 = (await store.getPointer("MALWARE"))!.version;
+
+ expect(await store.lookup([v1Hash])).toMatchObject({
+ status: "ok",
+ hits: [expect.anything(), expect.anything(), expect.anything()],
+ });
+
+ await store.writeVersion("MALWARE", [v2Hash.subarray(0, 4)], meta());
+ const v2 = (await store.getPointer("MALWARE"))!.version;
+ expect(v2).not.toBe(v1);
+
+ // New content served, old content gone from the MALWARE list.
+ const v2Lookup = await store.lookup([v2Hash]);
+ expect(v2Lookup).toMatchObject({ status: "ok" });
+ if (v2Lookup.status === "ok") {
+ expect(v2Lookup.hits.map(h => h.threatType)).toEqual(["MALWARE"]);
+ }
+ const v1Lookup = await store.lookup([v1Hash]);
+ if (v1Lookup.status === "ok") {
+ expect(v1Lookup.hits.map(h => h.threatType)).toEqual([
+ "SOCIAL_ENGINEERING",
+ "UNWANTED_SOFTWARE",
+ ]);
+ }
+
+ // The superseded version's data is retired (grace TTL), not readable as
+ // the current version.
+ expect((await store.getPointer("MALWARE"))!.version).toBe(v2);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/store.ts b/apps/api/src/lib/threat-protection/providers/web-risk/store.ts
new file mode 100644
index 000000000..a6eb61a7f
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/store.ts
@@ -0,0 +1,384 @@
+import { config } from "../../../../config";
+import { getRedisConnection } from "../../../../services/queue-service";
+
+// Redis-backed store for the locally synced Google Web Risk threat lists
+// (hash prefixes, Update API β€” see ./sync.ts). Design constraints:
+//
+// * Lives in the durable, NON-EVICTING Redis (config.REDIS_URL via
+// getRedisConnection()) β€” explicitly not the evict or rate-limit
+// connections: the lists are operational data that must not be evicted
+// under memory pressure. (This data is Google's threat lists, not
+// anything scrape-derived β€” storing it is ZDR-compatible.)
+//
+// * Memory-efficient layout: ~5M 4-byte prefixes as individual SET members
+// would balloon ~20MB of raw data into hundreds of MB of per-entry
+// overhead. Instead, prefixes are sharded by their leading 12 bits into
+// 4096 buckets, each bucket one binary string of sorted, concatenated
+// 4-byte prefixes. Membership = one MGET + a local binary search.
+// (Rare longer entries β€” the protocol allows 4-32 byte prefixes β€” go into
+// a per-version "long" key, checked by prefix comparison.)
+//
+// * Versioned namespaces with an atomic pointer swap: a sync writes every
+// bucket under a fresh version id, then flips the version pointer in a
+// single SET β€” a partially written version is never served. Old versions
+// are expired with a short grace period so in-flight readers that already
+// resolved the old pointer still find their buckets.
+
+export const WEB_RISK_THREAT_TYPES = [
+ "MALWARE",
+ "SOCIAL_ENGINEERING",
+ "UNWANTED_SOFTWARE",
+] as const;
+
+export type WebRiskThreatType = (typeof WEB_RISK_THREAT_TYPES)[number];
+
+const BUCKET_BITS = 12;
+const BUCKET_COUNT = 1 << BUCKET_BITS;
+// Refreshed on every sync; prevents orphaned lists from lingering forever if
+// syncing is disabled/abandoned. Staleness (hours) cuts in far earlier.
+const KEY_TTL_SECONDS = 7 * 24 * 60 * 60;
+// How long a superseded version's keys stay readable after the pointer swap.
+const OLD_VERSION_GRACE_SECONDS = 300;
+
+export interface ThreatListMeta {
+ /** Google's versionToken for the stored list state, base64. */
+ versionToken: string;
+ /** sha256 (base64) over the sorted concatenated entries, per protocol. */
+ checksum: string;
+ /** Number of stored entries (hash prefixes). */
+ count: number;
+ /** When this version was written (ISO timestamp). */
+ syncedAt: string;
+ /** Earliest time the next computeDiff should run (ISO timestamp). */
+ nextDiffAt: string;
+}
+
+interface ListPointer {
+ version: string;
+ meta: ThreatListMeta;
+}
+
+interface WebRiskPrefixHit {
+ threatType: WebRiskThreatType;
+ /** The stored entry that matched (a prefix of `fullHash`), 4-32 bytes. */
+ prefix: Buffer;
+ fullHash: Buffer;
+}
+
+export type WebRiskLookupResult =
+ | { status: "unavailable" }
+ | { status: "stale"; ageSeconds: number }
+ | { status: "ok"; hits: WebRiskPrefixHit[] };
+
+/** Minimal ioredis surface the store needs (kept narrow for unit testing). */
+export interface WebRiskRedis {
+ get(key: string): Promise<string | null>;
+ mget(...keys: string[]): Promise<(string | null)[]>;
+ mgetBuffer(...keys: string[]): Promise<(Buffer | null)[]>;
+ set(
+ key: string,
+ value: string | Buffer,
+ ...args: (string | number)[]
+ ): Promise<unknown>;
+ del(...keys: string[]): Promise<unknown>;
+ expire(key: string, seconds: number): Promise<unknown>;
+}
+
+const pointerKey = (type: WebRiskThreatType) => `threat_list:${type}:current`;
+const bucketKey = (type: WebRiskThreatType, version: string, bucket: number) =>
+ `threat_list:${type}:${version}:b:${bucket}`;
+const longKey = (type: WebRiskThreatType, version: string) =>
+ `threat_list:${type}:${version}:long`;
+
+function bucketIndex(entry: Buffer): number {
+ return (entry[0] << 4) | (entry[1] >> 4);
+}
+
+/** Binary search for a 4-byte prefix in a sorted concatenated bucket. */
+function bucketContains(bucket: Buffer, prefix: Buffer): boolean {
+ const target = prefix.readUInt32BE(0);
+ let lo = 0;
+ let hi = bucket.length / 4 - 1;
+ while (lo <= hi) {
+ const mid = (lo + hi) >> 1;
+ const value = bucket.readUInt32BE(mid * 4);
+ if (value === target) return true;
+ if (value < target) lo = mid + 1;
+ else hi = mid - 1;
+ }
+ return false;
+}
+
+/** Serialize length-prefixed long (>4 byte) entries. */
+function encodeLongEntries(entries: Buffer[]): Buffer {
+ return Buffer.concat(
+ entries.flatMap(entry => [Buffer.from([entry.length]), entry]),
+ );
+}
+
+function decodeLongEntries(blob: Buffer): Buffer[] {
+ const entries: Buffer[] = [];
+ let offset = 0;
+ while (offset < blob.length) {
+ const length = blob[offset];
+ entries.push(blob.subarray(offset + 1, offset + 1 + length));
+ offset += 1 + length;
+ }
+ return entries;
+}
+
+export class WebRiskListStore {
+ constructor(private readonly redis: WebRiskRedis) {}
+
+ private parsePointer(raw: string | null): ListPointer | null {
+ if (!raw) return null;
+ try {
+ const parsed = JSON.parse(raw) as ListPointer;
+ if (
+ typeof parsed?.version !== "string" ||
+ typeof parsed?.meta?.syncedAt !== "string"
+ ) {
+ return null;
+ }
+ return parsed;
+ } catch {
+ return null;
+ }
+ }
+
+ async getPointer(type: WebRiskThreatType): Promise<ListPointer | null> {
+ return this.parsePointer(await this.redis.get(pointerKey(type)));
+ }
+
+ async getPointers(): Promise<Map<WebRiskThreatType, ListPointer | null>> {
+ const raw = await this.redis.mget(
+ ...WEB_RISK_THREAT_TYPES.map(type => pointerKey(type)),
+ );
+ return new Map(
+ WEB_RISK_THREAT_TYPES.map((type, i) => [type, this.parsePointer(raw[i])]),
+ );
+ }
+
+ /**
+ * Loads the full sorted entry list for a stored version (used by the sync
+ * worker to apply an incremental diff β€” removal indices reference positions
+ * in this lexicographically sorted list).
+ */
+ async loadEntries(
+ type: WebRiskThreatType,
+ version: string,
+ ): Promise<Buffer[]> {
+ const keys: string[] = [];
+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
+ keys.push(bucketKey(type, version, bucket));
+ }
+ keys.push(longKey(type, version));
+
+ const CHUNK = 512;
+ const values: (Buffer | null)[] = [];
+ for (let i = 0; i < keys.length; i += CHUNK) {
+ values.push(
+ ...(await this.redis.mgetBuffer(...keys.slice(i, i + CHUNK))),
+ );
+ }
+
+ const shortEntries: Buffer[] = [];
+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
+ const blob = values[bucket];
+ if (!blob) continue;
+ for (let offset = 0; offset < blob.length; offset += 4) {
+ shortEntries.push(blob.subarray(offset, offset + 4));
+ }
+ }
+ const longBlob = values[BUCKET_COUNT];
+ const longEntries = longBlob ? decodeLongEntries(longBlob) : [];
+
+ // Buckets are globally sorted already; merge in the (sorted) long entries.
+ if (longEntries.length === 0) return shortEntries;
+ return [...shortEntries, ...longEntries].sort(Buffer.compare);
+ }
+
+ /**
+ * Writes a full list snapshot under a fresh version id and atomically swaps
+ * the version pointer to it. `entries` must be lexicographically sorted.
+ * Returns the new version id.
+ */
+ async writeVersion(
+ type: WebRiskThreatType,
+ entries: Buffer[],
+ meta: ThreatListMeta,
+ ): Promise<string> {
+ const previous = await this.getPointer(type);
+ const version = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+
+ const buckets: Buffer[][] = [];
+ const long: Buffer[] = [];
+ for (const entry of entries) {
+ if (entry.length === 4) {
+ const index = bucketIndex(entry);
+ (buckets[index] ??= []).push(entry);
+ } else {
+ long.push(entry);
+ }
+ }
+
+ // ioredis pipelines synchronously issued commands on the socket, so a
+ // Promise.all of SETs is one batched round trip in practice.
+ const writes: Promise<unknown>[] = [];
+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
+ const content = buckets[bucket];
+ if (!content || content.length === 0) continue;
+ writes.push(
+ this.redis.set(
+ bucketKey(type, version, bucket),
+ Buffer.concat(content),
+ "EX",
+ KEY_TTL_SECONDS,
+ ),
+ );
+ }
+ if (long.length > 0) {
+ writes.push(
+ this.redis.set(
+ longKey(type, version),
+ encodeLongEntries(long),
+ "EX",
+ KEY_TTL_SECONDS,
+ ),
+ );
+ }
+ await Promise.all(writes);
+
+ // Atomic pointer swap: the new version becomes visible in one SET.
+ await this.redis.set(
+ pointerKey(type),
+ JSON.stringify({ version, meta } satisfies ListPointer),
+ "EX",
+ KEY_TTL_SECONDS,
+ );
+
+ // Retire the superseded version with a grace window for in-flight reads.
+ if (previous && previous.version !== version) {
+ const expires: Promise<unknown>[] = [];
+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {
+ expires.push(
+ this.redis.expire(
+ bucketKey(type, previous.version, bucket),
+ OLD_VERSION_GRACE_SECONDS,
+ ),
+ );
+ }
+ expires.push(
+ this.redis.expire(
+ longKey(type, previous.version),
+ OLD_VERSION_GRACE_SECONDS,
+ ),
+ );
+ await Promise.all(expires);
+ }
+
+ return version;
+ }
+
+ /**
+ * Checks a set of full expression hashes against every threat list.
+ * Entirely local to our infrastructure: reads the synced buckets from
+ * Redis and binary-searches them β€” nothing is sent to Google here.
+ */
+ async lookup(fullHashes: Buffer[]): Promise<WebRiskLookupResult> {
+ const pointers = await this.getPointers();
+
+ let oldestAgeSeconds = 0;
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = pointers.get(type);
+ if (!pointer) return { status: "unavailable" };
+ const age = (Date.now() - Date.parse(pointer.meta.syncedAt)) / 1000;
+ if (!Number.isFinite(age)) return { status: "unavailable" };
+ oldestAgeSeconds = Math.max(oldestAgeSeconds, age);
+ }
+ if (oldestAgeSeconds > config.THREAT_LIST_STALENESS_SECONDS) {
+ return { status: "stale", ageSeconds: Math.round(oldestAgeSeconds) };
+ }
+
+ // Collect every bucket key we need across lists and hashes, fetch them
+ // in one MGET, then binary-search locally.
+ const keys: string[] = [];
+ const keyIndex = new Map<string, number>();
+ const keyFor = (key: string): number => {
+ let index = keyIndex.get(key);
+ if (index === undefined) {
+ index = keys.length;
+ keys.push(key);
+ keyIndex.set(key, index);
+ }
+ return index;
+ };
+
+ interface Probe {
+ type: WebRiskThreatType;
+ fullHash: Buffer;
+ bucketKeyIndex: number;
+ longKeyIndex: number;
+ }
+ const probes: Probe[] = [];
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = pointers.get(type)!;
+ const longIndex = keyFor(longKey(type, pointer.version));
+ for (const fullHash of fullHashes) {
+ probes.push({
+ type,
+ fullHash,
+ bucketKeyIndex: keyFor(
+ bucketKey(type, pointer.version, bucketIndex(fullHash)),
+ ),
+ longKeyIndex: longIndex,
+ });
+ }
+ }
+
+ const values = await this.redis.mgetBuffer(...keys);
+
+ const hits: WebRiskPrefixHit[] = [];
+ for (const probe of probes) {
+ const bucket = values[probe.bucketKeyIndex];
+ if (bucket && bucketContains(bucket, probe.fullHash.subarray(0, 4))) {
+ hits.push({
+ threatType: probe.type,
+ prefix: probe.fullHash.subarray(0, 4),
+ fullHash: probe.fullHash,
+ });
+ continue;
+ }
+ const longBlob = values[probe.longKeyIndex];
+ if (longBlob) {
+ for (const entry of decodeLongEntries(longBlob)) {
+ if (
+ entry.length <= probe.fullHash.length &&
+ entry.equals(probe.fullHash.subarray(0, entry.length))
+ ) {
+ hits.push({
+ threatType: probe.type,
+ prefix: entry,
+ fullHash: probe.fullHash,
+ });
+ break;
+ }
+ }
+ }
+ }
+
+ return { status: "ok", hits };
+ }
+}
+
+let defaultStore: WebRiskListStore | null = null;
+
+/** Store singleton on the durable (non-evict) Redis connection. */
+export function getWebRiskListStore(): WebRiskListStore {
+ if (!defaultStore) {
+ defaultStore = new WebRiskListStore(
+ getRedisConnection() as unknown as WebRiskRedis,
+ );
+ }
+ return defaultStore;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.ts
new file mode 100644
index 000000000..2fecc1adf
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.ts
@@ -0,0 +1,254 @@
+import { createHash } from "crypto";
+import http from "http";
+import { AddressInfo } from "net";
+import { config } from "../../../../config";
+import { runThreatListSyncPass } from "./sync";
+import { WEB_RISK_THREAT_TYPES, WebRiskListStore } from "./store";
+import { createFakeWebRiskRedis } from "./testing";
+
+// Sync-loop unit tests: a local HTTP server plays Google's
+// threatLists:computeDiff endpoint via the config URL override.
+
+type DiffHandler = (threatType: string, versionToken: string | null) => object;
+
+let server: http.Server;
+let baseUrl: string;
+let diffCalls: { threatType: string; versionToken: string | null }[] = [];
+let diffHandler: DiffHandler = () => ({});
+
+const originalConfig = {
+ url: config.GOOGLE_WEB_RISK_API_URL,
+ key: config.GOOGLE_WEB_RISK_API_KEY,
+};
+
+beforeAll(async () => {
+ await new Promise<void>(resolve => {
+ server = http.createServer((req, res) => {
+ const url = new URL(req.url ?? "/", "http://localhost");
+ if (url.pathname === "/v1/threatLists:computeDiff") {
+ const threatType = url.searchParams.get("threatType") ?? "";
+ const versionToken = url.searchParams.get("versionToken");
+ diffCalls.push({ threatType, versionToken });
+ res.setHeader("content-type", "application/json");
+ res.end(JSON.stringify(diffHandler(threatType, versionToken)));
+ } else {
+ res.statusCode = 404;
+ res.end("{}");
+ }
+ });
+ server.listen(0, "127.0.0.1", () => {
+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+ config.GOOGLE_WEB_RISK_API_URL = baseUrl;
+ config.GOOGLE_WEB_RISK_API_KEY = "test-key";
+ resolve();
+ });
+ });
+});
+
+afterAll(async () => {
+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.url;
+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.key;
+ await new Promise<void>(resolve => server.close(() => resolve()));
+});
+
+beforeEach(() => {
+ diffCalls = [];
+});
+
+function checksumOf(sortedEntries: Buffer[]): string {
+ const hash = createHash("sha256");
+ for (const entry of sortedEntries) hash.update(entry);
+ return hash.digest("base64");
+}
+
+function sortedPrefixes(hexes: string[]): Buffer[] {
+ return hexes.map(hex => Buffer.from(hex, "hex")).sort(Buffer.compare);
+}
+
+function resetResponse(entries: Buffer[], token: string): object {
+ return {
+ responseType: "RESET",
+ additions: {
+ rawHashes: [
+ { prefixSize: 4, rawHashes: Buffer.concat(entries).toString("base64") },
+ ],
+ },
+ newVersionToken: Buffer.from(token).toString("base64"),
+ checksum: { sha256: checksumOf(entries) },
+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),
+ };
+}
+
+describe("runThreatListSyncPass", () => {
+ it("bootstraps all lists from a RESET response and stores metadata", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ const entries = sortedPrefixes(["00000001", "7fffffff", "ffffff00"]);
+ diffHandler = () => resetResponse(entries, "v1");
+
+ const ran = await runThreatListSyncPass({}, store, redis);
+ expect(ran).toBe(true);
+ expect(diffCalls.map(c => c.threatType).sort()).toEqual(
+ [...WEB_RISK_THREAT_TYPES].sort(),
+ );
+ // First sync has no version token.
+ expect(diffCalls.every(c => c.versionToken === null)).toBe(true);
+
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = await store.getPointer(type);
+ expect(pointer).not.toBeNull();
+ expect(pointer!.meta.count).toBe(3);
+ expect(pointer!.meta.versionToken).toBe(
+ Buffer.from("v1").toString("base64"),
+ );
+ const stored = await store.loadEntries(type, pointer!.version);
+ expect(stored.map(e => e.toString("hex"))).toEqual(
+ entries.map(e => e.toString("hex")),
+ );
+ }
+ });
+
+ it("applies an incremental DIFF (removals by sorted index + additions)", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ const initial = sortedPrefixes([
+ "11111111",
+ "22222222",
+ "33333333",
+ "44444444",
+ ]);
+ diffHandler = () => resetResponse(initial, "v1");
+ await runThreatListSyncPass({}, store, redis);
+ diffCalls = [];
+
+ // Remove indices 1 and 3 (22222222, 44444444), add aaaaaaaa.
+ const expected = sortedPrefixes(["11111111", "33333333", "aaaaaaaa"]);
+ diffHandler = () => ({
+ responseType: "DIFF",
+ additions: {
+ rawHashes: [
+ {
+ prefixSize: 4,
+ rawHashes: Buffer.from("aaaaaaaa", "hex").toString("base64"),
+ },
+ ],
+ },
+ removals: { rawIndices: { indices: [1, 3] } },
+ newVersionToken: Buffer.from("v2").toString("base64"),
+ checksum: { sha256: checksumOf(expected) },
+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),
+ });
+
+ await runThreatListSyncPass({ force: true }, store, redis);
+
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = await store.getPointer(type);
+ expect(pointer!.meta.versionToken).toBe(
+ Buffer.from("v2").toString("base64"),
+ );
+ const stored = await store.loadEntries(type, pointer!.version);
+ expect(stored.map(e => e.toString("hex"))).toEqual(
+ expected.map(e => e.toString("hex")),
+ );
+ }
+ // The DIFF request carried the stored version token.
+ expect(
+ diffCalls.every(
+ c => c.versionToken === Buffer.from("v1").toString("base64"),
+ ),
+ ).toBe(true);
+ });
+
+ it("rejects a malformed prefixSize instead of spinning (remote JSON is untrusted)", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ diffHandler = () => ({
+ responseType: "RESET",
+ additions: {
+ rawHashes: [
+ {
+ prefixSize: 0,
+ rawHashes: Buffer.from("11111111", "hex").toString("base64"),
+ },
+ ],
+ },
+ newVersionToken: Buffer.from("v1").toString("base64"),
+ checksum: { sha256: checksumOf(sortedPrefixes(["11111111"])) },
+ });
+
+ // Per-list failures are logged and skipped (one bad list must not take
+ // down the pass) β€” the important part is: no infinite loop, and nothing
+ // gets published.
+ await runThreatListSyncPass({}, store, redis);
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ expect(await store.getPointer(type)).toBeNull();
+ }
+ });
+
+ it("recovers from a DIFF checksum mismatch by re-syncing from scratch", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ const initial = sortedPrefixes(["11111111"]);
+ diffHandler = () => resetResponse(initial, "v1");
+ await runThreatListSyncPass({}, store, redis);
+ diffCalls = [];
+
+ const fresh = sortedPrefixes(["55555555", "66666666"]);
+ diffHandler = (_type, versionToken) => {
+ if (versionToken) {
+ // Corrupt diff: checksum does not match what applying it produces.
+ return {
+ responseType: "DIFF",
+ additions: {
+ rawHashes: [
+ {
+ prefixSize: 4,
+ rawHashes: Buffer.from("99999999", "hex").toString("base64"),
+ },
+ ],
+ },
+ newVersionToken: Buffer.from("bad").toString("base64"),
+ checksum: { sha256: checksumOf(sortedPrefixes(["deadbeef"])) },
+ };
+ }
+ return resetResponse(fresh, "v2");
+ };
+
+ await runThreatListSyncPass({ force: true }, store, redis);
+
+ // Each list: one failed DIFF then one RESET recovery.
+ expect(diffCalls.length).toBe(WEB_RISK_THREAT_TYPES.length * 2);
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = await store.getPointer(type);
+ expect(pointer!.meta.versionToken).toBe(
+ Buffer.from("v2").toString("base64"),
+ );
+ const stored = await store.loadEntries(type, pointer!.version);
+ expect(stored.map(e => e.toString("hex"))).toEqual(
+ fresh.map(e => e.toString("hex")),
+ );
+ }
+ });
+
+ it("respects recommendedNextDiff (unforced pass is a no-op while fresh)", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ diffHandler = () => resetResponse(sortedPrefixes(["11111111"]), "v1");
+ await runThreatListSyncPass({}, store, redis);
+ diffCalls = [];
+
+ const ran = await runThreatListSyncPass({}, store, redis);
+ expect(ran).toBe(true); // lock acquired…
+ expect(diffCalls).toEqual([]); // …but nothing was due
+ });
+
+ it("returns false without syncing when another process holds the lock", async () => {
+ const redis = createFakeWebRiskRedis();
+ const store = new WebRiskListStore(redis);
+ await redis.set("threat_list_sync:lock", "someone-else", "EX", 60, "NX");
+
+ const ran = await runThreatListSyncPass({}, store, redis);
+ expect(ran).toBe(false);
+ expect(diffCalls).toEqual([]);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/sync.ts b/apps/api/src/lib/threat-protection/providers/web-risk/sync.ts
new file mode 100644
index 000000000..68d2f8035
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/sync.ts
@@ -0,0 +1,335 @@
+import { createHash, randomUUID } from "crypto";
+import { config } from "../../../../config";
+import { getRedisConnection } from "../../../../services/queue-service";
+import { logger as _logger } from "../../../logger";
+import {
+ WEB_RISK_THREAT_TYPES,
+ WebRiskListStore,
+ getWebRiskListStore,
+ type ThreatListMeta,
+ type WebRiskRedis,
+ type WebRiskThreatType,
+} from "./store";
+
+// Google Web Risk Update API sync worker (ZDR rework of "normal" mode).
+//
+// threatLists:computeDiff (free, unlike hashes:search) is polled per threat
+// list: with no stored versionToken Google sends the full list (RESET),
+// afterwards incremental additions/removals (DIFF). The updated list is
+// checksum-verified and written to the versioned bucket store (./store.ts)
+// with an atomic pointer swap.
+//
+// Topology: every API/worker process runs the loop, but a fleet-wide Redis
+// lock ensures only one process actually syncs at a time. The loop respects
+// Google's recommendedNextDiff (floored by
+// THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS). List age is logged every cycle so
+// staleness is monitorable; a list older than THREAT_LIST_STALENESS_SECONDS
+// is treated as unavailable by the check path (provider-failure semantics).
+
+const logger = _logger.child({ module: "web-risk-sync" });
+
+const LOCK_KEY = "threat_list_sync:lock";
+const LOCK_TTL_SECONDS = 300;
+const LOOP_TICK_MS = 30_000;
+const SYNC_FETCH_TIMEOUT_MS = 120_000;
+
+interface RawHashes {
+ prefixSize?: number;
+ rawHashes?: string;
+}
+
+interface ComputeDiffResponse {
+ responseType?: "RESET" | "DIFF" | "RESPONSE_TYPE_UNSPECIFIED";
+ additions?: { rawHashes?: RawHashes[] };
+ removals?: { rawIndices?: { indices?: number[] } };
+ newVersionToken?: string;
+ checksum?: { sha256?: string };
+ recommendedNextDiff?: string;
+}
+
+function isConfigured(): boolean {
+ return (
+ typeof config.GOOGLE_WEB_RISK_API_KEY === "string" &&
+ config.GOOGLE_WEB_RISK_API_KEY.trim().length > 0
+ );
+}
+
+async function computeDiff(
+ type: WebRiskThreatType,
+ versionToken: string | null,
+): Promise<ComputeDiffResponse> {
+ const params = new URLSearchParams();
+ params.append("threatType", type);
+ if (versionToken) params.append("versionToken", versionToken);
+ params.append("constraints.supportedCompressions", "RAW");
+ params.append("key", config.GOOGLE_WEB_RISK_API_KEY!);
+
+ const response = await fetch(
+ `${config.GOOGLE_WEB_RISK_API_URL}/v1/threatLists:computeDiff?${params.toString()}`,
+ { method: "GET", signal: AbortSignal.timeout(SYNC_FETCH_TIMEOUT_MS) },
+ );
+ if (!response.ok) {
+ throw new Error(
+ `Web Risk computeDiff for ${type} failed with status ${response.status}`,
+ );
+ }
+ return (await response.json()) as ComputeDiffResponse;
+}
+
+function decodeAdditions(additions: RawHashes[] | undefined): Buffer[] {
+ const entries: Buffer[] = [];
+ for (const group of additions ?? []) {
+ if (!group.rawHashes) continue;
+ const prefixSize = group.prefixSize ?? 4;
+ // The response is unvalidated remote JSON: a non-positive prefixSize
+ // would make the offset loop below spin forever and stall the fleet-wide
+ // sync for the whole lock TTL. Valid Web Risk prefixes are 4-32 bytes.
+ if (!Number.isInteger(prefixSize) || prefixSize < 4 || prefixSize > 32) {
+ throw new Error(
+ `Web Risk computeDiff returned invalid prefixSize ${prefixSize}`,
+ );
+ }
+ const blob = Buffer.from(group.rawHashes, "base64");
+ for (
+ let offset = 0;
+ offset + prefixSize <= blob.length;
+ offset += prefixSize
+ ) {
+ entries.push(blob.subarray(offset, offset + prefixSize));
+ }
+ }
+ return entries;
+}
+
+/** Protocol checksum: sha256 over the sorted concatenated entries, base64. */
+function listChecksum(sortedEntries: Buffer[]): string {
+ const hash = createHash("sha256");
+ for (const entry of sortedEntries) hash.update(entry);
+ return hash.digest("base64");
+}
+
+function nextDiffAt(recommendedNextDiff: string | undefined): string {
+ const floorMs =
+ Date.now() + config.THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS * 1000;
+ const recommendedMs = recommendedNextDiff
+ ? Date.parse(recommendedNextDiff)
+ : NaN;
+ return new Date(
+ Number.isFinite(recommendedMs) ? Math.max(recommendedMs, floorMs) : floorMs,
+ ).toISOString();
+}
+
+/**
+ * Syncs a single list: computeDiff against the stored version, apply, verify
+ * checksum, write + swap. On a checksum mismatch of an incremental diff the
+ * list is re-synced from scratch once (RESET).
+ */
+async function syncList(
+ store: WebRiskListStore,
+ type: WebRiskThreatType,
+ forceReset = false,
+): Promise<void> {
+ const pointer = forceReset ? null : await store.getPointer(type);
+ const versionToken = pointer?.meta.versionToken ?? null;
+
+ const diff = await computeDiff(type, versionToken);
+
+ let entries: Buffer[];
+ if (diff.responseType === "DIFF" && pointer) {
+ entries = await store.loadEntries(type, pointer.version);
+ const indices = diff.removals?.rawIndices?.indices ?? [];
+ if (indices.length > 0) {
+ const removed = new Set(indices);
+ entries = entries.filter((_, index) => !removed.has(index));
+ }
+ entries = entries.concat(decodeAdditions(diff.additions?.rawHashes));
+ } else {
+ // RESET (or first sync): the additions are the entire list.
+ entries = decodeAdditions(diff.additions?.rawHashes);
+ }
+
+ entries.sort(Buffer.compare);
+
+ const expectedChecksum = diff.checksum?.sha256;
+ if (expectedChecksum && listChecksum(entries) !== expectedChecksum) {
+ if (diff.responseType === "DIFF") {
+ logger.warn(
+ "Web Risk list checksum mismatch after diff; re-syncing from scratch",
+ { threatType: type },
+ );
+ return syncList(store, type, true);
+ }
+ throw new Error(`Web Risk ${type} full sync failed checksum verification`);
+ }
+
+ const meta: ThreatListMeta = {
+ versionToken: diff.newVersionToken ?? versionToken ?? "",
+ checksum: expectedChecksum ?? listChecksum(entries),
+ count: entries.length,
+ syncedAt: new Date().toISOString(),
+ nextDiffAt: nextDiffAt(diff.recommendedNextDiff),
+ };
+ await store.writeVersion(type, entries, meta);
+
+ logger.info("Web Risk threat list synced", {
+ canonicalLog: "threat-protection/list-sync",
+ threatType: type,
+ responseType: diff.responseType ?? "RESET",
+ entryCount: entries.length,
+ nextDiffAt: meta.nextDiffAt,
+ });
+}
+
+async function acquireLock(redis: WebRiskRedis): Promise<string | null> {
+ const token = randomUUID();
+ const result = await redis.set(LOCK_KEY, token, "EX", LOCK_TTL_SECONDS, "NX");
+ return result === "OK" ? token : null;
+}
+
+async function releaseLock(redis: WebRiskRedis, token: string): Promise<void> {
+ // Get-compare-del (not atomic; worst case two syncers overlap briefly,
+ // which the versioned store + atomic pointer swap makes harmless).
+ const current = await redis.get(LOCK_KEY);
+ if (current === token) await redis.del(LOCK_KEY);
+}
+
+function isListDue(
+ pointer: Awaited<ReturnType<WebRiskListStore["getPointer"]>>,
+ force: boolean,
+): boolean {
+ if (force || !pointer) return true;
+ const next = Date.parse(pointer.meta.nextDiffAt);
+ return !Number.isFinite(next) || next <= Date.now();
+}
+
+/**
+ * Runs one sync pass under the fleet-wide lock: every due (or, with `force`,
+ * every) list gets a computeDiff. Returns false when another process holds
+ * the lock. Throws when Web Risk is not configured.
+ */
+export async function runThreatListSyncPass(
+ options: { force?: boolean } = {},
+ store: WebRiskListStore = getWebRiskListStore(),
+ redis: WebRiskRedis = getRedisConnection() as unknown as WebRiskRedis,
+): Promise<boolean> {
+ if (!isConfigured()) {
+ throw new Error("Google Web Risk is not configured");
+ }
+
+ const token = await acquireLock(redis);
+ if (token === null) return false;
+
+ try {
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = await store.getPointer(type);
+ if (!isListDue(pointer, options.force ?? false)) continue;
+ try {
+ await syncList(store, type);
+ } catch (error) {
+ logger.error("Web Risk threat list sync failed", {
+ canonicalLog: "threat-protection/list-sync",
+ threatType: type,
+ error,
+ });
+ }
+ }
+ } finally {
+ await releaseLock(redis, token);
+ }
+ return true;
+}
+
+/** Logs the age of every stored list (staleness monitoring). */
+async function logListAges(store: WebRiskListStore): Promise<void> {
+ const pointers = await store.getPointers();
+ const ages: Record<string, number | null> = {};
+ let stale = false;
+ for (const type of WEB_RISK_THREAT_TYPES) {
+ const pointer = pointers.get(type);
+ const ageSeconds = pointer
+ ? Math.round((Date.now() - Date.parse(pointer.meta.syncedAt)) / 1000)
+ : null;
+ ages[type] = ageSeconds;
+ if (
+ ageSeconds === null ||
+ ageSeconds > config.THREAT_LIST_STALENESS_SECONDS
+ ) {
+ stale = true;
+ }
+ }
+ const log = stale ? logger.warn.bind(logger) : logger.info.bind(logger);
+ log("Web Risk threat list age", {
+ canonicalLog: "threat-protection/list-age",
+ ageSeconds: ages,
+ stale,
+ stalenessThresholdSeconds: config.THREAT_LIST_STALENESS_SECONDS,
+ });
+}
+
+let loopStarted = false;
+
+/**
+ * Starts the periodic sync loop for this process (idempotent, lazy β€” called
+ * on first Web Risk provider use so every API/worker process that performs
+ * checks also keeps the lists fresh). The Redis lock in
+ * runThreatListSyncPass makes the fleet-wide behavior single-syncer.
+ */
+export function ensureThreatListSyncLoop(): void {
+ if (loopStarted || !isConfigured()) return;
+ loopStarted = true;
+
+ let tick = 0;
+ const timer = setInterval(() => {
+ void (async () => {
+ try {
+ // Age metric every ~5min (stale lists log a warn from logListAges).
+ if (tick++ % 10 === 0) {
+ await logListAges(getWebRiskListStore());
+ }
+ await runThreatListSyncPass();
+ } catch (error) {
+ logger.warn("Web Risk sync loop tick failed", { error });
+ }
+ })();
+ }, LOOP_TICK_MS);
+ timer.unref();
+}
+
+let bootSyncPromise: Promise<void> | null = null;
+
+/**
+ * One forced sync pass per process lifetime, shared by all callers. Run
+ * before the first lookup so that (a) a fresh deployment bootstraps the
+ * lists and (b) a process never trusts a possibly-superseded stored state
+ * without having compared version tokens with Google once. If another
+ * process holds the sync lock, waits for it to finish instead.
+ */
+export function ensureThreatListBootSync(): Promise<void> {
+ if (!bootSyncPromise) {
+ bootSyncPromise = (async () => {
+ const redis = getRedisConnection() as unknown as WebRiskRedis;
+ const synced = await runThreatListSyncPass({ force: true });
+ if (synced) return;
+ // Another process is syncing right now β€” wait for the lock to clear,
+ // bounded by the lock TTL (plus padding) so we cover the longest
+ // possible sync: three sequential list fetches can exceed a minute.
+ // Callers stay responsive regardless: the provider awaits this behind
+ // the request's abort signal, and lists appearing mid-wait end it.
+ const store = getWebRiskListStore();
+ const deadline = Date.now() + LOCK_TTL_SECONDS * 1000 + 30_000;
+ while (Date.now() < deadline) {
+ await new Promise(resolve => setTimeout(resolve, 500));
+ if ((await redis.get(LOCK_KEY)) === null) return;
+ // The other process swaps pointers list-by-list; once every list is
+ // present we don't need to outwait its lock.
+ const pointers = await store.getPointers();
+ if ([...pointers.values()].every(pointer => pointer !== null)) return;
+ }
+ })().catch(error => {
+ // Do not cache a failed boot sync: the next check retries it.
+ bootSyncPromise = null;
+ throw error;
+ });
+ }
+ return bootSyncPromise;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/testing.ts b/apps/api/src/lib/threat-protection/providers/web-risk/testing.ts
new file mode 100644
index 000000000..acc8d42b3
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/web-risk/testing.ts
@@ -0,0 +1,180 @@
+import { createHash } from "crypto";
+import type http from "http";
+import { generateHostExpressions } from "./canonicalize";
+import { WEB_RISK_THREAT_TYPES, type WebRiskThreatType } from "./store";
+
+// Test helpers for the Web Risk Update API provider β€” shared by the unit
+// suites in this directory and the threat-protection snips (which mock the
+// Google endpoints with a local HTTP server). Not used in production code.
+// NOTE: the fake Redis lives in ./fake-redis.ts (imported by vi.mock
+// factories, which must not transitively import services/queue-service).
+
+export { createFakeWebRiskRedis } from "./fake-redis";
+
+export function sha256(data: string | Buffer): Buffer {
+ return createHash("sha256").update(data).digest();
+}
+
+/** SHA-256 of the primary (exact-host) lookup expression for a domain. */
+export function domainExpressionHash(domain: string): Buffer {
+ return sha256(generateHostExpressions(domain)[0]);
+}
+
+/**
+ * In-memory model of Google's threat lists, able to answer both mocked
+ * endpoints: threatLists:computeDiff (always a full RESET of the current
+ * state β€” exactly what a fresh syncer needs) and hashes:search.
+ */
+export class WebRiskMockDatabase {
+ private fullHashesByType = new Map<WebRiskThreatType, Buffer[]>();
+ /** Entries in the list with no confirmable full hash (collision cases). */
+ private prefixOnlyByType = new Map<WebRiskThreatType, Buffer[]>();
+
+ /** Flags a domain: its exact-host expression hash joins the list. */
+ addRiskyDomain(
+ domain: string,
+ threatType: WebRiskThreatType = "MALWARE",
+ ): void {
+ this.addFullHash(domainExpressionHash(domain), threatType);
+ }
+
+ addFullHash(fullHash: Buffer, threatType: WebRiskThreatType): void {
+ const hashes = this.fullHashesByType.get(threatType) ?? [];
+ hashes.push(fullHash);
+ this.fullHashesByType.set(threatType, hashes);
+ }
+
+ /**
+ * Puts only a 4-byte prefix in the list, whose hashes:search confirmation
+ * returns a full hash that does NOT match the caller's expression (i.e. a
+ * prefix collision): pass the full hash of some *other* expression that
+ * shares the target's 4-byte prefix.
+ */
+ addCollidingFullHash(fullHash: Buffer, threatType: WebRiskThreatType): void {
+ const hashes = this.prefixOnlyByType.get(threatType) ?? [];
+ hashes.push(fullHash);
+ this.prefixOnlyByType.set(threatType, hashes);
+ }
+
+ private entriesFor(threatType: WebRiskThreatType): Buffer[] {
+ const prefixes = [
+ ...(this.fullHashesByType.get(threatType) ?? []),
+ ...(this.prefixOnlyByType.get(threatType) ?? []),
+ ].map(hash => hash.subarray(0, 4));
+ const unique = new Map(prefixes.map(p => [p.toString("hex"), p]));
+ return [...unique.values()].sort(Buffer.compare);
+ }
+
+ /** RESET response body for a threatLists:computeDiff request. */
+ computeDiffResponse(threatType: string): object {
+ const entries = this.entriesFor(threatType as WebRiskThreatType);
+ const checksum = createHash("sha256");
+ for (const entry of entries) checksum.update(entry);
+ return {
+ responseType: "RESET",
+ additions:
+ entries.length > 0
+ ? {
+ rawHashes: [
+ {
+ prefixSize: 4,
+ rawHashes: Buffer.concat(entries).toString("base64"),
+ },
+ ],
+ }
+ : undefined,
+ newVersionToken: Buffer.from(
+ `token-${threatType}-${Date.now()}`,
+ ).toString("base64"),
+ checksum: { sha256: checksum.digest("base64") },
+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),
+ };
+ }
+
+ /** Response body for a hashes:search request. */
+ hashesSearchResponse(hashPrefixBase64: string): object {
+ const prefix = Buffer.from(hashPrefixBase64, "base64");
+ const threats: object[] = [];
+ for (const threatType of WEB_RISK_THREAT_TYPES) {
+ const candidates = [
+ ...(this.fullHashesByType.get(threatType) ?? []),
+ ...(this.prefixOnlyByType.get(threatType) ?? []),
+ ];
+ for (const fullHash of candidates) {
+ if (
+ prefix.length <= fullHash.length &&
+ fullHash.subarray(0, prefix.length).equals(prefix)
+ ) {
+ threats.push({
+ hash: fullHash.toString("base64"),
+ threatTypes: [threatType],
+ expireTime: new Date(Date.now() + 3600_000).toISOString(),
+ });
+ }
+ }
+ }
+ return {
+ ...(threats.length > 0 ? { threats } : {}),
+ negativeExpireTime: new Date(Date.now() + 3600_000).toISOString(),
+ };
+ }
+}
+
+interface WebRiskMockCounters {
+ computeDiffRequests: number;
+ hashesSearchRequests: number;
+ /** hashes:search requests per 4-byte prefix (hex). */
+ hashesSearchByPrefixHex: Map<string, number>;
+ hashesSearchRequestsForDomain(domain: string): number;
+}
+
+/**
+ * HTTP handler for the two mocked Web Risk endpoints. Returns true when the
+ * request was handled (so callers can chain their own routes).
+ */
+export function createWebRiskMockHandler(
+ db: WebRiskMockDatabase,
+ counters: WebRiskMockCounters,
+): (req: http.IncomingMessage, res: http.ServerResponse) => boolean {
+ return (req, res) => {
+ const url = new URL(req.url ?? "/", "http://localhost");
+ if (url.pathname === "/v1/threatLists:computeDiff") {
+ counters.computeDiffRequests++;
+ res.setHeader("Content-Type", "application/json");
+ res.end(
+ JSON.stringify(
+ db.computeDiffResponse(url.searchParams.get("threatType") ?? ""),
+ ),
+ );
+ return true;
+ }
+ if (url.pathname === "/v1/hashes:search") {
+ counters.hashesSearchRequests++;
+ const prefixB64 = url.searchParams.get("hashPrefix") ?? "";
+ const prefixHex = Buffer.from(prefixB64, "base64").toString("hex");
+ counters.hashesSearchByPrefixHex.set(
+ prefixHex,
+ (counters.hashesSearchByPrefixHex.get(prefixHex) ?? 0) + 1,
+ );
+ res.setHeader("Content-Type", "application/json");
+ res.end(JSON.stringify(db.hashesSearchResponse(prefixB64)));
+ return true;
+ }
+ return false;
+ };
+}
+
+export function createWebRiskMockCounters(): WebRiskMockCounters {
+ const counters: WebRiskMockCounters = {
+ computeDiffRequests: 0,
+ hashesSearchRequests: 0,
+ hashesSearchByPrefixHex: new Map(),
+ hashesSearchRequestsForDomain(domain: string): number {
+ const prefixHex = domainExpressionHash(domain)
+ .subarray(0, 4)
+ .toString("hex");
+ return counters.hashesSearchByPrefixHex.get(prefixHex) ?? 0;
+ },
+ };
+ return counters;
+}
diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.ts
new file mode 100644
index 000000000..f71e5014c
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/request.ts
@@ -0,0 +1,157 @@
+import type { TeamFlags } from "../../controllers/v2/types";
+import { getThreatProtection } from "../zdr-helpers";
+import { checkDomain, type ThreatCheckContext } from "./index";
+import {
+ getOrgIdForTeam,
+ getOrgThreatProtectionConfig,
+ resolveEffectivePolicy,
+ type OrgThreatProtectionConfig,
+} from "./store";
+import type { ThreatDecision, ThreatProtectionPolicy } from "./types";
+import { normalizeDomain } from "./verdict";
+
+// Controller-layer glue for threat protection enforcement. One helper
+// (resolveThreatProtection) is shared by every endpoint: it turns
+// (team flags + org config + per-request override) into an effective
+// ThreatProtectionPolicy β€” or null, meaning the feature is off for this
+// request (the enforcement hot path then does zero extra work).
+
+const SUPPORT_EMAIL = "[email protected]";
+
+export const THREAT_PROTECTION_NOT_ENABLED_MESSAGE = `Threat protection is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`;
+
+export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE =
+ "Per-request threat protection overrides are disabled by your organization's threat protection configuration.";
+
+export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE =
+ 'Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';
+
+interface ResolvedThreatProtection {
+ /** Set when the request must be rejected with a 403. */
+ error?: string;
+ /** Effective policy, or null when the feature is off for this request. */
+ policy: ThreatProtectionPolicy | null;
+ /** Loaded org config β€” pass to checkPermissions' third parameter. */
+ orgConfig: OrgThreatProtectionConfig | null;
+}
+
+/**
+ * Resolves the effective threat protection policy for an API request.
+ *
+ * - Team flag "disabled"/absent: any per-request override β†’ 403; otherwise the
+ * feature is entirely off (`policy: null`) with zero I/O.
+ * - Flag "allowed"/"forced": the org config (60s-cached) is loaded; a request
+ * override does field-level replacement IF the org allows overrides,
+ * otherwise β†’ 403.
+ * - Flag "forced": an override may never set `mode: "off"` β†’ 403.
+ * - Effective mode "off" resolves to `policy: null` so callers can skip all
+ * enforcement work.
+ */
+export async function resolveThreatProtection(args: {
+ teamId: string;
+ orgId?: string | null;
+ flags: TeamFlags;
+ override?: Partial<ThreatProtectionPolicy>;
+}): Promise<ResolvedThreatProtection> {
+ const flagMode = getThreatProtection(args.flags);
+
+ if (flagMode !== "allowed" && flagMode !== "forced") {
+ if (args.override !== undefined) {
+ return {
+ error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE,
+ policy: null,
+ orgConfig: null,
+ };
+ }
+ return { policy: null, orgConfig: null };
+ }
+
+ const orgId = args.orgId ?? (await getOrgIdForTeam(args.teamId));
+ const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;
+
+ if (args.override !== undefined) {
+ if (orgConfig && !orgConfig.allowRequestOverrides) {
+ return {
+ error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,
+ policy: null,
+ orgConfig,
+ };
+ }
+ if (flagMode === "forced" && args.override.mode === "off") {
+ return {
+ error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,
+ policy: null,
+ orgConfig,
+ };
+ }
+ }
+
+ const policy = resolveEffectivePolicy(orgConfig, args.override);
+ return {
+ policy: policy.mode === "off" ? null : policy,
+ orgConfig,
+ };
+}
+
+interface BlockedUrl {
+ url: string;
+ domain: string;
+ decision: ThreatDecision;
+}
+
+interface UrlPolicyCheckResult {
+ allowedUrls: string[];
+ blocked: BlockedUrl[];
+ /** All decisions, keyed by normalized domain β€” for logging/bookkeeping. */
+ decisionsByDomain: Map<string, ThreatDecision>;
+}
+
+// Crawls/searches fan out to many URLs on few domains: dedupe by domain and
+// check domains concurrently (bounded). Deduplication is strictly scoped to
+// this one batch (an in-memory map) β€” there is no cross-request or persisted
+// verdict reuse (ZDR).
+const DOMAIN_CHECK_CONCURRENCY = 16;
+
+/**
+ * Checks a list of URLs against a policy, deduplicated by domain. Never
+ * throws β€” `checkDomain` resolves provider failures via the policy's
+ * failurePolicy. Each call scans a given domain at most once (per-batch
+ * in-memory dedup; callers may pass their own `ctx.dedup` to widen the scope
+ * to a whole request/job).
+ */
+export async function checkUrlsAgainstThreatPolicy(
+ urls: string[],
+ policy: ThreatProtectionPolicy,
+ ctx: ThreatCheckContext,
+): Promise<UrlPolicyCheckResult> {
+ const domains = [...new Set(urls.map(url => normalizeDomain(url)))];
+ const dedupCtx: ThreatCheckContext = {
+ ...ctx,
+ dedup: ctx.dedup ?? new Map(),
+ };
+
+ const decisionsByDomain = new Map<string, ThreatDecision>();
+ for (let i = 0; i < domains.length; i += DOMAIN_CHECK_CONCURRENCY) {
+ const batch = domains.slice(i, i + DOMAIN_CHECK_CONCURRENCY);
+ const decisions = await Promise.all(
+ batch.map(domain => checkDomain(domain, policy, dedupCtx)),
+ );
+ batch.forEach((domain, index) =>
+ decisionsByDomain.set(domain, decisions[index]),
+ );
+ }
+
+ const allowedUrls: string[] = [];
+ const blocked: BlockedUrl[] = [];
+ for (const url of urls) {
+ const domain = normalizeDomain(url);
+ const decision = decisionsByDomain.get(domain);
+ if (decision && !decision.allowed) {
+ blocked.push({ url, domain, decision });
+ } else {
+ allowedUrls.push(url);
+ }
+ }
+
+ return { allowedUrls, blocked, decisionsByDomain };
+}
diff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.ts
new file mode 100644
index 000000000..d7987607f
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/store.test.ts
@@ -0,0 +1,212 @@
+import {
+ resolveEffectivePolicy,
+ rowToConfig,
+ OrgThreatProtectionConfig,
+} from "./store";
+import {
+ THREAT_PROTECTION_POLICY_DEFAULTS,
+ ThreatProtectionPolicy,
+} from "./types";
+
+const orgPolicy: ThreatProtectionPolicy = {
+ mode: "normal",
+ riskScoreThreshold: 60,
+ blacklist: ["*.bad.example"],
+ whitelist: ["example.com"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+};
+
+function makeOrgConfig(
+ overrides: Partial<OrgThreatProtectionConfig> = {},
+): OrgThreatProtectionConfig {
+ return {
+ orgId: "00000000-0000-0000-0000-000000000000",
+ policy: { ...orgPolicy },
+ allowRequestOverrides: true,
+ createdAt: null,
+ updatedAt: null,
+ ...overrides,
+ };
+}
+
+// =========================================
+// rowToConfig β€” mode column + config jsonb document
+// =========================================
+
+const ORG_ID = "00000000-0000-0000-0000-000000000001";
+
+function makeRow(mode: string, config: unknown) {
+ return {
+ id: "00000000-0000-0000-0000-00000000000a",
+ org_id: ORG_ID,
+ mode,
+ config,
+ created_at: "2026-01-01T00:00:00.000Z",
+ updated_at: "2026-01-02T00:00:00.000Z",
+ };
+}
+
+describe("rowToConfig", () => {
+ it("maps a full stored document", () => {
+ const config = rowToConfig(
+ makeRow("normal", {
+ riskScoreThreshold: 40,
+ blacklist: ["*.bad.example"],
+ whitelist: ["example.com"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+ allowRequestOverrides: false,
+ }),
+ );
+
+ expect(config).toEqual({
+ orgId: ORG_ID,
+ policy: {
+ mode: "normal",
+ riskScoreThreshold: 40,
+ blacklist: ["*.bad.example"],
+ whitelist: ["example.com"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+ },
+ allowRequestOverrides: false,
+ createdAt: "2026-01-01T00:00:00.000Z",
+ updatedAt: "2026-01-02T00:00:00.000Z",
+ });
+ });
+
+ it("falls back to defaults for an empty config document", () => {
+ const config = rowToConfig(makeRow("normal", {}));
+ expect(config.policy).toEqual({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ });
+ expect(config.allowRequestOverrides).toBe(true);
+ });
+
+ it("ignores unknown keys in the document", () => {
+ const config = rowToConfig(
+ makeRow("normal", {
+ riskScoreThreshold: 10,
+ someFutureField: { nested: true },
+ deniedCategories: ["Malicious"], // retired field, silently dropped
+ siem: { url: "https://siem.example.com" }, // retired field, silently dropped
+ }),
+ );
+ expect(config.policy.riskScoreThreshold).toBe(10);
+ expect(config.policy).not.toHaveProperty("deniedCategories");
+ expect(config.policy).not.toHaveProperty("someFutureField");
+ expect(config).not.toHaveProperty("siem");
+ });
+
+ it("fills defaults for a partial document", () => {
+ const config = rowToConfig(
+ makeRow("normal", { blacklist: ["bad.example"] }),
+ );
+ expect(config.policy).toEqual({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ blacklist: ["bad.example"],
+ });
+ });
+
+ it("never throws on field-level garbage β€” bad fields fall back to defaults", () => {
+ const config = rowToConfig(
+ makeRow("normal", {
+ riskScoreThreshold: "not-a-number",
+ blacklist: "not-an-array",
+ failurePolicy: "sideways",
+ allowRequestOverrides: "yes",
+ }),
+ );
+ expect(config.policy).toEqual({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ });
+ expect(config.allowRequestOverrides).toBe(true);
+ });
+
+ it("never throws when the document is not an object at all", () => {
+ for (const config of [null, "garbage", 42, ["array"]]) {
+ const parsed = rowToConfig(makeRow("normal", config));
+ expect(parsed.policy).toEqual({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ });
+ }
+ });
+
+ it("coerces unknown modes (including the retired enhanced) to off", () => {
+ expect(rowToConfig(makeRow("off", {})).policy.mode).toBe("off");
+ expect(rowToConfig(makeRow("enhanced", {})).policy.mode).toBe("off");
+ expect(rowToConfig(makeRow("garbage", {})).policy.mode).toBe("off");
+ });
+});
+
+describe("resolveEffectivePolicy", () => {
+ it("returns mode off with defaults when there is no org config", () => {
+ expect(resolveEffectivePolicy(null)).toEqual({
+ mode: "off",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ });
+ });
+
+ it("returns the org policy when there is no request override", () => {
+ expect(resolveEffectivePolicy(makeOrgConfig())).toEqual(orgPolicy);
+ });
+
+ it("does field-level replacement from the request override", () => {
+ const effective = resolveEffectivePolicy(makeOrgConfig(), {
+ riskScoreThreshold: 90,
+ blockedTlds: ["mov"],
+ });
+ expect(effective).toEqual({
+ ...orgPolicy,
+ riskScoreThreshold: 90,
+ blockedTlds: ["mov"],
+ });
+ });
+
+ it("replaces arrays wholesale instead of merging them", () => {
+ const effective = resolveEffectivePolicy(makeOrgConfig(), {
+ blacklist: ["*.other.example"],
+ });
+ expect(effective.blacklist).toEqual(["*.other.example"]);
+ });
+
+ it("ignores undefined fields in the override", () => {
+ const effective = resolveEffectivePolicy(makeOrgConfig(), {
+ riskScoreThreshold: undefined,
+ failurePolicy: "closed",
+ });
+ expect(effective.riskScoreThreshold).toBe(60);
+ expect(effective.failurePolicy).toBe("closed");
+ });
+
+ it("applies overrides on top of defaults when there is no org config", () => {
+ const effective = resolveEffectivePolicy(null, {
+ mode: "normal",
+ riskScoreThreshold: 10,
+ });
+ expect(effective).toEqual({
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ riskScoreThreshold: 10,
+ });
+ });
+
+ it("ignores the override when the org disables request overrides", () => {
+ const effective = resolveEffectivePolicy(
+ makeOrgConfig({ allowRequestOverrides: false }),
+ { mode: "off", riskScoreThreshold: 0 },
+ );
+ expect(effective).toEqual(orgPolicy);
+ });
+
+ it("does not mutate the org config", () => {
+ const orgConfig = makeOrgConfig();
+ resolveEffectivePolicy(orgConfig, { riskScoreThreshold: 1 });
+ expect(orgConfig.policy).toEqual(orgPolicy);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.ts
new file mode 100644
index 000000000..e7bde17b8
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/store.ts
@@ -0,0 +1,256 @@
+import { eq } from "drizzle-orm";
+import { z } from "zod";
+import { db, dbRr } from "../../db/connection";
+import * as schema from "../../db/schema";
+import { deleteKey, getValue, setValue } from "../../services/redis";
+import { logger as _logger } from "../logger";
+import {
+ THREAT_PROTECTION_POLICY_DEFAULTS,
+ type ThreatProtectionPolicy,
+} from "./types";
+import type { ThreatProtectionConfigInput } from "./config";
+
+const logger = _logger.child({ module: "threat-protection-store" });
+
+// Short TTL so config changes apply without a redeploy while keeping the
+// enforcement hot path off Postgres. Invalidated on write.
+//
+// ZDR boundary: this cache (and the team→org map below) stores the org's OWN
+// configuration β€” never scrape-derived data (no target domains, URLs, or
+// verdicts) β€” so it is compatible with zero-data-retention requirements and
+// deliberately survives the removal of the verdict cache (ENG-5004).
+const CACHE_TTL_SECONDS = 60;
+
+const cacheKey = (orgId: string) => `threat-protection-config:${orgId}`;
+
+/**
+ * Whether an error is Postgres 42P01 (undefined_table) for our config table.
+ * During the rollout window the enforcement code can be deployed before the
+ * `threat_protection_config` DDL has been applied β€” in that case the feature
+ * behaves as "not configured" instead of failing every flagged-team request.
+ * Any other database error still propagates (we do NOT silently fail open on
+ * transient failures).
+ */
+function isMissingTableError(error: unknown): boolean {
+ let current: unknown = error;
+ for (let depth = 0; depth < 5 && current; depth++) {
+ const code = (current as { code?: unknown }).code;
+ if (code === "42P01") return true;
+ current = (current as { cause?: unknown }).cause;
+ }
+ return false;
+}
+
+export interface OrgThreatProtectionConfig {
+ orgId: string;
+ policy: ThreatProtectionPolicy;
+ allowRequestOverrides: boolean;
+ createdAt: string | null;
+ updatedAt: string | null;
+}
+
+type ThreatProtectionConfigRow =
+ typeof schema.threat_protection_config.$inferSelect;
+
+/**
+ * Tolerant schema for the `config` jsonb document stored alongside `mode`.
+ * The document is written exclusively by {@link upsertOrgThreatProtectionConfig}
+ * (from API-validated input), but reads must survive anything: a missing or
+ * partial document, unknown keys, or field-level garbage all fall back to the
+ * field defaults instead of throwing β€” a bad row must never take down the
+ * enforcement hot path.
+ */
+const storedConfigDocumentSchema = z
+ .object({
+ riskScoreThreshold: z
+ .number()
+ .int()
+ .min(0)
+ .max(100)
+ .catch(THREAT_PROTECTION_POLICY_DEFAULTS.riskScoreThreshold),
+ blacklist: z.array(z.string()).catch([]),
+ whitelist: z.array(z.string()).catch([]),
+ blockedTlds: z.array(z.string()).catch([]),
+ failurePolicy: z
+ .enum(["open", "closed"])
+ .catch(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),
+ allowRequestOverrides: z.boolean().catch(true),
+ })
+ .catch({
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ allowRequestOverrides: true,
+ });
+
+/**
+ * Maps a `threat_protection_config` row (mode column + config jsonb document)
+ * to the runtime config shape. Tolerant by construction: never throws, even
+ * for empty/partial/unknown-keyed documents (see
+ * {@link storedConfigDocumentSchema}). Exported for tests.
+ */
+export function rowToConfig(
+ row: ThreatProtectionConfigRow,
+): OrgThreatProtectionConfig {
+ const doc = storedConfigDocumentSchema.parse(row.config ?? {});
+ return {
+ orgId: row.org_id,
+ policy: {
+ mode: row.mode === "normal" ? "normal" : "off",
+ riskScoreThreshold: doc.riskScoreThreshold,
+ blacklist: doc.blacklist,
+ whitelist: doc.whitelist,
+ blockedTlds: doc.blockedTlds,
+ failurePolicy: doc.failurePolicy,
+ },
+ allowRequestOverrides: doc.allowRequestOverrides,
+ createdAt: row.created_at,
+ updatedAt: row.updated_at,
+ };
+}
+
+/**
+ * Reads the org's threat protection config, with a short Redis cache
+ * (~60s TTL, negative results included) so the enforcement hot path
+ * doesn't hit Postgres per scrape. Invalidated by
+ * {@link upsertOrgThreatProtectionConfig}.
+ */
+export async function getOrgThreatProtectionConfig(
+ orgId: string,
+): Promise<OrgThreatProtectionConfig | null> {
+ const key = cacheKey(orgId);
+
+ try {
+ const cached = await getValue(key);
+ if (cached !== null) {
+ return JSON.parse(cached) as OrgThreatProtectionConfig | null;
+ }
+ } catch (error) {
+ logger.warn("Failed to read threat protection config cache", {
+ error,
+ orgId,
+ });
+ }
+
+ let rows: ThreatProtectionConfigRow[];
+ try {
+ rows = await dbRr
+ .select()
+ .from(schema.threat_protection_config)
+ .where(eq(schema.threat_protection_config.org_id, orgId))
+ .limit(1);
+ } catch (error) {
+ if (isMissingTableError(error)) {
+ logger.warn(
+ "threat_protection_config table does not exist yet; treating as unconfigured",
+ { orgId },
+ );
+ rows = [];
+ } else {
+ throw error;
+ }
+ }
+
+ const config = rows[0] ? rowToConfig(rows[0]) : null;
+
+ try {
+ await setValue(key, JSON.stringify(config), CACHE_TTL_SECONDS);
+ } catch (error) {
+ logger.warn("Failed to write threat protection config cache", {
+ error,
+ orgId,
+ });
+ }
+
+ return config;
+}
+
+/**
+ * Full-document upsert of the org's threat protection config. Invalidates
+ * the read cache on success.
+ */
+export async function upsertOrgThreatProtectionConfig(
+ orgId: string,
+ config: ThreatProtectionConfigInput,
+): Promise<OrgThreatProtectionConfig> {
+ const values = {
+ org_id: orgId,
+ mode: config.mode,
+ // Everything but the mode lives in the jsonb document.
+ config: {
+ riskScoreThreshold: config.riskScoreThreshold,
+ blacklist: config.blacklist,
+ whitelist: config.whitelist,
+ blockedTlds: config.blockedTlds,
+ failurePolicy: config.failurePolicy,
+ allowRequestOverrides: config.allowRequestOverrides,
+ },
+ };
+
+ const [row] = await db
+ .insert(schema.threat_protection_config)
+ .values(values)
+ .onConflictDoUpdate({
+ target: schema.threat_protection_config.org_id,
+ set: {
+ ...values,
+ updated_at: new Date().toISOString(),
+ },
+ })
+ .returning();
+
+ try {
+ await deleteKey(cacheKey(orgId));
+ } catch (error) {
+ logger.warn("Failed to invalidate threat protection config cache", {
+ error,
+ orgId,
+ });
+ }
+
+ return rowToConfig(row);
+}
+
+/**
+ * Resolves the org_id for a team. Used by the org-level config API when the
+ * auth chunk doesn't carry org_id.
+ */
+export async function getOrgIdForTeam(teamId: string): Promise<string | null> {
+ const rows = await dbRr
+ .select({ org_id: schema.teams.org_id })
+ .from(schema.teams)
+ .where(eq(schema.teams.id, teamId))
+ .limit(1);
+ return rows[0]?.org_id ?? null;
+}
+
+/**
+ * Computes the effective policy for a request.
+ *
+ * - No org config β†’ mode "off" with defaults.
+ * - A request override does field-level replacement on top of the org policy,
+ * unless the org locked overrides down (`allowRequestOverrides: false`),
+ * in which case it is ignored (the request should already have been
+ * rejected by `checkPermissions`; this is defense in depth).
+ */
+export function resolveEffectivePolicy(
+ orgConfig: OrgThreatProtectionConfig | null,
+ requestOverride?: Partial<ThreatProtectionPolicy>,
+): ThreatProtectionPolicy {
+ const base: ThreatProtectionPolicy = orgConfig
+ ? { ...orgConfig.policy }
+ : { mode: "off", ...THREAT_PROTECTION_POLICY_DEFAULTS };
+
+ if (!requestOverride || (orgConfig && !orgConfig.allowRequestOverrides)) {
+ return base;
+ }
+
+ for (const key of Object.keys(requestOverride) as Array<
+ keyof ThreatProtectionPolicy
+ >) {
+ const value = requestOverride[key];
+ if (value !== undefined) {
+ (base as unknown as Record<string, unknown>)[key] = value;
+ }
+ }
+
+ return base;
+}
diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.ts
new file mode 100644
index 000000000..b21282820
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/types.ts
@@ -0,0 +1,79 @@
+// Shared type contract for the threat protection feature.
+// NOTE: concurrent in-flight branches create this exact file β€” do not modify without coordinating.
+
+export type ThreatProtectionMode = "off" | "normal";
+
+export interface ThreatProtectionPolicy {
+ mode: ThreatProtectionMode;
+ /** Normalized 0-100; verdicts at or above this score are blocked. */
+ riskScoreThreshold: number;
+ /** Exact domains or globs like "*.example.com". Blocks without a provider call. */
+ blacklist: string[];
+ /** Exact domains or globs. Allows without a provider call; wins over everything. */
+ whitelist: string[];
+ /** Lowercase TLDs without leading dot, e.g. "zip". Blocks without a provider call. */
+ blockedTlds: string[];
+ /** Behavior when the provider is unavailable: "closed" blocks, "open" allows. */
+ failurePolicy: "open" | "closed";
+}
+
+export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<
+ ThreatProtectionPolicy,
+ "mode"
+> = {
+ riskScoreThreshold: 75,
+ blacklist: [],
+ whitelist: [],
+ blockedTlds: [],
+ failurePolicy: "closed",
+};
+
+// Only Google Web Risk today. The union (and the provider seam around it β€”
+// separate provider modules under ./providers, the mode→provider dispatch in
+// ./index.ts, and the provider-agnostic verdict engine in ./verdict.ts) is
+// deliberately kept so future partner classifiers slot in as new members.
+export type ThreatProvider = "google-web-risk";
+
+export interface RawVerdict {
+ provider: ThreatProvider;
+ /** Normalized 0-100, higher = riskier; null if the provider gave no score. */
+ riskScore: number | null;
+ /** Provider threat categories (Web Risk threat types map through here). */
+ categories: string[];
+ /**
+ * Always false: verdicts are never persisted (ZDR) β€” repeated checks within
+ * one request share the same in-flight decision via the request-scoped
+ * dedup handle instead of a cache. Kept for contract stability.
+ */
+ fromCache: boolean;
+ /** Raw provider payload, surfaced only in server logs. */
+ raw: unknown;
+}
+
+/**
+ * Request/job-scoped dedup handle for {@link import("./index").checkDomain}.
+ * Call sites create one per request or job (never shared across requests, and
+ * never persisted) so that repeated checks of the same domain within that
+ * scope β€” e.g. a scrape plus its redirect re-check, or one crawl-discovery
+ * batch β€” share a single in-flight decision instead of re-scanning. Keyed by
+ * normalized domain only: within one request the effective policy is
+ * constant, so the domain fully identifies the decision.
+ */
+export type ThreatCheckDedup = Map<string, Promise<ThreatDecision>>;
+
+export type ThreatDecisionRule =
+ | "whitelist"
+ | "blacklist"
+ | "blocked-tld"
+ | "risk-score"
+ | "provider-failure"
+ | "default-allow";
+
+export interface ThreatDecision {
+ allowed: boolean;
+ rule: ThreatDecisionRule;
+ /** True if a provider verdict (fresh OR cached) was consulted β€” this drives billing (+2 per scanned domain). */
+ providerConsulted: boolean;
+ verdict: RawVerdict | null;
+ mode: ThreatProtectionMode;
+}
diff --git a/apps/api/src/lib/threat-protection/verdict.test.ts b/apps/api/src/lib/threat-protection/verdict.test.ts
new file mode 100644
index 000000000..e089f3f2d
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/verdict.test.ts
@@ -0,0 +1,418 @@
+import {
+ RawVerdict,
+ THREAT_PROTECTION_POLICY_DEFAULTS,
+ ThreatDecisionRule,
+ ThreatProtectionPolicy,
+} from "./types";
+import { evaluatePolicy, localOnlyDecision, normalizeDomain } from "./verdict";
+
+function policy(
+ overrides: Partial<ThreatProtectionPolicy> = {},
+): ThreatProtectionPolicy {
+ return {
+ mode: "normal",
+ ...THREAT_PROTECTION_POLICY_DEFAULTS,
+ ...overrides,
+ };
+}
+
+function verdict(overrides: Partial<RawVerdict> = {}): RawVerdict {
+ return {
+ provider: "google-web-risk",
+ riskScore: 0,
+ categories: [],
+ fromCache: false,
+ raw: {},
+ ...overrides,
+ };
+}
+
+describe("evaluatePolicy", () => {
+ type Case = {
+ name: string;
+ domain: string;
+ verdict: RawVerdict | null;
+ policy: ThreatProtectionPolicy;
+ allowed: boolean;
+ rule: ThreatDecisionRule;
+ };
+
+ const cases: Case[] = [
+ // --- whitelist ---
+ {
+ name: "whitelist exact match allows",
+ domain: "example.com",
+ verdict: verdict(),
+ policy: policy({ whitelist: ["example.com"] }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist exact entry also matches subdomains",
+ domain: "deep.sub.example.com",
+ verdict: verdict(),
+ policy: policy({ whitelist: ["example.com"] }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist glob matches multi-label subdomains",
+ domain: "a.b.example.com",
+ verdict: verdict(),
+ policy: policy({ whitelist: ["*.example.com"] }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist glob does not match the apex domain",
+ domain: "example.com",
+ verdict: verdict({ riskScore: 100 }),
+ policy: policy({ whitelist: ["*.example.com"] }),
+ allowed: false,
+ rule: "risk-score",
+ },
+ {
+ name: "whitelist entries are case-insensitive",
+ domain: "example.com",
+ verdict: verdict(),
+ policy: policy({ whitelist: ["EXAMPLE.com"] }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist does not match unrelated suffix (notexample.com)",
+ domain: "notexample.com",
+ verdict: verdict({ riskScore: 100 }),
+ policy: policy({ whitelist: ["example.com"] }),
+ allowed: false,
+ rule: "risk-score",
+ },
+ // --- whitelist precedence: wins over everything ---
+ {
+ name: "whitelist wins over blacklist",
+ domain: "example.com",
+ verdict: verdict(),
+ policy: policy({
+ whitelist: ["example.com"],
+ blacklist: ["example.com"],
+ }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist wins over blocked TLD",
+ domain: "good.zip",
+ verdict: verdict(),
+ policy: policy({ whitelist: ["good.zip"], blockedTlds: ["zip"] }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ {
+ name: "whitelist wins over a maximally risky verdict",
+ domain: "example.com",
+ verdict: verdict({
+ riskScore: 100,
+ categories: ["MALWARE"],
+ }),
+ policy: policy({
+ whitelist: ["example.com"],
+ riskScoreThreshold: 10,
+ }),
+ allowed: true,
+ rule: "whitelist",
+ },
+ // --- blacklist ---
+ {
+ name: "blacklist exact match blocks",
+ domain: "bad.com",
+ verdict: verdict(),
+ policy: policy({ blacklist: ["bad.com"] }),
+ allowed: false,
+ rule: "blacklist",
+ },
+ {
+ name: "blacklist exact entry also blocks subdomains",
+ domain: "cdn.bad.com",
+ verdict: verdict(),
+ policy: policy({ blacklist: ["bad.com"] }),
+ allowed: false,
+ rule: "blacklist",
+ },
+ {
+ name: "blacklist glob blocks matching subdomains",
+ domain: "evil.bad.com",
+ verdict: verdict(),
+ policy: policy({ blacklist: ["*.bad.com"] }),
+ allowed: false,
+ rule: "blacklist",
+ },
+ {
+ name: "blacklist wins over blocked TLD (rule ordering)",
+ domain: "bad.zip",
+ verdict: verdict(),
+ policy: policy({ blacklist: ["bad.zip"], blockedTlds: ["zip"] }),
+ allowed: false,
+ rule: "blacklist",
+ },
+ // --- blocked-tld ---
+ {
+ name: "blocked TLD blocks matching domains",
+ domain: "archive.zip",
+ verdict: verdict(),
+ policy: policy({ blockedTlds: ["zip"] }),
+ allowed: false,
+ rule: "blocked-tld",
+ },
+ {
+ name: "blocked TLD respects label boundaries (foozip.com passes)",
+ domain: "foozip.com",
+ verdict: verdict(),
+ policy: policy({ blockedTlds: ["zip"] }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ {
+ name: "blocked TLD supports multi-label suffixes",
+ domain: "shop.co.uk",
+ verdict: verdict(),
+ policy: policy({ blockedTlds: ["co.uk"] }),
+ allowed: false,
+ rule: "blocked-tld",
+ },
+ {
+ name: "blocked TLD tolerates a leading dot in the entry",
+ domain: "archive.zip",
+ verdict: verdict(),
+ policy: policy({ blockedTlds: [".zip"] }),
+ allowed: false,
+ rule: "blocked-tld",
+ },
+ {
+ name: "blocked TLD wins over risk score (rule ordering)",
+ domain: "archive.zip",
+ verdict: verdict({ riskScore: 100 }),
+ policy: policy({ blockedTlds: ["zip"], riskScoreThreshold: 75 }),
+ allowed: false,
+ rule: "blocked-tld",
+ },
+ // --- risk-score ---
+ {
+ name: "score at the threshold is blocked",
+ domain: "example.com",
+ verdict: verdict({ riskScore: 75 }),
+ policy: policy({ riskScoreThreshold: 75 }),
+ allowed: false,
+ rule: "risk-score",
+ },
+ {
+ name: "score above the threshold is blocked",
+ domain: "example.com",
+ verdict: verdict({ riskScore: 100 }),
+ policy: policy({ riskScoreThreshold: 75 }),
+ allowed: false,
+ rule: "risk-score",
+ },
+ {
+ name: "score below the threshold is allowed",
+ domain: "example.com",
+ verdict: verdict({ riskScore: 74 }),
+ policy: policy({ riskScoreThreshold: 75 }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ {
+ name: "null score never triggers the risk-score rule",
+ domain: "example.com",
+ verdict: verdict({ riskScore: null }),
+ policy: policy({ riskScoreThreshold: 0 }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ {
+ name: "verdict categories alone never block (no category rules)",
+ domain: "example.com",
+ verdict: verdict({ riskScore: 0, categories: ["MALWARE"] }),
+ policy: policy({ riskScoreThreshold: 75 }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ // --- provider-failure ---
+ {
+ name: "provider failure with fail-closed blocks",
+ domain: "example.com",
+ verdict: null,
+ policy: policy({ failurePolicy: "closed" }),
+ allowed: false,
+ rule: "provider-failure",
+ },
+ {
+ name: "provider failure with fail-open allows",
+ domain: "example.com",
+ verdict: null,
+ policy: policy({ failurePolicy: "open" }),
+ allowed: true,
+ rule: "provider-failure",
+ },
+ {
+ name: "local rules still decide when the provider failed",
+ domain: "bad.com",
+ verdict: null,
+ policy: policy({ blacklist: ["bad.com"], failurePolicy: "open" }),
+ allowed: false,
+ rule: "blacklist",
+ },
+ {
+ name: "mode off with no verdict allows by default (not provider-failure)",
+ domain: "example.com",
+ verdict: null,
+ policy: policy({ mode: "off", failurePolicy: "closed" }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ // --- default-allow ---
+ {
+ name: "clean verdict passes every rule",
+ domain: "example.com",
+ verdict: verdict(),
+ policy: policy({
+ riskScoreThreshold: 50,
+ blockedTlds: ["zip"],
+ }),
+ allowed: true,
+ rule: "default-allow",
+ },
+ ];
+
+ test.each(cases)("$name", ({ domain, verdict, policy, allowed, rule }) => {
+ const decision = evaluatePolicy(domain, verdict, policy);
+ expect(decision.allowed).toBe(allowed);
+ expect(decision.rule).toBe(rule);
+ expect(decision.mode).toBe(policy.mode);
+ expect(decision.verdict).toBe(verdict);
+ // Billing invariant: providerConsulted iff a verdict (fresh or cached)
+ // was passed in β€” regardless of which rule decided.
+ expect(decision.providerConsulted).toBe(verdict !== null);
+ });
+});
+
+describe("localOnlyDecision", () => {
+ it("resolves whitelisted domains without a provider", () => {
+ const decision = localOnlyDecision(
+ "example.com",
+ policy({ whitelist: ["example.com"] }),
+ );
+ expect(decision).toEqual({
+ allowed: true,
+ rule: "whitelist",
+ providerConsulted: false,
+ verdict: null,
+ mode: "normal",
+ });
+ });
+
+ it("resolves blacklisted domains without a provider", () => {
+ const decision = localOnlyDecision(
+ "sub.bad.com",
+ policy({ blacklist: ["bad.com"] }),
+ );
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "blacklist",
+ providerConsulted: false,
+ verdict: null,
+ });
+ });
+
+ it("resolves blocked TLDs without a provider", () => {
+ const decision = localOnlyDecision(
+ "archive.zip",
+ policy({ blockedTlds: ["zip"] }),
+ );
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "blocked-tld",
+ providerConsulted: false,
+ });
+ });
+
+ it("prefers the whitelist over blacklist and blocked TLDs", () => {
+ const decision = localOnlyDecision(
+ "good.zip",
+ policy({
+ whitelist: ["good.zip"],
+ blacklist: ["*.zip", "good.zip"],
+ blockedTlds: ["zip"],
+ }),
+ );
+ expect(decision).toMatchObject({ allowed: true, rule: "whitelist" });
+ });
+
+ it("returns null when only provider-backed rules could decide", () => {
+ expect(
+ localOnlyDecision(
+ "example.com",
+ policy({
+ riskScoreThreshold: 10,
+ }),
+ ),
+ ).toBeNull();
+ });
+
+ it("normalizes URL-ish input before matching", () => {
+ const decision = localOnlyDecision(
+ "HTTPS://Sub.Example.COM/some/path?q=1",
+ policy({ blacklist: ["example.com"] }),
+ );
+ expect(decision).toMatchObject({ allowed: false, rule: "blacklist" });
+ });
+});
+
+describe("normalizeDomain", () => {
+ test.each([
+ ["Example.COM", "example.com"],
+ [" example.com ", "example.com"],
+ ["example.com.", "example.com"],
+ ["https://sub.example.com/path?q=1", "sub.example.com"],
+ ["example.com:8080", "example.com"],
+ ["example.com/path", "example.com"],
+ // inet_aton-style IP forms canonicalize to dotted-quad, matching the Web
+ // Risk provider (prevents blacklist bypass via alternate IP notation).
+ ["195.127.0.11", "195.127.0.11"],
+ ["3279880203", "195.127.0.11"],
+ ["http://3279880203/", "195.127.0.11"],
+ // IPv6 literals must not be truncated by port-stripping.
+ ["http://[2001:db8::1]/", "[2001:db8::1]"],
+ ["http://[2001:db8::1]:8080/", "[2001:db8::1]"],
+ ["[2001:db8::1]:8080", "[2001:db8::1]"],
+ ["2001:db8::1", "2001:db8::1"],
+ ])("normalizes %s to %s", (input, expected) => {
+ expect(normalizeDomain(input)).toBe(expected);
+ });
+
+ test("a blacklisted IPv6 literal is matched, not truncated to a wrong host", () => {
+ const p = policy({ blacklist: ["[2001:db8::1]"] });
+ expect(localOnlyDecision("http://[2001:db8::1]:8080/", p)).toMatchObject({
+ allowed: false,
+ rule: "blacklist",
+ });
+ });
+
+ test("a blacklisted dotted-quad IP is not bypassed by integer notation", () => {
+ const p = policy({ blacklist: ["195.127.0.11"] });
+ for (const host of ["http://3279880203/", "195.127.0.11"]) {
+ expect(localOnlyDecision(host, p)).toMatchObject({
+ allowed: false,
+ rule: "blacklist",
+ });
+ }
+ });
+
+ test("a blacklist entry in integer notation blocks the dotted-quad host", () => {
+ expect(
+ localOnlyDecision(
+ "http://195.127.0.11/",
+ policy({ blacklist: ["3279880203"] }),
+ ),
+ ).toMatchObject({ allowed: false, rule: "blacklist" });
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.ts
new file mode 100644
index 000000000..2e96247fa
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/verdict.ts
@@ -0,0 +1,174 @@
+import type {
+ RawVerdict,
+ ThreatDecision,
+ ThreatProtectionPolicy,
+} from "./types";
+// Shared host canonicalization (lowercase, dot handling, and inet_aton-style
+// IP normalization). The Web Risk provider already canonicalizes hosts this
+// way; local blacklist/whitelist rules MUST use the exact same function so
+// the two paths can never disagree on what host they're evaluating β€” e.g. so
+// a blacklist entry for "195.127.0.11" is not bypassed by "http://3279880203/".
+import { canonicalizeHost } from "./providers/web-risk/canonicalize";
+
+// Pure policy evaluation for threat protection. No I/O in this file β€” the
+// provider/cache orchestration lives in ./index.ts. Rule precedence (fixed):
+// whitelist β†’ blacklist β†’ blocked-tld β†’ risk-score β†’ provider-failure β†’
+// default-allow. The engine consumes the provider-agnostic RawVerdict shape
+// only (normalized riskScore + categories) β€” it never knows which provider
+// produced the verdict, so new providers slot in without touching it.
+
+/**
+ * Whether `domain` matches a single whitelist/blacklist entry.
+ *
+ * - Glob entries (containing `*`) match literally with `*` expanding to any
+ * run of characters, e.g. "*.example.com" matches "a.example.com" and
+ * "a.b.example.com" (but not the apex "example.com").
+ * - Exact entries match the domain itself AND its subdomains β€” consistent with
+ * how the global blocklist treats domains (see
+ * src/scraper/WebScraper/utils/blocklist.ts), and what users expect from
+ * listing "example.com".
+ */
+function domainMatchesEntry(domain: string, entry: string): boolean {
+ const trimmed = entry.trim().toLowerCase().replace(/\.$/, "");
+ if (!trimmed) return false;
+
+ if (trimmed.includes("*")) {
+ const pattern = trimmed
+ .split("*")
+ .map(part => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&"))
+ .join(".*");
+ try {
+ return new RegExp(`^${pattern}$`).test(domain);
+ } catch {
+ return false;
+ }
+ }
+
+ // Canonicalize non-glob entries the same way as the looked-up domain, so an
+ // entry written in a non-canonical IP form (e.g. "3279880203") still matches
+ // its dotted-quad host and vice versa.
+ const normalized = canonicalizeHost(trimmed);
+ return domain === normalized || domain.endsWith(`.${normalized}`);
+}
+
+function domainMatchesList(domain: string, entries: string[]): boolean {
+ return entries.some(entry => domainMatchesEntry(domain, entry));
+}
+
+/**
+ * Whether the domain's suffix matches a blocked TLD entry. Entries are
+ * lowercase without a leading dot ("zip"); multi-label suffixes ("co.uk")
+ * also work since we do a label-boundary suffix match.
+ */
+function matchesBlockedTld(domain: string, blockedTlds: string[]): boolean {
+ return blockedTlds.some(tld => {
+ const normalized = tld.trim().toLowerCase().replace(/^\./, "");
+ if (!normalized) return false;
+ return domain === normalized || domain.endsWith(`.${normalized}`);
+ });
+}
+
+/**
+ * Normalize a domain-ish input: strip URL parts/port, then apply the shared
+ * host canonicalization (lowercase, dot handling, inet_aton-style IP
+ * normalization) so local rule matching sees the same canonical host as the
+ * Web Risk provider β€” closing IP-form blacklist bypasses.
+ */
+export function normalizeDomain(input: string): string {
+ let domain = input.trim();
+ if (domain.includes("://")) {
+ try {
+ domain = new URL(domain).hostname;
+ } catch {
+ // Not a parseable URL β€” fall through with the raw string.
+ }
+ }
+ // Strip a path fragment, then the port β€” carefully, because IPv6 literals
+ // contain colons. A bracketed literal ("[2001:db8::1]") keeps everything
+ // through the closing bracket (dropping any ":port" after it); a plain
+ // host:port has exactly one colon; a bare IPv6 (multiple colons, no
+ // brackets) is left intact rather than truncated.
+ domain = domain.split("/")[0];
+ if (domain.startsWith("[")) {
+ const end = domain.indexOf("]");
+ if (end !== -1) domain = domain.slice(0, end + 1);
+ } else if (domain.split(":").length === 2) {
+ domain = domain.split(":")[0];
+ }
+ return canonicalizeHost(domain);
+}
+
+/**
+ * Resolve a decision using ONLY local policy rules (whitelist β†’ blacklist β†’
+ * blocked-tld), or null if a provider verdict is needed. Local decisions never
+ * consult a provider, so they never set `providerConsulted` (no billing).
+ */
+export function localOnlyDecision(
+ domain: string,
+ policy: ThreatProtectionPolicy,
+): ThreatDecision | null {
+ const normalized = normalizeDomain(domain);
+ const base = {
+ providerConsulted: false,
+ verdict: null,
+ mode: policy.mode,
+ } as const;
+
+ if (domainMatchesList(normalized, policy.whitelist)) {
+ return { allowed: true, rule: "whitelist", ...base };
+ }
+ if (domainMatchesList(normalized, policy.blacklist)) {
+ return { allowed: false, rule: "blacklist", ...base };
+ }
+ if (matchesBlockedTld(normalized, policy.blockedTlds)) {
+ return { allowed: false, rule: "blocked-tld", ...base };
+ }
+ return null;
+}
+
+/**
+ * Evaluate the full policy against a provider verdict. `verdict` is null when
+ * the provider failed (or was never called) β€” the org's failurePolicy then
+ * decides. `providerConsulted` reflects whether a verdict (fresh or cached)
+ * was used, which drives billing.
+ */
+export function evaluatePolicy(
+ domain: string,
+ verdict: RawVerdict | null,
+ policy: ThreatProtectionPolicy,
+): ThreatDecision {
+ const base = {
+ providerConsulted: verdict !== null,
+ verdict,
+ mode: policy.mode,
+ };
+
+ const local = localOnlyDecision(domain, policy);
+ if (local !== null) {
+ // Preserve the local rule but reflect any verdict we were given (billing
+ // still applies if a provider was consulted before evaluation).
+ return { allowed: local.allowed, rule: local.rule, ...base };
+ }
+
+ if (verdict !== null) {
+ if (
+ verdict.riskScore !== null &&
+ verdict.riskScore >= policy.riskScoreThreshold
+ ) {
+ return { allowed: false, rule: "risk-score", ...base };
+ }
+
+ return { allowed: true, rule: "default-allow", ...base };
+ }
+
+ // No verdict: the provider failed or was unavailable (mode "off" never
+ // reaches here via checkDomain). Fail open or closed per the org policy.
+ if (policy.mode === "off") {
+ return { allowed: true, rule: "default-allow", ...base };
+ }
+ return {
+ allowed: policy.failurePolicy === "open",
+ rule: "provider-failure",
+ ...base,
+ };
+}
diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.ts
index 2860d81be..578940dfd 100644
--- a/apps/api/src/lib/zdr-helpers.ts
+++ b/apps/api/src/lib/zdr-helpers.ts
@@ -68,6 +68,15 @@ export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {
return "disabled";
}
+/**
+ * Resolves the effective threat protection mode from team flags.
+ */
+export function getThreatProtection(flags: TeamFlags | undefined): OrgFlagMode {
+ if (flags?.threatProtection === "forced") return "forced";
+ if (flags?.threatProtection === "allowed") return "allowed";
+ return "disabled";
+}
+
/**
* Resolves the effective customRobotsAgent mode from team flags.
* Only supports "disabled" (default) and "allowed" β€” no "forced" mode.
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index f45bb0a68..f1e9bdfb7 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -60,6 +60,10 @@ import {
browserWebhookDestroyedController,
} from "../controllers/v2/browser";
import { activityController } from "../controllers/v1/activity";
+import {
+ getTeamThreatProtectionController,
+ putTeamThreatProtectionController,
+} from "../controllers/v2/team-threat-protection";
import { supportProxyController } from "../controllers/v2/support-proxy";
import { createResearchRouter } from "../controllers/v2/research-proxy";
import {
@@ -413,6 +417,18 @@ v2Router.get(
wrap(activityController),
);
+v2Router.get(
+ "/team/threat-protection",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(getTeamThreatProtectionController),
+);
+
+v2Router.put(
+ "/team/threat-protection",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(putTeamThreatProtectionController),
+);
+
v2Router.post(
"/monitor",
authMiddleware(RateLimiterMode.Crawl),
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 72d624b6c..6e235fe09 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -88,6 +88,14 @@ import path from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
import type { DataLayerScrapeMetadata } from "../../lib/data-layer";
+import {
+ checkDomain,
+ type ThreatCheckDedup,
+ type ThreatDecision,
+ type ThreatProtectionPolicy,
+} from "../../lib/threat-protection";
+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
+import { normalizeDomain } from "../../lib/threat-protection/verdict";
export type ScrapeUrlResponse =
| {
@@ -95,10 +103,18 @@ export type ScrapeUrlResponse =
document: Document;
unsupportedFeatures?: Set<FeatureFlag>;
dataLayer?: DataLayerScrapeMetadata;
+ /**
+ * Threat protection decisions made for this scrape (initial domain
+ * check + any redirect re-checks, in order). Read by the billing layer
+ * (`providerConsulted` drives +2/+3 credits) and the security-logging
+ * layer. Only set when a threat protection policy was active.
+ */
+ threatDecisions?: ThreatDecision[];
}
| {
success: false;
error: any;
+ threatDecisions?: ThreatDecision[];
};
export type BrowserCookie = {
@@ -157,6 +173,8 @@ export type Meta = {
winnerEngine?: Engine;
abortHandle?: NodeJS.Timeout;
audioCookies?: BrowserCookie[];
+ /** Threat protection decisions made during this scrape, in order (mutable, like logs). */
+ threatDecisions: ThreatDecision[];
};
function buildFeatureFlags(
@@ -451,6 +469,7 @@ async function buildMetaObject(
documentPrefetch,
fetchPrefetch,
costTracking,
+ threatDecisions: [],
};
}
@@ -475,6 +494,14 @@ export type InternalOptions = {
zeroDataRetention?: boolean;
teamFlags?: TeamFlags;
+ /**
+ * Effective threat protection policy for this scrape, resolved at the
+ * controller layer (org config + per-request override). When set (and mode
+ * is not "off"), the target domain is checked before any engine work, and
+ * redirect destinations are re-checked. Absent => zero enforcement overhead.
+ */
+ threatProtection?: ThreatProtectionPolicy;
+
v1Agent?: ScrapeOptionsV1["agent"];
v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];
v1JSONSystemPrompt?: string;
@@ -1084,6 +1111,37 @@ export async function scrapeURL(
meta.logger.info("scrapeURL entered");
+ // Threat protection: check the target domain BEFORE any engine selection
+ // or outbound fetch. The policy is resolved at the controller layer and
+ // threaded through internalOptions; absent policy = zero overhead.
+ // The dedup map is scoped to this one scrape: the initial check and any
+ // redirect re-check on the same domain share a single scan (one billable
+ // consulted decision); a cross-domain redirect is a second scan.
+ const threatPolicy = internalOptions.threatProtection;
+ const threatDedup: ThreatCheckDedup = new Map();
+ if (threatPolicy && threatPolicy.mode !== "off") {
+ const initialDomain = normalizeDomain(meta.rewrittenUrl ?? meta.url);
+ const decision = await checkDomain(initialDomain, threatPolicy, {
+ teamId: internalOptions.teamId,
+ dedup: threatDedup,
+ });
+ meta.threatDecisions.push(decision);
+ if (!decision.allowed) {
+ meta.logger.info("Domain blocked by threat protection policy", {
+ domain: initialDomain,
+ rule: decision.rule,
+ });
+ setSpanAttributes(span, {
+ "scrape.blocked_by_threat_protection": true,
+ });
+ return {
+ success: false,
+ error: new UnsafeDomainBlockedError(initialDomain, decision),
+ threatDecisions: meta.threatDecisions,
+ };
+ }
+ }
+
if (meta.rewrittenUrl) {
meta.logger.info("Rewriting URL");
setSpanAttributes(span, {
@@ -1269,6 +1327,36 @@ export async function scrapeURL(
}
}
+ // Threat protection: if the scrape ended up on a different domain than
+ // requested (redirect), re-check the destination domain. This closes
+ // the "clean domain redirects to a blocked domain" bypass vector.
+ if (threatPolicy && threatPolicy.mode !== "off" && result.success) {
+ const finalUrl = result.document.metadata.url;
+ const initialDomain = normalizeDomain(meta.rewrittenUrl ?? meta.url);
+ const finalDomain = finalUrl ? normalizeDomain(finalUrl) : null;
+ if (finalDomain !== null && finalDomain !== initialDomain) {
+ const decision = await checkDomain(finalDomain, threatPolicy, {
+ teamId: internalOptions.teamId,
+ dedup: threatDedup,
+ });
+ meta.threatDecisions.push(decision);
+ if (!decision.allowed) {
+ meta.logger.info(
+ "Redirect destination blocked by threat protection policy",
+ {
+ domain: finalDomain,
+ initialDomain,
+ rule: decision.rule,
+ },
+ );
+ setSpanAttributes(span, {
+ "scrape.blocked_by_threat_protection": true,
+ });
+ throw new UnsafeDomainBlockedError(finalDomain, decision);
+ }
+ }
+ }
+
meta.logger.debug("scrapeURL metrics", {
module: "scrapeURL/metrics",
timeTaken: Date.now() - startTime,
@@ -1315,7 +1403,9 @@ export async function scrapeURL(
result.success && result.document.metadata.cacheState === "hit",
});
- return result;
+ return meta.threatDecisions.length > 0
+ ? { ...result, threatDecisions: meta.threatDecisions }
+ : result;
} catch (error) {
// if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
// throw new FEPageLoadFailed();
@@ -1433,6 +1523,16 @@ export async function scrapeURL(
error,
retryStats: error.stats,
});
+ } else if (error instanceof UnsafeDomainBlockedError) {
+ errorType = "UnsafeDomainBlockedError";
+ meta.logger.warn(
+ "scrapeURL: Domain blocked by threat protection policy",
+ {
+ error,
+ domain: error.domain,
+ rule: error.decision.rule,
+ },
+ );
} else if (error instanceof AbortManagerThrownError) {
errorType = "AbortManagerThrownError";
throw error.inner;
@@ -1456,6 +1556,9 @@ export async function scrapeURL(
return {
success: false,
error,
+ ...(meta.threatDecisions.length > 0
+ ? { threatDecisions: meta.threatDecisions }
+ : {}),
};
}
});
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 88261c13f..2c80a200c 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -16,6 +16,10 @@ import {
import { applySearchHighlights, highlightsEnvReady } from "./highlights";
import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
import type { BillingMetadata } from "../services/billing/types";
+import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";
+import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";
+import { normalizeDomain } from "../lib/threat-protection/verdict";
+import { calculateThreatScanCredits } from "../lib/scrape-billing";
interface SearchOptions {
query: string;
@@ -48,6 +52,8 @@ interface SearchContext {
billing?: BillingMetadata;
agentIndexOnly?: boolean;
keylessReserved?: boolean;
+ /** Effective threat protection policy; blocked domains are removed from results entirely. */
+ threatProtectionPolicy?: ThreatProtectionPolicy | null;
}
interface SearchExecuteResult {
@@ -104,6 +110,48 @@ export async function executeSearch(
enterprise: options.enterprise,
})) as SearchV2Response;
+ // Threat protection: remove results on blocked domains entirely β€” before
+ // slicing/counting, before scraping, and before returning. Domain checks
+ // are deduped and Redis-cached. Every consulted decision (fresh or cached
+ // provider verdict) is a billable scan (+2 per scanned domain), charged
+ // as part of the search credits below.
+ let threatScanCredits = 0;
+ const threatPolicy = context.threatProtectionPolicy;
+ if (threatPolicy && threatPolicy.mode !== "off") {
+ const urlsToCheck = [
+ ...(searchResponse.web ?? []).map(x => x.url),
+ ...(searchResponse.news ?? []).map(x => x.url),
+ ...(searchResponse.images ?? []).map(x => x.url),
+ ].filter((x): x is string => !!x);
+
+ if (urlsToCheck.length > 0) {
+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(
+ urlsToCheck,
+ threatPolicy,
+ { teamId },
+ );
+ threatScanCredits = calculateThreatScanCredits(
+ decisionsByDomain.values(),
+ );
+ const isAllowed = (url: string | undefined | null): boolean => {
+ if (!url) return true;
+ const decision = decisionsByDomain.get(normalizeDomain(url));
+ return decision === undefined || decision.allowed;
+ };
+ if (searchResponse.web) {
+ searchResponse.web = searchResponse.web.filter(x => isAllowed(x.url));
+ }
+ if (searchResponse.news) {
+ searchResponse.news = searchResponse.news.filter(x => isAllowed(x.url));
+ }
+ if (searchResponse.images) {
+ searchResponse.images = searchResponse.images.filter(x =>
+ isAllowed(x.url),
+ );
+ }
+ }
+ }
+
if (searchResponse.web && searchResponse.web.length > 0) {
searchResponse.web = searchResponse.web.map(result => ({
...result,
@@ -145,8 +193,13 @@ export async function executeSearch(
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
+ // Threat protection scan fees ride on the search credits: they are part of
+ // serving the search itself (every result domain is scanned before
+ // filtering), so they bill against the same feature and show up in the
+ // request's creditsUsed.
const searchCredits =
- Math.ceil(totalResultsCount / 10) * creditsPerTenResults;
+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults +
+ threatScanCredits;
let scrapeCredits = 0;
const shouldScrape =
@@ -171,6 +224,7 @@ export async function executeSearch(
billing,
agentIndexOnly: context.agentIndexOnly,
keylessReserved: context.keylessReserved,
+ threatProtectionPolicy: threatPolicy ?? null,
};
const allDocsWithCostTracking = await scrapeSearchResults(
diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.ts
index 1c29fcc51..d14f4a3cb 100644
--- a/apps/api/src/search/scrape.ts
+++ b/apps/api/src/search/scrape.ts
@@ -10,6 +10,7 @@ import { ScrapeJobData } from "../types";
import { SearchV2Response } from "../lib/entities";
import type { BillingMetadata } from "../services/billing/types";
import { getScrapeZDR } from "../lib/zdr-helpers";
+import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";
export interface DocumentWithCostTracking {
document: Document;
@@ -40,6 +41,7 @@ interface ScrapeSearchOptions {
billing?: BillingMetadata;
agentIndexOnly?: boolean;
keylessReserved?: boolean;
+ threatProtectionPolicy?: ThreatProtectionPolicy | null;
}
async function scrapeSearchResultDirect(
@@ -81,6 +83,7 @@ async function scrapeSearchResultDirect(
zeroDataRetention,
teamFlags: flags,
agentIndexOnly: options.agentIndexOnly ?? false,
+ threatProtection: options.threatProtectionPolicy ?? undefined,
},
skipNuq: true,
origin: options.origin,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 6bb671d65..a9441b613 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -19,6 +19,7 @@ import {
addCrawlJobDone,
crawlToCrawler,
recordRobotsBlocked,
+ recordThreatBlocked,
finishCrawlKickoff,
generateURLPermutations,
getCrawl,
@@ -29,10 +30,13 @@ import {
setCrawlError,
StoredCrawl,
} from "../../lib/crawl-redis";
+import { checkUrlsAgainstThreatPolicy } from "../../lib/threat-protection/request";
+import type { ThreatDecision } from "../../lib/threat-protection/types";
import { redisEvictConnection } from "../redis";
import {
resolveBillingMetadata,
toAutumnBillingProperties,
+ type BillingMetadata,
} from "../billing/types";
import {
autumnService,
@@ -58,7 +62,11 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
import { WebCrawler } from "../../scraper/WebScraper/crawler";
-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
+import {
+ calculateCreditsToBeBilled,
+ calculateThreatScanCredits,
+} from "../../lib/scrape-billing";
+import { billTeam } from "../billing/credit_billing";
import { getBillingQueue } from "../queue-service";
import type { Logger } from "winston";
import {
@@ -112,6 +120,7 @@ async function billScrapeJob(
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
dataLayer?: DataLayerScrapeMetadata,
+ threatDecisions?: ThreatDecision[],
) {
let creditsToBeBilled: number | null = null;
const billing = resolveBillingMetadata({
@@ -140,6 +149,7 @@ async function billScrapeJob(
error,
unsupportedFeatures,
dataLayer,
+ threatDecisions,
);
// Charge the keyless free tier's per-IP daily credit budget unless this
@@ -215,6 +225,47 @@ async function billScrapeJob(
return creditsToBeBilled;
}
+/**
+ * Bills threat protection scan fees for crawl-discovered URLs that were
+ * blocked by the policy and therefore never become scrape jobs. Only blocked
+ * decisions bill here: allowed discoveries are billed by their own scrape
+ * job, which re-checks the (cached) verdict. Deduplicates by domain, and only
+ * decisions that consulted the classifier (fresh or cached provider verdict)
+ * carry a fee (+2 per scanned domain) β€” local-only blocks (e.g. blacklist)
+ * are free.
+ */
+function billThreatBlockedDiscoveries(
+ args: {
+ teamId: string;
+ apiKeyId: number | null;
+ billing: BillingMetadata;
+ bypassBilling: boolean;
+ },
+ blocked: { domain: string; decision: ThreatDecision }[],
+ logger: Logger,
+) {
+ if (args.bypassBilling) return;
+ const blockedDecisionsByDomain = new Map(
+ blocked.map(x => [x.domain, x.decision]),
+ );
+ const threatScanCredits = calculateThreatScanCredits(
+ blockedDecisionsByDomain.values(),
+ );
+ if (threatScanCredits <= 0) return;
+ billTeam(
+ args.teamId,
+ undefined,
+ threatScanCredits,
+ args.apiKeyId,
+ args.billing,
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${args.teamId} for ${threatScanCredits} threat scan credit(s)`,
+ { error },
+ );
+ });
+}
+
async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const logger = _logger.child({
module: "queue-worker",
@@ -244,6 +295,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
: undefined;
const signal = abortController.signal;
+ // Hoisted above the try so the catch path can read pipeline.threatDecisions
+ // for billing/logging even when the scrape failed.
+ let pipeline: ScrapeUrlResponse | null = null;
+
try {
if (remainingTime !== undefined && remainingTime < 0) {
throw new ScrapeJobTimeoutError();
@@ -256,7 +311,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
- let pipeline: ScrapeUrlResponse | null = null;
let timeoutHandle: NodeJS.Timeout | null = null;
try {
pipeline = await Promise.race([
@@ -458,7 +512,60 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
- for (const link of links.links) {
+ // Threat protection: silently skip discovered links on blocked
+ // domains (cross-domain links included) β€” the crawl continues.
+ // Skipped URLs + decisions are recorded for the security-logging
+ // layer. Domain checks are deduped and rely on the Redis verdict
+ // cache, so many links on few domains stay cheap.
+ let discoveredLinks = links.links;
+ const threatPolicy = job.data.internalOptions?.threatProtection;
+ if (
+ threatPolicy &&
+ threatPolicy.mode !== "off" &&
+ discoveredLinks.length > 0
+ ) {
+ const { allowedUrls, blocked } =
+ await checkUrlsAgainstThreatPolicy(
+ discoveredLinks,
+ threatPolicy,
+ { teamId: job.data.team_id },
+ );
+ discoveredLinks = allowedUrls;
+ for (const blockedUrl of blocked) {
+ await recordThreatBlocked(
+ job.data.crawl_id,
+ blockedUrl.url,
+ blockedUrl.decision,
+ );
+ }
+ if (blocked.length > 0) {
+ billThreatBlockedDiscoveries(
+ {
+ teamId: job.data.team_id,
+ apiKeyId: job.data.apiKeyId ?? null,
+ billing: resolveBillingMetadata({
+ billing: job.data.billing,
+ crawlId: job.data.crawl_id,
+ crawlerOptions: job.data.crawlerOptions,
+ }),
+ bypassBilling:
+ job.data.internalOptions?.bypassBilling ?? false,
+ },
+ blocked,
+ logger,
+ );
+ logger.info(
+ "Skipped " +
+ blocked.length +
+ " discovered link(s) blocked by threat protection",
+ {
+ blockedDomains: [...new Set(blocked.map(x => x.domain))],
+ },
+ );
+ }
+ }
+
+ for (const link of discoveredLinks) {
if (await lockURL(job.data.crawl_id, sc, link)) {
// This seems to work really welel
const jobPriority = await getJobPriority({
@@ -559,6 +666,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
undefined,
pipeline.unsupportedFeatures,
pipeline.dataLayer,
+ pipeline.threatDecisions,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;
@@ -650,6 +758,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
undefined,
pipeline.unsupportedFeatures,
pipeline.dataLayer,
+ pipeline.threatDecisions,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;
@@ -848,6 +957,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
costTracking,
(await getACUCTeam(job.data.team_id))?.flags ?? null,
error instanceof Error ? error : null,
+ undefined,
+ undefined,
+ pipeline?.threatDecisions,
);
logger.debug("Logging job to DB...");
@@ -1088,7 +1200,45 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
}
}
- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
+ let indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
+
+ // Threat protection: skip index-sourced discoveries on blocked domains.
+ const kickoffThreatPolicy = sc.internalOptions?.threatProtection;
+ if (
+ kickoffThreatPolicy &&
+ kickoffThreatPolicy.mode !== "off" &&
+ indexLinks.length > 0
+ ) {
+ const { allowedUrls, blocked } = await checkUrlsAgainstThreatPolicy(
+ indexLinks,
+ kickoffThreatPolicy,
+ { teamId: job.data.team_id },
+ );
+ indexLinks = allowedUrls;
+ for (const blockedUrl of blocked) {
+ await recordThreatBlocked(
+ job.data.crawl_id,
+ blockedUrl.url,
+ blockedUrl.decision,
+ );
+ }
+ if (blocked.length > 0) {
+ billThreatBlockedDiscoveries(
+ {
+ teamId: job.data.team_id,
+ apiKeyId: job.data.apiKeyId ?? null,
+ billing: resolveBillingMetadata({
+ billing: job.data.billing,
+ crawlId: job.data.crawl_id,
+ crawlerOptions: sc.crawlerOptions,
+ }),
+ bypassBilling: sc.internalOptions?.bypassBilling ?? false,
+ },
+ blocked,
+ logger,
+ );
+ }
+ }
if (indexLinks.length > 0) {
logger.debug("Using index links of length " + indexLinks.length, {
@@ -1201,7 +1351,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
isPreCrawl: sc.internalOptions?.isPreCrawl ?? false,
});
- const passingURLs = (
+ let passingURLs = (
await crawler.filterLinks(
results.urls.map(x => x.href),
Infinity,
@@ -1210,6 +1360,45 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
)
).links;
+ // Threat protection: skip sitemap entries on blocked domains β€” the crawl
+ // continues; skipped URLs + decisions are recorded for security logging.
+ const sitemapThreatPolicy = sc.internalOptions?.threatProtection;
+ if (
+ sitemapThreatPolicy &&
+ sitemapThreatPolicy.mode !== "off" &&
+ passingURLs.length > 0
+ ) {
+ const { allowedUrls, blocked } = await checkUrlsAgainstThreatPolicy(
+ passingURLs,
+ sitemapThreatPolicy,
+ { teamId: job.data.team_id },
+ );
+ passingURLs = allowedUrls;
+ for (const blockedUrl of blocked) {
+ await recordThreatBlocked(
+ job.data.crawl_id,
+ blockedUrl.url,
+ blockedUrl.decision,
+ );
+ }
+ if (blocked.length > 0) {
+ billThreatBlockedDiscoveries(
+ {
+ teamId: job.data.team_id,
+ apiKeyId: job.data.apiKeyId ?? null,
+ billing: resolveBillingMetadata({
+ billing: job.data.billing,
+ crawlId: job.data.crawl_id,
+ crawlerOptions: sc.crawlerOptions,
+ }),
+ bypassBilling: sc.internalOptions?.bypassBilling ?? false,
+ },
+ blocked,
+ logger,
+ );
+ }
+ }
+
if (passingURLs.length > 0) {
logger.debug("Using urls of length " + passingURLs.length, {
urlsLength: passingURLs.length,