Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): threat protection β enterprise domain risk blocking (#3949)
GergΕ MΓ³ricz6db599637f2c
diff --git a/apps/api/knip.config.ts b/apps/api/knip.config.tsindex 6c8f8de9c..e73777ef8 100644--- a/apps/api/knip.config.ts+++ b/apps/api/knip.config.ts@@ -15,6 +15,9 @@ const config: KnipConfig = {ignore: ["native/**","src/scraper/scrapeURL/engines/fire-engine/branding-script/**",+ // Shared type contract co-owned by concurrent threat-protection branches;+ // the provider/verdict types are consumed by the core-lib branch.+ "src/lib/threat-protection/types.ts",],ignoreDependencies: ["undici-types", "stripe"],};diff --git a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.tsnew file mode 100644index 000000000..d34552388--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts@@ -0,0 +1,167 @@+import request from "supertest";+import {+ describeIf,+ idmux,+ Identity,+ TEST_API_URL,+ TEST_PRODUCTION,+} from "../lib";+import { THREAT_PROTECTION_POLICY_DEFAULTS } from "../../../lib/threat-protection/types";++async function getConfigRaw(identity: Identity) {+ return await request(TEST_API_URL)+ .get("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++async function putConfigRaw(body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .put("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++// Requires DB authentication + idmux-provisioned team flags, which only exist+// in the production test configuration.+describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {+ describe("without the team flag", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "team-threat-protection/no-flag",+ });+ });++ it("GET returns 403", async () => {+ const res = await getConfigRaw(identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("enterprise feature");+ });++ it("PUT returns 403", async () => {+ const res = await putConfigRaw({ mode: "normal" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("enterprise feature");+ });+ });++ describe("with the team flag", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "team-threat-protection/flagged",+ flags: {+ threatProtection: "allowed",+ },+ });+ });++ it("GET returns the default (unconfigured) effective config", async () => {+ const res = await getConfigRaw(identity);+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.body.data).toMatchObject({+ mode: "off",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ allowRequestOverrides: true,+ configured: false,+ });+ expect(res.body.data).not.toHaveProperty("siem");+ });++ it("PUT round-trips a full config document", async () => {+ const doc = {+ mode: "normal",+ riskScoreThreshold: 60,+ blacklist: ["*.bad.example"],+ whitelist: ["firecrawl.dev"],+ blockedTlds: ["zip"],+ failurePolicy: "open",+ allowRequestOverrides: false,+ };++ const put = await putConfigRaw(doc, identity);+ expect(put.statusCode).toBe(200);+ expect(put.body.success).toBe(true);+ expect(put.body.data).toMatchObject({+ mode: "normal",+ riskScoreThreshold: 60,+ blacklist: ["*.bad.example"],+ whitelist: ["firecrawl.dev"],+ blockedTlds: ["zip"],+ failurePolicy: "open",+ allowRequestOverrides: false,+ configured: true,+ });+ // Retired policy fields must not appear in the served document.+ expect(put.body.data).not.toHaveProperty("siem");+ expect(put.body.data).not.toHaveProperty("deniedCategories");+ expect(put.body.data).not.toHaveProperty("maxDomainAgeDays");+ expect(put.body.data).not.toHaveProperty("blockedCountries");++ const get = await getConfigRaw(identity);+ expect(get.statusCode).toBe(200);+ expect(get.body.data).toMatchObject({+ mode: "normal",+ riskScoreThreshold: 60,+ configured: true,+ });+ expect(get.body.data).not.toHaveProperty("siem");+ expect(get.body.data).not.toHaveProperty("deniedCategories");+ expect(get.body.data).not.toHaveProperty("maxDomainAgeDays");+ expect(get.body.data).not.toHaveProperty("blockedCountries");+ });++ it("PUT is a full-document update (unspecified fields reset to defaults)", async () => {+ const put = await putConfigRaw({ mode: "normal" }, identity);+ expect(put.statusCode).toBe(200);+ expect(put.body.data).toMatchObject({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ allowRequestOverrides: true,+ configured: true,+ });+ expect(put.body.data).not.toHaveProperty("siem");+ });++ it("PUT rejects an invalid document with 400", async () => {+ const res = await putConfigRaw(+ {+ mode: "normal",+ riskScoreThreshold: 500,+ blacklist: ["https://not-a-domain"],+ },+ identity,+ );+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ });++ it('PUT rejects the retired "enhanced" mode with 400', async () => {+ const res = await putConfigRaw({ mode: "enhanced" }, identity);+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ });++ it("PUT rejects retired policy fields with 400", async () => {+ for (const retired of [+ { siem: { url: "https://siem.example.com/ingest" } },+ { deniedCategories: ["Malicious"] },+ { maxDomainAgeDays: 30 },+ { blockedCountries: ["KP"] },+ ]) {+ const res = await putConfigRaw(+ { mode: "normal", ...retired },+ identity,+ );+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ }+ });+ });+});diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.tsnew file mode 100644index 000000000..006b690e6--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/threat-protection-billing.test.ts@@ -0,0 +1,402 @@+import http from "http";+import {+ describeIf,+ idmux,+ Identity,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+ scrapeTimeout,+} from "../lib";+import {+ batchScrape,+ crawl,+ creditUsage,+ scrape,+ scrapeRaw,+ searchRaw,+} from "./lib";+import {+ createWebRiskMockCounters,+ createWebRiskMockHandler,+ WebRiskMockDatabase,+} from "../../../lib/threat-protection/providers/web-risk/testing";++// =========================================+// Threat protection billing (ENG-4985, ZDR rework ENG-5004)+//+// Billing rules under test:+// - +2 credits per domain scanned in "normal" mode (Google Web Risk)+// - a "scan" = a ThreatDecision with providerConsulted β +2 per consulted+// verdict. There is no verdict cache anymore (ZDR): every request scans+// afresh, and dedup only applies within one request/job.+// - blocked requests still bill the scan fee (no base scrape cost, matching+// how other failed scrapes bill)+// - local-only decisions (whitelist/blacklist/blocked-tld) never bill+//+// These tests need the mock provider below. Start the harness with the+// base-URL override pointing at it, e.g.:+//+// GOOGLE_WEB_RISK_API_KEY=test \+// GOOGLE_WEB_RISK_API_URL=http://localhost:4519 \+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-billing.test.ts+//+// Tests that need the provider self-skip when the override is not set to a+// local address. NOTE: this file binds the mock port itself β do not run it+// in the same pass as another file using the same port (e.g.+// threat-protection-enforcement.test.ts if given the same Web Risk port).+// =========================================++// Fixture domains. *.example.com is reserved (RFC 2606) β these domains are+// never actually fetched: blocked scrapes fail before any outbound request.+const BLACKLISTED_DOMAIN = "threat-billing-blacklisted.example.com";+const RISKY_DOMAIN = "threat-billing-risky.example.com";++const CLEAN_URL = TEST_SUITE_WEBSITE;++// A stable cross-hostname redirect: google.com 301s to www.google.com. The+// mock flags only the redirect target, so the scrape consults the provider+// twice (initial domain + redirect re-check) before being blocked.+// NOTE the two hostnames are different domains for the request-scoped dedup,+// so the redirect re-check is a second scan (two consulted decisions).+const REDIRECT_SOURCE_URL = "https://google.com/";+const REDIRECT_TARGET_DOMAIN = "www.google.com";++const MOCK_RISKY_DOMAINS = [RISKY_DOMAIN, REDIRECT_TARGET_DOMAIN];++const isLocalMock = (url: string) =>+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(url);++const webRiskMockUrl = process.env.GOOGLE_WEB_RISK_API_URL ?? "";+const HAS_WEB_RISK_MOCK =+ isLocalMock(webRiskMockUrl) && !!process.env.GOOGLE_WEB_RISK_API_KEY;++const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms));+// Credit deductions land through the batched billing queue; see billing.test.ts.+const sleepForBatchBilling = () => sleep(40000);++// Update API mock (ZDR rework): the API syncs the mock's threat lists via+// threatLists:computeDiff and confirms local prefix hits via hashes:search β+// clean domains resolve locally and never reach the mock at all.+const webRiskDb = new WebRiskMockDatabase();+for (const domain of MOCK_RISKY_DOMAINS) {+ webRiskDb.addRiskyDomain(domain, "MALWARE");+}+const webRiskCounters = createWebRiskMockCounters();+const webRiskHandler = createWebRiskMockHandler(webRiskDb, webRiskCounters);++let webRiskServer: http.Server | null = null;++/** hashes:search confirmations whose prefix belongs to `domain`. */+const webRiskHitsFor = (domain: string) =>+ webRiskCounters.hashesSearchRequestsForDomain(domain);++function startWebRiskMock(): Promise<void> {+ const port = Number(new URL(webRiskMockUrl).port);+ webRiskServer = http.createServer((req, res) => {+ if (!webRiskHandler(req, res)) {+ res.statusCode = 404;+ res.end("{}");+ }+ });+ return new Promise((resolve, reject) => {+ webRiskServer!.once("error", reject);+ webRiskServer!.listen(port, () => resolve());+ });+}++describeIf(TEST_PRODUCTION)("Threat protection billing", () => {+ beforeAll(async () => {+ if (HAS_WEB_RISK_MOCK) {+ await startWebRiskMock();+ }+ });++ afterAll(async () => {+ if (webRiskServer) {+ await new Promise(resolve => webRiskServer!.close(resolve));+ webRiskServer = null;+ }+ });++ describe("scrape", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-billing/scrape",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "normal-mode clean scrape bills base + 2",+ async () => {+ const doc = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(doc.metadata.creditsUsed).toBe(3);+ },+ scrapeTimeout,+ );++ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "a rescrape of the same domain is a fresh scan and bills + 2 again",+ async () => {+ // ZDR: there is no cross-request verdict cache β every request scans+ // afresh and bills per consulted verdict. A clean domain's scan is+ // entirely local (hash-prefix lookup against the synced list), so+ // Google is never contacted for it on either scrape.+ const cleanDomain = new URL(CLEAN_URL).hostname;++ const first = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(first.metadata.creditsUsed).toBe(3);++ const second = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(second.metadata.creditsUsed).toBe(3);++ // Clean domains never trigger a hashes:search confirmation.+ expect(webRiskHitsFor(cleanDomain)).toBe(0);+ },+ scrapeTimeout * 2,+ );++ it(+ "scan costs are visible in the document's creditsUsed metadata",+ async () => {+ // Local-only allow (whitelist): no provider consulted, so a plain+ // 1-credit scrape β proves the fee only appears when a scan happened.+ const cleanDomain = new URL(CLEAN_URL).hostname;+ const doc = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: {+ mode: "normal",+ whitelist: [cleanDomain],+ failurePolicy: "closed",+ },+ } as any,+ identity,+ );+ expect(doc.metadata.creditsUsed).toBe(1);+ },+ scrapeTimeout,+ );+ });++ describe("blocked scrapes (end-to-end credit deduction)", () => {+ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "bills scan fees for blocked scrapes, nothing for local-only blocks",+ async () => {+ // Dedicated team so the credit delta is exactly attributable.+ const identity = await idmux({+ name: "threat-protection-billing/blocked",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });++ // A fresh team's credit grant can land a moment after idmux returns+ // (and a small default allotment can show up before the 1M grant);+ // poll until the baseline read reflects the full grant so the delta+ // is exact.+ let before = 0;+ for (let i = 0; i < 30 && before < 100_000; i++) {+ before = (await creditUsage(identity)).remainingCredits;+ if (before < 100_000) await sleep(1000);+ }+ expect(before).toBeGreaterThanOrEqual(100_000);++ // 1. Blocked by provider verdict: 0 base + 2 scan.+ const blocked = await scrapeRaw(+ {+ url: `https://${RISKY_DOMAIN}/`,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(blocked.statusCode).toBe(403);+ expect(blocked.body.code).toBe("unsafe_domain_blocked");++ // 2. Blocked by blacklist (local-only): no scan fee, no base cost.+ const blacklisted = await scrapeRaw(+ {+ url: `https://${BLACKLISTED_DOMAIN}/page`,+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ } as any,+ identity,+ );+ expect(blacklisted.statusCode).toBe(403);+ expect(blacklisted.body.code).toBe("unsafe_domain_blocked");++ // 3. Clean domain redirecting to a blocked domain: two consulted+ // decisions (initial + redirect re-check) β two scan fees.+ const redirected = await scrapeRaw(+ {+ url: REDIRECT_SOURCE_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(redirected.statusCode).toBe(403);+ expect(redirected.body.code).toBe("unsafe_domain_blocked");+ expect(redirected.body.error).toContain(REDIRECT_TARGET_DOMAIN);++ await sleepForBatchBilling();++ const after = (await creditUsage(identity)).remainingCredits;+ // 2 (blocked scan) + 0 (blacklist) + 4 (redirect: 2 scans) = 6+ expect(before - after).toBe(6);+ },+ scrapeTimeout * 3 + 60000,+ );+ });++ describe("crawl and batch scrape (per-document scan fees)", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-billing/crawl-batch",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "batch scrape bills base + 2 per document (each job scans afresh)",+ async () => {+ const res = await batchScrape(+ {+ urls: [CLEAN_URL, `${CLEAN_URL}/blog`],+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(res.data.length).toBe(2);+ for (const doc of res.data) {+ expect(doc.metadata.creditsUsed).toBe(3);+ }+ },+ scrapeTimeout * 2,+ );++ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "crawl bills base + 2 per scraped document",+ async () => {+ const res = await crawl(+ {+ url: CLEAN_URL,+ limit: 2,+ scrapeOptions: {+ threatProtection: { mode: "normal", failurePolicy: "open" },+ },+ } as any,+ identity,+ );+ expect(res.status).toBe("completed");+ expect(res.data.length).toBeGreaterThan(0);+ for (const doc of res.data) {+ expect(doc.metadata.creditsUsed).toBe(3);+ }+ },+ scrapeTimeout * 5,+ );+ });++ describe("search", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-billing/search",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ (HAS_WEB_RISK_MOCK ? it : it.skip)(+ "bills a scan fee per result domain on top of normal search billing",+ async () => {+ const limit = 20;+ const res = await searchRaw(+ {+ query: "firecrawl",+ includeDomains: ["firecrawl.dev"],+ limit,+ timeout: 120000,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);++ const web: { url: string }[] = res.body.data.web ?? [];+ expect(web.length).toBeGreaterThan(0);++ const uniqueDomains = new Set(+ web.map(x => new URL(x.url).hostname.toLowerCase()),+ );+ const searchCredits = Math.ceil(web.length / 10) * 2;++ if (web.length < limit) {+ // Nothing was sliced off, so the returned set IS the scanned set:+ // exactly one +2 scan fee per unique result domain.+ expect(res.body.creditsUsed).toBe(+ searchCredits + 2 * uniqueDomains.size,+ );+ } else {+ // The provider over-fetched (limit * 2 buffer) and results were+ // sliced; scanned domains β returned domains.+ expect(res.body.creditsUsed).toBeGreaterThanOrEqual(+ searchCredits + 2 * uniqueDomains.size,+ );+ }+ },+ scrapeTimeout * 2,+ );++ it(+ "search without provider scans bills plain search credits",+ async () => {+ // Local-only policy (everything blacklist-missed resolves via+ // failurePolicy "open" without a provider configured => no consulted+ // decisions => no scan fees). With the mock provider configured this+ // still holds a stronger property: whitelisted domains short-circuit.+ const res = await searchRaw(+ {+ query: "firecrawl",+ includeDomains: ["firecrawl.dev"],+ limit: 5,+ timeout: 120000,+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ const web: { url: string }[] = res.body.data.web ?? [];+ expect(res.body.creditsUsed).toBe(Math.ceil(web.length / 10) * 2);+ },+ scrapeTimeout * 2,+ );+ });+});diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.tsnew file mode 100644index 000000000..796d5e613--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts@@ -0,0 +1,613 @@+import http from "http";+import request from "supertest";+import {+ describeIf,+ idmux,+ Identity,+ TEST_API_URL,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+ scrapeTimeout,+} from "../lib";+import { crawl, crawlStart, map, scrape, scrapeRaw, search } from "./lib";+import {+ createWebRiskMockCounters,+ createWebRiskMockHandler,+ WebRiskMockDatabase,+} from "../../../lib/threat-protection/providers/web-risk/testing";++// =========================================+// Threat protection enforcement (ENG-4982/4983/4984)+//+// Most tests here use per-request threatProtection overrides with local-only+// rules (blacklist/whitelist/risk threshold), which never hit the provider β+// they only need a team with the threatProtection flag (via idmux) and are+// gated on TEST_PRODUCTION.+//+// Provider-verdict tests (risky domain, redirect re-check) need the mock+// Google Web Risk server below. The API must be started with the base-URL+// override pointing at it, e.g.:+//+// GOOGLE_WEB_RISK_API_KEY=test \+// GOOGLE_WEB_RISK_API_URL=http://localhost:4517 \+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-enforcement.test.ts+//+// Those tests are skipped when the override is not set to a local address.+//+// Org-config-based tests (PUT /v2/team/threat-protection) additionally need+// the threat_protection_config table (DDL ships with the config PR); they+// self-skip with a warning when the config API is unavailable.+// =========================================++// Fixture domains. *.example.com is reserved (RFC 2606) β these domains are+// never actually fetched: blocked scrapes fail before any outbound request.+const BLACKLISTED_DOMAIN = "threat-blacklisted.example.com";+const RISKY_DOMAIN = "threat-risky.example.com";++// The scrape target for happy-path tests.+const CLEAN_URL = TEST_SUITE_WEBSITE;+const CLEAN_DOMAIN = new URL(TEST_SUITE_WEBSITE).hostname;++// A stable cross-hostname redirect: google.com 301s to www.google.com.+const REDIRECT_SOURCE_URL = "https://google.com/";+const REDIRECT_TARGET_DOMAIN = "www.google.com";++// Domains the mock provider's threat lists flag as MALWARE (risk score 100).+const MOCK_RISKY_DOMAINS = [RISKY_DOMAIN, REDIRECT_TARGET_DOMAIN];++const mockProviderUrl = process.env.GOOGLE_WEB_RISK_API_URL ?? "";+const HAS_MOCK_PROVIDER =+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(mockProviderUrl) &&+ !!process.env.GOOGLE_WEB_RISK_API_KEY;++// Update API mock (ZDR rework): the API syncs the mock's threat lists via+// threatLists:computeDiff and confirms local prefix hits via hashes:search β+// there is no uris:search anymore, and clean domains never reach the mock.+const webRiskDb = new WebRiskMockDatabase();+for (const domain of MOCK_RISKY_DOMAINS) {+ webRiskDb.addRiskyDomain(domain, "MALWARE");+}+const webRiskCounters = createWebRiskMockCounters();+const webRiskHandler = createWebRiskMockHandler(webRiskDb, webRiskCounters);++let mockServer: http.Server | null = null;++/** hashes:search confirmations whose prefix belongs to `domain`. */+const providerHitsFor = (domain: string) =>+ webRiskCounters.hashesSearchRequestsForDomain(domain);++function startMockProvider(): Promise<void> {+ const port = Number(new URL(mockProviderUrl).port);+ mockServer = http.createServer((req, res) => {+ if (!webRiskHandler(req, res)) {+ res.statusCode = 404;+ res.end("{}");+ }+ });+ return new Promise((resolve, reject) => {+ mockServer!.once("error", reject);+ mockServer!.listen(port, () => resolve());+ });+}++async function putThreatProtectionConfig(body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .put("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {+ beforeAll(async () => {+ if (HAS_MOCK_PROVIDER) {+ await startMockProvider();+ }+ });++ afterAll(async () => {+ if (mockServer) {+ await new Promise(resolve => mockServer!.close(resolve));+ mockServer = null;+ }+ });++ describe("without the team flag", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-enforcement/no-flag",+ });+ });++ it(+ "rejects any per-request threatProtection option with 403",+ async () => {+ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "normal" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("enterprise feature");+ },+ scrapeTimeout,+ );+ });++ describe("with the team flag (per-request overrides)", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-enforcement/allowed",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ it(+ "scrape: clean domain passes with an active policy",+ async () => {+ // failurePolicy "open" makes this deterministic whether or not a+ // provider is configured: clean verdict β allow; no provider β allow.+ const doc = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(doc.metadata.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "scrape: blacklisted domain is blocked with zero provider calls",+ async () => {+ const res = await scrapeRaw(+ {+ url: `https://${BLACKLISTED_DOMAIN}/some/page`,+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ expect(res.body.error).toContain(BLACKLISTED_DOMAIN);+ if (HAS_MOCK_PROVIDER) {+ expect(providerHitsFor(BLACKLISTED_DOMAIN)).toBe(0);+ }+ },+ scrapeTimeout,+ );++ it(+ "scrape: whitelisted domain passes without a provider call",+ async () => {+ // failurePolicy "closed" + no whitelist would block when the provider+ // is unavailable β so a success here proves the whitelist+ // short-circuited the check in environments without a provider, and+ // the hit counter proves it where the mock is running.+ const before = providerHitsFor(CLEAN_DOMAIN);+ const doc = await scrape(+ {+ url: CLEAN_URL,+ threatProtection: {+ mode: "normal",+ whitelist: [CLEAN_DOMAIN],+ failurePolicy: "closed",+ },+ } as any,+ identity,+ );+ expect(doc.metadata.statusCode).toBe(200);+ if (HAS_MOCK_PROVIDER) {+ expect(providerHitsFor(CLEAN_DOMAIN)).toBe(before);+ }+ },+ scrapeTimeout,+ );++ it(+ "scrape: risk threshold blocks (verdict or fail-closed, deterministic)",+ async () => {+ // riskScoreThreshold 0 blocks any domain with a verdict (score >= 0),+ // and failurePolicy "closed" blocks when no provider is configured β+ // so this asserts the block path in every environment.+ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: {+ mode: "normal",+ riskScoreThreshold: 0,+ failurePolicy: "closed",+ },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );++ it(+ "batch scrape: blocked URLs are rejected at enqueue, rest succeed",+ async () => {+ const start = await request(TEST_API_URL)+ .post("/v2/batch/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ urls: [CLEAN_URL, `https://${BLACKLISTED_DOMAIN}/page`],+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ failurePolicy: "open",+ },+ });++ expect(start.statusCode).toBe(200);+ expect(start.body.success).toBe(true);+ expect(start.body.invalidURLs).toContain(+ `https://${BLACKLISTED_DOMAIN}/page`,+ );++ let status: any;+ do {+ await new Promise(resolve => setTimeout(resolve, 500));+ const res = await request(TEST_API_URL)+ .get(`/v2/batch/scrape/${start.body.id}`)+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send();+ expect(res.statusCode).toBe(200);+ status = res.body;+ } while (status.status === "scraping");++ expect(status.status).toBe("completed");+ expect(status.data.length).toBe(1);+ expect(status.data[0].metadata.sourceURL).toBe(CLEAN_URL);+ },+ scrapeTimeout * 2,+ );++ it(+ "crawl: blocked seed URL is rejected at kickoff",+ async () => {+ const res = await crawlStart(+ {+ url: `https://${BLACKLISTED_DOMAIN}/`,+ limit: 2,+ scrapeOptions: {+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );++ it(+ "crawl: blocked discoveries are skipped and the crawl completes",+ async () => {+ const blockedDiscoveryDomain = "github.com";+ const res = await crawl(+ {+ url: CLEAN_URL,+ limit: 3,+ maxDiscoveryDepth: 1,+ scrapeOptions: {+ threatProtection: {+ mode: "normal",+ blacklist: [blockedDiscoveryDomain],+ failurePolicy: "open",+ },+ },+ } as any,+ identity,+ );+ expect(res.status).toBe("completed");+ expect(res.data.length).toBeGreaterThan(0);+ for (const doc of res.data) {+ const url = doc.metadata.url ?? doc.metadata.sourceURL;+ if (!url) continue;+ const hostname = new URL(url).hostname;+ expect(+ hostname === blockedDiscoveryDomain ||+ hostname.endsWith(`.${blockedDiscoveryDomain}`),+ ).toBe(false);+ }+ },+ scrapeTimeout * 5,+ );++ it(+ "map: blocked domains are filtered out of the returned URL list",+ async () => {+ const blockedSubdomain = `docs.${CLEAN_DOMAIN}`;+ const res = await map(+ {+ url: CLEAN_URL,+ limit: 100,+ includeSubdomains: true,+ threatProtection: {+ mode: "normal",+ blacklist: [blockedSubdomain],+ failurePolicy: "open",+ },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ for (const link of res.body.links) {+ const hostname = new URL(link.url).hostname;+ expect(+ hostname === blockedSubdomain ||+ hostname.endsWith(`.${blockedSubdomain}`),+ ).toBe(false);+ }+ },+ scrapeTimeout,+ );++ it(+ "search: blocked domains are removed from results entirely",+ async () => {+ const results = await search(+ {+ query: "firecrawl",+ limit: 10,+ threatProtection: {+ mode: "normal",+ blacklist: ["firecrawl.dev"],+ failurePolicy: "open",+ },+ } as any,+ identity,+ );+ for (const result of results.web ?? []) {+ const hostname = new URL(result.url).hostname;+ expect(+ hostname === "firecrawl.dev" || hostname.endsWith(".firecrawl.dev"),+ ).toBe(false);+ }+ },+ scrapeTimeout,+ );++ it(+ "extract: blocked target URLs are rejected before fetching",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v2/extract")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ urls: [`https://${BLACKLISTED_DOMAIN}/page`],+ prompt: "Extract the page title.",+ ignoreInvalidURLs: false,+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ });+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain(BLACKLISTED_DOMAIN);+ },+ scrapeTimeout,+ );+ });++ describe("provider verdicts (mock Google Web Risk)", () => {+ let identity: Identity;++ beforeAll(async () => {+ if (!HAS_MOCK_PROVIDER) return;+ identity = await idmux({+ name: "threat-protection-enforcement/provider",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ (HAS_MOCK_PROVIDER ? it : it.skip)(+ "scrape: risky domain is blocked by the provider verdict",+ async () => {+ const res = await scrapeRaw(+ {+ url: `https://${RISKY_DOMAIN}/`,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ // The domain's hash prefix is in the synced local list; the block+ // required a hashes:search confirmation against the mock.+ expect(res.body.error).toContain(RISKY_DOMAIN);+ expect(providerHitsFor(RISKY_DOMAIN)).toBeGreaterThanOrEqual(1);+ },+ scrapeTimeout,+ );++ (HAS_MOCK_PROVIDER ? it : it.skip)(+ "scrape: clean domain redirecting to a blocked domain is blocked",+ async () => {+ // google.com 301s to www.google.com; the mock flags only the+ // redirect target as risky, so the initial check passes and the+ // redirect re-check must catch it.+ const res = await scrapeRaw(+ {+ url: REDIRECT_SOURCE_URL,+ threatProtection: { mode: "normal", failurePolicy: "open" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ expect(res.body.error).toContain(REDIRECT_TARGET_DOMAIN);+ },+ scrapeTimeout,+ );+ });++ describe("with the team flag forced", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-enforcement/forced",+ flags: { threatProtection: "forced" },+ credits: 100_000,+ });+ });++ it(+ "rejects a per-request override that disables threat protection",+ async () => {+ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "off" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("cannot be disabled");+ },+ scrapeTimeout,+ );++ it(+ "still allows narrowing overrides (mode stays on)",+ async () => {+ const res = await scrapeRaw(+ {+ url: `https://${BLACKLISTED_DOMAIN}/`,+ threatProtection: {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );+ });++ describe("org config enforcement + override lockout", () => {+ let identity: Identity;+ let configApiAvailable = false;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-enforcement/org-config",+ flags: { threatProtection: "allowed" },+ credits: 100_000,+ });++ // The config API needs the threat_protection_config table (DDL ships+ // with the config PR). Skip these tests gracefully where it is not+ // provisioned yet.+ const res = await putThreatProtectionConfig(+ {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ failurePolicy: "open",+ allowRequestOverrides: true,+ },+ identity,+ );+ configApiAvailable = res.statusCode === 200;+ if (!configApiAvailable) {+ console.warn(+ "threat protection config API unavailable (missing DDL?); skipping org config tests",+ res.statusCode,+ res.body,+ );+ }+ });++ afterAll(async () => {+ if (configApiAvailable) {+ // Reset so reruns against the same idmux team start clean.+ await putThreatProtectionConfig(+ { mode: "off", allowRequestOverrides: true },+ identity,+ );+ }+ });++ it(+ "org policy applies without any per-request option",+ async () => {+ if (!configApiAvailable) return;+ const res = await scrapeRaw(+ { url: `https://${BLACKLISTED_DOMAIN}/` },+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );++ it(+ "request override is rejected when the org locks overrides down",+ async () => {+ if (!configApiAvailable) return;+ const lock = await putThreatProtectionConfig(+ {+ mode: "normal",+ blacklist: [BLACKLISTED_DOMAIN],+ failurePolicy: "open",+ allowRequestOverrides: false,+ },+ identity,+ );+ expect(lock.statusCode).toBe(200);++ // The org config read is cached for ~60s; the PUT invalidates it, so+ // the lockout applies immediately.+ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: { whitelist: [BLACKLISTED_DOMAIN] },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("overrides are disabled");+ },+ scrapeTimeout,+ );+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 0ce129200..35c082082 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -49,6 +49,33 @@ const configSchema = z.object({// TOR). Unset disables the check entirely (keyless behaves as before).SPUR_API_KEY: z.string().optional(),+ // Threat protection (enterprise domain risk blocking). "normal" mode uses+ // Google Web Risk. An unset key disables the provider (lookups then fail+ // per the org's failurePolicy).+ GOOGLE_WEB_RISK_API_KEY: z.string().optional(),+ GOOGLE_WEB_RISK_API_URL: z+ .string()+ .url()+ .default("https://webrisk.googleapis.com"),+ // Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run+ // against a locally synced hash-prefix database (threatLists:computeDiff)+ // instead of sending URLs to Google, and verdicts are never persisted.+ //+ // Floor for how often threatLists:computeDiff may run per list. Google's+ // recommendedNextDiff is respected when it is later than this floor.+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce+ .number()+ .int()+ .positive()+ .default(60),+ // A synced threat list older than this is treated as unavailable+ // (provider-failure semantics β the org's failurePolicy decides).+ THREAT_LIST_STALENESS_SECONDS: z.coerce+ .number()+ .int()+ .positive()+ .default(6 * 60 * 60),+// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),OPENAI_API_KEY: z.string().optional(),diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 2a30a540e..a45b7efc1 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -26,6 +26,13 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";import {actionTypesOf,checkKeyFormatRestriction,@@ -49,7 +56,22 @@ export async function batchScrapeController(req.body = batchScrapeRequestSchema.parse(req.body);}- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }++ const permissions = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });if (permissions.error) {return res.status(403).json({success: false,@@ -129,6 +151,68 @@ export async function batchScrapeController(}}+ // Threat protection: reject/report blocked URLs at enqueue time so they+ // never consume scrape slots (mirrors the isUrlBlocked handling above).+ if (threatProtection.policy) {+ const { blocked } = await checkUrlsAgainstThreatPolicy(+ urls,+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ if (blocked.length > 0) {+ // Blocked domains whose decision consulted the classifier (fresh or+ // cached verdict) bill the scan fee (+2 per scanned domain) even+ // though they will never be scraped β the scan already happened.+ // Allowed URLs are not billed here: their scrape jobs re-check the+ // cached verdict and bill there.+ const blockedDecisionsByDomain = new Map(+ blocked.map(x => [x.domain, x.decision]),+ );+ const threatScanCredits = calculateThreatScanCredits(+ blockedDecisionsByDomain.values(),+ );+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "batch_scrape", jobId: id },+ ).catch(error => {+ logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ if (req.body.ignoreInvalidURLs) {+ const blockedSet = new Set(blocked.map(x => x.url));+ const keptUnnormalized: string[] = [];+ const keptUrls: string[] = [];+ urls.forEach((u, i) => {+ if (blockedSet.has(u)) {+ invalidURLs!.push(unnormalizedURLs[i] ?? u);+ } else {+ keptUrls.push(u);+ keptUnnormalized.push(unnormalizedURLs[i]);+ }+ });+ urls = keptUrls;+ unnormalizedURLs = keptUnnormalized;+ } else {+ const first = blocked[0];+ const error = new UnsafeDomainBlockedError(+ first.domain,+ first.decision,+ );+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }+ }+ }+if (urls.length === 0) {return res.status(400).json({success: false,@@ -176,6 +260,7 @@ export async function batchScrapeController(: false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 57674d8f4..9c2c8702d 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -29,6 +29,12 @@ import {} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { resolveThreatProtection } from "../../lib/threat-protection/request";+import { checkDomain } from "../../lib/threat-protection";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { normalizeDomain } from "../../lib/threat-protection/verdict";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -37,9 +43,23 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.scrapeOptions?.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+const permissions = checkPermissions({ ...req.body, crawlerOptions: req.body },req.acuc?.flags,+ { threatProtectionOrgConfig: threatProtection.orgConfig },);if (permissions.error) {return res.status(403).json({@@ -48,6 +68,39 @@ export async function crawlController(});}+ // Threat protection: check the seed URL before kicking off the crawl.+ if (threatProtection.policy) {+ const seedDomain = normalizeDomain(req.body.url);+ const decision = await checkDomain(seedDomain, threatProtection.policy, {+ teamId: req.auth.team_id,+ });+ if (!decision.allowed) {+ // A blocked seed still bills the scan fee when the classifier was+ // consulted β the scan already happened. (An allowed seed is not billed+ // here: its scrape job re-checks the cached verdict and bills there.)+ const threatScanCredits = calculateThreatScanCredits([decision]);+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "crawl" },+ ).catch(error => {+ _logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ const error = new UnsafeDomainBlockedError(seedDomain, decision);+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }+ }+const keyRestriction = await checkKeyFormatRestriction(formatTypesOf(req.body.scrapeOptions?.formats),actionTypesOf(req.body.scrapeOptions?.actions),@@ -151,6 +204,7 @@ export async function crawlController(saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 5c498aa8c..6662f67ba 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -23,6 +23,13 @@ import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";import { config } from "../../config";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,res: Response<ExtractResponse>,@@ -133,6 +140,64 @@ export async function extractController(}}+ // Threat protection: check target URLs before fetching. Blocked URLs are+ // reported via invalidURLs (with ignoreInvalidURLs) or reject the request.+ // Discovered URLs are enforced in the scrape pipeline via the policy+ // threaded through the extract job.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {+ const { blocked, decisionsByDomain } = await checkUrlsAgainstThreatPolicy(+ req.body.urls ?? [],+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ // Every consulted decision (fresh or cached provider verdict) is a+ // billable scan (+2 per scanned domain) β including when the request is+ // rejected below: the scans already happened.+ const threatScanCredits = calculateThreatScanCredits(+ decisionsByDomain.values(),+ );+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "extract" },+ ).catch(error => {+ _logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ if (blocked.length > 0) {+ if (req.body.ignoreInvalidURLs) {+ invalidURLs.push(...blocked.map(x => x.url));+ } else {+ const first = blocked[0];+ const error = new UnsafeDomainBlockedError(+ first.domain,+ first.decision,+ );+ return res.status(403).json({+ success: false,+ error: error.message,+ });+ }+ }+ }+const extractId = uuidv7();_logger.info("Extract starting...", {diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex b9751c055..93aa6895e 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -31,6 +31,12 @@ import {import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { normalizeDomain } from "../../lib/threat-protection/verdict";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";configDotenv();const redis = new Redis(config.REDIS_URL!);@@ -379,7 +385,22 @@ export async function mapController(});}- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }++ const permissions = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });if (permissions.error) {return res.status(403).json({success: false,@@ -464,16 +485,34 @@ export async function mapController(}}+ // Threat protection: remove links on blocked domains from the returned+ // URL list entirely. Every consulted decision (fresh or cached provider+ // verdict) is a billable scan (+2 per scanned domain).+ let threatScanCredits = 0;+ if (threatProtection.policy && result.links.length > 0) {+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(+ result.links,+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ threatScanCredits = calculateThreatScanCredits(decisionsByDomain.values());+ result.links = result.links.filter(x => {+ const decision = decisionsByDomain.get(normalizeDomain(x));+ return decision === undefined || decision.allowed;+ });+ }+// Bill the team+ const creditsToBill = 1 + threatScanCredits;billTeam(req.auth.team_id,req.acuc?.sub_id,- 1,+ creditsToBill,req.acuc?.api_key_id ?? null,{ endpoint: "map", jobId: mapId },).catch(error => {logger.error(- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,+ `Failed to bill team ${req.auth.team_id} for ${creditsToBill} credit(s): ${error}`,);});@@ -494,7 +533,7 @@ export async function mapController(location: req.body.location,},results: result.links,- credits_cost: 1,+ credits_cost: creditsToBill,zeroDataRetention: false, // not supported});diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex f17ef0337..7fd083171 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -36,6 +36,7 @@ import {} from "../../lib/keyless";import { projectScrapeCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { resolveThreatProtection } from "../../lib/threat-protection/request";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -50,7 +51,22 @@ export async function scrapeController(const preNormalizedBody = { ...req.body };req.body = scrapeRequestSchema.parse(req.body);- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }++ const permissions = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });if (permissions.error) {return res.status(403).json({success: false,@@ -212,6 +228,7 @@ export async function scrapeController(zeroDataRetention,teamFlags: req.acuc?.flags ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,},skipNuq: true,origin,@@ -275,6 +292,14 @@ export async function scrapeController(});}+ if (e.code === "unsafe_domain_blocked") {+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 018698bb9..bbf4a09a4 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -22,6 +22,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { captureExceptionWithZdrCheck } from "../../services/sentry";import { z } from "zod";import { executeSearch } from "../../search/execute";+import { resolveThreatProtection } from "../../lib/threat-protection/request";import {DocumentWithCostTracking,scrapeSearchResults,@@ -151,6 +152,22 @@ export async function searchController(origin: req.body.origin,});+ // Threat protection: resolve the effective policy. Blocked domains are+ // removed from search results entirely.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override:+ req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+await logRequest({id: jobId,kind: "search",@@ -228,6 +245,7 @@ export async function searchController(zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,keylessReserved: reservedKeylessCredits > 0,+ threatProtectionPolicy: threatProtection.policy,},logger,);diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex eac175261..7e0db4d88 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -19,6 +19,7 @@ import { webhookSchema } from "../../services/webhook/schema";import { BrandingProfile } from "../../types/branding";import { ProductProfile } from "../../types/product";import { MenuProfile } from "../../types/menu";+import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";type Format =| "markdown"@@ -528,6 +529,9 @@ const baseScrapeOptions = z.strictObject({.gte(0).prefault(1 * 24 * 60 * 60 * 1000),storeInCache: z.boolean().prefault(true),+ // Enterprise: per-request field-level override of the org's threat+ // protection policy. Gated on the team flag + org config (checkPermissions).+ threatProtection: threatProtectionOverrideSchema.optional(),// @deprecated__experimental_cache: z.boolean().prefault(false).optional(),__searchPreviewToken: z.string().optional(),@@ -744,6 +748,7 @@ const extractV1Options = z__experimental_showCostTracking: z.boolean().prefault(false),ignoreInvalidURLs: z.boolean().prefault(false),webhook: webhookSchema.optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),}).refine(obj => obj.urls || obj.prompt, {error: "Either 'urls' or 'prompt' must be provided.",@@ -972,6 +977,7 @@ const mapRequestSchemaBase = crawlerOptionsignoreCache: z.boolean().prefault(false),location: locationSchema,headers: z.record(z.string(), z.string()).optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),});export const mapRequestSchema = mapRequestSchemaBase.strict();@@ -1302,6 +1308,7 @@ export type AuthCreditUsageChunk = {export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";+ threatProtection?: "disabled" | "allowed" | "forced";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;@@ -1559,6 +1566,7 @@ export const searchRequestSchema = ztimeout: z.int().positive().finite().prefault(60000),ignoreInvalidURLs: z.boolean().optional().prefault(false),__searchPreviewToken: z.string().optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),scrapeOptions: baseScrapeOptions.extend({formats: zdiff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex e478f4128..24b2572f6 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -11,6 +11,13 @@ import { logRequest } from "../../services/logging/log_job";import { config } from "../../config";import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";export async function agentController(req: RequestWithAuth<{}, AgentResponse, AgentRequest>,@@ -46,6 +53,63 @@ export async function agentController(zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});+ // Threat protection: check the agent's starting URLs before handing off to+ // the agent service. Content the agent fetches through the API+ // (agent-interop scrapes) is additionally enforced by the scrape pipeline's+ // org-policy resolution; in-page navigations performed by the remote+ // browser cannot be intercepted here.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {+ const { blocked } = await checkUrlsAgainstThreatPolicy(+ req.body.urls ?? [],+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ if (blocked.length > 0) {+ // Blocked domains whose decision consulted the classifier (fresh or+ // cached verdict) bill the scan fee (+2 per scanned domain) even+ // though the request is rejected β the scan already happened. Allowed+ // start URLs are not billed here: the agent's API-driven scrapes+ // re-check the policy in the scrape pipeline and bill there.+ const blockedDecisionsByDomain = new Map(+ blocked.map(x => [x.domain, x.decision]),+ );+ const threatScanCredits = calculateThreatScanCredits(+ blockedDecisionsByDomain.values(),+ );+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "agent", jobId: agentId },+ ).catch(error => {+ logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ const first = blocked[0];+ const error = new UnsafeDomainBlockedError(first.domain, first.decision);+ return res.status(403).json({+ success: false,+ error: error.message,+ });+ }+ }+if (!config.EXTRACT_V3_BETA_URL) {throw new Error("Agent beta is not enabled.");}diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex ad9df8c80..5d74d8894 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -38,6 +38,13 @@ import {import { logRequest } from "../../services/logging/log_job";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -50,7 +57,22 @@ export async function batchScrapeController(req.body = batchScrapeRequestSchema.parse(req.body);}- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }++ const permissions = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });if (permissions.error) {return res.status(403).json({success: false,@@ -150,6 +172,75 @@ export async function batchScrapeController(}}+ // Threat protection: reject/report blocked URLs at enqueue time so they+ // never consume scrape slots. Mirrors the isUrlBlocked handling above:+ // with ignoreInvalidURLs they are reported in invalidURLs; without it the+ // whole request is rejected. Any URL that slips through (e.g. via a+ // redirect) is still blocked in the scrape pipeline and its job document+ // gets the unsafe_domain_blocked error.+ if (threatProtection.policy) {+ const { blocked } = await checkUrlsAgainstThreatPolicy(+ urls,+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ if (blocked.length > 0) {+ // Blocked domains whose decision consulted the classifier (fresh or+ // cached verdict) bill the scan fee (+2 per scanned domain) even+ // though they will never be scraped β the scan already happened.+ // Allowed URLs are not billed here: their scrape jobs re-check the+ // cached verdict and bill there.+ const blockedDecisionsByDomain = new Map(+ blocked.map(x => [x.domain, x.decision]),+ );+ const threatScanCredits = calculateThreatScanCredits(+ blockedDecisionsByDomain.values(),+ );+ if (+ threatScanCredits > 0 &&+ (req.body.__agentInterop?.shouldBill ?? true)+ ) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ billing,+ ).catch(error => {+ logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ if (req.body.ignoreInvalidURLs) {+ const blockedSet = new Set(blocked.map(x => x.url));+ const keptUnnormalized: string[] = [];+ const keptUrls: string[] = [];+ urls.forEach((u, i) => {+ if (blockedSet.has(u)) {+ invalidURLs!.push(unnormalizedURLs[i] ?? u);+ } else {+ keptUrls.push(u);+ keptUnnormalized.push(unnormalizedURLs[i]);+ }+ });+ urls = keptUrls;+ unnormalizedURLs = keptUnnormalized;+ } else {+ const first = blocked[0];+ const error = new UnsafeDomainBlockedError(+ first.domain,+ first.decision,+ );+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }+ }+ }+if (urls.length === 0) {return res.status(400).json({success: false,@@ -191,6 +282,7 @@ export async function batchScrapeController(zeroDataRetention,bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 0568e9705..cb07fb6ca 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -31,6 +31,12 @@ import {} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { resolveThreatProtection } from "../../lib/threat-protection/request";+import { checkDomain } from "../../lib/threat-protection";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { normalizeDomain } from "../../lib/threat-protection/verdict";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -39,9 +45,23 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.scrapeOptions?.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+const permissions = checkPermissions({ ...req.body, crawlerOptions: req.body },req.acuc?.flags,+ { threatProtectionOrgConfig: threatProtection.orgConfig },);if (permissions.error) {return res.status(403).json({@@ -50,6 +70,41 @@ export async function crawlController(});}+ // Threat protection: check the seed URL before kicking off the crawl.+ // Blocked seed => request-level error. Discovered links are checked during+ // link discovery in the workers (blocked ones are silently skipped).+ if (threatProtection.policy) {+ const seedDomain = normalizeDomain(req.body.url);+ const decision = await checkDomain(seedDomain, threatProtection.policy, {+ teamId: req.auth.team_id,+ });+ if (!decision.allowed) {+ // A blocked seed still bills the scan fee when the classifier was+ // consulted β the scan already happened. (An allowed seed is not billed+ // here: its scrape job re-checks the cached verdict and bills there.)+ const threatScanCredits = calculateThreatScanCredits([decision]);+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "crawl" },+ ).catch(error => {+ _logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ const error = new UnsafeDomainBlockedError(seedDomain, decision);+ return res.status(403).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }+ }+const keyRestriction = await checkKeyFormatRestriction(formatTypesOf(req.body.scrapeOptions?.formats),actionTypesOf(req.body.scrapeOptions?.actions),@@ -210,6 +265,7 @@ export async function crawlController(saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,},team_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 5842dafd2..6bdbf8090 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -14,6 +14,13 @@ import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";import { config } from "../../config";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";+import { billTeam } from "../../services/billing/credit_billing";/*** Extracts data from the provided URLs based on the request parameters.@@ -74,6 +81,64 @@ export async function extractController(}}+ // Threat protection: check target URLs before fetching. Blocked URLs are+ // reported via invalidURLs (with ignoreInvalidURLs) or reject the request.+ // Discovered URLs are enforced in the scrape pipeline via the policy+ // threaded through the extract job.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+ if (threatProtection.policy && (req.body.urls?.length ?? 0) > 0) {+ const { blocked, decisionsByDomain } = await checkUrlsAgainstThreatPolicy(+ req.body.urls ?? [],+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ // Every consulted decision (fresh or cached provider verdict) is a+ // billable scan (+2 per scanned domain) β including when the request is+ // rejected below: the scans already happened.+ const threatScanCredits = calculateThreatScanCredits(+ decisionsByDomain.values(),+ );+ if (threatScanCredits > 0) {+ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ threatScanCredits,+ req.acuc?.api_key_id ?? null,+ { endpoint: "extract", jobId: extractId },+ ).catch(error => {+ _logger.error(+ `Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,+ );+ });+ }+ if (blocked.length > 0) {+ if (req.body.ignoreInvalidURLs) {+ invalidURLs.push(...blocked.map(x => x.url));+ } else {+ const first = blocked[0];+ const error = new UnsafeDomainBlockedError(+ first.domain,+ first.decision,+ );+ return res.status(403).json({+ success: false,+ error: error.message,+ });+ }+ }+ }+await logRequest({id: extractId,kind: "extract",diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 0728b662f..cc0203bfe 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -16,6 +16,12 @@ import { v7 as uuidv7 } from "uuid";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { getScrapeZDR } from "../../lib/zdr-helpers";import { resolveViaAvgrab } from "../../lib/avgrab-resolve";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../lib/threat-protection/request";+import { normalizeDomain } from "../../lib/threat-protection/verdict";+import { calculateThreatScanCredits } from "../../lib/scrape-billing";configDotenv();@@ -38,7 +44,22 @@ export async function mapController(const originalRequest = req.body;req.body = mapRequestSchema.parse(req.body);- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }++ const permissions = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });if (permissions.error) {return res.status(403).json({success: false,@@ -189,17 +210,37 @@ export async function mapController(}}+ // Threat protection: remove links on blocked domains from the returned+ // URL list entirely. Every consulted decision (fresh or cached provider+ // verdict) is a billable scan (+2 per scanned domain).+ let threatScanCredits = 0;+ if (threatProtection.policy && result.mapResults.length > 0) {+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(+ result.mapResults.map(x => x.url),+ threatProtection.policy,+ { teamId: req.auth.team_id },+ );+ threatScanCredits = calculateThreatScanCredits(decisionsByDomain.values());+ result.mapResults = result.mapResults.filter(x => {+ const decision = decisionsByDomain.get(normalizeDomain(x.url));+ return decision === undefined || decision.allowed;+ });+ }+// Bill the team+ const creditsToBill = 1 + threatScanCredits;billTeam(req.auth.team_id,req.acuc?.sub_id ?? undefined,- 1,+ creditsToBill,req.acuc?.api_key_id ?? null,{ endpoint: "map", jobId: mapId },).catch(error => {- logger.error(- `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,- );+ logger.error("Failed to bill team for map credits", {+ teamId: req.auth.team_id,+ creditsToBill,+ error,+ });});logMap({@@ -217,7 +258,7 @@ export async function mapController(location: req.body.location,},results: result.mapResults,- credits_cost: 1,+ credits_cost: creditsToBill,zeroDataRetention: false, // not supported}).catch(error => {logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 44823a2f7..4de9b9108 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -37,6 +37,7 @@ import {} from "../../lib/keyless";import { projectScrapeCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { resolveThreatProtection } from "../../lib/threat-protection/request";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -72,11 +73,33 @@ export async function scrapeController(});});+ // Threat protection: resolve the effective policy (org config ++ // per-request override). No-ops (null policy, zero I/O) for teams+ // without the flag.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override: req.body.threatProtection,+ });+ if (threatProtection.error) {+ setSpanAttributes(span, {+ "scrape.error": threatProtection.error,+ "scrape.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+// Permission check spanconst permissions = await withSpan("api.scrape.check_permissions",async permSpan => {- const perms = checkPermissions(req.body, req.acuc?.flags);+ const perms = checkPermissions(req.body, req.acuc?.flags, {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ });setSpanAttributes(permSpan, {"permissions.success": !perms.error,"permissions.error": perms.error,@@ -306,6 +329,7 @@ export async function scrapeController(zeroDataRetention,teamFlags: req.acuc?.flags ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,+ threatProtection: threatProtection.policy ?? undefined,},skipNuq: true,origin,@@ -416,6 +440,17 @@ export async function scrapeController(});}+ if (e.code === "unsafe_domain_blocked") {+ setSpanAttributes(span, {+ "scrape.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 7d2cdb43b..ffe8dd4b7 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -28,6 +28,7 @@ import type { BillingMetadata } from "../../services/billing/types";import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { resolveThreatProtection } from "../../lib/threat-protection/request";import {actionTypesOf,checkKeyFormatRestriction,@@ -100,6 +101,23 @@ export async function searchController(});}+ // Threat protection: resolve the effective policy. Blocked domains are+ // removed from search results entirely, and scraped results inherit the+ // policy through the scrape pipeline.+ const threatProtection = await resolveThreatProtection({+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ override:+ req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,+ });+ if (threatProtection.error) {+ return res.status(403).json({+ success: false,+ error: threatProtection.error,+ });+ }+const shouldBill = req.body.__agentInterop?.shouldBill ?? true;const agentRequestId = req.body.__agentInterop?.requestId ?? null;const billing: BillingMetadata = req.body.__agentInterop@@ -211,6 +229,7 @@ export async function searchController(billing,agentIndexOnly: (req as any).agentIndexOnly ?? false,keylessReserved: reservedKeylessCredits > 0,+ threatProtectionPolicy: threatProtection.policy,},logger,);diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.tsnew file mode 100644index 000000000..ea2528203--- /dev/null+++ b/apps/api/src/controllers/v2/team-threat-protection.ts@@ -0,0 +1,125 @@+import { Response } from "express";+import { z } from "zod";+import { logger as _logger } from "../../lib/logger";+import { RequestWithAuth } from "./types";+import { getThreatProtection } from "../../lib/zdr-helpers";+import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";+import {+ getOrgIdForTeam,+ getOrgThreatProtectionConfig,+ resolveEffectivePolicy,+ upsertOrgThreatProtectionConfig,+ type OrgThreatProtectionConfig,+} from "../../lib/threat-protection/store";++const logger = _logger.child({ module: "team-threat-protection" });++const SUPPORT_EMAIL = "[email protected]";++function rejectWithoutFlag(+ req: RequestWithAuth<any, any, any>,+ res: Response,+): boolean {+ const mode = getThreatProtection(req.acuc?.flags);+ if (mode !== "allowed" && mode !== "forced") {+ res.status(403).json({+ success: false,+ error: `Threat protection is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,+ });+ return true;+ }+ return false;+}++async function resolveOrgId(+ req: RequestWithAuth<any, any, any>,+ res: Response,+): Promise<string | null> {+ const orgId = req.auth.org_id ?? (await getOrgIdForTeam(req.auth.team_id));+ if (!orgId) {+ logger.error("Failed to resolve org for team", {+ teamId: req.auth.team_id,+ });+ res.status(500).json({+ success: false,+ error: "Failed to resolve the organization for this team.",+ });+ return null;+ }+ return orgId;+}++/**+ * Effective config document served by GET and PUT. Always includes every+ * policy field (defaults applied), so the dashboard can render the form+ * without knowing the defaults.+ */+function serializeConfig(orgConfig: OrgThreatProtectionConfig | null) {+ const policy = resolveEffectivePolicy(orgConfig);+ return {+ ...policy,+ allowRequestOverrides: orgConfig?.allowRequestOverrides ?? true,+ configured: orgConfig !== null,+ updatedAt: orgConfig?.updatedAt ?? null,+ };+}++function changedFields(+ previous: OrgThreatProtectionConfig | null,+ next: OrgThreatProtectionConfig,+): string[] {+ const before = serializeConfig(previous) as Record<string, unknown>;+ const after = serializeConfig(next) as Record<string, unknown>;+ return Object.keys(after).filter(+ key =>+ key !== "updatedAt" &&+ key !== "configured" &&+ JSON.stringify(before[key]) !== JSON.stringify(after[key]),+ );+}++export async function getTeamThreatProtectionController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;++ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ const orgConfig = await getOrgThreatProtectionConfig(orgId);++ res.status(200).json({+ success: true,+ data: serializeConfig(orgConfig),+ });+}++export async function putTeamThreatProtectionController(+ req: RequestWithAuth<{}, any, unknown>,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;++ const input = threatProtectionConfigSchema.parse(req.body);++ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ const previous = await getOrgThreatProtectionConfig(orgId);++ const updated = await upsertOrgThreatProtectionConfig(orgId, input);++ // Audit log β org-level security configuration change.+ logger.info("Threat protection config updated", {+ teamId: req.auth.team_id,+ orgId,+ mode: updated.policy.mode,+ changedFields: changedFields(previous, updated),+ });++ res.status(200).json({+ success: true,+ data: serializeConfig(updated),+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 61ee4ad23..37b95e62a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -28,6 +28,7 @@ import {import { BrandingProfile } from "../../types/branding";import { ProductProfile } from "../../types/product";import { MenuProfile } from "../../types/menu";+import { threatProtectionOverrideSchema } from "../../lib/threat-protection/config";// Base URL schema with common validation logicexport const URL = z.preprocess(@@ -692,6 +693,9 @@ const baseScrapeOptions = z.strictObject({storeInCache: z.boolean().prefault(true),lockdown: z.boolean().prefault(false),redactPII: redactPIISchema,+ // Enterprise: per-request field-level override of the org's threat+ // protection policy. Gated on the team flag + org config (checkPermissions).+ threatProtection: threatProtectionOverrideSchema.optional(),profile: z.object({@@ -880,6 +884,7 @@ const extractOptions = z__experimental_showCostTracking: z.boolean().prefault(false),ignoreInvalidURLs: z.boolean().prefault(true),webhook: webhookSchema.optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),}).refine(obj => obj.urls || obj.prompt, {error: "Either 'urls' or 'prompt' must be provided.",@@ -940,6 +945,7 @@ export const agentRequestSchema = z.strictObject({overrideWhitelist: z.string().optional(),model: z.enum(["spark-1-pro", "spark-1-mini"]).default("spark-1-pro"),+ threatProtection: threatProtectionOverrideSchema.optional(),});export type AgentRequest = z.infer<typeof agentRequestSchema>;@@ -1191,6 +1197,7 @@ const mapRequestSchemaBase = crawlerOptionsignoreCache: z.boolean().prefault(false),location: locationSchema,headers: z.record(z.string(), z.string()).optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),});export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);@@ -1534,6 +1541,7 @@ type Account = {export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";+ threatProtection?: "disabled" | "allowed" | "forced";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;@@ -1943,6 +1951,7 @@ export const searchRequestSchema = z// scrapeURL. Falls back to the provider snippet when the URL isn't indexed.highlights: z.boolean().optional().prefault(false),__searchPreviewToken: z.string().optional(),+ threatProtection: threatProtectionOverrideSchema.optional(),scrapeOptions: baseScrapeOptions.extend({formats: zdiff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 8fbdc3c4c..fa29eb7ae 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -654,6 +654,19 @@ export const teams = pgTable("teams", {org_id: uuid("org_id").notNull(),});+// Org-scoped threat protection configuration (enterprise feature, gated by the+// `threatProtection` team flag). One row per org; DDL is applied out-of-band.+// The policy document lives in the `config` jsonb column (parsed tolerantly in+// lib/threat-protection/store.ts); only `mode` is a dedicated column.+export const threat_protection_config = pgTable("threat_protection_config", {+ id: uuid("id").notNull().defaultRandom(),+ org_id: uuid("org_id").notNull().unique(),+ mode: varchar("mode").notNull().default("off"),+ config: jsonb("config").notNull().default({}),+ created_at: ts("created_at").notNull().defaultNow(),+ updated_at: ts("updated_at").notNull().defaultNow(),+});+export const user_notifications = pgTable("user_notifications", {id: uuid("id").notNull().defaultRandom(),team_id: uuid("team_id"),diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex f82c9feee..697fb774c 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -73,6 +73,28 @@ export async function recordRobotsBlocked(crawlId: string, url: string) {);}+/**+ * Records a URL that was silently skipped during crawl link discovery because+ * its domain was blocked by the team's threat protection policy. The crawl+ * continues without it; the record (url -> full ThreatDecision JSON) is kept+ * for the security-logging layer to read.+ *+ * ZDR posture: this hash follows the same rules as the rest of the transient+ * crawl bookkeeping in this file (crawl doc, visited sets, robots_blocked) β+ * it necessarily holds URLs while the crawl runs, lives in the evict Redis,+ * and expires after 24h like every other crawl key. Additionally, for+ * zero-data-retention crawls it is deleted eagerly in {@link finishCrawl}.+ */+export async function recordThreatBlocked(+ crawlId: string,+ url: string,+ decision: unknown,+) {+ const key = "crawl:" + crawlId + ":threat_blocked";+ await redisEvictConnection.hset(key, url, JSON.stringify(decision));+ await redisEvictConnection.expire(key, 24 * 60 * 60);+}+export async function markCrawlActive(id: string) {await redisEvictConnection.sadd("active_crawls", id);}@@ -331,6 +353,13 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {// Clear visited sets to save memoryawait redisEvictConnection.del("crawl:" + id + ":visited");await redisEvictConnection.del("crawl:" + id + ":visited_unique");++ // Eagerly drop the threat-protection bookkeeping (URL -> decision records+ // of silently skipped discoveries) instead of letting it ride out its 24h+ // TTL. Nothing reads it after the crawl finishes, and deleting it+ // unconditionally keeps the ZDR guarantee even when the crawl document is+ // missing or unreadable at finish time.+ await redisEvictConnection.del("crawl:" + id + ":threat_blocked");}export async function getCrawlJobs(id: string): Promise<string[]> {diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex cdf4ddf17..935ef612b 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -35,6 +35,7 @@ import {VideoUnsupportedUrlError,XTwitterConfigurationError,} from "../scraper/scrapeURL/error";+import { UnsafeDomainBlockedError } from "./threat-protection/error";// TODO: figure out correct typing for thisconst errorMap: Record<ErrorCodes, any> = {@@ -69,6 +70,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_VIDEO_UNSUPPORTED_URL: VideoUnsupportedUrlError,SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,MAP_FAILED: MapFailedError,+ unsafe_domain_blocked: UnsafeDomainBlockedError,// Zod errorsBAD_REQUEST: null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex d45ffdd5d..62df2572b 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -32,7 +32,10 @@ export type ErrorCodes =| "CRAWL_DENIAL"| "MAP_FAILED"| "BAD_REQUEST_INVALID_JSON"- | "BAD_REQUEST";+ | "BAD_REQUEST"+ // Threat protection (enterprise domain risk blocking). Lowercase by design:+ // this is the documented, user-facing error code for the feature.+ | "unsafe_domain_blocked";export class TransportableError extends Error {public readonly code: ErrorCodes;diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 0b884f62a..08a0b0b77 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -12,6 +12,7 @@ import { getJobPriority } from "../../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";import { scrapeQueue } from "../../../services/worker/nuq-router";+import type { ThreatProtectionPolicy } from "../../threat-protection/types";interface ScrapeDocumentOptions {url: string;@@ -22,6 +23,8 @@ interface ScrapeDocumentOptions {flags: TeamFlags | null;apiKeyId: number | null;requestId?: string;+ /** Effective threat protection policy; enforced in the scrape pipeline. */+ threatProtectionPolicy?: ThreatProtectionPolicy | null;}export async function scrapeDocument_F0(@@ -66,6 +69,7 @@ export async function scrapeDocument_F0(internalOptions: {teamId: options.teamId,bypassBilling: true,+ threatProtection: options.threatProtectionPolicy ?? undefined,},origin: options.origin,is_scrape: true,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex fecfa4b27..aeb3224a1 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -39,6 +39,7 @@ import {} from "./usage/llm-cost-f0";import { SourceTracker_F0 } from "./helpers/source-tracker-f0";import { getACUCTeam } from "../../../controllers/auth";+import { resolveThreatProtection } from "../../threat-protection/request";interface ExtractServiceOptions {request: ExtractRequest;@@ -92,6 +93,18 @@ export async function performExtraction_F0(const acuc = await getACUCTeam(teamId);+ // Threat protection: resolve the effective policy once for this extract+ // job (org config + the request's threatProtection override, which the+ // controller already validated). Threaded into every document scrape so+ // both user-provided and discovered URLs are enforced in the pipeline.+ const threatProtectionPolicy = (+ await resolveThreatProtection({+ teamId,+ flags: acuc?.flags ?? null,+ override: request.threatProtection,+ })+ ).policy;+const logger = _logger.child({module: "extract",method: "performExtraction",@@ -341,6 +354,7 @@ export async function performExtraction_F0(flags: acuc?.flags ?? null,apiKeyId,requestId: extractId,+ threatProtectionPolicy,},urlTraces,logger.child({@@ -636,6 +650,7 @@ export async function performExtraction_F0(flags: acuc?.flags ?? null,apiKeyId,requestId: extractId,+ threatProtectionPolicy,},urlTraces,logger.child({diff --git a/apps/api/src/lib/permissions.test.ts b/apps/api/src/lib/permissions.test.tsnew file mode 100644index 000000000..d97818301--- /dev/null+++ b/apps/api/src/lib/permissions.test.ts@@ -0,0 +1,55 @@+import { checkPermissions } from "./permissions";++describe("checkPermissions β threat protection", () => {+ const requestWithOption = { threatProtection: { mode: "normal" } };++ it("allows requests without a threatProtection option regardless of flags", () => {+ expect(checkPermissions({}, null)).toEqual({});+ expect(checkPermissions({}, { threatProtection: "disabled" })).toEqual({});+ });++ it("rejects a per-request option when the flag is missing or disabled", () => {+ expect(checkPermissions(requestWithOption, null).error).toMatch(+ /enterprise feature/,+ );+ expect(+ checkPermissions(requestWithOption, { threatProtection: "disabled" })+ .error,+ ).toMatch(/enterprise feature/);+ });++ it.each(["allowed", "forced"] as const)(+ "allows a per-request option when the flag is %s",+ mode => {+ expect(+ checkPermissions(requestWithOption, { threatProtection: mode }),+ ).toEqual({});+ },+ );++ it("rejects a per-request option when the org disables overrides", () => {+ const result = checkPermissions(+ requestWithOption,+ { threatProtection: "allowed" },+ { threatProtectionOrgConfig: { allowRequestOverrides: false } },+ );+ expect(result.error).toMatch(/overrides are disabled/);+ });++ it("allows a per-request option when the org config allows overrides", () => {+ expect(+ checkPermissions(+ requestWithOption,+ { threatProtection: "allowed" },+ { threatProtectionOrgConfig: { allowRequestOverrides: true } },+ ),+ ).toEqual({});+ expect(+ checkPermissions(+ requestWithOption,+ { threatProtection: "allowed" },+ { threatProtectionOrgConfig: null },+ ),+ ).toEqual({});+ });+});diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex 4f7478072..bb116f47a 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -3,20 +3,40 @@ import {getScrapeZDR,getIgnoreRobots,getCustomRobotsAgent,+ getThreatProtection,} from "./zdr-helpers";+import {+ THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,+ THREAT_PROTECTION_NOT_ENABLED_MESSAGE,+ THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,+} from "./threat-protection/request";type LocationOptions = { country?: string };+type ThreatProtectionOption = { mode?: string };interface APIRequest {zeroDataRetention?: boolean;location?: LocationOptions;scrapeOptions?: {location?: LocationOptions;+ threatProtection?: ThreatProtectionOption;};crawlerOptions?: {ignoreRobotsTxt?: boolean;robotsUserAgent?: string;};+ // Per-request threat protection policy override (field-level override of+ // the org config). Presence of any value gates on the team flag.+ threatProtection?: ThreatProtectionOption;+}++interface PermissionOptions {+ /**+ * Org-level threat protection config (or the relevant slice of it), if+ * already loaded. When the org disables request overrides, any per-request+ * threatProtection option is rejected.+ */+ threatProtectionOrgConfig?: { allowRequestOverrides: boolean } | null;}const SUPPORT_EMAIL = "[email protected]";@@ -24,6 +44,7 @@ const SUPPORT_EMAIL = "[email protected]";export function checkPermissions(request: APIRequest,flags?: TeamFlags,+ options?: PermissionOptions,): { error?: string } {// zdr perms β scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDRconst scrapeMode = getScrapeZDR(flags);@@ -59,6 +80,25 @@ export function checkPermissions(};}+ // threat protection perms β the flag must be 'allowed' or 'forced' for any+ // per-request threatProtection option, the org must not have locked down+ // request-level overrides, and a 'forced' team may never disable the+ // feature per-request.+ const threatProtectionOption =+ request.threatProtection ?? request.scrapeOptions?.threatProtection;+ if (threatProtectionOption !== undefined) {+ const threatMode = getThreatProtection(flags);+ if (threatMode !== "allowed" && threatMode !== "forced") {+ return { error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE };+ }+ if (options?.threatProtectionOrgConfig?.allowRequestOverrides === false) {+ return { error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE };+ }+ if (threatMode === "forced" && threatProtectionOption.mode === "off") {+ return { error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE };+ }+ }+// ip whitelist permsconst needsWhitelist =request.location?.country === "us-whitelist" ||diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex cdf49cb52..68d80fb5d 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -1,4 +1,9 @@-import { calculateCreditsToBeBilled } from "./scrape-billing";+import {+ calculateCreditsToBeBilled,+ calculateThreatScanCredits,+} from "./scrape-billing";+import { UnsafeDomainBlockedError } from "./threat-protection/error";+import type { ThreatDecision } from "./threat-protection/types";describe("calculateCreditsToBeBilled", () => {it("bills handled data layer successes at 15 credits", async () => {@@ -114,3 +119,190 @@ describe("calculateCreditsToBeBilled", () => {expect(credits).toBe(3);});});++// =========================================+// Threat protection scan fees (ENG-4985)+// =========================================++const consulted = (allowed = true): ThreatDecision => ({+ allowed,+ rule: allowed ? "default-allow" : "risk-score",+ providerConsulted: true,+ verdict: {+ provider: "google-web-risk",+ riskScore: allowed ? 0 : 100,+ categories: [],+ fromCache: false,+ raw: {},+ },+ mode: "normal",+});++const localOnly = (+ rule: ThreatDecision["rule"],+ allowed: boolean,+): ThreatDecision => ({+ allowed,+ rule,+ providerConsulted: false,+ verdict: null,+ mode: "normal",+});++const billWithDecisions = (args: {+ document: { metadata: Record<string, unknown> } | null;+ error?: Error | null;+ threatDecisions?: ThreatDecision[];+}) =>+ calculateCreditsToBeBilled(+ { formats: [{ type: "markdown" }] } as any,+ { teamId: "team-id" },+ args.document as any,+ { totalCost: 0 } as any,+ {} as any,+ args.error,+ undefined,+ undefined,+ args.threatDecisions,+ );++const successDocument = {+ metadata: { statusCode: 200, proxyUsed: "basic" },+};++describe("calculateThreatScanCredits", () => {+ it("bills nothing for no decisions", () => {+ expect(calculateThreatScanCredits([])).toBe(0);+ });++ it("bills +2 per consulted decision", () => {+ expect(calculateThreatScanCredits([consulted()])).toBe(2);+ expect(calculateThreatScanCredits([consulted(), consulted()])).toBe(4);+ });++ it("bills consulted decisions regardless of the allow/block outcome", () => {+ expect(calculateThreatScanCredits([consulted(false)])).toBe(2);+ });++ it("bills cached verdicts the same as fresh ones", () => {+ const cached = consulted();+ cached.verdict = { ...cached.verdict!, fromCache: true };+ expect(calculateThreatScanCredits([cached])).toBe(2);+ });++ it("never bills local-only decisions", () => {+ expect(+ calculateThreatScanCredits([+ localOnly("whitelist", true),+ localOnly("blacklist", false),+ localOnly("blocked-tld", false),+ localOnly("provider-failure", false),+ ]),+ ).toBe(0);+ });++ it("sums mixed decisions", () => {+ expect(+ calculateThreatScanCredits([+ consulted(),+ consulted(false),+ localOnly("blacklist", false),+ ]),+ ).toBe(4);+ });+});++describe("calculateCreditsToBeBilled β threat protection scan fees", () => {+ it("adds +2 to a successful scrape with a consulted decision", async () => {+ expect(+ await billWithDecisions({+ document: successDocument,+ threatDecisions: [consulted()],+ }),+ ).toBe(3);+ });++ it("bills each consulted decision on a redirect (two domains scanned)", async () => {+ expect(+ await billWithDecisions({+ document: successDocument,+ threatDecisions: [consulted(), consulted()],+ }),+ ).toBe(5);+ });++ it("adds nothing for local-only decisions on success", async () => {+ expect(+ await billWithDecisions({+ document: successDocument,+ threatDecisions: [localOnly("whitelist", true)],+ }),+ ).toBe(1);+ });++ it("bills the scan fee (and no base cost) for a blocked scrape", async () => {+ const decision = consulted(false);+ expect(+ await billWithDecisions({+ document: null,+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),+ threatDecisions: [decision],+ }),+ ).toBe(2);+ });++ it("bills a blocked scrape from the error decision when the decisions array is missing", async () => {+ const decision = consulted(false);+ expect(+ await billWithDecisions({+ document: null,+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),+ }),+ ).toBe(2);+ });++ it("does not double-bill when the error decision is also in the decisions array", async () => {+ const initial = consulted();+ const redirectBlock = consulted(false);+ expect(+ await billWithDecisions({+ document: null,+ error: new UnsafeDomainBlockedError(+ "blocked.example.com",+ redirectBlock,+ ),+ threatDecisions: [initial, redirectBlock],+ }),+ ).toBe(4);+ });++ it("bills nothing for a blocked scrape decided by local-only rules", async () => {+ const decision = localOnly("blacklist", false);+ expect(+ await billWithDecisions({+ document: null,+ error: new UnsafeDomainBlockedError("blocked.example.com", decision),+ threatDecisions: [decision],+ }),+ ).toBe(0);+ });++ it("bills nothing for unrelated failures without decisions", async () => {+ expect(+ await billWithDecisions({+ document: null,+ error: new Error("engine failure"),+ }),+ ).toBe(0);+ });++ it("adds the scan fee on top of other failure billing (scan happened before the failure)", async () => {+ expect(+ await billWithDecisions({+ document: null,+ error: new Error("engine failure"),+ threatDecisions: [consulted()],+ }),+ ).toBe(2);+ });+});diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 23b334c84..bef1ebdc9 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -14,6 +14,8 @@ import {DataLayerScrapeMetadata,getDataLayerSuccessCredits,} from "./data-layer";+import type { ThreatDecision } from "./threat-protection/types";+import { UnsafeDomainBlockedError } from "./threat-protection/error";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;@@ -23,6 +25,29 @@ const redactPIICostBonus = 4;// Each additional PDF page also gets redacted through fire-privacy, so// the per-page surcharge mirrors the +4 base β same tier as lockdown.const redactPIIPdfPageCostBonus = 4;+// Threat protection domain scans: +2 per scanned domain in "normal" mode+// (Google Web Risk). A "scan" is any ThreatDecision with providerConsulted+// set β +2 per consulted verdict. Checks deduplicated within one request/job+// (e.g. a scrape and its same-domain redirect re-check) share a single+// decision and therefore bill once; verdicts are never reused across+// requests (no verdict cache β ZDR). Local-only decisions (whitelist/+// blacklist/blocked-tld, mode off, provider failure) never bill.+const threatScanCost = 2;++/**+ * Sums the scan fees for a set of threat protection decisions. Only decisions+ * that consulted the provider bill; the fee is +2 per scanned domain.+ */+export function calculateThreatScanCredits(+ decisions: Iterable<ThreatDecision>,+): number {+ let credits = 0;+ for (const decision of decisions) {+ if (!decision.providerConsulted) continue;+ credits += threatScanCost;+ }+ return credits;+}export async function calculateCreditsToBeBilled(options: ScrapeOptions,@@ -33,10 +58,28 @@ export async function calculateCreditsToBeBilled(error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,dataLayer?: DataLayerScrapeMetadata,+ // Threat protection decisions for this scrape (initial + redirect checks,+ // in order). Each decision with `providerConsulted` bills a scan fee (+2+ // per scanned domain) on top of the scrape's own cost β on both success+ // and failure (a scrape blocked by threat protection still consulted the+ // classifier). For scrapes blocked by threat protection, the+ // UnsafeDomainBlockedError in `error` also carries its decision, which is+ // used as a fallback when the decisions array did not make it here.+ threatDecisions?: ThreatDecision[],) {const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;+ const effectiveThreatDecisions: ThreatDecision[] =+ threatDecisions && threatDecisions.length > 0+ ? threatDecisions+ : error instanceof UnsafeDomainBlockedError+ ? [error.decision]+ : [];+ const threatScanCredits = calculateThreatScanCredits(+ effectiveThreatDecisions,+ );+if (document === null) {// Failure -- check cost tracking if FIRE-1let creditsToBeBilled = 0;@@ -57,7 +100,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = 1;}- return creditsToBeBilled;+ // Failed scrapes bill no base cost (except the cases above), but threat+ // protection scans that already happened still bill β including scrapes+ // blocked by the policy itself.+ return creditsToBeBilled + threatScanCredits;}const dataLayerCredits = getDataLayerSuccessCredits({@@ -65,7 +111,7 @@ export async function calculateCreditsToBeBilled(statusCode: document.metadata?.statusCode,});if (dataLayerCredits !== null) {- return dataLayerCredits;+ return dataLayerCredits + threatScanCredits;}let creditsToBeBilled = 1; // Assuming 1 credit per document@@ -167,5 +213,7 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += unblockedDomainCostBonus;}+ creditsToBeBilled += threatScanCredits;+return creditsToBeBilled;}diff --git a/apps/api/src/lib/threat-protection/check-domain.test.ts b/apps/api/src/lib/threat-protection/check-domain.test.tsnew file mode 100644index 000000000..e0c0a2872--- /dev/null+++ b/apps/api/src/lib/threat-protection/check-domain.test.ts@@ -0,0 +1,279 @@+import http from "http";+import { AddressInfo } from "net";++// checkDomain is enforcement-only: it emits/exports no security events, so+// there are no event sinks to stub here.+// The Web Risk threat-list store lives on the durable Redis connection β+// swap in an in-memory fake. (fake-redis.ts has no runtime imports, so the+// factory cannot re-enter the module being mocked.)+vi.mock("../../services/queue-service", async () => {+ const { createFakeWebRiskRedis } = await import(+ "./providers/web-risk/fake-redis.js"+ );+ const client = createFakeWebRiskRedis();+ return { getRedisConnection: () => client };+});++import { config } from "../../config";+import {+ checkDomain,+ THREAT_PROTECTION_POLICY_DEFAULTS,+ ThreatCheckDedup,+ ThreatProtectionPolicy,+ UnsafeDomainBlockedError,+} from "./index";+import {+ createWebRiskMockCounters,+ createWebRiskMockHandler,+ WebRiskMockDatabase,+} from "./providers/web-risk/testing";++function policy(+ overrides: Partial<ThreatProtectionPolicy> = {},+): ThreatProtectionPolicy {+ return {+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ ...overrides,+ };+}++// The mock provider: a flagged fixture domain in the MALWARE list, served+// through the Update API endpoints (computeDiff for the local list sync,+// hashes:search for prefix-hit confirmation).+const RISKY_DOMAIN = "threat-risky.example";++const db = new WebRiskMockDatabase();+db.addRiskyDomain(RISKY_DOMAIN, "MALWARE");++const counters = createWebRiskMockCounters();+const webRiskHandler = createWebRiskMockHandler(db, counters);++// While > 0, hashes:search requests fail with 503 (decremented per request);+// Infinity = permanently down (provider-failure paths).+let failHashesSearches = 0;++let server: http.Server;++const originalConfig = {+ webRiskUrl: config.GOOGLE_WEB_RISK_API_URL,+ webRiskKey: config.GOOGLE_WEB_RISK_API_KEY,+};++beforeAll(async () => {+ await new Promise<void>(resolve => {+ server = http.createServer((req, res) => {+ if (+ failHashesSearches > 0 &&+ (req.url ?? "").startsWith("/v1/hashes:search")+ ) {+ failHashesSearches--;+ res.statusCode = 503;+ res.end("{}");+ return;+ }+ if (!webRiskHandler(req, res)) {+ res.statusCode = 404;+ res.end("{}");+ }+ });+ server.listen(0, "127.0.0.1", () => {+ const addr = server.address() as AddressInfo;+ config.GOOGLE_WEB_RISK_API_URL = `http://127.0.0.1:${addr.port}`;+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";+ resolve();+ });+ });+});++afterAll(async () => {+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.webRiskUrl;+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.webRiskKey;+ await new Promise<void>(resolve => server.close(() => resolve()));+});++beforeEach(() => {+ failHashesSearches = 0;+});++const riskyHits = () => counters.hashesSearchRequestsForDomain(RISKY_DOMAIN);++describe("checkDomain", () => {+ it("allows immediately when mode is off, with no provider call", async () => {+ const before = counters.hashesSearchRequests;+ const decision = await checkDomain("example.com", policy({ mode: "off" }), {+ teamId: "team-1",+ });++ expect(decision).toEqual({+ allowed: true,+ rule: "default-allow",+ providerConsulted: false,+ verdict: null,+ mode: "off",+ });+ expect(counters.hashesSearchRequests).toBe(before);+ });++ it("skips the provider scan when a local rule is decisive", async () => {+ const before = counters.hashesSearchRequests;+ const decision = await checkDomain(+ "cdn.blocked.com",+ policy({ blacklist: ["blocked.com"] }),+ { teamId: "team-1" },+ );++ expect(decision).toMatchObject({+ allowed: false,+ rule: "blacklist",+ providerConsulted: false,+ verdict: null,+ });+ expect(counters.hashesSearchRequests).toBe(before);+ });++ it("consults the provider and blocks a flagged domain (fresh scan)", async () => {+ const before = riskyHits();+ const decision = await checkDomain(RISKY_DOMAIN.toUpperCase(), policy(), {+ teamId: "team-1",+ });++ expect(decision).toMatchObject({+ allowed: false,+ rule: "risk-score",+ providerConsulted: true,+ mode: "normal",+ });+ expect(decision.verdict).toMatchObject({+ provider: "google-web-risk",+ riskScore: 100,+ categories: ["MALWARE"],+ fromCache: false,+ });+ // The threat list synced locally; the hit was confirmed by exactly one+ // hashes:search call carrying only the anonymized hash prefix.+ expect(counters.computeDiffRequests).toBeGreaterThanOrEqual(3);+ expect(riskyHits()).toBe(before + 1);+ });++ it("resolves clean domains locally with zero Google calls", async () => {+ const before = counters.hashesSearchRequests;+ const decision = await checkDomain("safe.example", policy(), {});++ expect(decision).toMatchObject({+ allowed: true,+ rule: "default-allow",+ providerConsulted: true,+ });+ expect(decision.verdict?.fromCache).toBe(false);+ expect(counters.hashesSearchRequests).toBe(before);+ });++ it("re-scans on a second lookup without a dedup handle (no verdict cache)", async () => {+ const before = riskyHits();++ const first = await checkDomain(RISKY_DOMAIN, policy(), {});+ const second = await checkDomain(RISKY_DOMAIN, policy(), {});++ expect(first).toMatchObject({ allowed: false, providerConsulted: true });+ expect(second).toMatchObject({ allowed: false, providerConsulted: true });+ expect(second).not.toBe(first);+ // Two independent scans, two confirmation calls β nothing was cached.+ expect(riskyHits()).toBe(before + 2);+ });++ it("shares one in-flight decision within a request via ctx.dedup", async () => {+ const before = riskyHits();+ const dedup: ThreatCheckDedup = new Map();++ // Concurrent + sequential re-checks of the same domain in one request.+ const [first, second] = await Promise.all([+ checkDomain(RISKY_DOMAIN, policy(), { dedup }),+ checkDomain(RISKY_DOMAIN, policy(), { dedup }),+ ]);+ const third = await checkDomain(`${RISKY_DOMAIN}.`, policy(), { dedup });++ // Same decision object β a single scan β a single billable decision.+ expect(second).toBe(first);+ expect(third).toBe(first); // normalization applies before dedup+ expect(riskyHits()).toBe(before + 1);+ });++ it("scans each distinct domain in a dedup scope separately", async () => {+ const dedup: ThreatCheckDedup = new Map();+ const before = riskyHits();++ const clean = await checkDomain("safe.example", policy(), { dedup });+ const risky = await checkDomain(RISKY_DOMAIN, policy(), { dedup });++ expect(clean.allowed).toBe(true);+ expect(risky.allowed).toBe(false);+ expect(riskyHits()).toBe(before + 1);+ });++ it("retries once and succeeds when the first confirmation attempt fails", async () => {+ failHashesSearches = 1; // exactly the first hashes:search attempt 503s++ const decision = await checkDomain(RISKY_DOMAIN, policy(), {});++ expect(decision).toMatchObject({+ allowed: false,+ rule: "risk-score",+ providerConsulted: true,+ });+ expect(failHashesSearches).toBe(0);+ });++ it("fails closed when confirmation is down and failurePolicy is closed", async () => {+ failHashesSearches = Infinity;++ const decision = await checkDomain(+ RISKY_DOMAIN,+ policy({ failurePolicy: "closed" }),+ { teamId: "team-1" },+ );++ expect(decision).toEqual({+ allowed: false,+ rule: "provider-failure",+ providerConsulted: false,+ verdict: null,+ mode: "normal",+ });+ });++ it("fails open when confirmation is down and failurePolicy is open", async () => {+ failHashesSearches = Infinity;++ const decision = await checkDomain(+ RISKY_DOMAIN,+ policy({ failurePolicy: "open" }),+ {},+ );++ expect(decision).toMatchObject({+ allowed: true,+ rule: "provider-failure",+ providerConsulted: false,+ verdict: null,+ });+ });+});++describe("UnsafeDomainBlockedError", () => {+ it("carries the decision and the unsafe_domain_blocked code", async () => {+ const decision = await checkDomain(+ "bad.example",+ policy({ blacklist: ["bad.example"] }),+ {},+ );+ const error = new UnsafeDomainBlockedError("bad.example", decision);++ expect(error.code).toBe("unsafe_domain_blocked");+ expect(error.name).toBe("UnsafeDomainBlockedError");+ expect(error.domain).toBe("bad.example");+ expect(error.decision).toBe(decision);+ expect(error.message).toContain("threat protection policy");+ expect(error.message).toContain("blacklist");+ });+});diff --git a/apps/api/src/lib/threat-protection/config.test.ts b/apps/api/src/lib/threat-protection/config.test.tsnew file mode 100644index 000000000..d43ddd820--- /dev/null+++ b/apps/api/src/lib/threat-protection/config.test.ts@@ -0,0 +1,152 @@+import {+ threatProtectionConfigSchema,+ threatProtectionOverrideSchema,+ threatProtectionPolicySchema,+} from "./config";+import { THREAT_PROTECTION_POLICY_DEFAULTS } from "./types";++describe("threatProtectionPolicySchema", () => {+ it("applies defaults for a minimal document", () => {+ const policy = threatProtectionPolicySchema.parse({ mode: "off" });+ expect(policy).toEqual({+ mode: "off",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ });+ });++ it("accepts a full valid document", () => {+ const policy = threatProtectionPolicySchema.parse({+ mode: "normal",+ riskScoreThreshold: 50,+ blacklist: ["bad.example.com", "*.malware.example"],+ whitelist: ["example.com", "*.example.org"],+ blockedTlds: ["zip", "mov"],+ failurePolicy: "open",+ });+ expect(policy.mode).toBe("normal");+ expect(policy.riskScoreThreshold).toBe(50);+ expect(policy.failurePolicy).toBe("open");+ });++ it("rejects an invalid mode", () => {+ expect(() =>+ threatProtectionPolicySchema.parse({ mode: "paranoid" }),+ ).toThrow();+ });++ it('rejects the retired "enhanced" mode', () => {+ expect(() =>+ threatProtectionPolicySchema.parse({ mode: "enhanced" }),+ ).toThrow();+ expect(() =>+ threatProtectionOverrideSchema.parse({ mode: "enhanced" }),+ ).toThrow();+ expect(() =>+ threatProtectionConfigSchema.parse({ mode: "enhanced" }),+ ).toThrow();+ });++ it("rejects unknown keys", () => {+ expect(() =>+ threatProtectionPolicySchema.parse({ mode: "off", nope: true }),+ ).toThrow();+ });++ it("rejects retired policy fields as unknown keys", () => {+ for (const retired of [+ { deniedCategories: ["Malicious"] },+ { maxDomainAgeDays: 30 },+ { blockedCountries: ["KP"] },+ ]) {+ expect(() =>+ threatProtectionPolicySchema.parse({ mode: "normal", ...retired }),+ ).toThrow();+ expect(() => threatProtectionOverrideSchema.parse(retired)).toThrow();+ }+ });++ describe("riskScoreThreshold", () => {+ it.each([0, 100, 75])("accepts %d", value => {+ expect(+ threatProtectionPolicySchema.parse({+ mode: "normal",+ riskScoreThreshold: value,+ }).riskScoreThreshold,+ ).toBe(value);+ });++ it.each([-1, 101, 50.5])("rejects %d", value => {+ expect(() =>+ threatProtectionPolicySchema.parse({+ mode: "normal",+ riskScoreThreshold: value,+ }),+ ).toThrow();+ });+ });++ describe("blacklist / whitelist globs", () => {+ it("normalizes case and whitespace", () => {+ const policy = threatProtectionPolicySchema.parse({+ mode: "normal",+ blacklist: [" Bad.Example.COM ", "*.Evil.Example"],+ });+ expect(policy.blacklist).toEqual(["bad.example.com", "*.evil.example"]);+ });++ it.each([+ "https://example.com",+ "example.com/path",+ "example.com:8080",+ "*.",+ "*",+ "*.*.example.com",+ "foo",+ "ex ample.com",+ "-bad-.example.com",+ "exa_mple.com",+ ])("rejects garbage entry %j with a clear message", entry => {+ expect(() =>+ threatProtectionPolicySchema.parse({+ mode: "normal",+ blacklist: [entry],+ }),+ ).toThrow(/Invalid domain entry/);+ });+ });++ describe("blockedTlds", () => {+ it("accepts and normalizes TLDs", () => {+ const policy = threatProtectionPolicySchema.parse({+ mode: "normal",+ blockedTlds: ["ZIP", " mov "],+ });+ expect(policy.blockedTlds).toEqual(["zip", "mov"]);+ });++ it.each([".zip", "z!p", "co.uk", ""])("rejects %j", entry => {+ expect(() =>+ threatProtectionPolicySchema.parse({+ mode: "normal",+ blockedTlds: [entry],+ }),+ ).toThrow(/Invalid TLD/);+ });+ });+});++describe("threatProtectionConfigSchema", () => {+ it("applies defaults for allowRequestOverrides", () => {+ const config = threatProtectionConfigSchema.parse({ mode: "normal" });+ expect(config.allowRequestOverrides).toBe(true);+ });++ it("rejects the retired siem field", () => {+ expect(() =>+ threatProtectionConfigSchema.parse({+ mode: "normal",+ siem: { url: "https://siem.example.com/ingest" },+ }),+ ).toThrow();+ });+});diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.tsnew file mode 100644index 000000000..34d439546--- /dev/null+++ b/apps/api/src/lib/threat-protection/config.ts@@ -0,0 +1,113 @@+import { z } from "zod";+import {+ THREAT_PROTECTION_POLICY_DEFAULTS,+ type ThreatProtectionPolicy,+} from "./types";++// =========================================+// Field schemas+// =========================================++// One DNS label: alphanumeric, optionally with inner hyphens, max 63 chars.+const DOMAIN_LABEL = "[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?";+// Plain domain ("example.com", "sub.example.co.uk") or a single leading+// wildcard label ("*.example.com"). No protocol, path, port, or inner "*".+const DOMAIN_GLOB_REGEX = new RegExp(+ `^(\\*\\.)?(?:${DOMAIN_LABEL}\\.)+${DOMAIN_LABEL}$`,+);++const domainGlobSchema = z+ .string()+ .trim()+ .toLowerCase()+ .refine(value => value.length <= 253 && DOMAIN_GLOB_REGEX.test(value), {+ error: iss =>+ `Invalid domain entry ${JSON.stringify(iss.input)}: must be a plain domain like "example.com" or a wildcard glob like "*.example.com" (no protocol, path, or port)`,+ });++const tldSchema = z+ .string()+ .trim()+ .toLowerCase()+ .refine(value => /^[a-z0-9]{1,63}$/.test(value), {+ error: iss =>+ `Invalid TLD ${JSON.stringify(iss.input)}: must be a lowercase alphanumeric TLD without the leading dot, e.g. "zip"`,+ });++// =========================================+// Policy + org config schemas+// =========================================++/**+ * Field-for-field zod schema for {@link ThreatProtectionPolicy}. Every field+ * except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.+ */+export const threatProtectionPolicySchema = z.strictObject({+ mode: z.enum(["off", "normal"]),+ riskScoreThreshold: z+ .number()+ .int()+ .min(0)+ .max(100)+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.riskScoreThreshold),+ blacklist: z+ .array(domainGlobSchema)+ .max(1000)+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.blacklist),+ whitelist: z+ .array(domainGlobSchema)+ .max(1000)+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.whitelist),+ blockedTlds: z+ .array(tldSchema)+ .max(1000)+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.blockedTlds),+ failurePolicy: z+ .enum(["open", "closed"])+ .prefault(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),+});++// Compile-time assertion that the schema output matches the shared contract+// in ./types.ts β fails to typecheck if the two drift apart.+const _policyContractCheck = (+ x: z.infer<typeof threatProtectionPolicySchema>,+): ThreatProtectionPolicy => x;+void _policyContractCheck;++/**+ * Per-request `threatProtection` option: a field-level override of the org+ * config. Mirrors {@link threatProtectionPolicySchema} but every field is+ * optional and NO defaults are injected β only fields the caller explicitly+ * provides replace the org policy's values (see `resolveEffectivePolicy`).+ */+export const threatProtectionOverrideSchema = z.strictObject({+ mode: z.enum(["off", "normal"]).optional(),+ riskScoreThreshold: z.number().int().min(0).max(100).optional(),+ blacklist: z.array(domainGlobSchema).max(1000).optional(),+ whitelist: z.array(domainGlobSchema).max(1000).optional(),+ blockedTlds: z.array(tldSchema).max(1000).optional(),+ failurePolicy: z.enum(["open", "closed"]).optional(),+});++type ThreatProtectionOverride = z.infer<typeof threatProtectionOverrideSchema>;++// Compile-time assertion that the override shape stays assignable to+// Partial<ThreatProtectionPolicy> (what `resolveEffectivePolicy` consumes).+const _overrideContractCheck = (+ x: ThreatProtectionOverride,+): Partial<ThreatProtectionPolicy> => x;+void _overrideContractCheck;++/**+ * Full org-level configuration document, as accepted by+ * `PUT /v2/team/threat-protection`.+ */+export const threatProtectionConfigSchema = threatProtectionPolicySchema.extend(+ {+ allowRequestOverrides: z.boolean().prefault(true),+ },+);++export type ThreatProtectionConfigInput = z.infer<+ typeof threatProtectionConfigSchema+>;diff --git a/apps/api/src/lib/threat-protection/error.ts b/apps/api/src/lib/threat-protection/error.tsnew file mode 100644index 000000000..bc12f70c6--- /dev/null+++ b/apps/api/src/lib/threat-protection/error.ts@@ -0,0 +1,40 @@+import { ErrorCodes, TransportableError } from "../error";+import type { ThreatDecision } from "./types";++// Error surfaced when threat protection blocks a domain. Wired into the shared+// TransportableError machinery (src/lib/error.ts + src/lib/error-serde.ts) so+// blocked scrapes surface as a clean, documented API error β both on sync+// responses (`code: "unsafe_domain_blocked"`) and on crawl/batch job document+// errors. The full ThreatDecision rides along in serialize() so the billing+// and security-logging layers can read it even across the job queue.++export class UnsafeDomainBlockedError extends TransportableError {+ constructor(+ public readonly domain: string,+ public readonly decision: ThreatDecision,+ ) {+ super(+ "unsafe_domain_blocked",+ `This domain (${domain}) is blocked by your organization's threat protection policy (rule: ${decision.rule}). ` ++ `If you believe this is a mistake, contact your organization administrator to adjust the policy (e.g. whitelist the domain).`,+ );+ this.name = "UnsafeDomainBlockedError";+ }++ serialize() {+ return {+ ...super.serialize(),+ domain: this.domain,+ decision: this.decision,+ };+ }++ static deserialize(+ _code: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new UnsafeDomainBlockedError(data.domain, data.decision);+ x.stack = data.stack;+ return x;+ }+}diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.tsnew file mode 100644index 000000000..b5f5e739a--- /dev/null+++ b/apps/api/src/lib/threat-protection/index.ts@@ -0,0 +1,184 @@+import { logger } from "../logger";+import { fetchGoogleWebRiskVerdict } from "./providers/google-web-risk";+import type {+ RawVerdict,+ ThreatCheckDedup,+ ThreatDecision,+ ThreatProtectionMode,+ ThreatProtectionPolicy,+} from "./types";+import { evaluatePolicy, localOnlyDecision, normalizeDomain } from "./verdict";++/**+ * Request context threaded into {@link checkDomain}. Enforcement-only: the+ * feature no longer emits or exports security events (both the ClickHouse+ * security log and the SIEM push were removed), so this carries just the+ * request/job-scoped dedup handle plus the team id used for server-side+ * logging. Everything is optional β call sites pass whatever is cheaply+ * available.+ */+export interface ThreatCheckContext {+ teamId?: string;+ /**+ * Request/job-scoped dedup handle (see {@link ThreatCheckDedup}). When set,+ * checkDomain() reuses the in-flight decision for a domain already checked+ * within this request instead of re-scanning it. Strictly in-memory and+ * request-scoped β never persisted, never shared across requests (ZDR).+ */+ dedup?: ThreatCheckDedup;+}++// Public entry point for the threat protection core library (enterprise+// domain risk blocking). Flow per domain:+// 1. mode "off" β allow, no provider, no billing+// 2. request-scoped dedup (ctx.dedup) β a domain already checked within+// this request/job reuses the same in-flight decision+// 3. local-only rules (whitelist/blacklist/blocked-tld) β decide without a+// provider call (no billing)+// 4. provider ("normal" = Google Web Risk local hash-prefix database),+// with a per-attempt timeout and one retry+// 5. evaluate the policy against the verdict; provider failure β the org's+// failurePolicy decides (fail-open allows, fail-closed blocks)+// Any decision backed by a verdict sets providerConsulted, which drives+// billing (+2 credits per scanned domain) in the enforcement layer. This+// module performs no billing or pipeline integration itself.+//+// ZDR boundary: verdicts are NEVER persisted β there is no cross-request+// verdict cache (scrape-target domains must not be stored at rest). The only+// reuse of a decision is via the caller-provided, request-scoped in-memory+// dedup map. The org-config cache in ./store.ts is unaffected: it caches the+// org's own settings, not scrape-derived data.+//+// This feature is enforcement-only: it emits and exports NO security events.+// (There is no built-in audit trail β both the ClickHouse security log and+// the SIEM push that once consumed decisions have been removed.)++// Policy evaluation helpers (evaluatePolicy, localOnlyDecision) are exported+// from ./verdict, and the shared contract types from ./types β import those+// directly; index.ts only re-exports what checkDomain callers need.+export { UnsafeDomainBlockedError } from "./error";+export * from "./types";++const PROVIDER_TIMEOUT_MS = 5000;+const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry++/**+ * Single modeβprovider dispatch point. Every provider is a separate module+ * under ./providers exporting `fetch<X>Verdict(domain) β RawVerdict`; this+ * switch is the only place that knows which mode maps to which classifier.+ * Deliberately kept as a dispatch even with one live branch β future partner+ * classifiers add a mode + a case here and nothing else changes.+ */+async function fetchProviderVerdict(+ domain: string,+ mode: Exclude<ThreatProtectionMode, "off">,+): Promise<RawVerdict> {+ let lastError: unknown;+ for (let attempt = 1; attempt <= PROVIDER_ATTEMPTS; attempt++) {+ try {+ const signal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS);+ switch (mode) {+ case "normal":+ return await fetchGoogleWebRiskVerdict(domain, { signal });+ }+ } catch (error) {+ lastError = error;+ logger.warn("Threat protection provider lookup failed", {+ canonicalLog: "threat-protection/provider",+ domain,+ mode,+ attempt,+ error,+ });+ }+ }+ throw lastError;+}++async function checkDomainFresh(+ normalized: string,+ policy: ThreatProtectionPolicy,+ ctx: ThreatCheckContext,+): Promise<ThreatDecision> {+ // Local rules first: when whitelist/blacklist/blocked-tld are decisive we+ // skip the paid provider scan entirely.+ const local = localOnlyDecision(normalized, policy);+ if (local !== null) {+ return local;+ }++ let verdict: RawVerdict | null = null;+ try {+ verdict = await fetchProviderVerdict(+ normalized,+ policy.mode as Exclude<ThreatProtectionMode, "off">,+ );+ } catch {+ // Already logged per-attempt; a null verdict routes the decision+ // through the org's failurePolicy below.+ verdict = null;+ }++ const decision = evaluatePolicy(normalized, verdict, policy);+ if (!decision.allowed || decision.rule === "provider-failure") {+ logger.info("Threat protection decision", {+ canonicalLog: "threat-protection/check",+ teamId: ctx.teamId,+ domain: normalized,+ mode: policy.mode,+ allowed: decision.allowed,+ rule: decision.rule,+ providerConsulted: decision.providerConsulted,+ riskScore: verdict?.riskScore ?? null,+ categories: verdict?.categories ?? [],+ rawVerdict: verdict?.raw,+ });+ }+ return decision;+}++/**+ * Classify a domain against an org's threat protection policy. Never throws:+ * provider failures resolve through the policy's failurePolicy+ * ("provider-failure" rule). Callers bill +2 credits per scanned domain when+ * the returned decision has `providerConsulted` set.+ *+ * When `ctx.dedup` is provided (a request/job-scoped map created by the call+ * site), repeated checks of the same domain within that scope share one+ * in-flight decision: one scan, one billable consulted verdict. There is+ * deliberately no cross-request or persisted verdict reuse (ZDR:+ * scrape-target domains are never stored at rest).+ *+ * Enforcement-only: no security event is emitted or exported for a scan+ * (neither to a ClickHouse log nor to a SIEM destination β both were+ * removed). There is no built-in audit trail.+ */+export async function checkDomain(+ domain: string,+ policy: ThreatProtectionPolicy,+ ctx: ThreatCheckContext,+): Promise<ThreatDecision> {+ const normalized = normalizeDomain(domain);++ if (policy.mode === "off") {+ return {+ allowed: true,+ rule: "default-allow",+ providerConsulted: false,+ verdict: null,+ mode: "off",+ };+ }++ if (ctx.dedup) {+ const existing = ctx.dedup.get(normalized);+ if (existing !== undefined) {+ return existing;+ }+ const pending = checkDomainFresh(normalized, policy, ctx);+ ctx.dedup.set(normalized, pending);+ return pending;+ }++ return checkDomainFresh(normalized, policy, ctx);+}diff --git a/apps/api/src/lib/threat-protection/providers/google-web-risk.ts b/apps/api/src/lib/threat-protection/providers/google-web-risk.tsnew file mode 100644index 000000000..cd4867cfd--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/google-web-risk.ts@@ -0,0 +1,200 @@+import { createHash } from "crypto";+import { config } from "../../../config";+import type { RawVerdict, ThreatProvider } from "../types";+import { generateHostExpressions } from "./web-risk/canonicalize";+import {+ getWebRiskListStore,+ WEB_RISK_THREAT_TYPES,+ type WebRiskLookupResult,+ type WebRiskThreatType,+} from "./web-risk/store";+import {+ ensureThreatListBootSync,+ ensureThreatListSyncLoop,+} from "./web-risk/sync";++// Google Web Risk provider ("normal" threat protection mode), built on the+// Update API for ZDR compliance:+//+// 1. The domain is canonicalized into host-suffix expressions per the Safe+// Browsing spec, each expression is SHA-256 hashed, and the 4-byte hash+// prefixes are checked against the locally synced threat lists+// (./web-risk/store.ts + ./web-risk/sync.ts). No network I/O, and the+// target domain is never transmitted anywhere.+// 2. Only when a local prefix matches (rare) is Google consulted:+// GET /v1/hashes:search with the matched (anonymized, non-reversible)+// hash prefix. The returned full hashes are compared locally β a full+// match confirms the threat, a mismatch was a prefix collision (clean).+// The response's ttl/negativeExpireTime metadata is deliberately ignored:+// verdicts are never persisted (ZDR).+//+// Cost note: hashes:search is ~100x the price of the old uris:search per+// call, so it must ONLY ever run to confirm a local prefix hit β never per+// check. computeDiff (the sync path) is free.+//+// Web Risk gives a boolean-ish signal (flagged for a threat type or not), not+// a granular score β so we normalize any confirmed threat to riskScore 100 and+// a clean lookup to 0, and surface the threat types as category strings.++const PROVIDER: ThreatProvider = "google-web-risk";++interface HashesSearchThreat {+ hash?: string;+ threatTypes?: string[];+ expireTime?: string;+}++type HashesSearchResponse = {+ threats?: HashesSearchThreat[];+ negativeExpireTime?: string;+};++function isGoogleWebRiskConfigured(): boolean {+ return (+ typeof config.GOOGLE_WEB_RISK_API_KEY === "string" &&+ config.GOOGLE_WEB_RISK_API_KEY.trim().length > 0+ );+}++async function searchHashPrefix(+ prefix: Buffer,+ signal: AbortSignal | undefined,+): Promise<HashesSearchResponse> {+ const params = new URLSearchParams();+ params.append("hashPrefix", prefix.toString("base64"));+ for (const threatType of WEB_RISK_THREAT_TYPES) {+ params.append("threatTypes", threatType);+ }+ params.append("key", config.GOOGLE_WEB_RISK_API_KEY!);++ const response = await fetch(+ `${config.GOOGLE_WEB_RISK_API_URL}/v1/hashes:search?${params.toString()}`,+ { method: "GET", signal },+ );+ if (!response.ok) {+ throw new Error(+ `Google Web Risk hash search failed with status ${response.status}`,+ );+ }+ return (await response.json()) as HashesSearchResponse;+}++/**+ * Look up a domain against Google Web Risk. Throws on any transport/API+ * error, and when the local threat lists are unavailable or stale, so the+ * caller can apply the org's failurePolicy.+ */+export async function fetchGoogleWebRiskVerdict(+ domain: string,+ options?: { signal?: AbortSignal },+): Promise<RawVerdict> {+ if (!isGoogleWebRiskConfigured()) {+ throw new Error("Google Web Risk is not configured");+ }++ ensureThreatListSyncLoop();++ const expressions = generateHostExpressions(domain);+ if (expressions.length === 0) {+ throw new Error(`Cannot canonicalize domain for Web Risk lookup`);+ }+ const fullHashes = expressions.map(expression =>+ createHash("sha256").update(expression).digest(),+ );++ const store = getWebRiskListStore();++ // Wait for the once-per-process boot sync (bounded by the caller's timeout+ // signal) so a fresh deployment doesn't fail every check while the lists+ // download; afterwards this resolves instantly.+ await abortable(ensureThreatListBootSync(), options?.signal);++ const lookup: WebRiskLookupResult = await store.lookup(fullHashes);+ if (lookup.status !== "ok") {+ throw new Error(`Web Risk threat lists are ${lookup.status}`);+ }++ if (lookup.hits.length === 0) {+ // The overwhelmingly common case: clean domain, zero Google calls,+ // nothing transmitted anywhere.+ return {+ provider: PROVIDER,+ riskScore: 0,+ categories: [],+ fromCache: false,+ raw: { localPrefixMatch: false },+ };+ }++ // Local prefix hit: confirm with hashes:search (only the anonymized hash+ // prefix leaves our infrastructure), then compare full hashes locally.+ const uniquePrefixes = new Map<string, Buffer>();+ for (const hit of lookup.hits) {+ uniquePrefixes.set(hit.prefix.toString("base64"), hit.prefix);+ }+ const responses = await Promise.all(+ [...uniquePrefixes.values()].map(prefix =>+ searchHashPrefix(prefix, options?.signal),+ ),+ );++ const fullHashSet = new Set(fullHashes.map(hash => hash.toString("base64")));+ const confirmedThreatTypes = new Set<string>();+ for (const response of responses) {+ for (const threat of response.threats ?? []) {+ if (threat.hash && fullHashSet.has(threat.hash)) {+ for (const threatType of threat.threatTypes ?? []) {+ if (+ (WEB_RISK_THREAT_TYPES as readonly string[]).includes(threatType)+ ) {+ confirmedThreatTypes.add(threatType);+ }+ }+ }+ }+ }++ const categories = [...confirmedThreatTypes] as WebRiskThreatType[];+ return {+ provider: PROVIDER,+ // No confirmed full-hash match = prefix collision = clean.+ riskScore: categories.length > 0 ? 100 : 0,+ categories,+ fromCache: false,+ raw: {+ localPrefixMatch: true,+ prefixesSearched: uniquePrefixes.size,+ threats: responses.flatMap(response =>+ (response.threats ?? []).map(threat => ({+ hash: threat.hash,+ threatTypes: threat.threatTypes,+ })),+ ),+ },+ };+}++/** Await a promise, rejecting early if the signal aborts. */+function abortable<T>(+ promise: Promise<T>,+ signal: AbortSignal | undefined,+): Promise<T> {+ if (!signal) return promise;+ if (signal.aborted) {+ return Promise.reject(new Error("Web Risk lookup aborted"));+ }+ return new Promise<T>((resolve, reject) => {+ const onAbort = () => reject(new Error("Web Risk lookup aborted"));+ signal.addEventListener("abort", onAbort, { once: true });+ promise.then(+ value => {+ signal.removeEventListener("abort", onAbort);+ resolve(value);+ },+ error => {+ signal.removeEventListener("abort", onAbort);+ reject(error);+ },+ );+ });+}diff --git a/apps/api/src/lib/threat-protection/providers/providers.test.ts b/apps/api/src/lib/threat-protection/providers/providers.test.tsnew file mode 100644index 000000000..a2d5ae811--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/providers.test.ts@@ -0,0 +1,225 @@+import http from "http";+import { AddressInfo } from "net";++// The Web Risk threat-list store lives on the durable Redis connection β+// swap in an in-memory fake. (fake-redis.ts has no runtime imports, so the+// factory cannot re-enter the module being mocked.)+vi.mock("../../../services/queue-service", async () => {+ const { createFakeWebRiskRedis } = await import("./web-risk/fake-redis.js");+ const client = createFakeWebRiskRedis();+ return { getRedisConnection: () => client };+});++import { config } from "../../../config";+import { fetchGoogleWebRiskVerdict } from "./google-web-risk";+import { domainExpressionHash, WebRiskMockDatabase } from "./web-risk/testing";++// Mocked-HTTP provider tests: a local http server stands in for the real+// provider API via the config URL override (same pattern as+// src/lib/fire-privacy-client.test.ts). For Google Web Risk the mock serves+// the Update API endpoints (threatLists:computeDiff + hashes:search); the+// old uris:search endpoint intentionally no longer exists.++type SeenRequest = { url: string; method: string; body: unknown };++let server: http.Server;+let baseUrl: string;+let seenRequests: SeenRequest[] = [];++// Fixture threat lists, fixed for the whole file (the local list is synced+// once per process by the provider's boot sync).+const CONFIRMED_DOMAIN = "malware.example";+const COLLISION_DOMAIN = "collision.example";++const webRiskDb = new WebRiskMockDatabase();+webRiskDb.addRiskyDomain(CONFIRMED_DOMAIN, "MALWARE");+webRiskDb.addRiskyDomain(CONFIRMED_DOMAIN, "SOCIAL_ENGINEERING");+// A list entry that shares the 4-byte prefix of COLLISION_DOMAIN's expression+// hash but is a different full hash β local hit, unconfirmed by hashes:search.+webRiskDb.addCollidingFullHash(+ Buffer.concat([+ domainExpressionHash(COLLISION_DOMAIN).subarray(0, 4),+ Buffer.alloc(28, 0xab),+ ]),+ "UNWANTED_SOFTWARE",+);++// While > 0, hashes:search requests fail with 503 (decremented per request).+let failHashesSearches = 0;++const originalConfig = {+ webRiskUrl: config.GOOGLE_WEB_RISK_API_URL,+ webRiskKey: config.GOOGLE_WEB_RISK_API_KEY,+};++beforeAll(async () => {+ await new Promise<void>(resolve => {+ server = http.createServer((req, res) => {+ const chunks: Buffer[] = [];+ req.on("data", chunk => chunks.push(chunk));+ req.on("end", () => {+ const rawBody = Buffer.concat(chunks).toString("utf8");+ let body: unknown = null;+ try {+ body = rawBody ? JSON.parse(rawBody) : null;+ } catch {}+ seenRequests.push({+ url: req.url ?? "",+ method: req.method ?? "",+ body,+ });++ const url = new URL(req.url ?? "/", "http://localhost");+ const path = url.pathname;++ // Web Risk Update API endpoints.+ if (path === "/v1/threatLists:computeDiff") {+ res.setHeader("content-type", "application/json");+ res.end(+ JSON.stringify(+ webRiskDb.computeDiffResponse(+ url.searchParams.get("threatType") ?? "",+ ),+ ),+ );+ return;+ }+ if (path === "/v1/hashes:search") {+ if (failHashesSearches > 0) {+ failHashesSearches--;+ res.statusCode = 503;+ res.end("{}");+ return;+ }+ res.setHeader("content-type", "application/json");+ res.end(+ JSON.stringify(+ webRiskDb.hashesSearchResponse(+ url.searchParams.get("hashPrefix") ?? "",+ ),+ ),+ );+ return;+ }++ res.statusCode = 404;+ res.end("{}");+ });+ });+ server.listen(0, "127.0.0.1", () => {+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ config.GOOGLE_WEB_RISK_API_URL = baseUrl;+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";+ resolve();+ });+ });+});++afterAll(async () => {+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.webRiskUrl;+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.webRiskKey;+ await new Promise<void>(resolve => server.close(() => resolve()));+});++beforeEach(() => {+ seenRequests = [];+ failHashesSearches = 0;+});++const hashesSearchRequests = () =>+ seenRequests.filter(r => r.url.startsWith("/v1/hashes:search"));+const urisSearchRequests = () =>+ seenRequests.filter(r => r.url.startsWith("/v1/uris:search"));++describe("fetchGoogleWebRiskVerdict", () => {+ it("confirms a local prefix hit via hashes:search β riskScore 100 with categories", async () => {+ const verdict = await fetchGoogleWebRiskVerdict(CONFIRMED_DOMAIN);++ expect(verdict).toMatchObject({+ provider: "google-web-risk",+ riskScore: 100,+ fromCache: false,+ });+ expect([...verdict.categories].sort()).toEqual([+ "MALWARE",+ "SOCIAL_ENGINEERING",+ ]);++ // Exactly one confirmation call, carrying ONLY the anonymized 4-byte+ // hash prefix (never the domain or URL), plus the API key.+ const confirmations = hashesSearchRequests();+ expect(confirmations).toHaveLength(1);+ const url = new URL(baseUrl + confirmations[0].url);+ expect(confirmations[0].method).toBe("GET");+ expect(url.searchParams.get("hashPrefix")).toBe(+ domainExpressionHash(CONFIRMED_DOMAIN).subarray(0, 4).toString("base64"),+ );+ expect(url.searchParams.getAll("threatTypes")).toEqual([+ "MALWARE",+ "SOCIAL_ENGINEERING",+ "UNWANTED_SOFTWARE",+ ]);+ expect(url.searchParams.get("key")).toBe("test-web-risk-key");+ expect(confirmations[0].url).not.toContain(CONFIRMED_DOMAIN);+ // The legacy full-URL lookup endpoint is never used.+ expect(urisSearchRequests()).toHaveLength(0);+ });++ it("flags subdomains of a listed domain through host-suffix expressions", async () => {+ const verdict = await fetchGoogleWebRiskVerdict(+ `cdn.assets.${CONFIRMED_DOMAIN}`,+ );++ expect(verdict.riskScore).toBe(100);+ expect(verdict.categories).toContain("MALWARE");+ });++ it("resolves clean domains locally with zero Google calls", async () => {+ const verdict = await fetchGoogleWebRiskVerdict("safe.example");++ expect(verdict).toMatchObject({+ provider: "google-web-risk",+ riskScore: 0,+ categories: [],+ fromCache: false,+ raw: { localPrefixMatch: false },+ });+ // The common case transmits nothing: no hashes:search, no uris:search.+ expect(hashesSearchRequests()).toHaveLength(0);+ expect(urisSearchRequests()).toHaveLength(0);+ });++ it("treats an unconfirmed prefix hit (collision) as clean", async () => {+ const verdict = await fetchGoogleWebRiskVerdict(COLLISION_DOMAIN);++ expect(verdict).toMatchObject({+ provider: "google-web-risk",+ riskScore: 0,+ categories: [],+ });+ // The collision DID require a confirmation round tripβ¦+ expect(hashesSearchRequests()).toHaveLength(1);+ // β¦whose returned full hash didn't match any of our expression hashes.+ expect(verdict.raw).toMatchObject({ localPrefixMatch: true });+ });++ it("throws on hashes:search errors so failurePolicy can apply", async () => {+ failHashesSearches = Infinity;++ await expect(fetchGoogleWebRiskVerdict(CONFIRMED_DOMAIN)).rejects.toThrow(+ /status 503/,+ );+ });++ it("throws when the API key is not configured", async () => {+ config.GOOGLE_WEB_RISK_API_KEY = undefined;+ try {+ await expect(fetchGoogleWebRiskVerdict("safe.example")).rejects.toThrow(+ /not configured/,+ );+ } finally {+ config.GOOGLE_WEB_RISK_API_KEY = "test-web-risk-key";+ }+ expect(seenRequests).toHaveLength(0);+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.tsnew file mode 100644index 000000000..54234e5fe--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.test.ts@@ -0,0 +1,119 @@+import {+ canonicalizeHost,+ canonicalizeUrl,+ generateHostExpressions,+} from "./canonicalize";++// Google's published canonicalization test vectors, verbatim:+// https://cloud.google.com/web-risk/docs/urls-hashing#canonicalization+// (identical to the Safe Browsing v4 vectors).+const CANONICALIZATION_VECTORS: [string, string][] = [+ ["http://host/%25%32%35", "http://host/%25"],+ ["http://host/%25%32%35%25%32%35", "http://host/%25%25"],+ ["http://host/%2525252525252525", "http://host/%25"],+ ["http://host/asdf%25%32%35asd", "http://host/asdf%25asd"],+ ["http://host/%%%25%32%35asd%%", "http://host/%25%25%25asd%25%25"],+ ["http://www.google.com/", "http://www.google.com/"],+ [+ "http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",+ "http://168.188.99.26/.secure/www.ebay.com/",+ ],+ [+ "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",+ "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",+ ],+ [+ "http://host%23.com/%257Ea%2521b%2540c%2523d%2524e%25f%255E00%252611%252A22%252833%252944_55%252B",+ "http://host%23.com/~a!b@c%23d$e%25f^00&11*22(33)44_55+",+ ],+ ["http://3279880203/blah", "http://195.127.0.11/blah"],+ ["http://www.google.com/blah/..", "http://www.google.com/"],+ ["www.google.com/", "http://www.google.com/"],+ ["www.google.com", "http://www.google.com/"],+ ["http://www.evil.com/blah#frag", "http://www.evil.com/blah"],+ ["http://www.GOOgle.com/", "http://www.google.com/"],+ ["http://www.google.com.../", "http://www.google.com/"],+ [+ "http://www.google.com/foo\tbar\rbaz\n2",+ "http://www.google.com/foobarbaz2",+ ],+ ["http://www.google.com/q?", "http://www.google.com/q?"],+ ["http://www.google.com/q?r?", "http://www.google.com/q?r?"],+ ["http://www.google.com/q?r?s", "http://www.google.com/q?r?s"],+ ["http://evil.com/foo#bar#baz", "http://evil.com/foo"],+ ["http://evil.com/foo;", "http://evil.com/foo;"],+ ["http://evil.com/foo?bar;", "http://evil.com/foo?bar;"],+ ["http://\x01\x80.com/", "http://%01%80.com/"],+ ["http://notrailingslash.com", "http://notrailingslash.com/"],+ ["http://www.gotaport.com:1234/", "http://www.gotaport.com/"],+ [" http://www.google.com/ ", "http://www.google.com/"],+ ["http:// leadingspace.com/", "http://%20leadingspace.com/"],+ ["http://%20leadingspace.com/", "http://%20leadingspace.com/"],+ ["%20leadingspace.com/", "http://%20leadingspace.com/"],+ ["https://www.securesite.com/", "https://www.securesite.com/"],+ ["http://host.com/ab%23cd", "http://host.com/ab%23cd"],+ [+ "http://host.com//twoslashes?more//slashes",+ "http://host.com/twoslashes?more//slashes",+ ],+];++describe("canonicalizeUrl", () => {+ it.each(CANONICALIZATION_VECTORS)("canonicalizes %j", (input, expected) => {+ expect(canonicalizeUrl(input)).toBe(expected);+ });+});++describe("canonicalizeHost", () => {+ it("normalizes dots and case", () => {+ expect(canonicalizeHost(".Example..COM.")).toBe("example.com");+ });++ it("normalizes integer, octal and hex IP forms", () => {+ expect(canonicalizeHost("3279880203")).toBe("195.127.0.11");+ expect(canonicalizeHost("0303.0177.0.013")).toBe("195.127.0.11");+ expect(canonicalizeHost("0xc3.0x7f.0x0.0xb")).toBe("195.127.0.11");+ expect(canonicalizeHost("195.127.11")).toBe("195.127.0.11");+ });++ it("leaves regular hostnames that merely look numeric-ish alone", () => {+ expect(canonicalizeHost("1234x.com")).toBe("1234x.com");+ expect(canonicalizeHost("256.256.256.256")).toBe("256.256.256.256");+ });+});++describe("generateHostExpressions", () => {+ // Spec example (host component of "http://a.b.c.d.e.f.g/1.html"): the+ // exact host plus the four suffixes built from the last five components.+ it("walks at most the last five host components", () => {+ expect(generateHostExpressions("a.b.c.d.e.f.g")).toEqual([+ "a.b.c.d.e.f.g/",+ "c.d.e.f.g/",+ "d.e.f.g/",+ "e.f.g/",+ "f.g/",+ ]);+ });++ it("generates exact host + registrable suffixes for short hosts", () => {+ expect(generateHostExpressions("a.b.c")).toEqual(["a.b.c/", "b.c/"]);+ expect(generateHostExpressions("example.com")).toEqual(["example.com/"]);+ expect(generateHostExpressions("www.phishing.example.com")).toEqual([+ "www.phishing.example.com/",+ "phishing.example.com/",+ "example.com/",+ ]);+ });++ it("generates a single expression for IP hosts (no suffix walk)", () => {+ expect(generateHostExpressions("195.127.0.11")).toEqual(["195.127.0.11/"]);+ expect(generateHostExpressions("3279880203")).toEqual(["195.127.0.11/"]);+ });++ it("canonicalizes the host before expanding", () => {+ expect(generateHostExpressions("WWW.Example.COM.")).toEqual([+ "www.example.com/",+ "example.com/",+ ]);+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.ts b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.tsnew file mode 100644index 000000000..f150a68a2--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/canonicalize.ts@@ -0,0 +1,281 @@+// URL canonicalization and lookup-expression generation for the Google Web+// Risk / Safe Browsing hash-prefix protocol.+// https://cloud.google.com/web-risk/docs/urls-hashing+//+// The threat lists store SHA-256 hash prefixes of *canonicalized expressions*+// (host[/path] strings). To check a target locally we must produce byte-exact+// expressions per the published spec β the canonicalization rules here are+// verified against Google's published test vectors (see canonicalize.test.ts).+//+// Everything in this file is pure string/byte manipulation: nothing is+// transmitted anywhere (ZDR β the check path only ever sends anonymized hash+// prefixes to Google, and only on a local prefix hit).++/**+ * Treats a JS string as a byte sequence: code units <= 0xFF are bytes,+ * anything above is UTF-8 encoded. Matches how reference implementations+ * operate on raw URL bytes.+ */+function toBytes(input: string): number[] {+ const bytes: number[] = [];+ for (const char of input) {+ const code = char.codePointAt(0)!;+ if (code <= 0xff) {+ bytes.push(code);+ } else {+ bytes.push(...Buffer.from(char, "utf8"));+ }+ }+ return bytes;+}++function fromBytes(bytes: number[]): string {+ return bytes.map(b => String.fromCharCode(b)).join("");+}++const HEX = "0123456789ABCDEF";++function isHexDigit(char: string): boolean {+ return /^[0-9a-fA-F]$/.test(char);+}++/** One pass of percent-unescaping (only valid %XX sequences). */+function unescapeOnce(input: string): string {+ let out = "";+ for (let i = 0; i < input.length; i++) {+ if (+ input[i] === "%" &&+ i + 2 < input.length &&+ isHexDigit(input[i + 1]) &&+ isHexDigit(input[i + 2])+ ) {+ out += String.fromCharCode(parseInt(input.slice(i + 1, i + 3), 16));+ i += 2;+ } else {+ out += input[i];+ }+ }+ return out;+}++/** Repeatedly percent-unescape until the string stops changing (spec step). */+function fullyUnescape(input: string): string {+ let prev = input;+ // Bounded to defend against adversarial inputs; real URLs converge fast.+ for (let i = 0; i < 1024; i++) {+ const next = unescapeOnce(prev);+ if (next === prev) return next;+ prev = next;+ }+ return prev;+}++/**+ * Percent-escape bytes <= 0x20, >= 0x7F, '#' and '%', uppercase hex β the+ * spec's escape character class.+ */+function escapeBytes(input: string): string {+ const bytes = toBytes(input);+ let out = "";+ for (const byte of bytes) {+ if (+ byte <= 0x20 ||+ byte >= 0x7f ||+ byte === 0x23 /* # */ ||+ byte === 0x25 /* % */+ ) {+ out += "%" + HEX[byte >> 4] + HEX[byte & 0xf];+ } else {+ out += String.fromCharCode(byte);+ }+ }+ return out;+}++/**+ * inet_aton-style IP parsing: 1-4 dot-separated components, each decimal,+ * octal (leading 0) or hex (0x); the last component fills the remaining+ * bytes. Returns the normalized dotted-quad, or null if not an IP.+ */+function canonicalizeIp(host: string): string | null {+ if (+ !/^(?:0x[0-9a-f]+|[0-9]+)(?:\.(?:0x[0-9a-f]+|[0-9]+)){0,3}$/i.test(host)+ ) {+ return null;+ }+ const parts = host.split(".");+ const values: number[] = [];+ for (const part of parts) {+ let value: number;+ if (/^0x/i.test(part)) {+ value = parseInt(part.slice(2), 16);+ } else if (/^0[0-7]*$/.test(part)) {+ value = parseInt(part, 8);+ } else if (/^[0-9]+$/.test(part)) {+ value = parseInt(part, 10);+ } else {+ return null;+ }+ if (!Number.isFinite(value) || value < 0) return null;+ values.push(value);+ }++ // All but the last component must fit in one byte; the last fills the rest.+ const tailBytes = 4 - (values.length - 1);+ const last = values[values.length - 1];+ if (last >= 2 ** (8 * tailBytes)) return null;+ for (let i = 0; i < values.length - 1; i++) {+ if (values[i] > 0xff) return null;+ }++ const bytes: number[] = values.slice(0, -1);+ for (let i = tailBytes - 1; i >= 0; i--) {+ bytes.push((last >> (8 * i)) & 0xff);+ }+ return bytes.join(".");+}++/**+ * Canonicalize a hostname per the spec: fully unescape, remove leading /+ * trailing dots, collapse consecutive dots, normalize IP forms to+ * dotted-quad, lowercase, then re-escape.+ */+export function canonicalizeHost(host: string): string {+ let out = fullyUnescape(host);+ out = out+ .replace(/^\.+/, "")+ .replace(/\.+$/, "")+ .replace(/\.{2,}/g, ".");+ out = out.toLowerCase();+ const ip = canonicalizeIp(out);+ if (ip !== null) out = ip;+ return escapeBytes(out);+}++/** Resolve "/./" and "/../" segments and collapse runs of slashes. */+function canonicalizePath(path: string): string {+ const segments = path.split("/");+ const out: string[] = [];+ for (let i = 0; i < segments.length; i++) {+ const segment = segments[i];+ if (segment === "" || segment === ".") {+ // Empty segments are runs of slashes (or the leading slash) β skip.+ continue;+ }+ if (segment === "..") {+ out.pop();+ continue;+ }+ out.push(segment);+ }+ const endsWithSlash =+ path.endsWith("/") ||+ path.endsWith("/.") ||+ path.endsWith("/..") ||+ out.length === 0;+ return "/" + out.join("/") + (endsWithSlash && out.length > 0 ? "/" : "");+}++interface ParsedUrl {+ scheme: string;+ host: string;+ path: string;+ query: string | null;+}++/**+ * Lenient URL splitter (real URL parsers reject hosts the spec requires us to+ * handle, e.g. embedded spaces or raw control bytes).+ */+function splitUrl(input: string): ParsedUrl {+ let rest = input;+ let scheme = "http";+ const schemeMatch = rest.match(/^([a-zA-Z][a-zA-Z0-9+.-]*):\/\//);+ if (schemeMatch) {+ scheme = schemeMatch[1].toLowerCase();+ rest = rest.slice(schemeMatch[0].length);+ }++ let hostEnd = rest.length;+ for (let i = 0; i < rest.length; i++) {+ if (rest[i] === "/" || rest[i] === "?") {+ hostEnd = i;+ break;+ }+ }+ let authority = rest.slice(0, hostEnd);+ const afterHost = rest.slice(hostEnd);++ // Strip userinfo and port from the authority.+ const at = authority.lastIndexOf("@");+ if (at !== -1) authority = authority.slice(at + 1);+ const portMatch = authority.match(/^(.*):(\d*)$/);+ if (portMatch) authority = portMatch[1];++ let path = "/";+ let query: string | null = null;+ if (afterHost.length > 0) {+ const q = afterHost.indexOf("?");+ if (q !== -1) {+ path = q === 0 ? "/" : afterHost.slice(0, q);+ query = afterHost.slice(q + 1);+ } else {+ path = afterHost;+ }+ }++ return { scheme, host: authority, path, query };+}++/**+ * Full URL canonicalization per the Web Risk / Safe Browsing spec. Verified+ * against Google's published test vectors.+ */+export function canonicalizeUrl(input: string): string {+ // Leading/trailing whitespace, then embedded tab/CR/LF, then the fragment.+ let url = input.trim().replace(/[\t\r\n]/g, "");+ const hash = url.indexOf("#");+ if (hash !== -1) url = url.slice(0, hash);++ const { scheme, host, path, query } = splitUrl(url);++ const canonicalHost = canonicalizeHost(host);+ const canonicalPath = escapeBytes(canonicalizePath(fullyUnescape(path)));+ const canonicalQuery = query === null ? null : escapeBytes(query);++ return (+ `${scheme}://${canonicalHost}${canonicalPath}` ++ (canonicalQuery !== null ? `?${canonicalQuery}` : "")+ );+}++/** Whether a canonicalized host is a (dotted-quad) IP address. */+function isIpHost(host: string): boolean {+ return /^\d+\.\d+\.\d+\.\d+$/.test(host);+}++/**+ * Host-suffix lookup expressions for a domain-level check, per the spec: the+ * exact hostname plus up to four suffixes formed from the last five host+ * components, successively dropping the leading component (never the bare+ * TLD), each with the "/" root path. IP hosts get a single expression.+ *+ * Firecrawl's threat protection is domain-level (we classify the target+ * domain, not individual URLs), so path variants are intentionally not+ * generated β this matches the previous uris:search behavior of looking up+ * `http://<domain>/`.+ */+export function generateHostExpressions(domain: string): string[] {+ const host = canonicalizeHost(domain.trim());+ if (host.length === 0) return [];+ if (isIpHost(host)) return [host + "/"];++ const expressions = [host + "/"];+ const parts = host.split(".");+ const maxSuffix = Math.min(5, parts.length - 1);+ for (let suffixLen = maxSuffix; suffixLen >= 2; suffixLen--) {+ const suffix = parts.slice(parts.length - suffixLen).join(".");+ if (suffix !== host) expressions.push(suffix + "/");+ }+ return expressions;+}diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.ts b/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.tsnew file mode 100644index 000000000..262b1c506--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/fake-redis.ts@@ -0,0 +1,73 @@+import type { WebRiskRedis } from "./store";++// In-memory stand-in for the durable Redis connection used by the threat-list+// store and sync lock (supports EX/NX set semantics and the buffer read+// variants the store uses). Test helper only.+//+// Lives in its own module with a type-only import so that vi.mock factories+// for services/queue-service can import it without re-entering the module+// being mocked (testing.ts β store.ts β queue-service would deadlock).++export function createFakeWebRiskRedis(): WebRiskRedis & {+ dump(): Map<string, Buffer>;+} {+ const store = new Map<string, { value: Buffer; expiresAt: number | null }>();++ const live = (key: string): Buffer | null => {+ const entry = store.get(key);+ if (!entry) return null;+ if (entry.expiresAt !== null && entry.expiresAt <= Date.now()) {+ store.delete(key);+ return null;+ }+ return entry.value;+ };++ return {+ async get(key) {+ const value = live(key);+ return value === null ? null : value.toString("utf8");+ },+ async mget(...keys) {+ return keys.map(key => {+ const value = live(key);+ return value === null ? null : value.toString("utf8");+ });+ },+ async mgetBuffer(...keys) {+ return keys.map(key => live(key));+ },+ async set(key, value, ...args) {+ const upper = args.map(a => String(a).toUpperCase());+ if (upper.includes("NX") && live(key) !== null) return null;+ const exIndex = upper.indexOf("EX");+ const expiresAt =+ exIndex !== -1 ? Date.now() + Number(args[exIndex + 1]) * 1000 : null;+ store.set(key, {+ value: Buffer.isBuffer(value) ? value : Buffer.from(String(value)),+ expiresAt,+ });+ return "OK";+ },+ async del(...keys) {+ let deleted = 0;+ for (const key of keys) {+ if (store.delete(key)) deleted++;+ }+ return deleted;+ },+ async expire(key, seconds) {+ const entry = store.get(key);+ if (!entry || live(key) === null) return 0;+ entry.expiresAt = Date.now() + seconds * 1000;+ return 1;+ },+ dump() {+ return new Map(+ [...store.entries()]+ .filter(([key]) => live(key) !== null)+ .map(([key, entry]) => [key, entry.value]),+ );+ },+ };+}diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/store.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/store.test.tsnew file mode 100644index 000000000..d58380dab--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/store.test.ts@@ -0,0 +1,148 @@+import { config } from "../../../../config";+import {+ WEB_RISK_THREAT_TYPES,+ WebRiskListStore,+ type ThreatListMeta,+} from "./store";+import { createFakeWebRiskRedis, sha256 } from "./testing";++function meta(overrides: Partial<ThreatListMeta> = {}): ThreatListMeta {+ return {+ versionToken: "dG9rZW4=",+ checksum: "",+ count: 0,+ syncedAt: new Date().toISOString(),+ nextDiffAt: new Date(Date.now() + 60_000).toISOString(),+ ...overrides,+ };+}++function sorted(entries: Buffer[]): Buffer[] {+ return [...entries].sort(Buffer.compare);+}++describe("WebRiskListStore", () => {+ it("round-trips entries through the bucketed layout", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);++ // Hashes spread across many buckets, plus one long (8-byte) entry.+ const fullHashes = Array.from({ length: 500 }, (_, i) =>+ sha256(`domain-${i}.example/`),+ );+ const entries = sorted([+ ...fullHashes.map(hash => hash.subarray(0, 4)),+ sha256("long-entry.example/").subarray(0, 8),+ ]);++ for (const type of WEB_RISK_THREAT_TYPES) {+ await store.writeVersion(type, entries, meta({ count: entries.length }));+ }++ const roundTripped = await store.loadEntries(+ "MALWARE",+ (await store.getPointer("MALWARE"))!.version,+ );+ expect(roundTripped.map(e => e.toString("hex"))).toEqual(+ entries.map(e => e.toString("hex")),+ );++ // Membership: a listed 4-byte prefix hits, an unlisted hash misses.+ const hit = await store.lookup([fullHashes[123]]);+ expect(hit).toMatchObject({ status: "ok" });+ if (hit.status === "ok") {+ expect(hit.hits.length).toBe(3); // present in every list+ expect(hit.hits[0].prefix.equals(fullHashes[123].subarray(0, 4))).toBe(+ true,+ );+ }++ const miss = await store.lookup([sha256("clean.example/")]);+ expect(miss).toMatchObject({ status: "ok", hits: [] });++ // Long entries match by prefix comparison against the full hash.+ const longHit = await store.lookup([sha256("long-entry.example/")]);+ expect(longHit).toMatchObject({ status: "ok" });+ if (longHit.status === "ok") {+ expect(longHit.hits[0].prefix.length).toBe(8);+ }+ });++ it("reports unavailable until every list has a version", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);++ expect(await store.lookup([sha256("x/")])).toEqual({+ status: "unavailable",+ });++ await store.writeVersion("MALWARE", [], meta());+ expect(await store.lookup([sha256("x/")])).toEqual({+ status: "unavailable",+ });++ await store.writeVersion("SOCIAL_ENGINEERING", [], meta());+ await store.writeVersion("UNWANTED_SOFTWARE", [], meta());+ expect(await store.lookup([sha256("x/")])).toMatchObject({ status: "ok" });+ });++ it("reports stale when a list exceeds the staleness threshold", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ const staleSyncedAt = new Date(+ Date.now() - (config.THREAT_LIST_STALENESS_SECONDS + 60) * 1000,+ ).toISOString();++ await store.writeVersion("MALWARE", [], meta({ syncedAt: staleSyncedAt }));+ await store.writeVersion("SOCIAL_ENGINEERING", [], meta());+ await store.writeVersion("UNWANTED_SOFTWARE", [], meta());++ const result = await store.lookup([sha256("x/")]);+ expect(result).toMatchObject({ status: "stale" });+ if (result.status === "stale") {+ expect(result.ageSeconds).toBeGreaterThan(+ config.THREAT_LIST_STALENESS_SECONDS,+ );+ }+ });++ it("swaps the version pointer atomically and serves the new content", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);++ const v1Hash = sha256("v1-only.example/");+ const v2Hash = sha256("v2-only.example/");++ for (const type of WEB_RISK_THREAT_TYPES) {+ await store.writeVersion(type, [v1Hash.subarray(0, 4)], meta());+ }+ const v1 = (await store.getPointer("MALWARE"))!.version;++ expect(await store.lookup([v1Hash])).toMatchObject({+ status: "ok",+ hits: [expect.anything(), expect.anything(), expect.anything()],+ });++ await store.writeVersion("MALWARE", [v2Hash.subarray(0, 4)], meta());+ const v2 = (await store.getPointer("MALWARE"))!.version;+ expect(v2).not.toBe(v1);++ // New content served, old content gone from the MALWARE list.+ const v2Lookup = await store.lookup([v2Hash]);+ expect(v2Lookup).toMatchObject({ status: "ok" });+ if (v2Lookup.status === "ok") {+ expect(v2Lookup.hits.map(h => h.threatType)).toEqual(["MALWARE"]);+ }+ const v1Lookup = await store.lookup([v1Hash]);+ if (v1Lookup.status === "ok") {+ expect(v1Lookup.hits.map(h => h.threatType)).toEqual([+ "SOCIAL_ENGINEERING",+ "UNWANTED_SOFTWARE",+ ]);+ }++ // The superseded version's data is retired (grace TTL), not readable as+ // the current version.+ expect((await store.getPointer("MALWARE"))!.version).toBe(v2);+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/store.ts b/apps/api/src/lib/threat-protection/providers/web-risk/store.tsnew file mode 100644index 000000000..a6eb61a7f--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/store.ts@@ -0,0 +1,384 @@+import { config } from "../../../../config";+import { getRedisConnection } from "../../../../services/queue-service";++// Redis-backed store for the locally synced Google Web Risk threat lists+// (hash prefixes, Update API β see ./sync.ts). Design constraints:+//+// * Lives in the durable, NON-EVICTING Redis (config.REDIS_URL via+// getRedisConnection()) β explicitly not the evict or rate-limit+// connections: the lists are operational data that must not be evicted+// under memory pressure. (This data is Google's threat lists, not+// anything scrape-derived β storing it is ZDR-compatible.)+//+// * Memory-efficient layout: ~5M 4-byte prefixes as individual SET members+// would balloon ~20MB of raw data into hundreds of MB of per-entry+// overhead. Instead, prefixes are sharded by their leading 12 bits into+// 4096 buckets, each bucket one binary string of sorted, concatenated+// 4-byte prefixes. Membership = one MGET + a local binary search.+// (Rare longer entries β the protocol allows 4-32 byte prefixes β go into+// a per-version "long" key, checked by prefix comparison.)+//+// * Versioned namespaces with an atomic pointer swap: a sync writes every+// bucket under a fresh version id, then flips the version pointer in a+// single SET β a partially written version is never served. Old versions+// are expired with a short grace period so in-flight readers that already+// resolved the old pointer still find their buckets.++export const WEB_RISK_THREAT_TYPES = [+ "MALWARE",+ "SOCIAL_ENGINEERING",+ "UNWANTED_SOFTWARE",+] as const;++export type WebRiskThreatType = (typeof WEB_RISK_THREAT_TYPES)[number];++const BUCKET_BITS = 12;+const BUCKET_COUNT = 1 << BUCKET_BITS;+// Refreshed on every sync; prevents orphaned lists from lingering forever if+// syncing is disabled/abandoned. Staleness (hours) cuts in far earlier.+const KEY_TTL_SECONDS = 7 * 24 * 60 * 60;+// How long a superseded version's keys stay readable after the pointer swap.+const OLD_VERSION_GRACE_SECONDS = 300;++export interface ThreatListMeta {+ /** Google's versionToken for the stored list state, base64. */+ versionToken: string;+ /** sha256 (base64) over the sorted concatenated entries, per protocol. */+ checksum: string;+ /** Number of stored entries (hash prefixes). */+ count: number;+ /** When this version was written (ISO timestamp). */+ syncedAt: string;+ /** Earliest time the next computeDiff should run (ISO timestamp). */+ nextDiffAt: string;+}++interface ListPointer {+ version: string;+ meta: ThreatListMeta;+}++interface WebRiskPrefixHit {+ threatType: WebRiskThreatType;+ /** The stored entry that matched (a prefix of `fullHash`), 4-32 bytes. */+ prefix: Buffer;+ fullHash: Buffer;+}++export type WebRiskLookupResult =+ | { status: "unavailable" }+ | { status: "stale"; ageSeconds: number }+ | { status: "ok"; hits: WebRiskPrefixHit[] };++/** Minimal ioredis surface the store needs (kept narrow for unit testing). */+export interface WebRiskRedis {+ get(key: string): Promise<string | null>;+ mget(...keys: string[]): Promise<(string | null)[]>;+ mgetBuffer(...keys: string[]): Promise<(Buffer | null)[]>;+ set(+ key: string,+ value: string | Buffer,+ ...args: (string | number)[]+ ): Promise<unknown>;+ del(...keys: string[]): Promise<unknown>;+ expire(key: string, seconds: number): Promise<unknown>;+}++const pointerKey = (type: WebRiskThreatType) => `threat_list:${type}:current`;+const bucketKey = (type: WebRiskThreatType, version: string, bucket: number) =>+ `threat_list:${type}:${version}:b:${bucket}`;+const longKey = (type: WebRiskThreatType, version: string) =>+ `threat_list:${type}:${version}:long`;++function bucketIndex(entry: Buffer): number {+ return (entry[0] << 4) | (entry[1] >> 4);+}++/** Binary search for a 4-byte prefix in a sorted concatenated bucket. */+function bucketContains(bucket: Buffer, prefix: Buffer): boolean {+ const target = prefix.readUInt32BE(0);+ let lo = 0;+ let hi = bucket.length / 4 - 1;+ while (lo <= hi) {+ const mid = (lo + hi) >> 1;+ const value = bucket.readUInt32BE(mid * 4);+ if (value === target) return true;+ if (value < target) lo = mid + 1;+ else hi = mid - 1;+ }+ return false;+}++/** Serialize length-prefixed long (>4 byte) entries. */+function encodeLongEntries(entries: Buffer[]): Buffer {+ return Buffer.concat(+ entries.flatMap(entry => [Buffer.from([entry.length]), entry]),+ );+}++function decodeLongEntries(blob: Buffer): Buffer[] {+ const entries: Buffer[] = [];+ let offset = 0;+ while (offset < blob.length) {+ const length = blob[offset];+ entries.push(blob.subarray(offset + 1, offset + 1 + length));+ offset += 1 + length;+ }+ return entries;+}++export class WebRiskListStore {+ constructor(private readonly redis: WebRiskRedis) {}++ private parsePointer(raw: string | null): ListPointer | null {+ if (!raw) return null;+ try {+ const parsed = JSON.parse(raw) as ListPointer;+ if (+ typeof parsed?.version !== "string" ||+ typeof parsed?.meta?.syncedAt !== "string"+ ) {+ return null;+ }+ return parsed;+ } catch {+ return null;+ }+ }++ async getPointer(type: WebRiskThreatType): Promise<ListPointer | null> {+ return this.parsePointer(await this.redis.get(pointerKey(type)));+ }++ async getPointers(): Promise<Map<WebRiskThreatType, ListPointer | null>> {+ const raw = await this.redis.mget(+ ...WEB_RISK_THREAT_TYPES.map(type => pointerKey(type)),+ );+ return new Map(+ WEB_RISK_THREAT_TYPES.map((type, i) => [type, this.parsePointer(raw[i])]),+ );+ }++ /**+ * Loads the full sorted entry list for a stored version (used by the sync+ * worker to apply an incremental diff β removal indices reference positions+ * in this lexicographically sorted list).+ */+ async loadEntries(+ type: WebRiskThreatType,+ version: string,+ ): Promise<Buffer[]> {+ const keys: string[] = [];+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {+ keys.push(bucketKey(type, version, bucket));+ }+ keys.push(longKey(type, version));++ const CHUNK = 512;+ const values: (Buffer | null)[] = [];+ for (let i = 0; i < keys.length; i += CHUNK) {+ values.push(+ ...(await this.redis.mgetBuffer(...keys.slice(i, i + CHUNK))),+ );+ }++ const shortEntries: Buffer[] = [];+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {+ const blob = values[bucket];+ if (!blob) continue;+ for (let offset = 0; offset < blob.length; offset += 4) {+ shortEntries.push(blob.subarray(offset, offset + 4));+ }+ }+ const longBlob = values[BUCKET_COUNT];+ const longEntries = longBlob ? decodeLongEntries(longBlob) : [];++ // Buckets are globally sorted already; merge in the (sorted) long entries.+ if (longEntries.length === 0) return shortEntries;+ return [...shortEntries, ...longEntries].sort(Buffer.compare);+ }++ /**+ * Writes a full list snapshot under a fresh version id and atomically swaps+ * the version pointer to it. `entries` must be lexicographically sorted.+ * Returns the new version id.+ */+ async writeVersion(+ type: WebRiskThreatType,+ entries: Buffer[],+ meta: ThreatListMeta,+ ): Promise<string> {+ const previous = await this.getPointer(type);+ const version = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;++ const buckets: Buffer[][] = [];+ const long: Buffer[] = [];+ for (const entry of entries) {+ if (entry.length === 4) {+ const index = bucketIndex(entry);+ (buckets[index] ??= []).push(entry);+ } else {+ long.push(entry);+ }+ }++ // ioredis pipelines synchronously issued commands on the socket, so a+ // Promise.all of SETs is one batched round trip in practice.+ const writes: Promise<unknown>[] = [];+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {+ const content = buckets[bucket];+ if (!content || content.length === 0) continue;+ writes.push(+ this.redis.set(+ bucketKey(type, version, bucket),+ Buffer.concat(content),+ "EX",+ KEY_TTL_SECONDS,+ ),+ );+ }+ if (long.length > 0) {+ writes.push(+ this.redis.set(+ longKey(type, version),+ encodeLongEntries(long),+ "EX",+ KEY_TTL_SECONDS,+ ),+ );+ }+ await Promise.all(writes);++ // Atomic pointer swap: the new version becomes visible in one SET.+ await this.redis.set(+ pointerKey(type),+ JSON.stringify({ version, meta } satisfies ListPointer),+ "EX",+ KEY_TTL_SECONDS,+ );++ // Retire the superseded version with a grace window for in-flight reads.+ if (previous && previous.version !== version) {+ const expires: Promise<unknown>[] = [];+ for (let bucket = 0; bucket < BUCKET_COUNT; bucket++) {+ expires.push(+ this.redis.expire(+ bucketKey(type, previous.version, bucket),+ OLD_VERSION_GRACE_SECONDS,+ ),+ );+ }+ expires.push(+ this.redis.expire(+ longKey(type, previous.version),+ OLD_VERSION_GRACE_SECONDS,+ ),+ );+ await Promise.all(expires);+ }++ return version;+ }++ /**+ * Checks a set of full expression hashes against every threat list.+ * Entirely local to our infrastructure: reads the synced buckets from+ * Redis and binary-searches them β nothing is sent to Google here.+ */+ async lookup(fullHashes: Buffer[]): Promise<WebRiskLookupResult> {+ const pointers = await this.getPointers();++ let oldestAgeSeconds = 0;+ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = pointers.get(type);+ if (!pointer) return { status: "unavailable" };+ const age = (Date.now() - Date.parse(pointer.meta.syncedAt)) / 1000;+ if (!Number.isFinite(age)) return { status: "unavailable" };+ oldestAgeSeconds = Math.max(oldestAgeSeconds, age);+ }+ if (oldestAgeSeconds > config.THREAT_LIST_STALENESS_SECONDS) {+ return { status: "stale", ageSeconds: Math.round(oldestAgeSeconds) };+ }++ // Collect every bucket key we need across lists and hashes, fetch them+ // in one MGET, then binary-search locally.+ const keys: string[] = [];+ const keyIndex = new Map<string, number>();+ const keyFor = (key: string): number => {+ let index = keyIndex.get(key);+ if (index === undefined) {+ index = keys.length;+ keys.push(key);+ keyIndex.set(key, index);+ }+ return index;+ };++ interface Probe {+ type: WebRiskThreatType;+ fullHash: Buffer;+ bucketKeyIndex: number;+ longKeyIndex: number;+ }+ const probes: Probe[] = [];+ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = pointers.get(type)!;+ const longIndex = keyFor(longKey(type, pointer.version));+ for (const fullHash of fullHashes) {+ probes.push({+ type,+ fullHash,+ bucketKeyIndex: keyFor(+ bucketKey(type, pointer.version, bucketIndex(fullHash)),+ ),+ longKeyIndex: longIndex,+ });+ }+ }++ const values = await this.redis.mgetBuffer(...keys);++ const hits: WebRiskPrefixHit[] = [];+ for (const probe of probes) {+ const bucket = values[probe.bucketKeyIndex];+ if (bucket && bucketContains(bucket, probe.fullHash.subarray(0, 4))) {+ hits.push({+ threatType: probe.type,+ prefix: probe.fullHash.subarray(0, 4),+ fullHash: probe.fullHash,+ });+ continue;+ }+ const longBlob = values[probe.longKeyIndex];+ if (longBlob) {+ for (const entry of decodeLongEntries(longBlob)) {+ if (+ entry.length <= probe.fullHash.length &&+ entry.equals(probe.fullHash.subarray(0, entry.length))+ ) {+ hits.push({+ threatType: probe.type,+ prefix: entry,+ fullHash: probe.fullHash,+ });+ break;+ }+ }+ }+ }++ return { status: "ok", hits };+ }+}++let defaultStore: WebRiskListStore | null = null;++/** Store singleton on the durable (non-evict) Redis connection. */+export function getWebRiskListStore(): WebRiskListStore {+ if (!defaultStore) {+ defaultStore = new WebRiskListStore(+ getRedisConnection() as unknown as WebRiskRedis,+ );+ }+ return defaultStore;+}diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.ts b/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.tsnew file mode 100644index 000000000..2fecc1adf--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/sync.test.ts@@ -0,0 +1,254 @@+import { createHash } from "crypto";+import http from "http";+import { AddressInfo } from "net";+import { config } from "../../../../config";+import { runThreatListSyncPass } from "./sync";+import { WEB_RISK_THREAT_TYPES, WebRiskListStore } from "./store";+import { createFakeWebRiskRedis } from "./testing";++// Sync-loop unit tests: a local HTTP server plays Google's+// threatLists:computeDiff endpoint via the config URL override.++type DiffHandler = (threatType: string, versionToken: string | null) => object;++let server: http.Server;+let baseUrl: string;+let diffCalls: { threatType: string; versionToken: string | null }[] = [];+let diffHandler: DiffHandler = () => ({});++const originalConfig = {+ url: config.GOOGLE_WEB_RISK_API_URL,+ key: config.GOOGLE_WEB_RISK_API_KEY,+};++beforeAll(async () => {+ await new Promise<void>(resolve => {+ server = http.createServer((req, res) => {+ const url = new URL(req.url ?? "/", "http://localhost");+ if (url.pathname === "/v1/threatLists:computeDiff") {+ const threatType = url.searchParams.get("threatType") ?? "";+ const versionToken = url.searchParams.get("versionToken");+ diffCalls.push({ threatType, versionToken });+ res.setHeader("content-type", "application/json");+ res.end(JSON.stringify(diffHandler(threatType, versionToken)));+ } else {+ res.statusCode = 404;+ res.end("{}");+ }+ });+ server.listen(0, "127.0.0.1", () => {+ baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+ config.GOOGLE_WEB_RISK_API_URL = baseUrl;+ config.GOOGLE_WEB_RISK_API_KEY = "test-key";+ resolve();+ });+ });+});++afterAll(async () => {+ config.GOOGLE_WEB_RISK_API_URL = originalConfig.url;+ config.GOOGLE_WEB_RISK_API_KEY = originalConfig.key;+ await new Promise<void>(resolve => server.close(() => resolve()));+});++beforeEach(() => {+ diffCalls = [];+});++function checksumOf(sortedEntries: Buffer[]): string {+ const hash = createHash("sha256");+ for (const entry of sortedEntries) hash.update(entry);+ return hash.digest("base64");+}++function sortedPrefixes(hexes: string[]): Buffer[] {+ return hexes.map(hex => Buffer.from(hex, "hex")).sort(Buffer.compare);+}++function resetResponse(entries: Buffer[], token: string): object {+ return {+ responseType: "RESET",+ additions: {+ rawHashes: [+ { prefixSize: 4, rawHashes: Buffer.concat(entries).toString("base64") },+ ],+ },+ newVersionToken: Buffer.from(token).toString("base64"),+ checksum: { sha256: checksumOf(entries) },+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),+ };+}++describe("runThreatListSyncPass", () => {+ it("bootstraps all lists from a RESET response and stores metadata", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ const entries = sortedPrefixes(["00000001", "7fffffff", "ffffff00"]);+ diffHandler = () => resetResponse(entries, "v1");++ const ran = await runThreatListSyncPass({}, store, redis);+ expect(ran).toBe(true);+ expect(diffCalls.map(c => c.threatType).sort()).toEqual(+ [...WEB_RISK_THREAT_TYPES].sort(),+ );+ // First sync has no version token.+ expect(diffCalls.every(c => c.versionToken === null)).toBe(true);++ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = await store.getPointer(type);+ expect(pointer).not.toBeNull();+ expect(pointer!.meta.count).toBe(3);+ expect(pointer!.meta.versionToken).toBe(+ Buffer.from("v1").toString("base64"),+ );+ const stored = await store.loadEntries(type, pointer!.version);+ expect(stored.map(e => e.toString("hex"))).toEqual(+ entries.map(e => e.toString("hex")),+ );+ }+ });++ it("applies an incremental DIFF (removals by sorted index + additions)", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ const initial = sortedPrefixes([+ "11111111",+ "22222222",+ "33333333",+ "44444444",+ ]);+ diffHandler = () => resetResponse(initial, "v1");+ await runThreatListSyncPass({}, store, redis);+ diffCalls = [];++ // Remove indices 1 and 3 (22222222, 44444444), add aaaaaaaa.+ const expected = sortedPrefixes(["11111111", "33333333", "aaaaaaaa"]);+ diffHandler = () => ({+ responseType: "DIFF",+ additions: {+ rawHashes: [+ {+ prefixSize: 4,+ rawHashes: Buffer.from("aaaaaaaa", "hex").toString("base64"),+ },+ ],+ },+ removals: { rawIndices: { indices: [1, 3] } },+ newVersionToken: Buffer.from("v2").toString("base64"),+ checksum: { sha256: checksumOf(expected) },+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),+ });++ await runThreatListSyncPass({ force: true }, store, redis);++ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = await store.getPointer(type);+ expect(pointer!.meta.versionToken).toBe(+ Buffer.from("v2").toString("base64"),+ );+ const stored = await store.loadEntries(type, pointer!.version);+ expect(stored.map(e => e.toString("hex"))).toEqual(+ expected.map(e => e.toString("hex")),+ );+ }+ // The DIFF request carried the stored version token.+ expect(+ diffCalls.every(+ c => c.versionToken === Buffer.from("v1").toString("base64"),+ ),+ ).toBe(true);+ });++ it("rejects a malformed prefixSize instead of spinning (remote JSON is untrusted)", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ diffHandler = () => ({+ responseType: "RESET",+ additions: {+ rawHashes: [+ {+ prefixSize: 0,+ rawHashes: Buffer.from("11111111", "hex").toString("base64"),+ },+ ],+ },+ newVersionToken: Buffer.from("v1").toString("base64"),+ checksum: { sha256: checksumOf(sortedPrefixes(["11111111"])) },+ });++ // Per-list failures are logged and skipped (one bad list must not take+ // down the pass) β the important part is: no infinite loop, and nothing+ // gets published.+ await runThreatListSyncPass({}, store, redis);+ for (const type of WEB_RISK_THREAT_TYPES) {+ expect(await store.getPointer(type)).toBeNull();+ }+ });++ it("recovers from a DIFF checksum mismatch by re-syncing from scratch", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ const initial = sortedPrefixes(["11111111"]);+ diffHandler = () => resetResponse(initial, "v1");+ await runThreatListSyncPass({}, store, redis);+ diffCalls = [];++ const fresh = sortedPrefixes(["55555555", "66666666"]);+ diffHandler = (_type, versionToken) => {+ if (versionToken) {+ // Corrupt diff: checksum does not match what applying it produces.+ return {+ responseType: "DIFF",+ additions: {+ rawHashes: [+ {+ prefixSize: 4,+ rawHashes: Buffer.from("99999999", "hex").toString("base64"),+ },+ ],+ },+ newVersionToken: Buffer.from("bad").toString("base64"),+ checksum: { sha256: checksumOf(sortedPrefixes(["deadbeef"])) },+ };+ }+ return resetResponse(fresh, "v2");+ };++ await runThreatListSyncPass({ force: true }, store, redis);++ // Each list: one failed DIFF then one RESET recovery.+ expect(diffCalls.length).toBe(WEB_RISK_THREAT_TYPES.length * 2);+ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = await store.getPointer(type);+ expect(pointer!.meta.versionToken).toBe(+ Buffer.from("v2").toString("base64"),+ );+ const stored = await store.loadEntries(type, pointer!.version);+ expect(stored.map(e => e.toString("hex"))).toEqual(+ fresh.map(e => e.toString("hex")),+ );+ }+ });++ it("respects recommendedNextDiff (unforced pass is a no-op while fresh)", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ diffHandler = () => resetResponse(sortedPrefixes(["11111111"]), "v1");+ await runThreatListSyncPass({}, store, redis);+ diffCalls = [];++ const ran = await runThreatListSyncPass({}, store, redis);+ expect(ran).toBe(true); // lock acquiredβ¦+ expect(diffCalls).toEqual([]); // β¦but nothing was due+ });++ it("returns false without syncing when another process holds the lock", async () => {+ const redis = createFakeWebRiskRedis();+ const store = new WebRiskListStore(redis);+ await redis.set("threat_list_sync:lock", "someone-else", "EX", 60, "NX");++ const ran = await runThreatListSyncPass({}, store, redis);+ expect(ran).toBe(false);+ expect(diffCalls).toEqual([]);+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/sync.ts b/apps/api/src/lib/threat-protection/providers/web-risk/sync.tsnew file mode 100644index 000000000..68d2f8035--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/sync.ts@@ -0,0 +1,335 @@+import { createHash, randomUUID } from "crypto";+import { config } from "../../../../config";+import { getRedisConnection } from "../../../../services/queue-service";+import { logger as _logger } from "../../../logger";+import {+ WEB_RISK_THREAT_TYPES,+ WebRiskListStore,+ getWebRiskListStore,+ type ThreatListMeta,+ type WebRiskRedis,+ type WebRiskThreatType,+} from "./store";++// Google Web Risk Update API sync worker (ZDR rework of "normal" mode).+//+// threatLists:computeDiff (free, unlike hashes:search) is polled per threat+// list: with no stored versionToken Google sends the full list (RESET),+// afterwards incremental additions/removals (DIFF). The updated list is+// checksum-verified and written to the versioned bucket store (./store.ts)+// with an atomic pointer swap.+//+// Topology: every API/worker process runs the loop, but a fleet-wide Redis+// lock ensures only one process actually syncs at a time. The loop respects+// Google's recommendedNextDiff (floored by+// THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS). List age is logged every cycle so+// staleness is monitorable; a list older than THREAT_LIST_STALENESS_SECONDS+// is treated as unavailable by the check path (provider-failure semantics).++const logger = _logger.child({ module: "web-risk-sync" });++const LOCK_KEY = "threat_list_sync:lock";+const LOCK_TTL_SECONDS = 300;+const LOOP_TICK_MS = 30_000;+const SYNC_FETCH_TIMEOUT_MS = 120_000;++interface RawHashes {+ prefixSize?: number;+ rawHashes?: string;+}++interface ComputeDiffResponse {+ responseType?: "RESET" | "DIFF" | "RESPONSE_TYPE_UNSPECIFIED";+ additions?: { rawHashes?: RawHashes[] };+ removals?: { rawIndices?: { indices?: number[] } };+ newVersionToken?: string;+ checksum?: { sha256?: string };+ recommendedNextDiff?: string;+}++function isConfigured(): boolean {+ return (+ typeof config.GOOGLE_WEB_RISK_API_KEY === "string" &&+ config.GOOGLE_WEB_RISK_API_KEY.trim().length > 0+ );+}++async function computeDiff(+ type: WebRiskThreatType,+ versionToken: string | null,+): Promise<ComputeDiffResponse> {+ const params = new URLSearchParams();+ params.append("threatType", type);+ if (versionToken) params.append("versionToken", versionToken);+ params.append("constraints.supportedCompressions", "RAW");+ params.append("key", config.GOOGLE_WEB_RISK_API_KEY!);++ const response = await fetch(+ `${config.GOOGLE_WEB_RISK_API_URL}/v1/threatLists:computeDiff?${params.toString()}`,+ { method: "GET", signal: AbortSignal.timeout(SYNC_FETCH_TIMEOUT_MS) },+ );+ if (!response.ok) {+ throw new Error(+ `Web Risk computeDiff for ${type} failed with status ${response.status}`,+ );+ }+ return (await response.json()) as ComputeDiffResponse;+}++function decodeAdditions(additions: RawHashes[] | undefined): Buffer[] {+ const entries: Buffer[] = [];+ for (const group of additions ?? []) {+ if (!group.rawHashes) continue;+ const prefixSize = group.prefixSize ?? 4;+ // The response is unvalidated remote JSON: a non-positive prefixSize+ // would make the offset loop below spin forever and stall the fleet-wide+ // sync for the whole lock TTL. Valid Web Risk prefixes are 4-32 bytes.+ if (!Number.isInteger(prefixSize) || prefixSize < 4 || prefixSize > 32) {+ throw new Error(+ `Web Risk computeDiff returned invalid prefixSize ${prefixSize}`,+ );+ }+ const blob = Buffer.from(group.rawHashes, "base64");+ for (+ let offset = 0;+ offset + prefixSize <= blob.length;+ offset += prefixSize+ ) {+ entries.push(blob.subarray(offset, offset + prefixSize));+ }+ }+ return entries;+}++/** Protocol checksum: sha256 over the sorted concatenated entries, base64. */+function listChecksum(sortedEntries: Buffer[]): string {+ const hash = createHash("sha256");+ for (const entry of sortedEntries) hash.update(entry);+ return hash.digest("base64");+}++function nextDiffAt(recommendedNextDiff: string | undefined): string {+ const floorMs =+ Date.now() + config.THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS * 1000;+ const recommendedMs = recommendedNextDiff+ ? Date.parse(recommendedNextDiff)+ : NaN;+ return new Date(+ Number.isFinite(recommendedMs) ? Math.max(recommendedMs, floorMs) : floorMs,+ ).toISOString();+}++/**+ * Syncs a single list: computeDiff against the stored version, apply, verify+ * checksum, write + swap. On a checksum mismatch of an incremental diff the+ * list is re-synced from scratch once (RESET).+ */+async function syncList(+ store: WebRiskListStore,+ type: WebRiskThreatType,+ forceReset = false,+): Promise<void> {+ const pointer = forceReset ? null : await store.getPointer(type);+ const versionToken = pointer?.meta.versionToken ?? null;++ const diff = await computeDiff(type, versionToken);++ let entries: Buffer[];+ if (diff.responseType === "DIFF" && pointer) {+ entries = await store.loadEntries(type, pointer.version);+ const indices = diff.removals?.rawIndices?.indices ?? [];+ if (indices.length > 0) {+ const removed = new Set(indices);+ entries = entries.filter((_, index) => !removed.has(index));+ }+ entries = entries.concat(decodeAdditions(diff.additions?.rawHashes));+ } else {+ // RESET (or first sync): the additions are the entire list.+ entries = decodeAdditions(diff.additions?.rawHashes);+ }++ entries.sort(Buffer.compare);++ const expectedChecksum = diff.checksum?.sha256;+ if (expectedChecksum && listChecksum(entries) !== expectedChecksum) {+ if (diff.responseType === "DIFF") {+ logger.warn(+ "Web Risk list checksum mismatch after diff; re-syncing from scratch",+ { threatType: type },+ );+ return syncList(store, type, true);+ }+ throw new Error(`Web Risk ${type} full sync failed checksum verification`);+ }++ const meta: ThreatListMeta = {+ versionToken: diff.newVersionToken ?? versionToken ?? "",+ checksum: expectedChecksum ?? listChecksum(entries),+ count: entries.length,+ syncedAt: new Date().toISOString(),+ nextDiffAt: nextDiffAt(diff.recommendedNextDiff),+ };+ await store.writeVersion(type, entries, meta);++ logger.info("Web Risk threat list synced", {+ canonicalLog: "threat-protection/list-sync",+ threatType: type,+ responseType: diff.responseType ?? "RESET",+ entryCount: entries.length,+ nextDiffAt: meta.nextDiffAt,+ });+}++async function acquireLock(redis: WebRiskRedis): Promise<string | null> {+ const token = randomUUID();+ const result = await redis.set(LOCK_KEY, token, "EX", LOCK_TTL_SECONDS, "NX");+ return result === "OK" ? token : null;+}++async function releaseLock(redis: WebRiskRedis, token: string): Promise<void> {+ // Get-compare-del (not atomic; worst case two syncers overlap briefly,+ // which the versioned store + atomic pointer swap makes harmless).+ const current = await redis.get(LOCK_KEY);+ if (current === token) await redis.del(LOCK_KEY);+}++function isListDue(+ pointer: Awaited<ReturnType<WebRiskListStore["getPointer"]>>,+ force: boolean,+): boolean {+ if (force || !pointer) return true;+ const next = Date.parse(pointer.meta.nextDiffAt);+ return !Number.isFinite(next) || next <= Date.now();+}++/**+ * Runs one sync pass under the fleet-wide lock: every due (or, with `force`,+ * every) list gets a computeDiff. Returns false when another process holds+ * the lock. Throws when Web Risk is not configured.+ */+export async function runThreatListSyncPass(+ options: { force?: boolean } = {},+ store: WebRiskListStore = getWebRiskListStore(),+ redis: WebRiskRedis = getRedisConnection() as unknown as WebRiskRedis,+): Promise<boolean> {+ if (!isConfigured()) {+ throw new Error("Google Web Risk is not configured");+ }++ const token = await acquireLock(redis);+ if (token === null) return false;++ try {+ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = await store.getPointer(type);+ if (!isListDue(pointer, options.force ?? false)) continue;+ try {+ await syncList(store, type);+ } catch (error) {+ logger.error("Web Risk threat list sync failed", {+ canonicalLog: "threat-protection/list-sync",+ threatType: type,+ error,+ });+ }+ }+ } finally {+ await releaseLock(redis, token);+ }+ return true;+}++/** Logs the age of every stored list (staleness monitoring). */+async function logListAges(store: WebRiskListStore): Promise<void> {+ const pointers = await store.getPointers();+ const ages: Record<string, number | null> = {};+ let stale = false;+ for (const type of WEB_RISK_THREAT_TYPES) {+ const pointer = pointers.get(type);+ const ageSeconds = pointer+ ? Math.round((Date.now() - Date.parse(pointer.meta.syncedAt)) / 1000)+ : null;+ ages[type] = ageSeconds;+ if (+ ageSeconds === null ||+ ageSeconds > config.THREAT_LIST_STALENESS_SECONDS+ ) {+ stale = true;+ }+ }+ const log = stale ? logger.warn.bind(logger) : logger.info.bind(logger);+ log("Web Risk threat list age", {+ canonicalLog: "threat-protection/list-age",+ ageSeconds: ages,+ stale,+ stalenessThresholdSeconds: config.THREAT_LIST_STALENESS_SECONDS,+ });+}++let loopStarted = false;++/**+ * Starts the periodic sync loop for this process (idempotent, lazy β called+ * on first Web Risk provider use so every API/worker process that performs+ * checks also keeps the lists fresh). The Redis lock in+ * runThreatListSyncPass makes the fleet-wide behavior single-syncer.+ */+export function ensureThreatListSyncLoop(): void {+ if (loopStarted || !isConfigured()) return;+ loopStarted = true;++ let tick = 0;+ const timer = setInterval(() => {+ void (async () => {+ try {+ // Age metric every ~5min (stale lists log a warn from logListAges).+ if (tick++ % 10 === 0) {+ await logListAges(getWebRiskListStore());+ }+ await runThreatListSyncPass();+ } catch (error) {+ logger.warn("Web Risk sync loop tick failed", { error });+ }+ })();+ }, LOOP_TICK_MS);+ timer.unref();+}++let bootSyncPromise: Promise<void> | null = null;++/**+ * One forced sync pass per process lifetime, shared by all callers. Run+ * before the first lookup so that (a) a fresh deployment bootstraps the+ * lists and (b) a process never trusts a possibly-superseded stored state+ * without having compared version tokens with Google once. If another+ * process holds the sync lock, waits for it to finish instead.+ */+export function ensureThreatListBootSync(): Promise<void> {+ if (!bootSyncPromise) {+ bootSyncPromise = (async () => {+ const redis = getRedisConnection() as unknown as WebRiskRedis;+ const synced = await runThreatListSyncPass({ force: true });+ if (synced) return;+ // Another process is syncing right now β wait for the lock to clear,+ // bounded by the lock TTL (plus padding) so we cover the longest+ // possible sync: three sequential list fetches can exceed a minute.+ // Callers stay responsive regardless: the provider awaits this behind+ // the request's abort signal, and lists appearing mid-wait end it.+ const store = getWebRiskListStore();+ const deadline = Date.now() + LOCK_TTL_SECONDS * 1000 + 30_000;+ while (Date.now() < deadline) {+ await new Promise(resolve => setTimeout(resolve, 500));+ if ((await redis.get(LOCK_KEY)) === null) return;+ // The other process swaps pointers list-by-list; once every list is+ // present we don't need to outwait its lock.+ const pointers = await store.getPointers();+ if ([...pointers.values()].every(pointer => pointer !== null)) return;+ }+ })().catch(error => {+ // Do not cache a failed boot sync: the next check retries it.+ bootSyncPromise = null;+ throw error;+ });+ }+ return bootSyncPromise;+}diff --git a/apps/api/src/lib/threat-protection/providers/web-risk/testing.ts b/apps/api/src/lib/threat-protection/providers/web-risk/testing.tsnew file mode 100644index 000000000..acc8d42b3--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/web-risk/testing.ts@@ -0,0 +1,180 @@+import { createHash } from "crypto";+import type http from "http";+import { generateHostExpressions } from "./canonicalize";+import { WEB_RISK_THREAT_TYPES, type WebRiskThreatType } from "./store";++// Test helpers for the Web Risk Update API provider β shared by the unit+// suites in this directory and the threat-protection snips (which mock the+// Google endpoints with a local HTTP server). Not used in production code.+// NOTE: the fake Redis lives in ./fake-redis.ts (imported by vi.mock+// factories, which must not transitively import services/queue-service).++export { createFakeWebRiskRedis } from "./fake-redis";++export function sha256(data: string | Buffer): Buffer {+ return createHash("sha256").update(data).digest();+}++/** SHA-256 of the primary (exact-host) lookup expression for a domain. */+export function domainExpressionHash(domain: string): Buffer {+ return sha256(generateHostExpressions(domain)[0]);+}++/**+ * In-memory model of Google's threat lists, able to answer both mocked+ * endpoints: threatLists:computeDiff (always a full RESET of the current+ * state β exactly what a fresh syncer needs) and hashes:search.+ */+export class WebRiskMockDatabase {+ private fullHashesByType = new Map<WebRiskThreatType, Buffer[]>();+ /** Entries in the list with no confirmable full hash (collision cases). */+ private prefixOnlyByType = new Map<WebRiskThreatType, Buffer[]>();++ /** Flags a domain: its exact-host expression hash joins the list. */+ addRiskyDomain(+ domain: string,+ threatType: WebRiskThreatType = "MALWARE",+ ): void {+ this.addFullHash(domainExpressionHash(domain), threatType);+ }++ addFullHash(fullHash: Buffer, threatType: WebRiskThreatType): void {+ const hashes = this.fullHashesByType.get(threatType) ?? [];+ hashes.push(fullHash);+ this.fullHashesByType.set(threatType, hashes);+ }++ /**+ * Puts only a 4-byte prefix in the list, whose hashes:search confirmation+ * returns a full hash that does NOT match the caller's expression (i.e. a+ * prefix collision): pass the full hash of some *other* expression that+ * shares the target's 4-byte prefix.+ */+ addCollidingFullHash(fullHash: Buffer, threatType: WebRiskThreatType): void {+ const hashes = this.prefixOnlyByType.get(threatType) ?? [];+ hashes.push(fullHash);+ this.prefixOnlyByType.set(threatType, hashes);+ }++ private entriesFor(threatType: WebRiskThreatType): Buffer[] {+ const prefixes = [+ ...(this.fullHashesByType.get(threatType) ?? []),+ ...(this.prefixOnlyByType.get(threatType) ?? []),+ ].map(hash => hash.subarray(0, 4));+ const unique = new Map(prefixes.map(p => [p.toString("hex"), p]));+ return [...unique.values()].sort(Buffer.compare);+ }++ /** RESET response body for a threatLists:computeDiff request. */+ computeDiffResponse(threatType: string): object {+ const entries = this.entriesFor(threatType as WebRiskThreatType);+ const checksum = createHash("sha256");+ for (const entry of entries) checksum.update(entry);+ return {+ responseType: "RESET",+ additions:+ entries.length > 0+ ? {+ rawHashes: [+ {+ prefixSize: 4,+ rawHashes: Buffer.concat(entries).toString("base64"),+ },+ ],+ }+ : undefined,+ newVersionToken: Buffer.from(+ `token-${threatType}-${Date.now()}`,+ ).toString("base64"),+ checksum: { sha256: checksum.digest("base64") },+ recommendedNextDiff: new Date(Date.now() + 30 * 60 * 1000).toISOString(),+ };+ }++ /** Response body for a hashes:search request. */+ hashesSearchResponse(hashPrefixBase64: string): object {+ const prefix = Buffer.from(hashPrefixBase64, "base64");+ const threats: object[] = [];+ for (const threatType of WEB_RISK_THREAT_TYPES) {+ const candidates = [+ ...(this.fullHashesByType.get(threatType) ?? []),+ ...(this.prefixOnlyByType.get(threatType) ?? []),+ ];+ for (const fullHash of candidates) {+ if (+ prefix.length <= fullHash.length &&+ fullHash.subarray(0, prefix.length).equals(prefix)+ ) {+ threats.push({+ hash: fullHash.toString("base64"),+ threatTypes: [threatType],+ expireTime: new Date(Date.now() + 3600_000).toISOString(),+ });+ }+ }+ }+ return {+ ...(threats.length > 0 ? { threats } : {}),+ negativeExpireTime: new Date(Date.now() + 3600_000).toISOString(),+ };+ }+}++interface WebRiskMockCounters {+ computeDiffRequests: number;+ hashesSearchRequests: number;+ /** hashes:search requests per 4-byte prefix (hex). */+ hashesSearchByPrefixHex: Map<string, number>;+ hashesSearchRequestsForDomain(domain: string): number;+}++/**+ * HTTP handler for the two mocked Web Risk endpoints. Returns true when the+ * request was handled (so callers can chain their own routes).+ */+export function createWebRiskMockHandler(+ db: WebRiskMockDatabase,+ counters: WebRiskMockCounters,+): (req: http.IncomingMessage, res: http.ServerResponse) => boolean {+ return (req, res) => {+ const url = new URL(req.url ?? "/", "http://localhost");+ if (url.pathname === "/v1/threatLists:computeDiff") {+ counters.computeDiffRequests++;+ res.setHeader("Content-Type", "application/json");+ res.end(+ JSON.stringify(+ db.computeDiffResponse(url.searchParams.get("threatType") ?? ""),+ ),+ );+ return true;+ }+ if (url.pathname === "/v1/hashes:search") {+ counters.hashesSearchRequests++;+ const prefixB64 = url.searchParams.get("hashPrefix") ?? "";+ const prefixHex = Buffer.from(prefixB64, "base64").toString("hex");+ counters.hashesSearchByPrefixHex.set(+ prefixHex,+ (counters.hashesSearchByPrefixHex.get(prefixHex) ?? 0) + 1,+ );+ res.setHeader("Content-Type", "application/json");+ res.end(JSON.stringify(db.hashesSearchResponse(prefixB64)));+ return true;+ }+ return false;+ };+}++export function createWebRiskMockCounters(): WebRiskMockCounters {+ const counters: WebRiskMockCounters = {+ computeDiffRequests: 0,+ hashesSearchRequests: 0,+ hashesSearchByPrefixHex: new Map(),+ hashesSearchRequestsForDomain(domain: string): number {+ const prefixHex = domainExpressionHash(domain)+ .subarray(0, 4)+ .toString("hex");+ return counters.hashesSearchByPrefixHex.get(prefixHex) ?? 0;+ },+ };+ return counters;+}diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.tsnew file mode 100644index 000000000..f71e5014c--- /dev/null+++ b/apps/api/src/lib/threat-protection/request.ts@@ -0,0 +1,157 @@+import type { TeamFlags } from "../../controllers/v2/types";+import { getThreatProtection } from "../zdr-helpers";+import { checkDomain, type ThreatCheckContext } from "./index";+import {+ getOrgIdForTeam,+ getOrgThreatProtectionConfig,+ resolveEffectivePolicy,+ type OrgThreatProtectionConfig,+} from "./store";+import type { ThreatDecision, ThreatProtectionPolicy } from "./types";+import { normalizeDomain } from "./verdict";++// Controller-layer glue for threat protection enforcement. One helper+// (resolveThreatProtection) is shared by every endpoint: it turns+// (team flags + org config + per-request override) into an effective+// ThreatProtectionPolicy β or null, meaning the feature is off for this+// request (the enforcement hot path then does zero extra work).++const SUPPORT_EMAIL = "[email protected]";++export const THREAT_PROTECTION_NOT_ENABLED_MESSAGE = `Threat protection is an enterprise feature and is not enabled for your team. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`;++export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE =+ "Per-request threat protection overrides are disabled by your organization's threat protection configuration.";++export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE =+ 'Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';++interface ResolvedThreatProtection {+ /** Set when the request must be rejected with a 403. */+ error?: string;+ /** Effective policy, or null when the feature is off for this request. */+ policy: ThreatProtectionPolicy | null;+ /** Loaded org config β pass to checkPermissions' third parameter. */+ orgConfig: OrgThreatProtectionConfig | null;+}++/**+ * Resolves the effective threat protection policy for an API request.+ *+ * - Team flag "disabled"/absent: any per-request override β 403; otherwise the+ * feature is entirely off (`policy: null`) with zero I/O.+ * - Flag "allowed"/"forced": the org config (60s-cached) is loaded; a request+ * override does field-level replacement IF the org allows overrides,+ * otherwise β 403.+ * - Flag "forced": an override may never set `mode: "off"` β 403.+ * - Effective mode "off" resolves to `policy: null` so callers can skip all+ * enforcement work.+ */+export async function resolveThreatProtection(args: {+ teamId: string;+ orgId?: string | null;+ flags: TeamFlags;+ override?: Partial<ThreatProtectionPolicy>;+}): Promise<ResolvedThreatProtection> {+ const flagMode = getThreatProtection(args.flags);++ if (flagMode !== "allowed" && flagMode !== "forced") {+ if (args.override !== undefined) {+ return {+ error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE,+ policy: null,+ orgConfig: null,+ };+ }+ return { policy: null, orgConfig: null };+ }++ const orgId = args.orgId ?? (await getOrgIdForTeam(args.teamId));+ const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;++ if (args.override !== undefined) {+ if (orgConfig && !orgConfig.allowRequestOverrides) {+ return {+ error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE,+ policy: null,+ orgConfig,+ };+ }+ if (flagMode === "forced" && args.override.mode === "off") {+ return {+ error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,+ policy: null,+ orgConfig,+ };+ }+ }++ const policy = resolveEffectivePolicy(orgConfig, args.override);+ return {+ policy: policy.mode === "off" ? null : policy,+ orgConfig,+ };+}++interface BlockedUrl {+ url: string;+ domain: string;+ decision: ThreatDecision;+}++interface UrlPolicyCheckResult {+ allowedUrls: string[];+ blocked: BlockedUrl[];+ /** All decisions, keyed by normalized domain β for logging/bookkeeping. */+ decisionsByDomain: Map<string, ThreatDecision>;+}++// Crawls/searches fan out to many URLs on few domains: dedupe by domain and+// check domains concurrently (bounded). Deduplication is strictly scoped to+// this one batch (an in-memory map) β there is no cross-request or persisted+// verdict reuse (ZDR).+const DOMAIN_CHECK_CONCURRENCY = 16;++/**+ * Checks a list of URLs against a policy, deduplicated by domain. Never+ * throws β `checkDomain` resolves provider failures via the policy's+ * failurePolicy. Each call scans a given domain at most once (per-batch+ * in-memory dedup; callers may pass their own `ctx.dedup` to widen the scope+ * to a whole request/job).+ */+export async function checkUrlsAgainstThreatPolicy(+ urls: string[],+ policy: ThreatProtectionPolicy,+ ctx: ThreatCheckContext,+): Promise<UrlPolicyCheckResult> {+ const domains = [...new Set(urls.map(url => normalizeDomain(url)))];+ const dedupCtx: ThreatCheckContext = {+ ...ctx,+ dedup: ctx.dedup ?? new Map(),+ };++ const decisionsByDomain = new Map<string, ThreatDecision>();+ for (let i = 0; i < domains.length; i += DOMAIN_CHECK_CONCURRENCY) {+ const batch = domains.slice(i, i + DOMAIN_CHECK_CONCURRENCY);+ const decisions = await Promise.all(+ batch.map(domain => checkDomain(domain, policy, dedupCtx)),+ );+ batch.forEach((domain, index) =>+ decisionsByDomain.set(domain, decisions[index]),+ );+ }++ const allowedUrls: string[] = [];+ const blocked: BlockedUrl[] = [];+ for (const url of urls) {+ const domain = normalizeDomain(url);+ const decision = decisionsByDomain.get(domain);+ if (decision && !decision.allowed) {+ blocked.push({ url, domain, decision });+ } else {+ allowedUrls.push(url);+ }+ }++ return { allowedUrls, blocked, decisionsByDomain };+}diff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.tsnew file mode 100644index 000000000..d7987607f--- /dev/null+++ b/apps/api/src/lib/threat-protection/store.test.ts@@ -0,0 +1,212 @@+import {+ resolveEffectivePolicy,+ rowToConfig,+ OrgThreatProtectionConfig,+} from "./store";+import {+ THREAT_PROTECTION_POLICY_DEFAULTS,+ ThreatProtectionPolicy,+} from "./types";++const orgPolicy: ThreatProtectionPolicy = {+ mode: "normal",+ riskScoreThreshold: 60,+ blacklist: ["*.bad.example"],+ whitelist: ["example.com"],+ blockedTlds: ["zip"],+ failurePolicy: "open",+};++function makeOrgConfig(+ overrides: Partial<OrgThreatProtectionConfig> = {},+): OrgThreatProtectionConfig {+ return {+ orgId: "00000000-0000-0000-0000-000000000000",+ policy: { ...orgPolicy },+ allowRequestOverrides: true,+ createdAt: null,+ updatedAt: null,+ ...overrides,+ };+}++// =========================================+// rowToConfig β mode column + config jsonb document+// =========================================++const ORG_ID = "00000000-0000-0000-0000-000000000001";++function makeRow(mode: string, config: unknown) {+ return {+ id: "00000000-0000-0000-0000-00000000000a",+ org_id: ORG_ID,+ mode,+ config,+ created_at: "2026-01-01T00:00:00.000Z",+ updated_at: "2026-01-02T00:00:00.000Z",+ };+}++describe("rowToConfig", () => {+ it("maps a full stored document", () => {+ const config = rowToConfig(+ makeRow("normal", {+ riskScoreThreshold: 40,+ blacklist: ["*.bad.example"],+ whitelist: ["example.com"],+ blockedTlds: ["zip"],+ failurePolicy: "open",+ allowRequestOverrides: false,+ }),+ );++ expect(config).toEqual({+ orgId: ORG_ID,+ policy: {+ mode: "normal",+ riskScoreThreshold: 40,+ blacklist: ["*.bad.example"],+ whitelist: ["example.com"],+ blockedTlds: ["zip"],+ failurePolicy: "open",+ },+ allowRequestOverrides: false,+ createdAt: "2026-01-01T00:00:00.000Z",+ updatedAt: "2026-01-02T00:00:00.000Z",+ });+ });++ it("falls back to defaults for an empty config document", () => {+ const config = rowToConfig(makeRow("normal", {}));+ expect(config.policy).toEqual({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ });+ expect(config.allowRequestOverrides).toBe(true);+ });++ it("ignores unknown keys in the document", () => {+ const config = rowToConfig(+ makeRow("normal", {+ riskScoreThreshold: 10,+ someFutureField: { nested: true },+ deniedCategories: ["Malicious"], // retired field, silently dropped+ siem: { url: "https://siem.example.com" }, // retired field, silently dropped+ }),+ );+ expect(config.policy.riskScoreThreshold).toBe(10);+ expect(config.policy).not.toHaveProperty("deniedCategories");+ expect(config.policy).not.toHaveProperty("someFutureField");+ expect(config).not.toHaveProperty("siem");+ });++ it("fills defaults for a partial document", () => {+ const config = rowToConfig(+ makeRow("normal", { blacklist: ["bad.example"] }),+ );+ expect(config.policy).toEqual({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ blacklist: ["bad.example"],+ });+ });++ it("never throws on field-level garbage β bad fields fall back to defaults", () => {+ const config = rowToConfig(+ makeRow("normal", {+ riskScoreThreshold: "not-a-number",+ blacklist: "not-an-array",+ failurePolicy: "sideways",+ allowRequestOverrides: "yes",+ }),+ );+ expect(config.policy).toEqual({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ });+ expect(config.allowRequestOverrides).toBe(true);+ });++ it("never throws when the document is not an object at all", () => {+ for (const config of [null, "garbage", 42, ["array"]]) {+ const parsed = rowToConfig(makeRow("normal", config));+ expect(parsed.policy).toEqual({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ });+ }+ });++ it("coerces unknown modes (including the retired enhanced) to off", () => {+ expect(rowToConfig(makeRow("off", {})).policy.mode).toBe("off");+ expect(rowToConfig(makeRow("enhanced", {})).policy.mode).toBe("off");+ expect(rowToConfig(makeRow("garbage", {})).policy.mode).toBe("off");+ });+});++describe("resolveEffectivePolicy", () => {+ it("returns mode off with defaults when there is no org config", () => {+ expect(resolveEffectivePolicy(null)).toEqual({+ mode: "off",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ });+ });++ it("returns the org policy when there is no request override", () => {+ expect(resolveEffectivePolicy(makeOrgConfig())).toEqual(orgPolicy);+ });++ it("does field-level replacement from the request override", () => {+ const effective = resolveEffectivePolicy(makeOrgConfig(), {+ riskScoreThreshold: 90,+ blockedTlds: ["mov"],+ });+ expect(effective).toEqual({+ ...orgPolicy,+ riskScoreThreshold: 90,+ blockedTlds: ["mov"],+ });+ });++ it("replaces arrays wholesale instead of merging them", () => {+ const effective = resolveEffectivePolicy(makeOrgConfig(), {+ blacklist: ["*.other.example"],+ });+ expect(effective.blacklist).toEqual(["*.other.example"]);+ });++ it("ignores undefined fields in the override", () => {+ const effective = resolveEffectivePolicy(makeOrgConfig(), {+ riskScoreThreshold: undefined,+ failurePolicy: "closed",+ });+ expect(effective.riskScoreThreshold).toBe(60);+ expect(effective.failurePolicy).toBe("closed");+ });++ it("applies overrides on top of defaults when there is no org config", () => {+ const effective = resolveEffectivePolicy(null, {+ mode: "normal",+ riskScoreThreshold: 10,+ });+ expect(effective).toEqual({+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ riskScoreThreshold: 10,+ });+ });++ it("ignores the override when the org disables request overrides", () => {+ const effective = resolveEffectivePolicy(+ makeOrgConfig({ allowRequestOverrides: false }),+ { mode: "off", riskScoreThreshold: 0 },+ );+ expect(effective).toEqual(orgPolicy);+ });++ it("does not mutate the org config", () => {+ const orgConfig = makeOrgConfig();+ resolveEffectivePolicy(orgConfig, { riskScoreThreshold: 1 });+ expect(orgConfig.policy).toEqual(orgPolicy);+ });+});diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.tsnew file mode 100644index 000000000..e7bde17b8--- /dev/null+++ b/apps/api/src/lib/threat-protection/store.ts@@ -0,0 +1,256 @@+import { eq } from "drizzle-orm";+import { z } from "zod";+import { db, dbRr } from "../../db/connection";+import * as schema from "../../db/schema";+import { deleteKey, getValue, setValue } from "../../services/redis";+import { logger as _logger } from "../logger";+import {+ THREAT_PROTECTION_POLICY_DEFAULTS,+ type ThreatProtectionPolicy,+} from "./types";+import type { ThreatProtectionConfigInput } from "./config";++const logger = _logger.child({ module: "threat-protection-store" });++// Short TTL so config changes apply without a redeploy while keeping the+// enforcement hot path off Postgres. Invalidated on write.+//+// ZDR boundary: this cache (and the teamβorg map below) stores the org's OWN+// configuration β never scrape-derived data (no target domains, URLs, or+// verdicts) β so it is compatible with zero-data-retention requirements and+// deliberately survives the removal of the verdict cache (ENG-5004).+const CACHE_TTL_SECONDS = 60;++const cacheKey = (orgId: string) => `threat-protection-config:${orgId}`;++/**+ * Whether an error is Postgres 42P01 (undefined_table) for our config table.+ * During the rollout window the enforcement code can be deployed before the+ * `threat_protection_config` DDL has been applied β in that case the feature+ * behaves as "not configured" instead of failing every flagged-team request.+ * Any other database error still propagates (we do NOT silently fail open on+ * transient failures).+ */+function isMissingTableError(error: unknown): boolean {+ let current: unknown = error;+ for (let depth = 0; depth < 5 && current; depth++) {+ const code = (current as { code?: unknown }).code;+ if (code === "42P01") return true;+ current = (current as { cause?: unknown }).cause;+ }+ return false;+}++export interface OrgThreatProtectionConfig {+ orgId: string;+ policy: ThreatProtectionPolicy;+ allowRequestOverrides: boolean;+ createdAt: string | null;+ updatedAt: string | null;+}++type ThreatProtectionConfigRow =+ typeof schema.threat_protection_config.$inferSelect;++/**+ * Tolerant schema for the `config` jsonb document stored alongside `mode`.+ * The document is written exclusively by {@link upsertOrgThreatProtectionConfig}+ * (from API-validated input), but reads must survive anything: a missing or+ * partial document, unknown keys, or field-level garbage all fall back to the+ * field defaults instead of throwing β a bad row must never take down the+ * enforcement hot path.+ */+const storedConfigDocumentSchema = z+ .object({+ riskScoreThreshold: z+ .number()+ .int()+ .min(0)+ .max(100)+ .catch(THREAT_PROTECTION_POLICY_DEFAULTS.riskScoreThreshold),+ blacklist: z.array(z.string()).catch([]),+ whitelist: z.array(z.string()).catch([]),+ blockedTlds: z.array(z.string()).catch([]),+ failurePolicy: z+ .enum(["open", "closed"])+ .catch(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),+ allowRequestOverrides: z.boolean().catch(true),+ })+ .catch({+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ allowRequestOverrides: true,+ });++/**+ * Maps a `threat_protection_config` row (mode column + config jsonb document)+ * to the runtime config shape. Tolerant by construction: never throws, even+ * for empty/partial/unknown-keyed documents (see+ * {@link storedConfigDocumentSchema}). Exported for tests.+ */+export function rowToConfig(+ row: ThreatProtectionConfigRow,+): OrgThreatProtectionConfig {+ const doc = storedConfigDocumentSchema.parse(row.config ?? {});+ return {+ orgId: row.org_id,+ policy: {+ mode: row.mode === "normal" ? "normal" : "off",+ riskScoreThreshold: doc.riskScoreThreshold,+ blacklist: doc.blacklist,+ whitelist: doc.whitelist,+ blockedTlds: doc.blockedTlds,+ failurePolicy: doc.failurePolicy,+ },+ allowRequestOverrides: doc.allowRequestOverrides,+ createdAt: row.created_at,+ updatedAt: row.updated_at,+ };+}++/**+ * Reads the org's threat protection config, with a short Redis cache+ * (~60s TTL, negative results included) so the enforcement hot path+ * doesn't hit Postgres per scrape. Invalidated by+ * {@link upsertOrgThreatProtectionConfig}.+ */+export async function getOrgThreatProtectionConfig(+ orgId: string,+): Promise<OrgThreatProtectionConfig | null> {+ const key = cacheKey(orgId);++ try {+ const cached = await getValue(key);+ if (cached !== null) {+ return JSON.parse(cached) as OrgThreatProtectionConfig | null;+ }+ } catch (error) {+ logger.warn("Failed to read threat protection config cache", {+ error,+ orgId,+ });+ }++ let rows: ThreatProtectionConfigRow[];+ try {+ rows = await dbRr+ .select()+ .from(schema.threat_protection_config)+ .where(eq(schema.threat_protection_config.org_id, orgId))+ .limit(1);+ } catch (error) {+ if (isMissingTableError(error)) {+ logger.warn(+ "threat_protection_config table does not exist yet; treating as unconfigured",+ { orgId },+ );+ rows = [];+ } else {+ throw error;+ }+ }++ const config = rows[0] ? rowToConfig(rows[0]) : null;++ try {+ await setValue(key, JSON.stringify(config), CACHE_TTL_SECONDS);+ } catch (error) {+ logger.warn("Failed to write threat protection config cache", {+ error,+ orgId,+ });+ }++ return config;+}++/**+ * Full-document upsert of the org's threat protection config. Invalidates+ * the read cache on success.+ */+export async function upsertOrgThreatProtectionConfig(+ orgId: string,+ config: ThreatProtectionConfigInput,+): Promise<OrgThreatProtectionConfig> {+ const values = {+ org_id: orgId,+ mode: config.mode,+ // Everything but the mode lives in the jsonb document.+ config: {+ riskScoreThreshold: config.riskScoreThreshold,+ blacklist: config.blacklist,+ whitelist: config.whitelist,+ blockedTlds: config.blockedTlds,+ failurePolicy: config.failurePolicy,+ allowRequestOverrides: config.allowRequestOverrides,+ },+ };++ const [row] = await db+ .insert(schema.threat_protection_config)+ .values(values)+ .onConflictDoUpdate({+ target: schema.threat_protection_config.org_id,+ set: {+ ...values,+ updated_at: new Date().toISOString(),+ },+ })+ .returning();++ try {+ await deleteKey(cacheKey(orgId));+ } catch (error) {+ logger.warn("Failed to invalidate threat protection config cache", {+ error,+ orgId,+ });+ }++ return rowToConfig(row);+}++/**+ * Resolves the org_id for a team. Used by the org-level config API when the+ * auth chunk doesn't carry org_id.+ */+export async function getOrgIdForTeam(teamId: string): Promise<string | null> {+ const rows = await dbRr+ .select({ org_id: schema.teams.org_id })+ .from(schema.teams)+ .where(eq(schema.teams.id, teamId))+ .limit(1);+ return rows[0]?.org_id ?? null;+}++/**+ * Computes the effective policy for a request.+ *+ * - No org config β mode "off" with defaults.+ * - A request override does field-level replacement on top of the org policy,+ * unless the org locked overrides down (`allowRequestOverrides: false`),+ * in which case it is ignored (the request should already have been+ * rejected by `checkPermissions`; this is defense in depth).+ */+export function resolveEffectivePolicy(+ orgConfig: OrgThreatProtectionConfig | null,+ requestOverride?: Partial<ThreatProtectionPolicy>,+): ThreatProtectionPolicy {+ const base: ThreatProtectionPolicy = orgConfig+ ? { ...orgConfig.policy }+ : { mode: "off", ...THREAT_PROTECTION_POLICY_DEFAULTS };++ if (!requestOverride || (orgConfig && !orgConfig.allowRequestOverrides)) {+ return base;+ }++ for (const key of Object.keys(requestOverride) as Array<+ keyof ThreatProtectionPolicy+ >) {+ const value = requestOverride[key];+ if (value !== undefined) {+ (base as unknown as Record<string, unknown>)[key] = value;+ }+ }++ return base;+}diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.tsnew file mode 100644index 000000000..b21282820--- /dev/null+++ b/apps/api/src/lib/threat-protection/types.ts@@ -0,0 +1,79 @@+// Shared type contract for the threat protection feature.+// NOTE: concurrent in-flight branches create this exact file β do not modify without coordinating.++export type ThreatProtectionMode = "off" | "normal";++export interface ThreatProtectionPolicy {+ mode: ThreatProtectionMode;+ /** Normalized 0-100; verdicts at or above this score are blocked. */+ riskScoreThreshold: number;+ /** Exact domains or globs like "*.example.com". Blocks without a provider call. */+ blacklist: string[];+ /** Exact domains or globs. Allows without a provider call; wins over everything. */+ whitelist: string[];+ /** Lowercase TLDs without leading dot, e.g. "zip". Blocks without a provider call. */+ blockedTlds: string[];+ /** Behavior when the provider is unavailable: "closed" blocks, "open" allows. */+ failurePolicy: "open" | "closed";+}++export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<+ ThreatProtectionPolicy,+ "mode"+> = {+ riskScoreThreshold: 75,+ blacklist: [],+ whitelist: [],+ blockedTlds: [],+ failurePolicy: "closed",+};++// Only Google Web Risk today. The union (and the provider seam around it β+// separate provider modules under ./providers, the modeβprovider dispatch in+// ./index.ts, and the provider-agnostic verdict engine in ./verdict.ts) is+// deliberately kept so future partner classifiers slot in as new members.+export type ThreatProvider = "google-web-risk";++export interface RawVerdict {+ provider: ThreatProvider;+ /** Normalized 0-100, higher = riskier; null if the provider gave no score. */+ riskScore: number | null;+ /** Provider threat categories (Web Risk threat types map through here). */+ categories: string[];+ /**+ * Always false: verdicts are never persisted (ZDR) β repeated checks within+ * one request share the same in-flight decision via the request-scoped+ * dedup handle instead of a cache. Kept for contract stability.+ */+ fromCache: boolean;+ /** Raw provider payload, surfaced only in server logs. */+ raw: unknown;+}++/**+ * Request/job-scoped dedup handle for {@link import("./index").checkDomain}.+ * Call sites create one per request or job (never shared across requests, and+ * never persisted) so that repeated checks of the same domain within that+ * scope β e.g. a scrape plus its redirect re-check, or one crawl-discovery+ * batch β share a single in-flight decision instead of re-scanning. Keyed by+ * normalized domain only: within one request the effective policy is+ * constant, so the domain fully identifies the decision.+ */+export type ThreatCheckDedup = Map<string, Promise<ThreatDecision>>;++export type ThreatDecisionRule =+ | "whitelist"+ | "blacklist"+ | "blocked-tld"+ | "risk-score"+ | "provider-failure"+ | "default-allow";++export interface ThreatDecision {+ allowed: boolean;+ rule: ThreatDecisionRule;+ /** True if a provider verdict (fresh OR cached) was consulted β this drives billing (+2 per scanned domain). */+ providerConsulted: boolean;+ verdict: RawVerdict | null;+ mode: ThreatProtectionMode;+}diff --git a/apps/api/src/lib/threat-protection/verdict.test.ts b/apps/api/src/lib/threat-protection/verdict.test.tsnew file mode 100644index 000000000..e089f3f2d--- /dev/null+++ b/apps/api/src/lib/threat-protection/verdict.test.ts@@ -0,0 +1,418 @@+import {+ RawVerdict,+ THREAT_PROTECTION_POLICY_DEFAULTS,+ ThreatDecisionRule,+ ThreatProtectionPolicy,+} from "./types";+import { evaluatePolicy, localOnlyDecision, normalizeDomain } from "./verdict";++function policy(+ overrides: Partial<ThreatProtectionPolicy> = {},+): ThreatProtectionPolicy {+ return {+ mode: "normal",+ ...THREAT_PROTECTION_POLICY_DEFAULTS,+ ...overrides,+ };+}++function verdict(overrides: Partial<RawVerdict> = {}): RawVerdict {+ return {+ provider: "google-web-risk",+ riskScore: 0,+ categories: [],+ fromCache: false,+ raw: {},+ ...overrides,+ };+}++describe("evaluatePolicy", () => {+ type Case = {+ name: string;+ domain: string;+ verdict: RawVerdict | null;+ policy: ThreatProtectionPolicy;+ allowed: boolean;+ rule: ThreatDecisionRule;+ };++ const cases: Case[] = [+ // --- whitelist ---+ {+ name: "whitelist exact match allows",+ domain: "example.com",+ verdict: verdict(),+ policy: policy({ whitelist: ["example.com"] }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist exact entry also matches subdomains",+ domain: "deep.sub.example.com",+ verdict: verdict(),+ policy: policy({ whitelist: ["example.com"] }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist glob matches multi-label subdomains",+ domain: "a.b.example.com",+ verdict: verdict(),+ policy: policy({ whitelist: ["*.example.com"] }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist glob does not match the apex domain",+ domain: "example.com",+ verdict: verdict({ riskScore: 100 }),+ policy: policy({ whitelist: ["*.example.com"] }),+ allowed: false,+ rule: "risk-score",+ },+ {+ name: "whitelist entries are case-insensitive",+ domain: "example.com",+ verdict: verdict(),+ policy: policy({ whitelist: ["EXAMPLE.com"] }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist does not match unrelated suffix (notexample.com)",+ domain: "notexample.com",+ verdict: verdict({ riskScore: 100 }),+ policy: policy({ whitelist: ["example.com"] }),+ allowed: false,+ rule: "risk-score",+ },+ // --- whitelist precedence: wins over everything ---+ {+ name: "whitelist wins over blacklist",+ domain: "example.com",+ verdict: verdict(),+ policy: policy({+ whitelist: ["example.com"],+ blacklist: ["example.com"],+ }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist wins over blocked TLD",+ domain: "good.zip",+ verdict: verdict(),+ policy: policy({ whitelist: ["good.zip"], blockedTlds: ["zip"] }),+ allowed: true,+ rule: "whitelist",+ },+ {+ name: "whitelist wins over a maximally risky verdict",+ domain: "example.com",+ verdict: verdict({+ riskScore: 100,+ categories: ["MALWARE"],+ }),+ policy: policy({+ whitelist: ["example.com"],+ riskScoreThreshold: 10,+ }),+ allowed: true,+ rule: "whitelist",+ },+ // --- blacklist ---+ {+ name: "blacklist exact match blocks",+ domain: "bad.com",+ verdict: verdict(),+ policy: policy({ blacklist: ["bad.com"] }),+ allowed: false,+ rule: "blacklist",+ },+ {+ name: "blacklist exact entry also blocks subdomains",+ domain: "cdn.bad.com",+ verdict: verdict(),+ policy: policy({ blacklist: ["bad.com"] }),+ allowed: false,+ rule: "blacklist",+ },+ {+ name: "blacklist glob blocks matching subdomains",+ domain: "evil.bad.com",+ verdict: verdict(),+ policy: policy({ blacklist: ["*.bad.com"] }),+ allowed: false,+ rule: "blacklist",+ },+ {+ name: "blacklist wins over blocked TLD (rule ordering)",+ domain: "bad.zip",+ verdict: verdict(),+ policy: policy({ blacklist: ["bad.zip"], blockedTlds: ["zip"] }),+ allowed: false,+ rule: "blacklist",+ },+ // --- blocked-tld ---+ {+ name: "blocked TLD blocks matching domains",+ domain: "archive.zip",+ verdict: verdict(),+ policy: policy({ blockedTlds: ["zip"] }),+ allowed: false,+ rule: "blocked-tld",+ },+ {+ name: "blocked TLD respects label boundaries (foozip.com passes)",+ domain: "foozip.com",+ verdict: verdict(),+ policy: policy({ blockedTlds: ["zip"] }),+ allowed: true,+ rule: "default-allow",+ },+ {+ name: "blocked TLD supports multi-label suffixes",+ domain: "shop.co.uk",+ verdict: verdict(),+ policy: policy({ blockedTlds: ["co.uk"] }),+ allowed: false,+ rule: "blocked-tld",+ },+ {+ name: "blocked TLD tolerates a leading dot in the entry",+ domain: "archive.zip",+ verdict: verdict(),+ policy: policy({ blockedTlds: [".zip"] }),+ allowed: false,+ rule: "blocked-tld",+ },+ {+ name: "blocked TLD wins over risk score (rule ordering)",+ domain: "archive.zip",+ verdict: verdict({ riskScore: 100 }),+ policy: policy({ blockedTlds: ["zip"], riskScoreThreshold: 75 }),+ allowed: false,+ rule: "blocked-tld",+ },+ // --- risk-score ---+ {+ name: "score at the threshold is blocked",+ domain: "example.com",+ verdict: verdict({ riskScore: 75 }),+ policy: policy({ riskScoreThreshold: 75 }),+ allowed: false,+ rule: "risk-score",+ },+ {+ name: "score above the threshold is blocked",+ domain: "example.com",+ verdict: verdict({ riskScore: 100 }),+ policy: policy({ riskScoreThreshold: 75 }),+ allowed: false,+ rule: "risk-score",+ },+ {+ name: "score below the threshold is allowed",+ domain: "example.com",+ verdict: verdict({ riskScore: 74 }),+ policy: policy({ riskScoreThreshold: 75 }),+ allowed: true,+ rule: "default-allow",+ },+ {+ name: "null score never triggers the risk-score rule",+ domain: "example.com",+ verdict: verdict({ riskScore: null }),+ policy: policy({ riskScoreThreshold: 0 }),+ allowed: true,+ rule: "default-allow",+ },+ {+ name: "verdict categories alone never block (no category rules)",+ domain: "example.com",+ verdict: verdict({ riskScore: 0, categories: ["MALWARE"] }),+ policy: policy({ riskScoreThreshold: 75 }),+ allowed: true,+ rule: "default-allow",+ },+ // --- provider-failure ---+ {+ name: "provider failure with fail-closed blocks",+ domain: "example.com",+ verdict: null,+ policy: policy({ failurePolicy: "closed" }),+ allowed: false,+ rule: "provider-failure",+ },+ {+ name: "provider failure with fail-open allows",+ domain: "example.com",+ verdict: null,+ policy: policy({ failurePolicy: "open" }),+ allowed: true,+ rule: "provider-failure",+ },+ {+ name: "local rules still decide when the provider failed",+ domain: "bad.com",+ verdict: null,+ policy: policy({ blacklist: ["bad.com"], failurePolicy: "open" }),+ allowed: false,+ rule: "blacklist",+ },+ {+ name: "mode off with no verdict allows by default (not provider-failure)",+ domain: "example.com",+ verdict: null,+ policy: policy({ mode: "off", failurePolicy: "closed" }),+ allowed: true,+ rule: "default-allow",+ },+ // --- default-allow ---+ {+ name: "clean verdict passes every rule",+ domain: "example.com",+ verdict: verdict(),+ policy: policy({+ riskScoreThreshold: 50,+ blockedTlds: ["zip"],+ }),+ allowed: true,+ rule: "default-allow",+ },+ ];++ test.each(cases)("$name", ({ domain, verdict, policy, allowed, rule }) => {+ const decision = evaluatePolicy(domain, verdict, policy);+ expect(decision.allowed).toBe(allowed);+ expect(decision.rule).toBe(rule);+ expect(decision.mode).toBe(policy.mode);+ expect(decision.verdict).toBe(verdict);+ // Billing invariant: providerConsulted iff a verdict (fresh or cached)+ // was passed in β regardless of which rule decided.+ expect(decision.providerConsulted).toBe(verdict !== null);+ });+});++describe("localOnlyDecision", () => {+ it("resolves whitelisted domains without a provider", () => {+ const decision = localOnlyDecision(+ "example.com",+ policy({ whitelist: ["example.com"] }),+ );+ expect(decision).toEqual({+ allowed: true,+ rule: "whitelist",+ providerConsulted: false,+ verdict: null,+ mode: "normal",+ });+ });++ it("resolves blacklisted domains without a provider", () => {+ const decision = localOnlyDecision(+ "sub.bad.com",+ policy({ blacklist: ["bad.com"] }),+ );+ expect(decision).toMatchObject({+ allowed: false,+ rule: "blacklist",+ providerConsulted: false,+ verdict: null,+ });+ });++ it("resolves blocked TLDs without a provider", () => {+ const decision = localOnlyDecision(+ "archive.zip",+ policy({ blockedTlds: ["zip"] }),+ );+ expect(decision).toMatchObject({+ allowed: false,+ rule: "blocked-tld",+ providerConsulted: false,+ });+ });++ it("prefers the whitelist over blacklist and blocked TLDs", () => {+ const decision = localOnlyDecision(+ "good.zip",+ policy({+ whitelist: ["good.zip"],+ blacklist: ["*.zip", "good.zip"],+ blockedTlds: ["zip"],+ }),+ );+ expect(decision).toMatchObject({ allowed: true, rule: "whitelist" });+ });++ it("returns null when only provider-backed rules could decide", () => {+ expect(+ localOnlyDecision(+ "example.com",+ policy({+ riskScoreThreshold: 10,+ }),+ ),+ ).toBeNull();+ });++ it("normalizes URL-ish input before matching", () => {+ const decision = localOnlyDecision(+ "HTTPS://Sub.Example.COM/some/path?q=1",+ policy({ blacklist: ["example.com"] }),+ );+ expect(decision).toMatchObject({ allowed: false, rule: "blacklist" });+ });+});++describe("normalizeDomain", () => {+ test.each([+ ["Example.COM", "example.com"],+ [" example.com ", "example.com"],+ ["example.com.", "example.com"],+ ["https://sub.example.com/path?q=1", "sub.example.com"],+ ["example.com:8080", "example.com"],+ ["example.com/path", "example.com"],+ // inet_aton-style IP forms canonicalize to dotted-quad, matching the Web+ // Risk provider (prevents blacklist bypass via alternate IP notation).+ ["195.127.0.11", "195.127.0.11"],+ ["3279880203", "195.127.0.11"],+ ["http://3279880203/", "195.127.0.11"],+ // IPv6 literals must not be truncated by port-stripping.+ ["http://[2001:db8::1]/", "[2001:db8::1]"],+ ["http://[2001:db8::1]:8080/", "[2001:db8::1]"],+ ["[2001:db8::1]:8080", "[2001:db8::1]"],+ ["2001:db8::1", "2001:db8::1"],+ ])("normalizes %s to %s", (input, expected) => {+ expect(normalizeDomain(input)).toBe(expected);+ });++ test("a blacklisted IPv6 literal is matched, not truncated to a wrong host", () => {+ const p = policy({ blacklist: ["[2001:db8::1]"] });+ expect(localOnlyDecision("http://[2001:db8::1]:8080/", p)).toMatchObject({+ allowed: false,+ rule: "blacklist",+ });+ });++ test("a blacklisted dotted-quad IP is not bypassed by integer notation", () => {+ const p = policy({ blacklist: ["195.127.0.11"] });+ for (const host of ["http://3279880203/", "195.127.0.11"]) {+ expect(localOnlyDecision(host, p)).toMatchObject({+ allowed: false,+ rule: "blacklist",+ });+ }+ });++ test("a blacklist entry in integer notation blocks the dotted-quad host", () => {+ expect(+ localOnlyDecision(+ "http://195.127.0.11/",+ policy({ blacklist: ["3279880203"] }),+ ),+ ).toMatchObject({ allowed: false, rule: "blacklist" });+ });+});diff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.tsnew file mode 100644index 000000000..2e96247fa--- /dev/null+++ b/apps/api/src/lib/threat-protection/verdict.ts@@ -0,0 +1,174 @@+import type {+ RawVerdict,+ ThreatDecision,+ ThreatProtectionPolicy,+} from "./types";+// Shared host canonicalization (lowercase, dot handling, and inet_aton-style+// IP normalization). The Web Risk provider already canonicalizes hosts this+// way; local blacklist/whitelist rules MUST use the exact same function so+// the two paths can never disagree on what host they're evaluating β e.g. so+// a blacklist entry for "195.127.0.11" is not bypassed by "http://3279880203/".+import { canonicalizeHost } from "./providers/web-risk/canonicalize";++// Pure policy evaluation for threat protection. No I/O in this file β the+// provider/cache orchestration lives in ./index.ts. Rule precedence (fixed):+// whitelist β blacklist β blocked-tld β risk-score β provider-failure β+// default-allow. The engine consumes the provider-agnostic RawVerdict shape+// only (normalized riskScore + categories) β it never knows which provider+// produced the verdict, so new providers slot in without touching it.++/**+ * Whether `domain` matches a single whitelist/blacklist entry.+ *+ * - Glob entries (containing `*`) match literally with `*` expanding to any+ * run of characters, e.g. "*.example.com" matches "a.example.com" and+ * "a.b.example.com" (but not the apex "example.com").+ * - Exact entries match the domain itself AND its subdomains β consistent with+ * how the global blocklist treats domains (see+ * src/scraper/WebScraper/utils/blocklist.ts), and what users expect from+ * listing "example.com".+ */+function domainMatchesEntry(domain: string, entry: string): boolean {+ const trimmed = entry.trim().toLowerCase().replace(/\.$/, "");+ if (!trimmed) return false;++ if (trimmed.includes("*")) {+ const pattern = trimmed+ .split("*")+ .map(part => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&"))+ .join(".*");+ try {+ return new RegExp(`^${pattern}$`).test(domain);+ } catch {+ return false;+ }+ }++ // Canonicalize non-glob entries the same way as the looked-up domain, so an+ // entry written in a non-canonical IP form (e.g. "3279880203") still matches+ // its dotted-quad host and vice versa.+ const normalized = canonicalizeHost(trimmed);+ return domain === normalized || domain.endsWith(`.${normalized}`);+}++function domainMatchesList(domain: string, entries: string[]): boolean {+ return entries.some(entry => domainMatchesEntry(domain, entry));+}++/**+ * Whether the domain's suffix matches a blocked TLD entry. Entries are+ * lowercase without a leading dot ("zip"); multi-label suffixes ("co.uk")+ * also work since we do a label-boundary suffix match.+ */+function matchesBlockedTld(domain: string, blockedTlds: string[]): boolean {+ return blockedTlds.some(tld => {+ const normalized = tld.trim().toLowerCase().replace(/^\./, "");+ if (!normalized) return false;+ return domain === normalized || domain.endsWith(`.${normalized}`);+ });+}++/**+ * Normalize a domain-ish input: strip URL parts/port, then apply the shared+ * host canonicalization (lowercase, dot handling, inet_aton-style IP+ * normalization) so local rule matching sees the same canonical host as the+ * Web Risk provider β closing IP-form blacklist bypasses.+ */+export function normalizeDomain(input: string): string {+ let domain = input.trim();+ if (domain.includes("://")) {+ try {+ domain = new URL(domain).hostname;+ } catch {+ // Not a parseable URL β fall through with the raw string.+ }+ }+ // Strip a path fragment, then the port β carefully, because IPv6 literals+ // contain colons. A bracketed literal ("[2001:db8::1]") keeps everything+ // through the closing bracket (dropping any ":port" after it); a plain+ // host:port has exactly one colon; a bare IPv6 (multiple colons, no+ // brackets) is left intact rather than truncated.+ domain = domain.split("/")[0];+ if (domain.startsWith("[")) {+ const end = domain.indexOf("]");+ if (end !== -1) domain = domain.slice(0, end + 1);+ } else if (domain.split(":").length === 2) {+ domain = domain.split(":")[0];+ }+ return canonicalizeHost(domain);+}++/**+ * Resolve a decision using ONLY local policy rules (whitelist β blacklist β+ * blocked-tld), or null if a provider verdict is needed. Local decisions never+ * consult a provider, so they never set `providerConsulted` (no billing).+ */+export function localOnlyDecision(+ domain: string,+ policy: ThreatProtectionPolicy,+): ThreatDecision | null {+ const normalized = normalizeDomain(domain);+ const base = {+ providerConsulted: false,+ verdict: null,+ mode: policy.mode,+ } as const;++ if (domainMatchesList(normalized, policy.whitelist)) {+ return { allowed: true, rule: "whitelist", ...base };+ }+ if (domainMatchesList(normalized, policy.blacklist)) {+ return { allowed: false, rule: "blacklist", ...base };+ }+ if (matchesBlockedTld(normalized, policy.blockedTlds)) {+ return { allowed: false, rule: "blocked-tld", ...base };+ }+ return null;+}++/**+ * Evaluate the full policy against a provider verdict. `verdict` is null when+ * the provider failed (or was never called) β the org's failurePolicy then+ * decides. `providerConsulted` reflects whether a verdict (fresh or cached)+ * was used, which drives billing.+ */+export function evaluatePolicy(+ domain: string,+ verdict: RawVerdict | null,+ policy: ThreatProtectionPolicy,+): ThreatDecision {+ const base = {+ providerConsulted: verdict !== null,+ verdict,+ mode: policy.mode,+ };++ const local = localOnlyDecision(domain, policy);+ if (local !== null) {+ // Preserve the local rule but reflect any verdict we were given (billing+ // still applies if a provider was consulted before evaluation).+ return { allowed: local.allowed, rule: local.rule, ...base };+ }++ if (verdict !== null) {+ if (+ verdict.riskScore !== null &&+ verdict.riskScore >= policy.riskScoreThreshold+ ) {+ return { allowed: false, rule: "risk-score", ...base };+ }++ return { allowed: true, rule: "default-allow", ...base };+ }++ // No verdict: the provider failed or was unavailable (mode "off" never+ // reaches here via checkDomain). Fail open or closed per the org policy.+ if (policy.mode === "off") {+ return { allowed: true, rule: "default-allow", ...base };+ }+ return {+ allowed: policy.failurePolicy === "open",+ rule: "provider-failure",+ ...base,+ };+}diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex 2860d81be..578940dfd 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -68,6 +68,15 @@ export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {return "disabled";}+/**+ * Resolves the effective threat protection mode from team flags.+ */+export function getThreatProtection(flags: TeamFlags | undefined): OrgFlagMode {+ if (flags?.threatProtection === "forced") return "forced";+ if (flags?.threatProtection === "allowed") return "allowed";+ return "disabled";+}+/*** Resolves the effective customRobotsAgent mode from team flags.* Only supports "disabled" (default) and "allowed" β no "forced" mode.diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex f45bb0a68..f1e9bdfb7 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -60,6 +60,10 @@ import {browserWebhookDestroyedController,} from "../controllers/v2/browser";import { activityController } from "../controllers/v1/activity";+import {+ getTeamThreatProtectionController,+ putTeamThreatProtectionController,+} from "../controllers/v2/team-threat-protection";import { supportProxyController } from "../controllers/v2/support-proxy";import { createResearchRouter } from "../controllers/v2/research-proxy";import {@@ -413,6 +417,18 @@ v2Router.get(wrap(activityController),);+v2Router.get(+ "/team/threat-protection",+ authMiddleware(RateLimiterMode.Account),+ wrap(getTeamThreatProtectionController),+);++v2Router.put(+ "/team/threat-protection",+ authMiddleware(RateLimiterMode.Account),+ wrap(putTeamThreatProtectionController),+);+v2Router.post("/monitor",authMiddleware(RateLimiterMode.Crawl),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 72d624b6c..6e235fe09 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -88,6 +88,14 @@ import path from "node:path";import { tmpdir } from "node:os";import { randomUUID } from "node:crypto";import type { DataLayerScrapeMetadata } from "../../lib/data-layer";+import {+ checkDomain,+ type ThreatCheckDedup,+ type ThreatDecision,+ type ThreatProtectionPolicy,+} from "../../lib/threat-protection";+import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";+import { normalizeDomain } from "../../lib/threat-protection/verdict";export type ScrapeUrlResponse =| {@@ -95,10 +103,18 @@ export type ScrapeUrlResponse =document: Document;unsupportedFeatures?: Set<FeatureFlag>;dataLayer?: DataLayerScrapeMetadata;+ /**+ * Threat protection decisions made for this scrape (initial domain+ * check + any redirect re-checks, in order). Read by the billing layer+ * (`providerConsulted` drives +2/+3 credits) and the security-logging+ * layer. Only set when a threat protection policy was active.+ */+ threatDecisions?: ThreatDecision[];}| {success: false;error: any;+ threatDecisions?: ThreatDecision[];};export type BrowserCookie = {@@ -157,6 +173,8 @@ export type Meta = {winnerEngine?: Engine;abortHandle?: NodeJS.Timeout;audioCookies?: BrowserCookie[];+ /** Threat protection decisions made during this scrape, in order (mutable, like logs). */+ threatDecisions: ThreatDecision[];};function buildFeatureFlags(@@ -451,6 +469,7 @@ async function buildMetaObject(documentPrefetch,fetchPrefetch,costTracking,+ threatDecisions: [],};}@@ -475,6 +494,14 @@ export type InternalOptions = {zeroDataRetention?: boolean;teamFlags?: TeamFlags;+ /**+ * Effective threat protection policy for this scrape, resolved at the+ * controller layer (org config + per-request override). When set (and mode+ * is not "off"), the target domain is checked before any engine work, and+ * redirect destinations are re-checked. Absent => zero enforcement overhead.+ */+ threatProtection?: ThreatProtectionPolicy;+v1Agent?: ScrapeOptionsV1["agent"];v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];v1JSONSystemPrompt?: string;@@ -1084,6 +1111,37 @@ export async function scrapeURL(meta.logger.info("scrapeURL entered");+ // Threat protection: check the target domain BEFORE any engine selection+ // or outbound fetch. The policy is resolved at the controller layer and+ // threaded through internalOptions; absent policy = zero overhead.+ // The dedup map is scoped to this one scrape: the initial check and any+ // redirect re-check on the same domain share a single scan (one billable+ // consulted decision); a cross-domain redirect is a second scan.+ const threatPolicy = internalOptions.threatProtection;+ const threatDedup: ThreatCheckDedup = new Map();+ if (threatPolicy && threatPolicy.mode !== "off") {+ const initialDomain = normalizeDomain(meta.rewrittenUrl ?? meta.url);+ const decision = await checkDomain(initialDomain, threatPolicy, {+ teamId: internalOptions.teamId,+ dedup: threatDedup,+ });+ meta.threatDecisions.push(decision);+ if (!decision.allowed) {+ meta.logger.info("Domain blocked by threat protection policy", {+ domain: initialDomain,+ rule: decision.rule,+ });+ setSpanAttributes(span, {+ "scrape.blocked_by_threat_protection": true,+ });+ return {+ success: false,+ error: new UnsafeDomainBlockedError(initialDomain, decision),+ threatDecisions: meta.threatDecisions,+ };+ }+ }+if (meta.rewrittenUrl) {meta.logger.info("Rewriting URL");setSpanAttributes(span, {@@ -1269,6 +1327,36 @@ export async function scrapeURL(}}+ // Threat protection: if the scrape ended up on a different domain than+ // requested (redirect), re-check the destination domain. This closes+ // the "clean domain redirects to a blocked domain" bypass vector.+ if (threatPolicy && threatPolicy.mode !== "off" && result.success) {+ const finalUrl = result.document.metadata.url;+ const initialDomain = normalizeDomain(meta.rewrittenUrl ?? meta.url);+ const finalDomain = finalUrl ? normalizeDomain(finalUrl) : null;+ if (finalDomain !== null && finalDomain !== initialDomain) {+ const decision = await checkDomain(finalDomain, threatPolicy, {+ teamId: internalOptions.teamId,+ dedup: threatDedup,+ });+ meta.threatDecisions.push(decision);+ if (!decision.allowed) {+ meta.logger.info(+ "Redirect destination blocked by threat protection policy",+ {+ domain: finalDomain,+ initialDomain,+ rule: decision.rule,+ },+ );+ setSpanAttributes(span, {+ "scrape.blocked_by_threat_protection": true,+ });+ throw new UnsafeDomainBlockedError(finalDomain, decision);+ }+ }+ }+meta.logger.debug("scrapeURL metrics", {module: "scrapeURL/metrics",timeTaken: Date.now() - startTime,@@ -1315,7 +1403,9 @@ export async function scrapeURL(result.success && result.document.metadata.cacheState === "hit",});- return result;+ return meta.threatDecisions.length > 0+ ? { ...result, threatDecisions: meta.threatDecisions }+ : result;} catch (error) {// if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {// throw new FEPageLoadFailed();@@ -1433,6 +1523,16 @@ export async function scrapeURL(error,retryStats: error.stats,});+ } else if (error instanceof UnsafeDomainBlockedError) {+ errorType = "UnsafeDomainBlockedError";+ meta.logger.warn(+ "scrapeURL: Domain blocked by threat protection policy",+ {+ error,+ domain: error.domain,+ rule: error.decision.rule,+ },+ );} else if (error instanceof AbortManagerThrownError) {errorType = "AbortManagerThrownError";throw error.inner;@@ -1456,6 +1556,9 @@ export async function scrapeURL(return {success: false,error,+ ...(meta.threatDecisions.length > 0+ ? { threatDecisions: meta.threatDecisions }+ : {}),};}});diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 88261c13f..2c80a200c 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -16,6 +16,10 @@ import {import { applySearchHighlights, highlightsEnvReady } from "./highlights";import { trackSearchResults, trackSearchRequest } from "../lib/tracking";import type { BillingMetadata } from "../services/billing/types";+import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";+import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";+import { normalizeDomain } from "../lib/threat-protection/verdict";+import { calculateThreatScanCredits } from "../lib/scrape-billing";interface SearchOptions {query: string;@@ -48,6 +52,8 @@ interface SearchContext {billing?: BillingMetadata;agentIndexOnly?: boolean;keylessReserved?: boolean;+ /** Effective threat protection policy; blocked domains are removed from results entirely. */+ threatProtectionPolicy?: ThreatProtectionPolicy | null;}interface SearchExecuteResult {@@ -104,6 +110,48 @@ export async function executeSearch(enterprise: options.enterprise,})) as SearchV2Response;+ // Threat protection: remove results on blocked domains entirely β before+ // slicing/counting, before scraping, and before returning. Domain checks+ // are deduped and Redis-cached. Every consulted decision (fresh or cached+ // provider verdict) is a billable scan (+2 per scanned domain), charged+ // as part of the search credits below.+ let threatScanCredits = 0;+ const threatPolicy = context.threatProtectionPolicy;+ if (threatPolicy && threatPolicy.mode !== "off") {+ const urlsToCheck = [+ ...(searchResponse.web ?? []).map(x => x.url),+ ...(searchResponse.news ?? []).map(x => x.url),+ ...(searchResponse.images ?? []).map(x => x.url),+ ].filter((x): x is string => !!x);++ if (urlsToCheck.length > 0) {+ const { decisionsByDomain } = await checkUrlsAgainstThreatPolicy(+ urlsToCheck,+ threatPolicy,+ { teamId },+ );+ threatScanCredits = calculateThreatScanCredits(+ decisionsByDomain.values(),+ );+ const isAllowed = (url: string | undefined | null): boolean => {+ if (!url) return true;+ const decision = decisionsByDomain.get(normalizeDomain(url));+ return decision === undefined || decision.allowed;+ };+ if (searchResponse.web) {+ searchResponse.web = searchResponse.web.filter(x => isAllowed(x.url));+ }+ if (searchResponse.news) {+ searchResponse.news = searchResponse.news.filter(x => isAllowed(x.url));+ }+ if (searchResponse.images) {+ searchResponse.images = searchResponse.images.filter(x =>+ isAllowed(x.url),+ );+ }+ }+ }+if (searchResponse.web && searchResponse.web.length > 0) {searchResponse.web = searchResponse.web.map(result => ({...result,@@ -145,8 +193,13 @@ export async function executeSearch(const isZDR = options.enterprise?.includes("zdr");const creditsPerTenResults = isZDR ? 10 : 2;+ // Threat protection scan fees ride on the search credits: they are part of+ // serving the search itself (every result domain is scanned before+ // filtering), so they bill against the same feature and show up in the+ // request's creditsUsed.const searchCredits =- Math.ceil(totalResultsCount / 10) * creditsPerTenResults;+ Math.ceil(totalResultsCount / 10) * creditsPerTenResults ++ threatScanCredits;let scrapeCredits = 0;const shouldScrape =@@ -171,6 +224,7 @@ export async function executeSearch(billing,agentIndexOnly: context.agentIndexOnly,keylessReserved: context.keylessReserved,+ threatProtectionPolicy: threatPolicy ?? null,};const allDocsWithCostTracking = await scrapeSearchResults(diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex 1c29fcc51..d14f4a3cb 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -10,6 +10,7 @@ import { ScrapeJobData } from "../types";import { SearchV2Response } from "../lib/entities";import type { BillingMetadata } from "../services/billing/types";import { getScrapeZDR } from "../lib/zdr-helpers";+import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";export interface DocumentWithCostTracking {document: Document;@@ -40,6 +41,7 @@ interface ScrapeSearchOptions {billing?: BillingMetadata;agentIndexOnly?: boolean;keylessReserved?: boolean;+ threatProtectionPolicy?: ThreatProtectionPolicy | null;}async function scrapeSearchResultDirect(@@ -81,6 +83,7 @@ async function scrapeSearchResultDirect(zeroDataRetention,teamFlags: flags,agentIndexOnly: options.agentIndexOnly ?? false,+ threatProtection: options.threatProtectionPolicy ?? undefined,},skipNuq: true,origin: options.origin,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 6bb671d65..a9441b613 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -19,6 +19,7 @@ import {addCrawlJobDone,crawlToCrawler,recordRobotsBlocked,+ recordThreatBlocked,finishCrawlKickoff,generateURLPermutations,getCrawl,@@ -29,10 +30,13 @@ import {setCrawlError,StoredCrawl,} from "../../lib/crawl-redis";+import { checkUrlsAgainstThreatPolicy } from "../../lib/threat-protection/request";+import type { ThreatDecision } from "../../lib/threat-protection/types";import { redisEvictConnection } from "../redis";import {resolveBillingMetadata,toAutumnBillingProperties,+ type BillingMetadata,} from "../billing/types";import {autumnService,@@ -58,7 +62,11 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { generateURLSplits, queryIndexAtSplitLevel } from "../index";import { WebCrawler } from "../../scraper/WebScraper/crawler";-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";+import {+ calculateCreditsToBeBilled,+ calculateThreatScanCredits,+} from "../../lib/scrape-billing";+import { billTeam } from "../billing/credit_billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";import {@@ -112,6 +120,7 @@ async function billScrapeJob(error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,dataLayer?: DataLayerScrapeMetadata,+ threatDecisions?: ThreatDecision[],) {let creditsToBeBilled: number | null = null;const billing = resolveBillingMetadata({@@ -140,6 +149,7 @@ async function billScrapeJob(error,unsupportedFeatures,dataLayer,+ threatDecisions,);// Charge the keyless free tier's per-IP daily credit budget unless this@@ -215,6 +225,47 @@ async function billScrapeJob(return creditsToBeBilled;}+/**+ * Bills threat protection scan fees for crawl-discovered URLs that were+ * blocked by the policy and therefore never become scrape jobs. Only blocked+ * decisions bill here: allowed discoveries are billed by their own scrape+ * job, which re-checks the (cached) verdict. Deduplicates by domain, and only+ * decisions that consulted the classifier (fresh or cached provider verdict)+ * carry a fee (+2 per scanned domain) β local-only blocks (e.g. blacklist)+ * are free.+ */+function billThreatBlockedDiscoveries(+ args: {+ teamId: string;+ apiKeyId: number | null;+ billing: BillingMetadata;+ bypassBilling: boolean;+ },+ blocked: { domain: string; decision: ThreatDecision }[],+ logger: Logger,+) {+ if (args.bypassBilling) return;+ const blockedDecisionsByDomain = new Map(+ blocked.map(x => [x.domain, x.decision]),+ );+ const threatScanCredits = calculateThreatScanCredits(+ blockedDecisionsByDomain.values(),+ );+ if (threatScanCredits <= 0) return;+ billTeam(+ args.teamId,+ undefined,+ threatScanCredits,+ args.apiKeyId,+ args.billing,+ ).catch(error => {+ logger.error(+ `Failed to bill team ${args.teamId} for ${threatScanCredits} threat scan credit(s)`,+ { error },+ );+ });+}+async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const logger = _logger.child({module: "queue-worker",@@ -244,6 +295,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {: undefined;const signal = abortController.signal;+ // Hoisted above the try so the catch path can read pipeline.threatDecisions+ // for billing/logging even when the scrape failed.+ let pipeline: ScrapeUrlResponse | null = null;+try {if (remainingTime !== undefined && remainingTime < 0) {throw new ScrapeJobTimeoutError();@@ -256,7 +311,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}- let pipeline: ScrapeUrlResponse | null = null;let timeoutHandle: NodeJS.Timeout | null = null;try {pipeline = await Promise.race([@@ -458,7 +512,60 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}- for (const link of links.links) {+ // Threat protection: silently skip discovered links on blocked+ // domains (cross-domain links included) β the crawl continues.+ // Skipped URLs + decisions are recorded for the security-logging+ // layer. Domain checks are deduped and rely on the Redis verdict+ // cache, so many links on few domains stay cheap.+ let discoveredLinks = links.links;+ const threatPolicy = job.data.internalOptions?.threatProtection;+ if (+ threatPolicy &&+ threatPolicy.mode !== "off" &&+ discoveredLinks.length > 0+ ) {+ const { allowedUrls, blocked } =+ await checkUrlsAgainstThreatPolicy(+ discoveredLinks,+ threatPolicy,+ { teamId: job.data.team_id },+ );+ discoveredLinks = allowedUrls;+ for (const blockedUrl of blocked) {+ await recordThreatBlocked(+ job.data.crawl_id,+ blockedUrl.url,+ blockedUrl.decision,+ );+ }+ if (blocked.length > 0) {+ billThreatBlockedDiscoveries(+ {+ teamId: job.data.team_id,+ apiKeyId: job.data.apiKeyId ?? null,+ billing: resolveBillingMetadata({+ billing: job.data.billing,+ crawlId: job.data.crawl_id,+ crawlerOptions: job.data.crawlerOptions,+ }),+ bypassBilling:+ job.data.internalOptions?.bypassBilling ?? false,+ },+ blocked,+ logger,+ );+ logger.info(+ "Skipped " ++ blocked.length ++ " discovered link(s) blocked by threat protection",+ {+ blockedDomains: [...new Set(blocked.map(x => x.domain))],+ },+ );+ }+ }++ for (const link of discoveredLinks) {if (await lockURL(job.data.crawl_id, sc, link)) {// This seems to work really welelconst jobPriority = await getJobPriority({@@ -559,6 +666,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {undefined,pipeline.unsupportedFeatures,pipeline.dataLayer,+ pipeline.threatDecisions,);doc.metadata.creditsUsed = credits_billed ?? undefined;@@ -650,6 +758,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {undefined,pipeline.unsupportedFeatures,pipeline.dataLayer,+ pipeline.threatDecisions,);doc.metadata.creditsUsed = credits_billed ?? undefined;@@ -848,6 +957,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {costTracking,(await getACUCTeam(job.data.team_id))?.flags ?? null,error instanceof Error ? error : null,+ undefined,+ undefined,+ pipeline?.threatDecisions,);logger.debug("Logging job to DB...");@@ -1088,7 +1200,45 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {}}- const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);+ let indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);++ // Threat protection: skip index-sourced discoveries on blocked domains.+ const kickoffThreatPolicy = sc.internalOptions?.threatProtection;+ if (+ kickoffThreatPolicy &&+ kickoffThreatPolicy.mode !== "off" &&+ indexLinks.length > 0+ ) {+ const { allowedUrls, blocked } = await checkUrlsAgainstThreatPolicy(+ indexLinks,+ kickoffThreatPolicy,+ { teamId: job.data.team_id },+ );+ indexLinks = allowedUrls;+ for (const blockedUrl of blocked) {+ await recordThreatBlocked(+ job.data.crawl_id,+ blockedUrl.url,+ blockedUrl.decision,+ );+ }+ if (blocked.length > 0) {+ billThreatBlockedDiscoveries(+ {+ teamId: job.data.team_id,+ apiKeyId: job.data.apiKeyId ?? null,+ billing: resolveBillingMetadata({+ billing: job.data.billing,+ crawlId: job.data.crawl_id,+ crawlerOptions: sc.crawlerOptions,+ }),+ bypassBilling: sc.internalOptions?.bypassBilling ?? false,+ },+ blocked,+ logger,+ );+ }+ }if (indexLinks.length > 0) {logger.debug("Using index links of length " + indexLinks.length, {@@ -1201,7 +1351,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {isPreCrawl: sc.internalOptions?.isPreCrawl ?? false,});- const passingURLs = (+ let passingURLs = (await crawler.filterLinks(results.urls.map(x => x.href),Infinity,@@ -1210,6 +1360,45 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {)).links;+ // Threat protection: skip sitemap entries on blocked domains β the crawl+ // continues; skipped URLs + decisions are recorded for security logging.+ const sitemapThreatPolicy = sc.internalOptions?.threatProtection;+ if (+ sitemapThreatPolicy &&+ sitemapThreatPolicy.mode !== "off" &&+ passingURLs.length > 0+ ) {+ const { allowedUrls, blocked } = await checkUrlsAgainstThreatPolicy(+ passingURLs,+ sitemapThreatPolicy,+ { teamId: job.data.team_id },+ );+ passingURLs = allowedUrls;+ for (const blockedUrl of blocked) {+ await recordThreatBlocked(+ job.data.crawl_id,+ blockedUrl.url,+ blockedUrl.decision,+ );+ }+ if (blocked.length > 0) {+ billThreatBlockedDiscoveries(+ {+ teamId: job.data.team_id,+ apiKeyId: job.data.apiKeyId ?? null,+ billing: resolveBillingMetadata({+ billing: job.data.billing,+ crawlId: job.data.crawl_id,+ crawlerOptions: sc.crawlerOptions,+ }),+ bypassBilling: sc.internalOptions?.bypassBilling ?? false,+ },+ blocked,+ logger,+ );+ }+ }+if (passingURLs.length > 0) {logger.debug("Using urls of length " + passingURLs.length, {urlsLength: passingURLs.length,