Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(threat-protection): Zscaler ZIA provider (zscaler mode) (#4161)
Gergő Móriczf18fbe46ffc2
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.tsnew file mode 100644index 000000000..8eccbde21--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.ts@@ -0,0 +1,364 @@+import http from "http";+import request from "supertest";+import {+ describeIf,+ idmux,+ Identity,+ TEST_API_URL,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+ scrapeTimeout,+} from "../lib";+import { scrape, scrapeRaw } from "./lib";+import {+ createZscalerMockCounters,+ createZscalerMockHandler,+ type ZscalerMockDatabase,+} from "../../../lib/threat-protection/providers/zscaler/testing";++// =========================================+// Threat protection "zscaler" mode+//+// Config-API tests need a team with the threatProtection flag (idmux) and+// are gated on TEST_PRODUCTION; the ones that store a client secret also+// need SIEM_LOGGING_ENCRYPTION_KEY (shared org-secret encryption key).+//+// Enforcement tests additionally need the mock ZIA server below. The API+// must be started with the base-URL overrides pointing at it, e.g.:+//+// SIEM_LOGGING_ENCRYPTION_KEY=<32-byte hex> \+// ZSCALER_TOKEN_URL_OVERRIDE=http://localhost:4519/oauth2/v1/token \+// ZSCALER_API_URL_OVERRIDE=http://localhost:4519 \+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-zscaler.test.ts+//+// Those tests self-skip when the override is not set to a local address.+// =========================================++const HAS_ENCRYPTION_KEY = !!process.env.SIEM_LOGGING_ENCRYPTION_KEY;++const mockApiUrl = process.env.ZSCALER_API_URL_OVERRIDE ?? "";+const HAS_MOCK_PROVIDER =+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(mockApiUrl) &&+ HAS_ENCRYPTION_KEY;++// Fixture domains. *.example.com is reserved (RFC 2606) — blocked scrapes+// fail before any outbound request, so these never need to exist.+const CASINO_DOMAIN = "threat-zscaler-casino.example.com";+const VENDOR_DOMAIN = "threat-zscaler-vendor.example.com";+const UNKNOWN_DOMAIN = "threat-zscaler-unknown.example.com";+const CLEAN_URL = TEST_SUITE_WEBSITE;++const MOCK_CLIENT_ID = "firecrawl-test-client";+const MOCK_CLIENT_SECRET = "firecrawl-test-secret";++const zscalerDb: ZscalerMockDatabase = {+ clientId: MOCK_CLIENT_ID,+ clientSecret: MOCK_CLIENT_SECRET,+ categories: [+ { id: "GAMBLING" },+ { id: "NEWS_AND_MEDIA" },+ {+ id: "CUSTOM_01",+ configuredName: "Blocked Vendors",+ customCategory: true,+ urls: [VENDOR_DOMAIN],+ },+ ],+ classifications: {+ [CASINO_DOMAIN]: {+ urlClassifications: ["GAMBLING"],+ urlClassificationsWithSecurityAlert: [],+ },+ },+};+const zscalerCounters = createZscalerMockCounters();++let mockServer: http.Server | null = null;++function startMockProvider(): Promise<void> {+ const port = Number(new URL(mockApiUrl).port);+ mockServer = http.createServer(+ createZscalerMockHandler(zscalerDb, zscalerCounters),+ );+ return new Promise((resolve, reject) => {+ mockServer!.once("error", reject);+ mockServer!.listen(port, () => resolve());+ });+}++async function getConfigRaw(identity: Identity) {+ return await request(TEST_API_URL)+ .get("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++async function putConfigRaw(body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .put("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++async function postRaw(path: string, body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .post(path)+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send((body ?? {}) as object);+}++function zscalerConfigDoc(overrides: Record<string, unknown> = {}) {+ return {+ mode: "zscaler",+ failurePolicy: "closed",+ zscaler: {+ clientId: MOCK_CLIENT_ID,+ clientSecret: MOCK_CLIENT_SECRET,+ vanityDomain: "testtenant",+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ ...overrides,+ },+ };+}++describeIf(TEST_PRODUCTION)("Threat protection zscaler config API", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-zscaler/config",+ flags: { threatProtection: "allowed" },+ });+ });++ it('PUT rejects mode "zscaler" without a connection block', async () => {+ const res = await putConfigRaw({ mode: "zscaler" }, identity);+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("zscaler");+ });++ it("test-connection without stored or inline credentials returns 400", async () => {+ const res = await postRaw(+ "/v2/team/threat-protection/zscaler/test-connection",+ {},+ identity,+ );+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ });++ it("categories endpoint without a connection returns 400", async () => {+ const res = await request(TEST_API_URL)+ .get("/v2/team/threat-protection/zscaler/categories")+ .set("Authorization", `Bearer ${identity.apiKey}`);+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ });++ describeIf(HAS_ENCRYPTION_KEY)("with the encryption key", () => {+ it("stores the connection, never serves the secret back", async () => {+ const put = await putConfigRaw(zscalerConfigDoc(), identity);+ expect(put.statusCode).toBe(200);+ expect(put.body.success).toBe(true);+ expect(put.body.data.mode).toBe("zscaler");+ expect(put.body.data.zscaler).toMatchObject({+ clientId: MOCK_CLIENT_ID,+ hasClientSecret: true,+ vanityDomain: "testtenant",+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ syncIntervalMinutes: 60,+ });+ expect(JSON.stringify(put.body)).not.toContain(MOCK_CLIENT_SECRET);++ const get = await getConfigRaw(identity);+ expect(get.statusCode).toBe(200);+ expect(get.body.data.zscaler).toMatchObject({+ clientId: MOCK_CLIENT_ID,+ hasClientSecret: true,+ });+ expect(JSON.stringify(get.body)).not.toContain(MOCK_CLIENT_SECRET);+ });++ it("keeps the stored secret when the update omits it", async () => {+ const doc = zscalerConfigDoc({ deniedCategories: ["GAMBLING"] });+ delete (doc.zscaler as Record<string, unknown>).clientSecret;+ const put = await putConfigRaw(doc, identity);+ expect(put.statusCode).toBe(200);+ expect(put.body.data.zscaler).toMatchObject({+ hasClientSecret: true,+ deniedCategories: ["GAMBLING"],+ });+ });++ it("requires a new secret when the client ID changes", async () => {+ const doc = zscalerConfigDoc({ clientId: "different-client" });+ delete (doc.zscaler as Record<string, unknown>).clientSecret;+ const put = await putConfigRaw(doc, identity);+ expect(put.statusCode).toBe(400);+ expect(put.body.success).toBe(false);+ expect(put.body.error).toContain("clientSecret");+ });++ it("clears the connection when the block is removed", async () => {+ const put = await putConfigRaw({ mode: "off" }, identity);+ expect(put.statusCode).toBe(200);+ expect(put.body.data.zscaler).toBeNull();++ // Re-configuring after a clear needs the secret again.+ const doc = zscalerConfigDoc();+ delete (doc.zscaler as Record<string, unknown>).clientSecret;+ const rePut = await putConfigRaw(doc, identity);+ expect(rePut.statusCode).toBe(400);+ });+ });+});++describeIf(TEST_PRODUCTION && HAS_MOCK_PROVIDER)(+ "Threat protection zscaler enforcement (mock ZIA)",+ () => {+ let identity: Identity;++ beforeAll(async () => {+ await startMockProvider();+ identity = await idmux({+ name: "threat-protection-zscaler/enforcement",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });++ const put = await putConfigRaw(zscalerConfigDoc(), identity);+ expect(put.statusCode).toBe(200);++ // Deterministic setup: force the custom-list sync before enforcement.+ const sync = await postRaw(+ "/v2/team/threat-protection/zscaler/sync",+ {},+ identity,+ );+ expect(sync.statusCode).toBe(200);+ expect(sync.body.data.syncStatus.lastSyncedAt).not.toBeNull();+ expect(sync.body.data.syncStatus.ruleCount).toBeGreaterThanOrEqual(1);+ }, 60_000);++ afterAll(async () => {+ if (mockServer) {+ await new Promise(resolve => mockServer!.close(resolve));+ mockServer = null;+ }+ });++ beforeEach(() => {+ zscalerDb.rateLimitLookups = false;+ });++ it("connection test passes against the mock tenant", async () => {+ const res = await postRaw(+ "/v2/team/threat-protection/zscaler/test-connection",+ {},+ identity,+ );+ expect(res.statusCode).toBe(200);+ expect(res.body.data).toMatchObject({+ ok: true,+ tokenOk: true,+ categoriesOk: true,+ lookupOk: true,+ });+ });++ it("connection test distinguishes bad credentials", async () => {+ const res = await postRaw(+ "/v2/team/threat-protection/zscaler/test-connection",+ {+ zscaler: {+ clientId: "wrong-client",+ clientSecret: "wrong-secret",+ vanityDomain: "testtenant",+ },+ },+ identity,+ );+ expect(res.statusCode).toBe(200);+ expect(res.body.data).toMatchObject({+ ok: false,+ tokenOk: false,+ error: { kind: "auth" },+ });+ });++ it("serves the synced taxonomy for the category picker", async () => {+ const res = await request(TEST_API_URL)+ .get("/v2/team/threat-protection/zscaler/categories")+ .set("Authorization", `Bearer ${identity.apiKey}`);+ expect(res.statusCode).toBe(200);+ const ids = res.body.data.categories.map(+ (category: { id: string }) => category.id,+ );+ expect(ids).toContain("GAMBLING");+ expect(ids).toContain("CUSTOM_01");+ const custom = res.body.data.categories.find(+ (category: { id: string }) => category.id === "CUSTOM_01",+ );+ expect(custom).toMatchObject({ name: "Blocked Vendors", custom: true });+ });++ it(+ "scrape: a denied Zscaler-defined category blocks via urlLookup",+ async () => {+ const res = await scrapeRaw(+ { url: `https://${CASINO_DOMAIN}/` } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ expect(res.body.error).toContain(CASINO_DOMAIN);+ },+ scrapeTimeout,+ );++ it(+ "scrape: a denied custom-list domain blocks from synced rules, no lookup",+ async () => {+ const lookupCallsBefore = zscalerCounters.lookupCalls;+ const res = await scrapeRaw(+ { url: `https://${VENDOR_DOMAIN}/any/page` } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ expect(res.body.error).toContain(VENDOR_DOMAIN);+ // Custom-list decisions never spend the tenant's lookup budget.+ expect(zscalerCounters.lookupCalls).toBe(lookupCallsBefore);+ },+ scrapeTimeout,+ );++ it(+ "scrape: a clean domain passes with the lookup consulted",+ async () => {+ const lookupUrlsBefore = zscalerCounters.lookupUrls;+ const doc = await scrape({ url: CLEAN_URL } as any, identity);+ expect(doc.metadata.statusCode).toBe(200);+ expect(zscalerCounters.lookupUrls).toBeGreaterThan(lookupUrlsBefore);+ },+ scrapeTimeout,+ );++ it(+ "scrape: provider failure resolves through failurePolicy closed",+ async () => {+ zscalerDb.rateLimitLookups = true;+ const res = await scrapeRaw(+ { url: `https://${UNKNOWN_DOMAIN}/` } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );+ },+);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 827e8bf06..15080288d 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -81,6 +81,35 @@ const configSchema = z.object({.positive().default(6 * 60 * 60),+ // Zscaler ZIA provider ("zscaler" threat protection mode). Base-URL+ // overrides exist for tests (mock ZIA server); production always uses the+ // real endpoints derived from the org's vanity domain and cloud name.+ ZSCALER_TOKEN_URL_OVERRIDE: z.string().url().optional(),+ ZSCALER_API_URL_OVERRIDE: z.string().url().optional(),+ // Per-org hourly budget for POST /urlLookup calls. ZIA allows 400/hour;+ // the default keeps headroom for connection tests.+ ZSCALER_LOOKUP_HOURLY_BUDGET: z.coerce.number().int().positive().default(380),+ // Per-org hourly budget for GET /urlCategories calls (ZIA allows 1,000/hour).+ ZSCALER_CATEGORIES_HOURLY_BUDGET: z.coerce+ .number()+ .int()+ .positive()+ .default(900),+ // How long one queued urlLookup may wait for its batched verdict before the+ // org's failurePolicy decides.+ ZSCALER_LOOKUP_TIMEOUT_MS: z.coerce.number().int().positive().default(15000),+ // Queued-URL depth per org above which new lookups fail fast into the+ // failurePolicy instead of waiting in a line they cannot exit.+ ZSCALER_LOOKUP_MAX_QUEUE_DEPTH: z.coerce+ .number()+ .int()+ .positive()+ .default(500),+ // Effective scrape concurrency cap for teams in "zscaler" mode, keeping URL+ // demand near the classification budget (≤100 URLs/s burst, ~11 URLs/s+ // sustained at 400 lookups/hour).+ ZSCALER_MODE_CONCURRENCY_CAP: z.coerce.number().int().positive().default(15),+// Organization SIEM logging delivery. The encryption key must decode to// exactly 32 bytes; validation happens when a secret is encrypted/decrypted// so self-hosted deployments that do not use this feature need no key.diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex b9ec77ecd..2b46f79ac 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -490,12 +490,20 @@ export async function mapController(// Threat protection: remove blocked links from the returned URL list// entirely. Checks are URL-level; scan fees bill +2 per unique scanned// URL (see calculateThreatScanCredits).+ //+ // "zscaler" mode evaluates map results against local rules only, same as+ // the v2 map controller: one map can return thousands of URLs, and inline+ // classification would burn the tenant's 400/hour urlLookup budget on+ // links that may never be fetched.let threatScanCredits = 0;if (threatProtection.policy && result.links.length > 0) {const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(result.links,threatProtection.policy,- { teamId: req.auth.team_id },+ {+ teamId: req.auth.team_id,+ localRulesOnly: threatProtection.policy.mode === "zscaler",+ },);threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());result.links = result.links.filter(x => {diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex eb02d2ae9..e6a2165b5 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -210,12 +210,21 @@ export async function mapController(// Threat protection: remove blocked links from the returned URL list// entirely. Checks are URL-level; scan fees bill +2 per unique scanned// URL (see calculateThreatScanCredits).+ //+ // "zscaler" mode evaluates map results against local rules only (org+ // lists + synced custom categories): one map can return thousands of+ // URLs, and inline classification would burn the tenant's 400/hour+ // urlLookup budget on links that may never be fetched. Every URL still+ // gets the full provider check when a scrape of it starts.let threatScanCredits = 0;if (threatProtection.policy && result.mapResults.length > 0) {const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(result.mapResults.map(x => x.url),threatProtection.policy,- { teamId: req.auth.team_id },+ {+ teamId: req.auth.team_id,+ localRulesOnly: threatProtection.policy.mode === "zscaler",+ },);threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());result.mapResults = result.mapResults.filter(x => {diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.tsindex db2f7012c..6378ea3ed 100644--- a/apps/api/src/controllers/v2/team-threat-protection.ts+++ b/apps/api/src/controllers/v2/team-threat-protection.ts@@ -3,15 +3,29 @@ import { z } from "zod";import { logger as _logger } from "../../lib/logger";import { RequestWithAuth } from "./types";import { getThreatProtection } from "../../lib/zdr-helpers";-import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";+import {+ threatProtectionConfigSchema,+ zscalerConfigSchema,+} from "../../lib/threat-protection/config";import { THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE } from "../../lib/threat-protection/request";import {getOrgIdForTeam,getOrgThreatProtectionConfig,+ getOrgZscalerCredentials,resolveEffectivePolicy,upsertOrgThreatProtectionConfig,+ ZscalerSecretRequiredError,type OrgThreatProtectionConfig,} from "../../lib/threat-protection/store";+import { zscalerTestConnection } from "../../lib/threat-protection/providers/zscaler/client";+import { consumeZscalerLookupBudget } from "../../lib/threat-protection/providers/zscaler/lookup-queue";+import {+ clearZscalerSyncState,+ consumeZscalerCategoriesBudget,+ getZscalerSyncDocument,+ syncOrgZscalerRules,+ type ZscalerSyncDocument,+} from "../../lib/threat-protection/providers/zscaler/sync";const logger = _logger.child({ module: "team-threat-protection" });@@ -50,16 +64,46 @@ async function resolveOrgId(return orgId;}+/**+ * Dashboard view of the sync plane: last success, last attempt, staleness,+ * and the last failure. Derived from the synced document — no tenant call.+ */+function serializeSyncStatus(doc: ZscalerSyncDocument | null) {+ return {+ lastSyncedAt: doc?.syncedAt ?? null,+ lastAttemptedAt: doc?.attemptedAt ?? null,+ error: doc?.status === "error" ? doc.error : null,+ ruleCount: doc?.rules.length ?? 0,+ categoryCount: doc?.taxonomy.length ?? 0,+ };+}+/*** Effective config document served by GET and PUT. Always includes every* policy field (defaults applied), so the dashboard can render the form- * without knowing the defaults.+ * without knowing the defaults. The Zscaler client secret is write-only and+ * never serialized; `hasClientSecret` tells the dashboard one is stored.*/-function serializeConfig(orgConfig: OrgThreatProtectionConfig | null) {- const policy = resolveEffectivePolicy(orgConfig);+function serializeConfig(+ orgConfig: OrgThreatProtectionConfig | null,+ syncDoc: ZscalerSyncDocument | null = null,+) {+ const { zscaler: _zscalerPolicy, ...policy } =+ resolveEffectivePolicy(orgConfig);return {...policy,allowRequestOverrides: orgConfig?.allowRequestOverrides ?? true,+ zscaler: orgConfig?.zscaler+ ? {+ clientId: orgConfig.zscaler.clientId,+ hasClientSecret: true,+ vanityDomain: orgConfig.zscaler.vanityDomain,+ cloud: orgConfig.zscaler.cloud,+ deniedCategories: orgConfig.zscaler.deniedCategories,+ syncIntervalMinutes: orgConfig.zscaler.syncIntervalMinutes,+ syncStatus: serializeSyncStatus(syncDoc),+ }+ : null,configured: orgConfig !== null,updatedAt: orgConfig?.updatedAt ?? null,};@@ -89,10 +133,13 @@ export async function getTeamThreatProtectionController(if (!orgId) return;const orgConfig = await getOrgThreatProtectionConfig(orgId);+ const syncDoc = orgConfig?.zscaler+ ? await getZscalerSyncDocument(orgId)+ : null;res.status(200).json({success: true,- data: serializeConfig(orgConfig),+ data: serializeConfig(orgConfig, syncDoc),});}@@ -117,23 +164,248 @@ export async function putTeamThreatProtectionController(return;}+ if (input.mode === "zscaler" && !input.zscaler) {+ res.status(400).json({+ success: false,+ error:+ 'Threat protection mode "zscaler" requires the zscaler connection block.',+ });+ return;+ }+const orgId = await resolveOrgId(req, res);if (!orgId) return;const previous = await getOrgThreatProtectionConfig(orgId);- const updated = await upsertOrgThreatProtectionConfig(orgId, input);+ let updated: OrgThreatProtectionConfig;+ try {+ updated = await upsertOrgThreatProtectionConfig(orgId, input);+ } catch (error) {+ if (error instanceof ZscalerSecretRequiredError) {+ res.status(400).json({ success: false, error: error.message });+ return;+ }+ throw error;+ }++ // Keep the sync plane consistent with the connection: a removed connection+ // drops the synced rules, and so does a change of connection identity — the+ // old tenant's custom rules must not keep enforcing while the new tenant's+ // first sync runs. A (re)configured connection syncs in the background so+ // custom-list rules and the category picker are ready shortly after save.+ const connectionIdentityChanged =+ previous?.zscaler &&+ updated.zscaler &&+ (previous.zscaler.clientId !== updated.zscaler.clientId ||+ previous.zscaler.vanityDomain !== updated.zscaler.vanityDomain ||+ previous.zscaler.cloud !== updated.zscaler.cloud);+ if (previous?.zscaler && (!updated.zscaler || connectionIdentityChanged)) {+ await clearZscalerSyncState(orgId).catch(error => {+ logger.warn("Failed to clear Zscaler sync state", { error, orgId });+ });+ }+ if (updated.zscaler) {+ syncOrgZscalerRules(orgId).catch(error => {+ logger.warn("Background Zscaler sync after config save failed", {+ error,+ orgId,+ });+ });+ }- // Audit log — org-level security configuration change.+ // Audit log — org-level security configuration change. The serialized view+ // never carries the secret, so a rotation gets an explicit marker.+ const auditChangedFields = changedFields(previous, updated);+ if (input.zscaler?.clientSecret) {+ auditChangedFields.push("zscaler.clientSecret");+ }logger.info("Threat protection config updated", {teamId: req.auth.team_id,orgId,mode: updated.policy.mode,- changedFields: changedFields(previous, updated),+ changedFields: auditChangedFields,+ });++ res.status(200).json({+ success: true,+ data: serializeConfig(+ updated,+ updated.zscaler ? await getZscalerSyncDocument(orgId) : null,+ ),+ });+}++// Connection test accepts either inline credentials (pre-save validation+// from the dashboard) or none (test the stored connection). The inline shape+// requires the secret — testing brand-new credentials without one is+// meaningless — except when the client ID matches the stored connection, in+// which case the stored secret fills in (dashboards round-trip the config+// without the write-only secret).+const testConnectionBodySchema = z+ .strictObject({+ zscaler: zscalerConfigSchema+ .pick({+ clientId: true,+ clientSecret: true,+ vanityDomain: true,+ cloud: true,+ })+ .optional(),+ })+ .optional();++export async function testTeamZscalerConnectionController(+ req: RequestWithAuth<{}, any, unknown>,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;++ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ // Primitive JSON bodies (0, false, "x") must not silently become "test the+ // stored connection" — only an object body (or none) is valid.+ if (+ req.body !== undefined &&+ (typeof req.body !== "object" ||+ req.body === null ||+ Array.isArray(req.body))+ ) {+ res.status(400).json({+ success: false,+ error: "The request body must be a JSON object.",+ });+ return;+ }+ const body = testConnectionBodySchema.parse(+ req.body && Object.keys(req.body as object).length > 0+ ? req.body+ : undefined,+ );++ const stored = await getOrgZscalerCredentials(orgId);+ let credentials = stored;+ if (body?.zscaler) {+ const inline = body.zscaler;+ let clientSecret = inline.clientSecret ?? null;+ if (!clientSecret && stored && stored.clientId === inline.clientId) {+ clientSecret = stored.clientSecret;+ }+ if (!clientSecret) {+ res.status(400).json({+ success: false,+ error:+ "zscaler.clientSecret is required to test credentials that are not stored yet",+ });+ return;+ }+ credentials = {+ clientId: inline.clientId,+ clientSecret,+ vanityDomain: inline.vanityDomain,+ cloud: inline.cloud,+ };+ }++ if (!credentials) {+ res.status(400).json({+ success: false,+ error:+ "No Zscaler connection is configured; provide credentials in the request body to test them.",+ });+ return;+ }++ // The probe spends one urlCategories call and one urlLookup call of the+ // tenant's quota. Each stage charges the shared budget right before its+ // own API call, so repeated tests can never starve real scrapes — and a+ // test that dies earlier (bad credentials) spends nothing.+ const result = await zscalerTestConnection(credentials, {+ signal: AbortSignal.timeout(30_000),+ beforeCategories: () => consumeZscalerCategoriesBudget(credentials),+ beforeLookup: () => consumeZscalerLookupBudget(credentials),+ });++ logger.info("Zscaler connection test", {+ teamId: req.auth.team_id,+ orgId,+ ok: result.ok,+ tokenOk: result.tokenOk,+ categoriesOk: result.categoriesOk,+ lookupOk: result.lookupOk,+ errorKind: result.error?.kind ?? null,+ });++ res.status(200).json({ success: true, data: result });+}++/**+ * Category taxonomy for the dashboard's denied-categories picker: Zscaler-+ * defined and custom categories, from the synced document. Syncs inline+ * when no document exists yet (first visit after connecting).+ */+export async function getTeamZscalerCategoriesController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;++ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ const orgConfig = await getOrgThreatProtectionConfig(orgId);+ if (!orgConfig?.zscaler) {+ res.status(400).json({+ success: false,+ error: "No Zscaler connection is configured for this organization.",+ });+ return;+ }++ let doc = await getZscalerSyncDocument(orgId);+ if (!doc || doc.taxonomy.length === 0) {+ doc = await syncOrgZscalerRules(orgId);+ }++ res.status(200).json({+ success: true,+ data: {+ categories: doc?.taxonomy ?? [],+ syncStatus: serializeSyncStatus(doc),+ },+ });+}++/** Explicit "Sync now" — bypasses the interval, still lock- and budget-guarded. */+export async function syncTeamZscalerController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ if (rejectWithoutFlag(req, res)) return;++ const orgId = await resolveOrgId(req, res);+ if (!orgId) return;++ const orgConfig = await getOrgThreatProtectionConfig(orgId);+ if (!orgConfig?.zscaler) {+ res.status(400).json({+ success: false,+ error: "No Zscaler connection is configured for this organization.",+ });+ return;+ }++ const doc = await syncOrgZscalerRules(orgId);++ logger.info("Zscaler manual sync", {+ teamId: req.auth.team_id,+ orgId,+ status: doc?.status ?? "unknown",});res.status(200).json({success: true,- data: serializeConfig(updated),+ data: { syncStatus: serializeSyncStatus(doc) },});}diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 246e376f0..6b8ae5c73 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -19,6 +19,7 @@ import {removeConcurrencyLimitActiveJob,} from "./concurrency-redis";import { autumnService } from "../services/autumn/autumn.service";+import { getThreatProtectionConcurrencyCap } from "./threat-protection/store";// Fallback when Autumn can't give us a concurrency value.const DEFAULT_CONCURRENCY_LIMIT = 2;@@ -28,13 +29,22 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;* balance. Autumn is authoritative; when the entity is missing we fall back to* the low default of 2. When Autumn errors, getConcurrencyLimit already returns* a high fail-open value, so that carries through here.+ *+ * Threat protection can clamp the result: "zscaler" mode caps concurrency so+ * URL demand stays near the tenant's classification budget — otherwise a+ * large crawl outruns the 400-lookups/hour budget and every scrape resolves+ * through the failurePolicy instead of a verdict.*/export async function getEffectiveConcurrencyLimit(teamId: string,orgId?: string | null,): Promise<number> {const autumnValue = await autumnService.getConcurrencyLimit(teamId, orgId);- return autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;+ const limit = autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;+ const threatProtectionCap = await getThreatProtectionConcurrencyCap(teamId);+ return threatProtectionCap !== null+ ? Math.min(limit, threatProtectionCap)+ : limit;}const constructKey = constructConcurrencyLimitKey;diff --git a/apps/api/src/lib/siem-logging/crypto.test.ts b/apps/api/src/lib/org-secret-crypto.test.tssimilarity index 60%rename from apps/api/src/lib/siem-logging/crypto.test.tsrename to apps/api/src/lib/org-secret-crypto.test.tsindex ea0c98a14..e478f992f 100644--- a/apps/api/src/lib/siem-logging/crypto.test.ts+++ b/apps/api/src/lib/org-secret-crypto.test.ts@@ -1,7 +1,7 @@import crypto from "crypto";import { afterEach, describe, expect, it } from "vitest";-import { config } from "../../config";-import { decryptSiemSecret, encryptSiemSecret } from "./crypto";+import { config } from "../config";+import { decryptOrgSecret, encryptOrgSecret } from "./org-secret-crypto";const originalKey = config.SIEM_LOGGING_ENCRYPTION_KEY;@@ -9,25 +9,25 @@ afterEach(() => {config.SIEM_LOGGING_ENCRYPTION_KEY = originalKey;});-describe("SIEM secret encryption", () => {+describe("Org secret encryption", () => {it("round-trips with AES-256-GCM without storing plaintext", () => {config.SIEM_LOGGING_ENCRYPTION_KEY = crypto.randomBytes(32).toString("hex");- const encrypted = encryptSiemSecret("client-secret-value", "org-one");+ const encrypted = encryptOrgSecret("client-secret-value", "org-one");expect(encrypted).toMatch(/^gcm:/);expect(encrypted).not.toContain("client-secret-value");- expect(decryptSiemSecret(encrypted, "org-one")).toBe("client-secret-value");- expect(() => decryptSiemSecret(encrypted, "org-two")).toThrow();+ expect(decryptOrgSecret(encrypted, "org-one")).toBe("client-secret-value");+ expect(() => decryptOrgSecret(encrypted, "org-two")).toThrow();});it("refuses to store a secret without a valid external key", () => {config.SIEM_LOGGING_ENCRYPTION_KEY = undefined;- expect(() => encryptSiemSecret("secret", "org-one")).toThrow(+ expect(() => encryptOrgSecret("secret", "org-one")).toThrow("SIEM_LOGGING_ENCRYPTION_KEY is not configured",);config.SIEM_LOGGING_ENCRYPTION_KEY = "too-short";- expect(() => encryptSiemSecret("secret", "org-one")).toThrow(+ expect(() => encryptOrgSecret("secret", "org-one")).toThrow("must be a 32-byte",);});diff --git a/apps/api/src/lib/siem-logging/crypto.ts b/apps/api/src/lib/org-secret-crypto.tssimilarity index 72%rename from apps/api/src/lib/siem-logging/crypto.tsrename to apps/api/src/lib/org-secret-crypto.tsindex 04cd3fc4e..39d0dc7a0 100644--- a/apps/api/src/lib/siem-logging/crypto.ts+++ b/apps/api/src/lib/org-secret-crypto.ts@@ -1,5 +1,10 @@import crypto from "crypto";-import { config } from "../../config";+import { config } from "../config";++// AES-256-GCM encryption for org-scoped partner credentials at rest (SIEM+// destination secrets, Zscaler OAuth client secrets). The key lives outside+// the database (env), and the org ID is bound in as AAD so a ciphertext+// pasted onto another org's row fails authentication instead of decrypting.const GCM_PREFIX = "gcm:";@@ -25,7 +30,7 @@ function getEncryptionKey(): Buffer {return key;}-export function encryptSiemSecret(secret: string, orgId: string): string {+export function encryptOrgSecret(secret: string, orgId: string): string {const key = getEncryptionKey();const iv = crypto.randomBytes(12);const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);@@ -38,16 +43,16 @@ export function encryptSiemSecret(secret: string, orgId: string): string {return `${GCM_PREFIX}${iv.toString("base64")}:${tag.toString("base64")}:${ciphertext.toString("base64")}`;}-export function decryptSiemSecret(stored: string, orgId: string): string {+export function decryptOrgSecret(stored: string, orgId: string): string {if (!stored.startsWith(GCM_PREFIX)) {- throw new Error("SIEM client secret is not encrypted with AES-256-GCM");+ throw new Error("Stored secret is not encrypted with AES-256-GCM");}const [ivB64, tagB64, ciphertextB64] = stored.slice(GCM_PREFIX.length).split(":");if (!ivB64 || !tagB64 || !ciphertextB64) {- throw new Error("Malformed encrypted SIEM client secret");+ throw new Error("Malformed encrypted secret");}const decipher = crypto.createDecipheriv(diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.tsindex 33984e953..e87f78800 100644--- a/apps/api/src/lib/siem-logging/event.ts+++ b/apps/api/src/lib/siem-logging/event.ts@@ -96,6 +96,9 @@ function threatForDecisions(rule: decision.rule,provider: decision.verdict?.provider ?? null,categories: [...new Set(decision.verdict?.categories ?? [])],+ ...(typeof decision.verdict?.securityAlert === "boolean"+ ? { security_alert: decision.verdict.securityAlert }+ : {}),};}diff --git a/apps/api/src/lib/siem-logging/store.ts b/apps/api/src/lib/siem-logging/store.tsindex 4c69885f3..35a59ca09 100644--- a/apps/api/src/lib/siem-logging/store.ts+++ b/apps/api/src/lib/siem-logging/store.ts@@ -4,7 +4,7 @@ import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { getRedisConnection } from "../../services/queue-service";import { logger as _logger } from "../logger";-import { decryptSiemSecret, encryptSiemSecret } from "./crypto";+import { decryptOrgSecret, encryptOrgSecret } from "../org-secret-crypto";import type { OrgSiemLoggingConfig, SiemLoggingConfigInput } from "./types";const CACHE_TTL_MS = 60_000;@@ -47,7 +47,7 @@ function rowToConfig(row: SiemConfigRow): OrgSiemLoggingConfig {enabled: row.enabled,destination: {...destination,- clientSecret: decryptSiemSecret(row.secret_ciphertext, row.org_id),+ clientSecret: decryptOrgSecret(row.secret_ciphertext, row.org_id),},createdAt: row.created_at,updatedAt: row.updated_at,@@ -164,7 +164,7 @@ export async function upsertOrgSiemLoggingConfig(}const secretCiphertext = input.destination.clientSecret- ? encryptSiemSecret(input.destination.clientSecret, orgId)+ ? encryptOrgSecret(input.destination.clientSecret, orgId): existingCiphertext!;const values = {org_id: orgId,diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.tsindex 688f57e0a..7bbd9f38f 100644--- a/apps/api/src/lib/siem-logging/types.ts+++ b/apps/api/src/lib/siem-logging/types.ts@@ -87,6 +87,13 @@ export interface ScrapeActivityThreat {rule: ThreatDecisionRule;provider: ThreatProvider | null;categories: string[];+ /**+ * Present only for providers with the concept (Zscaler): true when the+ * URL carried a security-alert classification+ * (`urlClassificationsWithSecurityAlert`). Only this normalized flag is+ * exported — never the provider's raw response payload.+ */+ security_alert?: boolean;}export interface ScrapeActivityEvent {diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.tsindex 34d439546..9891ad963 100644--- a/apps/api/src/lib/threat-protection/config.ts+++ b/apps/api/src/lib/threat-protection/config.ts@@ -34,6 +34,62 @@ const tldSchema = z`Invalid TLD ${JSON.stringify(iss.input)}: must be a lowercase alphanumeric TLD without the leading dot, e.g. "zip"`,});+// A single DNS label, as used for the Zidentity vanity domain+// (<vanityDomain>.zslogin.net) and the optional OneAPI cloud name+// (api.<cloud>.zsapi.net).+const dnsLabelSchema = z+ .string()+ .trim()+ .toLowerCase()+ .refine(value => new RegExp(`^${DOMAIN_LABEL}$`).test(value), {+ error: iss =>+ `Invalid value ${JSON.stringify(iss.input)}: must be a single DNS label (letters, digits, inner hyphens)`,+ });++/**+ * Zscaler category IDs: predefined ("GAMBLING", "OTHER_ADULT_MATERIAL") and+ * custom ("CUSTOM_01") IDs are uppercase with digits/underscores.+ */+const zscalerCategoryIdSchema = z+ .string()+ .trim()+ .refine(value => /^[A-Z0-9_]{1,128}$/.test(value), {+ error: iss =>+ `Invalid category ID ${JSON.stringify(iss.input)}: must be an uppercase Zscaler category ID like "GAMBLING" or "CUSTOM_01"`,+ });++/**+ * Zscaler connection + policy block, accepted by+ * `PUT /v2/team/threat-protection` when the org uses mode "zscaler".+ *+ * `clientSecret` is write-only: it is encrypted at rest, never serialized+ * back, and may be omitted on update to keep the stored secret (only when+ * `clientId` is unchanged — pairing a new client ID with an old secret is+ * always a mistake).+ */+export const zscalerConfigSchema = z.strictObject({+ clientId: z.string().trim().min(1).max(256),+ // Not trimmed (the secret is stored verbatim), but a whitespace-only value+ // must not be persisted as a "configured" secret that can never mint tokens.+ clientSecret: z+ .string()+ .min(1)+ .max(1024)+ .refine(value => value.trim().length > 0, {+ error: "clientSecret must not be blank",+ })+ .optional(),+ /** Zidentity vanity domain: tokens come from https://<vanityDomain>.zslogin.net */+ vanityDomain: dnsLabelSchema,+ /** Optional OneAPI cloud name (api.<cloud>.zsapi.net); null = default cloud. */+ cloud: dnsLabelSchema.nullable().prefault(null),+ deniedCategories: z.array(zscalerCategoryIdSchema).max(512).prefault([]),+ /** How often the category taxonomy + custom lists re-sync. */+ syncIntervalMinutes: z.number().int().min(5).max(1440).prefault(60),+});++type ZscalerConfigInput = z.infer<typeof zscalerConfigSchema>;+// =========================================// Policy + org config schemas// =========================================@@ -43,7 +99,7 @@ const tldSchema = z* except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.*/export const threatProtectionPolicySchema = z.strictObject({- mode: z.enum(["off", "normal"]),+ mode: z.enum(["off", "normal", "zscaler"]),riskScoreThreshold: z.number().int()@@ -81,7 +137,7 @@ void _policyContractCheck;* provides replace the org policy's values (see `resolveEffectivePolicy`).*/export const threatProtectionOverrideSchema = z.strictObject({- mode: z.enum(["off", "normal"]).optional(),+ mode: z.enum(["off", "normal", "zscaler"]).optional(),riskScoreThreshold: z.number().int().min(0).max(100).optional(),blacklist: z.array(domainGlobSchema).max(1000).optional(),whitelist: z.array(domainGlobSchema).max(1000).optional(),@@ -91,11 +147,14 @@ export const threatProtectionOverrideSchema = z.strictObject({type ThreatProtectionOverride = z.infer<typeof threatProtectionOverrideSchema>;-// Compile-time assertion that the override shape stays assignable to-// Partial<ThreatProtectionPolicy> (what `resolveEffectivePolicy` consumes).+// Compile-time assertion that the override shape stays assignable to what+// `resolveEffectivePolicy` consumes. The Omit is deliberate: the Zscaler+// connection is org-owned and never request-settable, and typing it out of+// the override surface makes passing one a compile error instead of a+// silently ignored field.const _overrideContractCheck = (x: ThreatProtectionOverride,-): Partial<ThreatProtectionPolicy> => x;+): Partial<Omit<ThreatProtectionPolicy, "zscaler">> => x;void _overrideContractCheck;/**@@ -105,6 +164,13 @@ void _overrideContractCheck;export const threatProtectionConfigSchema = threatProtectionPolicySchema.extend({allowRequestOverrides: z.boolean().prefault(true),+ /**+ * Zscaler connection block. Absent = no Zscaler connection (clears any+ * stored one, consistent with the full-document PUT semantics). Required+ * when mode is "zscaler" — enforced in the controller, where the stored+ * secret is available for the omitted-secret update path.+ */+ zscaler: zscalerConfigSchema.optional(),},);diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.tsindex f38a87ff2..74b5e38b5 100644--- a/apps/api/src/lib/threat-protection/index.ts+++ b/apps/api/src/lib/threat-protection/index.ts@@ -1,6 +1,9 @@+import { config } from "../../config";import { logger } from "../logger";import { fetchGoogleWebRiskVerdict } from "./providers/google-web-risk";import { canonicalizeUrl } from "./providers/web-risk/canonicalize";+import { fetchZscalerVerdict } from "./providers/zscaler";+import { evaluateZscalerSyncedRules } from "./providers/zscaler/sync";import type {RawVerdict,ThreatCheckDedup,@@ -27,6 +30,15 @@ export interface ThreatCheckContext {* request-scoped — never persisted, never shared across requests (ZDR).*/dedup?: ThreatCheckDedup;+ /**+ * Evaluate local rules only (whitelist/blacklist/blocked-TLD and, in+ * "zscaler" mode, the synced custom-list rules) and default-allow instead+ * of consulting the provider. Used for map results in "zscaler" mode: one+ * map can return thousands of URLs, and classifying them inline would+ * burn the tenant's 400-lookups/hour budget on links that may never be+ * scraped — each URL still gets its full check when a scrape of it starts.+ */+ localRulesOnly?: boolean;}// Public entry point for the threat protection core library (enterprise@@ -36,13 +48,18 @@ export interface ThreatCheckContext {// this request/job reuses the same in-flight decision// 3. local-only rules (whitelist/blacklist/blocked-tld, evaluated against// the URL's host) → decide without a provider call (no billing)-// 4. provider ("normal" = Google Web Risk local hash-prefix database),-// with a per-attempt timeout and one retry. Lookups are URL-level:-// host-suffix × path-prefix expressions per the Safe Browsing spec, so-// a listing that flags only a specific page is caught even when its-// domain is otherwise clean.-// 5. evaluate the policy against the verdict; provider failure → the org's-// failurePolicy decides (fail-open allows, fail-closed blocks)+// 4. "zscaler" mode only: the tenant's synced custom-list rules (custom+// categories, keywords, customer category additions — urlLookup never+// returns those) → a decisive match skips the provider call+// 5. provider ("normal" = Google Web Risk local hash-prefix database,+// "zscaler" = batched ZIA urlLookup through the shared per-org queue),+// with a per-attempt timeout; "normal" gets one retry. Web Risk lookups+// are URL-level: host-suffix × path-prefix expressions per the Safe+// Browsing spec, so a listing that flags only a specific page is caught+// even when its domain is otherwise clean.+// 6. evaluate the policy against the verdict (zscaler: the org's denied+// categories); provider failure → the org's failurePolicy decides+// (fail-open allows, fail-closed blocks)// Any decision backed by a verdict sets providerConsulted, which drives// billing in the enforcement layer (+2 credits per unique scanned URL per// billing scope — see calculateThreatScanCredits). This module performs no@@ -71,20 +88,37 @@ const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry* Single mode→provider dispatch point. Every provider is a separate module* under ./providers exporting `fetch<X>Verdict(url) → RawVerdict`; this* switch is the only place that knows which mode maps to which classifier.- * Deliberately kept as a dispatch even with one live branch — future partner- * classifiers add a mode + a case here and nothing else changes.+ *+ * Per-mode transport behavior: "normal" (Web Risk) is a local hash-prefix+ * check, so it gets a short timeout and one retry. "zscaler" waits on a+ * batched remote lookup whose pacing the queue already owns — a longer+ * timeout and NO retry (a retry would spend a second slice of the tenant's+ * 400/hour budget on the same URL).*/async function fetchProviderVerdict(url: string,mode: Exclude<ThreatProtectionMode, "off">,+ policy: ThreatProtectionPolicy,): Promise<RawVerdict> {+ const attempts = mode === "zscaler" ? 1 : PROVIDER_ATTEMPTS;+ const timeoutMs =+ mode === "zscaler" ? config.ZSCALER_LOOKUP_TIMEOUT_MS : PROVIDER_TIMEOUT_MS;+let lastError: unknown;- for (let attempt = 1; attempt <= PROVIDER_ATTEMPTS; attempt++) {+ for (let attempt = 1; attempt <= attempts; attempt++) {try {- const signal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS);+ const signal = AbortSignal.timeout(timeoutMs);switch (mode) {case "normal":return await fetchGoogleWebRiskVerdict(url, { signal });+ case "zscaler": {+ if (!policy.zscaler) {+ throw new Error(+ "Threat protection mode is zscaler but the org has no Zscaler connection configured",+ );+ }+ return await fetchZscalerVerdict(url, policy.zscaler, { signal });+ }}} catch (error) {lastError = error;@@ -112,11 +146,43 @@ async function checkUrlFresh(return local;}+ // Zscaler mode: the tenant's synced custom-list rules next. urlLookup+ // never returns custom classifications, so these can only be decided+ // here — and a decisive match skips the budgeted provider call entirely.+ if (policy.mode === "zscaler" && policy.zscaler) {+ const synced = await evaluateZscalerSyncedRules(+ canonicalUrl,+ policy.zscaler,+ policy.failurePolicy,+ {+ url: canonicalUrl,+ domain: normalizeDomain(canonicalUrl),+ mode: policy.mode,+ },+ );+ if (synced !== null) {+ return synced;+ }+ }++ if (ctx.localRulesOnly) {+ return {+ allowed: true,+ rule: "default-allow",+ url: canonicalUrl,+ domain: normalizeDomain(canonicalUrl),+ providerConsulted: false,+ verdict: null,+ mode: policy.mode,+ };+ }+let verdict: RawVerdict | null = null;try {verdict = await fetchProviderVerdict(canonicalUrl,policy.mode as Exclude<ThreatProtectionMode, "off">,+ policy,);} catch {// Already logged per-attempt; a null verdict routes the decisiondiff --git a/apps/api/src/lib/threat-protection/providers/zscaler/client.test.ts b/apps/api/src/lib/threat-protection/providers/zscaler/client.test.tsnew file mode 100644index 000000000..be383451e--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/client.test.ts@@ -0,0 +1,201 @@+import http from "http";+import { config } from "../../../../config";+import {+ toLookupUrl,+ ZscalerError,+ zscalerGetUrlCategories,+ zscalerLookupUrls,+ zscalerTestConnection,+ type ZscalerCredentials,+} from "./client";+import {+ createZscalerMockCounters,+ createZscalerMockHandler,+ type ZscalerMockDatabase,+} from "./testing";++const counters = createZscalerMockCounters();+const db: ZscalerMockDatabase = {+ clientId: "test-client",+ clientSecret: "test-secret",+ categories: [+ { id: "GAMBLING" },+ {+ id: "CUSTOM_01",+ configuredName: "Blocked Vendors",+ customCategory: true,+ urls: ["vendor.example.com"],+ },+ ],+ classifications: {+ "casino.example.com": {+ urlClassifications: ["GAMBLING"],+ urlClassificationsWithSecurityAlert: [],+ },+ "malware.example.com": {+ urlClassifications: [],+ urlClassificationsWithSecurityAlert: ["MALWARE_SITE"],+ },+ },+};++let server: http.Server;+let originalTokenOverride: string | undefined;+let originalApiOverride: string | undefined;++function credentials(+ overrides: Partial<ZscalerCredentials> = {},+): ZscalerCredentials {+ return {+ clientId: "test-client",+ clientSecret: "test-secret",+ vanityDomain: "testtenant",+ cloud: null,+ ...overrides,+ };+}++beforeAll(async () => {+ server = http.createServer(createZscalerMockHandler(db, counters));+ await new Promise<void>((resolve, reject) => {+ server.once("error", reject);+ server.listen(0, "127.0.0.1", resolve);+ });+ const address = server.address();+ if (address === null || typeof address !== "object") {+ throw new Error("Mock server did not bind");+ }+ const base = `http://127.0.0.1:${address.port}`;+ originalTokenOverride = config.ZSCALER_TOKEN_URL_OVERRIDE;+ originalApiOverride = config.ZSCALER_API_URL_OVERRIDE;+ config.ZSCALER_TOKEN_URL_OVERRIDE = `${base}/oauth2/v1/token`;+ config.ZSCALER_API_URL_OVERRIDE = base;+});++afterAll(async () => {+ config.ZSCALER_TOKEN_URL_OVERRIDE = originalTokenOverride;+ config.ZSCALER_API_URL_OVERRIDE = originalApiOverride;+ await new Promise(resolve => server.close(resolve));+});++beforeEach(() => {+ db.denyLookupScope = false;+ db.rateLimitLookups = false;+});++describe("zscalerLookupUrls", () => {+ it("classifies a batch and normalizes the response", async () => {+ const results = await zscalerLookupUrls(credentials(), [+ "casino.example.com",+ "malware.example.com",+ "unknown.example.com",+ ]);+ expect(results).toEqual([+ {+ url: "casino.example.com",+ urlClassifications: ["GAMBLING"],+ urlClassificationsWithSecurityAlert: [],+ },+ {+ url: "malware.example.com",+ urlClassifications: [],+ urlClassificationsWithSecurityAlert: ["MALWARE_SITE"],+ },+ {+ url: "unknown.example.com",+ urlClassifications: ["MISCELLANEOUS_OR_UNKNOWN"],+ urlClassificationsWithSecurityAlert: [],+ },+ ]);+ });++ it("reuses the cached token across calls", async () => {+ // Prime the cache explicitly instead of relying on test order.+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);+ const tokenCallsBefore = counters.tokenCalls;+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);+ expect(counters.tokenCalls).toBe(tokenCallsBefore);+ });++ it("refuses more than 100 URLs per call", async () => {+ const urls = Array.from({ length: 101 }, (_, i) => `u${i}.example.com`);+ await expect(zscalerLookupUrls(credentials(), urls)).rejects.toMatchObject({+ kind: "api",+ });+ });++ it("maps a tenant-side 429 to a rate-limit error with Retry-After", async () => {+ db.rateLimitLookups = true;+ let caught: unknown;+ try {+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);+ } catch (error) {+ caught = error;+ }+ expect(caught).toBeInstanceOf(ZscalerError);+ expect(caught).toMatchObject({ kind: "rate-limit", status: 429 });+ expect((caught as ZscalerError).retryAfterMs).toBeGreaterThan(0);+ });+});++describe("authentication errors", () => {+ it("maps rejected credentials to an auth error", async () => {+ await expect(+ zscalerGetUrlCategories(credentials({ clientSecret: "wrong-secret" })),+ ).rejects.toMatchObject({ kind: "auth" });+ });+});++describe("zscalerTestConnection", () => {+ it("passes all three stages with a full-access client", async () => {+ const result = await zscalerTestConnection(credentials());+ expect(result).toMatchObject({+ ok: true,+ tokenOk: true,+ categoriesOk: true,+ lookupOk: true,+ categoryCount: 2,+ error: null,+ });+ });++ it("distinguishes bad credentials from a scope problem", async () => {+ const badCredentials = await zscalerTestConnection(+ credentials({ clientSecret: "wrong-secret" }),+ );+ expect(badCredentials).toMatchObject({+ ok: false,+ tokenOk: false,+ error: { kind: "auth" },+ });++ // View Only-style role: taxonomy reads fine, the lookup probe is denied.+ db.denyLookupScope = true;+ const scopeProblem = await zscalerTestConnection(credentials());+ expect(scopeProblem).toMatchObject({+ ok: false,+ tokenOk: true,+ categoriesOk: true,+ lookupOk: false,+ error: { kind: "scope" },+ });+ });+});++describe("toLookupUrl", () => {+ test.each([+ ["https://Example.com/Path?q=1#frag", "example.com/Path?q=1"],+ ["http://example.com/", "example.com"],+ ["example.com", "example.com"],+ ["https://example.com:8080/x", "example.com:8080/x"],+ ])("shapes %s into %s", (input, expected) => {+ expect(toLookupUrl(input)).toBe(expected);+ });++ it("rejects entries over the ZIA limit instead of truncating them", () => {+ const long = `example.com/${"a".repeat(2000)}`;+ // Truncating would classify a different URL; the failurePolicy decides.+ expect(() => toLookupUrl(long)).toThrow(ZscalerError);+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/client.ts b/apps/api/src/lib/threat-protection/providers/zscaler/client.tsnew file mode 100644index 000000000..74257d5e7--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/client.ts@@ -0,0 +1,488 @@+import { createHash } from "crypto";+import { ProxyAgent, type Dispatcher } from "undici";+import { config } from "../../../../config";++// Minimal Zscaler OneAPI client for the ZIA endpoints the "zscaler" threat+// protection mode needs: OAuth2 client-credentials tokens from Zidentity,+// GET /urlCategories (taxonomy + custom-list sync), and POST /urlLookup+// (inline classification). All traffic goes through the partner egress proxy+// (PARTNER_EGRESS_PROXY_URL) when configured, so the customer only has to+// allowlist our published static IP.+//+// Tokens are cached in memory only — never in Redis or Postgres. The lookup+// batcher and the sync path are the only sustained callers, and both run+// behind per-org distributed locks, so at any moment only a handful of+// processes hold a token. Zscaler supports two active client secrets, which+// makes rotation zero-downtime: verdicts keep flowing on the old secret's+// tokens until the new secret is saved here.++export interface ZscalerCredentials {+ clientId: string;+ clientSecret: string;+ /** Zidentity vanity domain: tokens come from https://<vanityDomain>.zslogin.net */+ vanityDomain: string;+ /** OneAPI cloud name (api.<cloud>.zsapi.net); null = default (api.zsapi.net). */+ cloud: string | null;+}++type ZscalerErrorKind =+ /** Token endpoint rejected the client (bad ID/secret, revoked client). */+ | "auth"+ /** Authenticated, but the role/scope does not allow the resource (403). */+ | "scope"+ /** 429 from the API — the tenant-side rate limit, not our local budget. */+ | "rate-limit"+ /** Our own per-org hourly budget is exhausted (no API call was made). */+ | "quota"+ /** Anything else: transport failures, 5xx, malformed responses. */+ | "api";++export class ZscalerError extends Error {+ constructor(+ public readonly kind: ZscalerErrorKind,+ message: string,+ public readonly status?: number,+ public readonly retryAfterMs?: number,+ ) {+ super(message);+ this.name = "ZscalerError";+ }+}++export interface ZscalerUrlLookupResult {+ url: string;+ urlClassifications: string[];+ urlClassificationsWithSecurityAlert: string[];+}++/**+ * One entry of GET /urlCategories. Field names follow the ZIA API (see the+ * official zscaler-sdk-go/py `urlcategories` services): custom categories+ * carry `urls`/`keywords`, and `dbCategorizedUrls` /+ * `keywordsRetainingParentCategory` hold the "retaining parent category"+ * variants (the URL/keyword keeps its Zscaler-defined classification AND+ * gains the custom category).+ */+export interface ZscalerUrlCategory {+ id: string;+ configuredName?: string;+ customCategory?: boolean;+ urls?: string[];+ dbCategorizedUrls?: string[];+ keywords?: string[];+ keywordsRetainingParentCategory?: string[];+}++const TOKEN_EXPIRY_SKEW_MS = 60_000;+const MAX_TOKEN_CACHE_ENTRIES = 100;++interface CachedToken {+ token: string;+ expiresAt: number;+}++const tokenCache = new Map<string, CachedToken>();+let proxyDispatcher: ProxyAgent | undefined;+let proxyDispatcherUrl: string | undefined;++function getDispatcher(): Dispatcher | undefined {+ if (!config.PARTNER_EGRESS_PROXY_URL) return undefined;+ if (+ !proxyDispatcher ||+ proxyDispatcherUrl !== config.PARTNER_EGRESS_PROXY_URL+ ) {+ proxyDispatcher = new ProxyAgent(config.PARTNER_EGRESS_PROXY_URL);+ proxyDispatcherUrl = config.PARTNER_EGRESS_PROXY_URL;+ }+ return proxyDispatcher;+}++function tokenUrl(credentials: ZscalerCredentials): string {+ return (+ config.ZSCALER_TOKEN_URL_OVERRIDE ??+ `https://${credentials.vanityDomain}.zslogin.net/oauth2/v1/token`+ );+}++function apiBaseUrl(credentials: ZscalerCredentials): string {+ const base =+ config.ZSCALER_API_URL_OVERRIDE ??+ (credentials.cloud+ ? `https://api.${credentials.cloud}.zsapi.net`+ : "https://api.zsapi.net");+ return `${base.replace(/\/+$/, "")}/zia/api/v1`;+}++function tokenCacheKey(credentials: ZscalerCredentials): string {+ const secretDigest = createHash("sha256")+ .update(credentials.clientSecret)+ .digest("base64url");+ return `${credentials.vanityDomain}:${credentials.clientId}:${secretDigest}`;+}++function cacheToken(cacheKey: string, token: CachedToken): void {+ const now = Date.now();+ for (const [key, cached] of tokenCache) {+ if (cached.expiresAt <= now) tokenCache.delete(key);+ }+ tokenCache.delete(cacheKey);+ while (tokenCache.size >= MAX_TOKEN_CACHE_ENTRIES) {+ const oldestKey = tokenCache.keys().next().value;+ if (oldestKey === undefined) break;+ tokenCache.delete(oldestKey);+ }+ tokenCache.set(cacheKey, token);+}++function retryAfterMs(response: Response): number | undefined {+ const value = response.headers.get("retry-after");+ if (!value) return undefined;+ const seconds = Number(value);+ if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);+ const date = Date.parse(value);+ if (Number.isNaN(date)) return undefined;+ return Math.max(0, date - Date.now());+}++// Error responses must have their bodies consumed, or undici cannot reuse+// the connection and sockets pile up under repeated failures.+function drainBody(response: Response): void {+ response.body?.cancel().catch(() => {});+}++async function fetchAccessToken(+ credentials: ZscalerCredentials,+ signal: AbortSignal | undefined,+): Promise<string> {+ const cacheKey = tokenCacheKey(credentials);+ const cached = tokenCache.get(cacheKey);+ if (cached && cached.expiresAt > Date.now()) {+ return cached.token;+ }++ const body = new URLSearchParams({+ grant_type: "client_credentials",+ client_id: credentials.clientId,+ client_secret: credentials.clientSecret,+ audience: "https://api.zscaler.com",+ });++ let response: Response;+ try {+ response = await fetch(tokenUrl(credentials), {+ method: "POST",+ headers: { "Content-Type": "application/x-www-form-urlencoded" },+ body: body.toString(),+ signal,+ // undici extension: route through the partner egress proxy.+ dispatcher: getDispatcher(),+ } as RequestInit);+ } catch (error) {+ throw new ZscalerError(+ "api",+ `Failed to reach the Zscaler token endpoint: ${error instanceof Error ? error.message : String(error)}`,+ );+ }++ if (response.status === 400 || response.status === 401) {+ drainBody(response);+ throw new ZscalerError(+ "auth",+ "Zscaler rejected the OAuth client credentials",+ response.status,+ );+ }+ if (response.status === 429) {+ drainBody(response);+ throw new ZscalerError(+ "rate-limit",+ "Zscaler token endpoint rate limit hit",+ response.status,+ retryAfterMs(response),+ );+ }+ if (!response.ok) {+ drainBody(response);+ throw new ZscalerError(+ "api",+ `Zscaler token endpoint returned status ${response.status}`,+ response.status,+ );+ }++ let payload: { access_token?: string; expires_in?: number };+ try {+ payload = (await response.json()) as typeof payload;+ } catch {+ throw new ZscalerError("api", "Zscaler token response was not JSON");+ }+ if (!payload.access_token) {+ throw new ZscalerError("api", "Zscaler token response had no access_token");+ }++ const expiresInMs =+ (typeof payload.expires_in === "number" && payload.expires_in > 0+ ? payload.expires_in * 1000+ : 3600_000) - TOKEN_EXPIRY_SKEW_MS;+ cacheToken(cacheKey, {+ token: payload.access_token,+ expiresAt: Date.now() + Math.max(expiresInMs, 10_000),+ });+ return payload.access_token;+}++async function apiRequest<T>(+ credentials: ZscalerCredentials,+ method: "GET" | "POST",+ path: string,+ body: unknown | undefined,+ signal: AbortSignal | undefined,+): Promise<T> {+ let token = await fetchAccessToken(credentials, signal);++ for (let attempt = 1; ; attempt++) {+ let response: Response;+ try {+ response = await fetch(`${apiBaseUrl(credentials)}${path}`, {+ method,+ headers: {+ Authorization: `Bearer ${token}`,+ ...(body !== undefined ? { "Content-Type": "application/json" } : {}),+ },+ body: body !== undefined ? JSON.stringify(body) : undefined,+ signal,+ dispatcher: getDispatcher(),+ } as RequestInit);+ } catch (error) {+ throw new ZscalerError(+ "api",+ `Failed to reach the Zscaler API: ${error instanceof Error ? error.message : String(error)}`,+ );+ }++ if (response.status === 401 && attempt === 1) {+ // Token expired or was revoked mid-flight: drop it and retry once with+ // a fresh one. A second 401 is a real auth problem.+ drainBody(response);+ tokenCache.delete(tokenCacheKey(credentials));+ token = await fetchAccessToken(credentials, signal);+ continue;+ }+ if (response.status === 401) {+ drainBody(response);+ throw new ZscalerError(+ "auth",+ "Zscaler API rejected the access token",+ response.status,+ );+ }+ if (response.status === 403) {+ drainBody(response);+ throw new ZscalerError(+ "scope",+ `Zscaler API denied access to ${path} — the API client's role does not cover it`,+ response.status,+ );+ }+ if (response.status === 429) {+ drainBody(response);+ throw new ZscalerError(+ "rate-limit",+ `Zscaler API rate limit hit on ${path}`,+ response.status,+ retryAfterMs(response),+ );+ }+ if (!response.ok) {+ drainBody(response);+ throw new ZscalerError(+ "api",+ `Zscaler API returned status ${response.status} for ${path}`,+ response.status,+ );+ }++ try {+ return (await response.json()) as T;+ } catch {+ throw new ZscalerError(+ "api",+ `Zscaler API response for ${path} was not JSON`,+ );+ }+ }+}++/** ZIA rejects lookup entries longer than this. */+const MAX_LOOKUP_URL_LENGTH = 1024;++/**+ * Shape a URL the way POST /urlLookup expects: no scheme, no fragment,+ * lowercased host. Accepts anything checkUrl's canonicalizer produces.+ * URLs over the ZIA length limit throw (a truncated URL would classify a+ * DIFFERENT URL); the caller resolves that through the failurePolicy.+ */+export function toLookupUrl(url: string): string {+ let value = url.trim();+ value = value.replace(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//, "");+ const hash = value.indexOf("#");+ if (hash !== -1) value = value.slice(0, hash);+ const slash = value.indexOf("/");+ const host = (slash === -1 ? value : value.slice(0, slash)).toLowerCase();+ const rest = slash === -1 ? "" : value.slice(slash);+ // A bare host needs no trailing slash; strip one so "example.com/" and+ // "example.com" share a verdict key.+ const combined = `${host}${rest === "/" ? "" : rest}`;+ if (combined.length > MAX_LOOKUP_URL_LENGTH) {+ throw new ZscalerError(+ "api",+ `URL exceeds the ${MAX_LOOKUP_URL_LENGTH}-character ZIA lookup limit`,+ );+ }+ return combined;+}++/**+ * Classify up to 100 URLs. Returns one entry per distinct URL; entries the+ * tenant has not categorized come back with `MISCELLANEOUS_OR_UNKNOWN`.+ * NOTE (ZIA behavior): custom URL classifications are NOT returned by this+ * endpoint — custom-list decisions come from the synced local rules instead.+ */+export async function zscalerLookupUrls(+ credentials: ZscalerCredentials,+ urls: string[],+ options?: { signal?: AbortSignal },+): Promise<ZscalerUrlLookupResult[]> {+ if (urls.length === 0) return [];+ if (urls.length > 100) {+ throw new ZscalerError("api", "urlLookup accepts at most 100 URLs");+ }+ const results = await apiRequest<+ Array<{+ url?: string;+ urlClassifications?: string[];+ urlClassificationsWithSecurityAlert?: string[];+ }>+ >(credentials, "POST", "/urlLookup", urls, options?.signal);+ if (!Array.isArray(results)) {+ throw new ZscalerError("api", "urlLookup response was not an array");+ }+ return results.map(entry => ({+ url: entry.url ?? "",+ urlClassifications: Array.isArray(entry.urlClassifications)+ ? entry.urlClassifications.filter(c => typeof c === "string")+ : [],+ urlClassificationsWithSecurityAlert: Array.isArray(+ entry.urlClassificationsWithSecurityAlert,+ )+ ? entry.urlClassificationsWithSecurityAlert.filter(+ c => typeof c === "string",+ )+ : [],+ }));+}++/**+ * Fetch the tenant's full category taxonomy: Zscaler-defined categories,+ * custom categories with their URL lists and keywords, and customer+ * additions to Zscaler-defined categories (`dbCategorizedUrls`).+ */+export async function zscalerGetUrlCategories(+ credentials: ZscalerCredentials,+ options?: { signal?: AbortSignal },+): Promise<ZscalerUrlCategory[]> {+ const results = await apiRequest<ZscalerUrlCategory[]>(+ credentials,+ "GET",+ "/urlCategories",+ undefined,+ options?.signal,+ );+ if (!Array.isArray(results)) {+ throw new ZscalerError("api", "urlCategories response was not an array");+ }+ return results.filter(+ entry => entry && typeof entry === "object" && typeof entry.id === "string",+ );+}++interface ZscalerConnectionTestResult {+ ok: boolean;+ /** Token endpoint accepted the client credentials. */+ tokenOk: boolean;+ /** GET /urlCategories succeeded (taxonomy readable). */+ categoriesOk: boolean;+ /** POST /urlLookup succeeded (inline classification permitted). */+ lookupOk: boolean;+ categoryCount: number | null;+ /** Set when ok is false: which stage failed and why. */+ error: { kind: ZscalerErrorKind; message: string } | null;+}++/**+ * Three-stage connection test: token (distinguishes bad credentials),+ * taxonomy read, and a lookup probe (a View Only role can read the taxonomy+ * but may be refused the read-semantic POST /urlLookup — surfacing that here+ * beats surfacing it on the first scrape).+ *+ * The optional stage hooks run immediately before their API call and abort+ * the stage by throwing — the caller charges its shared tenant budgets there,+ * so a test that dies at an earlier stage never spends points for calls that+ * were never made.+ */+export async function zscalerTestConnection(+ credentials: ZscalerCredentials,+ options?: {+ signal?: AbortSignal;+ beforeCategories?: () => Promise<void>;+ beforeLookup?: () => Promise<void>;+ },+): Promise<ZscalerConnectionTestResult> {+ const result: ZscalerConnectionTestResult = {+ ok: false,+ tokenOk: false,+ categoriesOk: false,+ lookupOk: false,+ categoryCount: null,+ error: null,+ };++ const fail = (error: unknown): ZscalerConnectionTestResult => {+ const zscalerError =+ error instanceof ZscalerError+ ? error+ : new ZscalerError(+ "api",+ error instanceof Error ? error.message : String(error),+ );+ result.error = { kind: zscalerError.kind, message: zscalerError.message };+ return result;+ };++ try {+ await fetchAccessToken(credentials, options?.signal);+ result.tokenOk = true;+ } catch (error) {+ return fail(error);+ }++ try {+ await options?.beforeCategories?.();+ const categories = await zscalerGetUrlCategories(credentials, options);+ result.categoriesOk = true;+ result.categoryCount = categories.length;+ } catch (error) {+ return fail(error);+ }++ try {+ await options?.beforeLookup?.();+ await zscalerLookupUrls(credentials, ["example.com"], options);+ result.lookupOk = true;+ } catch (error) {+ return fail(error);+ }++ result.ok = true;+ return result;+}diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/index.ts b/apps/api/src/lib/threat-protection/providers/zscaler/index.tsnew file mode 100644index 000000000..b1a68bafb--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/index.ts@@ -0,0 +1,56 @@+import type { RawVerdict, ZscalerPolicySettings } from "../../types";+import { toLookupUrl } from "./client";+import { enqueueZscalerLookup } from "./lookup-queue";++// Zscaler ZIA provider ("zscaler" threat protection mode): inline+// classification via batched POST /urlLookup calls (see ./lookup-queue.ts).+// Custom categories and customer additions never come back from urlLookup —+// those are evaluated locally from the synced rules BEFORE this provider is+// consulted (see ./sync.ts and checkUrlFresh in ../../index.ts).+//+// Verdict normalization into the provider-agnostic contract:+// - categories = urlClassifications ∪ urlClassificationsWithSecurityAlert;+// the org's denied-categories selection decides in evaluatePolicy.+// - securityAlert = the URL carries a security-alert classification.+// - riskScore stays null: ZIA is categorical, so the score threshold rule+// never fires for this provider.+// - raw carries the split lists for server logs only — never for SIEM+// export (the SIEM event builder reads only the normalized fields).++/**+ * Classify a URL through the org's Zscaler tenant. Throws on quota+ * exhaustion, queue overflow, timeout, and API failures — the caller+ * resolves those through the org's failurePolicy.+ */+export async function fetchZscalerVerdict(+ url: string,+ zscaler: ZscalerPolicySettings,+ options?: { signal?: AbortSignal },+): Promise<RawVerdict> {+ const lookupUrl = toLookupUrl(url);+ const result = await enqueueZscalerLookup(+ zscaler.orgId,+ lookupUrl,+ options?.signal,+ );++ const categories = [+ ...new Set([+ ...result.urlClassifications,+ ...result.urlClassificationsWithSecurityAlert,+ ]),+ ];++ return {+ provider: "zscaler-zia",+ riskScore: null,+ categories,+ securityAlert: result.urlClassificationsWithSecurityAlert.length > 0,+ fromCache: false,+ raw: {+ urlClassifications: result.urlClassifications,+ urlClassificationsWithSecurityAlert:+ result.urlClassificationsWithSecurityAlert,+ },+ };+}diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.tsnew file mode 100644index 000000000..7cb0ae4df--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts@@ -0,0 +1,465 @@+import { randomUUID } from "crypto";+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";+import { config } from "../../../../config";+import { logger as _logger } from "../../../logger";+import { redisRateLimitClient } from "../../../../services/rate-limiter";+import { redlock } from "../../../../services/redlock";+import { getOrgZscalerCredentials } from "../../store";+import {+ ZscalerError,+ zscalerLookupUrls,+ type ZscalerCredentials,+ type ZscalerUrlLookupResult,+} from "./client";++// Inline classification plane for the Zscaler ZIA provider.+//+// POST /urlLookup accepts up to 100 URLs per call, but the tenant budget is+// 1 call/second and 400 calls/hour — and the budget counts CALLS, not URLs.+// With scrape workers running on hundreds of pods, per-pod batching would+// fragment the batches and waste the scarce budget, so lookups are funneled+// through one shared per-org queue in Redis with a single drainer:+//+// 1. A waiting scrape RPUSHes {id, url} onto the org's queue and polls its+// reply key.+// 2. Whichever process notices work tries to become the org's drainer via+// a non-blocking distributed lock; losers just keep polling.+// 3. The drainer pops up to 100 entries, spends 1 point of the org's+// hourly budget and 1 point of the 1/second pacer (both enforced in+// Redis, so they hold across all pods and drainer failovers), calls+// urlLookup once, and writes each entry's verdict to its reply key.+//+// Fail-fast beats waiting: when the hourly budget is exhausted or the queue+// is too deep to drain before the lookup timeout, the caller gets an+// immediate error and the org's failurePolicy decides — a scrape must never+// sit in a line it cannot exit. Verdicts live only in the short-TTL reply+// keys consumed moments later; there is no verdict cache (ZDR).+//+// Capacity note: the sustained ceiling is ~11 URLs/second per org (400+// calls/hour × 100 URLs). The "zscaler"-mode concurrency cap (see+// getThreatProtectionConcurrencyCap) keeps URL demand near this budget;+// this queue only absorbs bursts.++const logger = _logger.child({ module: "threat-protection-zscaler-lookup" });++const queueKey = (orgId: string) =>+ `threat-protection:zscaler:lookup-queue:${orgId}`;+const replyKey = (id: string) => `threat-protection:zscaler:lookup-reply:${id}`;+const drainLockKey = (orgId: string) =>+ `threat-protection:zscaler:drain-lock:${orgId}`;++const BATCH_SIZE = 100;+const REPLY_TTL_SECONDS = 120;+const REPLY_POLL_INTERVAL_MS = 150;+/** Re-attempt to become the drainer this often while waiting (drainer death recovery). */+const DRAINER_REENSURE_INTERVAL_MS = 2000;+/** One idle re-check before the drainer exits, so a straggler doesn't wait a full re-ensure cycle. */+const DRAIN_IDLE_LINGER_MS = 200;+const LOOKUP_CALL_TIMEOUT_MS = 30_000;+// Must comfortably exceed one full iteration (pacer wait + the lookup call+// timeout), or the lock expires mid-iteration and a second drainer starts+// while the first is still replying.+const DRAIN_LOCK_TTL_MS = LOOKUP_CALL_TIMEOUT_MS + 15_000;++interface QueueEntry {+ id: string;+ url: string;+ /** Enqueue time (epoch ms): the drainer drops entries whose requester has given up. */+ at: number;+}++/**+ * Budget keys are tenant-scoped, not org-scoped: ZIA rate limits apply to the+ * customer's tenant, and several Firecrawl orgs may share one.+ */+export function zscalerBudgetKey(credentials: ZscalerCredentials): string {+ return `${credentials.vanityDomain}:${credentials.cloud ?? "default"}`;+}++type ReplyPayload =+ | {+ status: "ok";+ urlClassifications: string[];+ urlClassificationsWithSecurityAlert: string[];+ }+ | { status: "quota" }+ | { status: "error"; message: string };++let hourlyBudget: RateLimiterRedis | undefined;+function getHourlyBudget(): RateLimiterRedis {+ if (!hourlyBudget) {+ hourlyBudget = new RateLimiterRedis({+ storeClient: redisRateLimitClient,+ keyPrefix: "threat-protection:zscaler:lookup-budget",+ points: config.ZSCALER_LOOKUP_HOURLY_BUDGET,+ duration: 3600,+ });+ }+ return hourlyBudget;+}++let perSecondPacer: RateLimiterRedis | undefined;+function getPerSecondPacer(): RateLimiterRedis {+ if (!perSecondPacer) {+ perSecondPacer = new RateLimiterRedis({+ storeClient: redisRateLimitClient,+ keyPrefix: "threat-protection:zscaler:lookup-pacer",+ points: 1,+ duration: 1,+ });+ }+ return perSecondPacer;+}++function sleep(ms: number): Promise<void> {+ return new Promise(resolve => setTimeout(resolve, ms));+}++/**+ * Spend one point of the tenant's urlLookup budget outside the queue (the+ * connection test's lookup probe). Throws a quota ZscalerError when the+ * budget is exhausted, so ad-hoc tests can never starve real scrapes.+ */+export async function consumeZscalerLookupBudget(+ credentials: ZscalerCredentials,+): Promise<void> {+ try {+ await getHourlyBudget().consume(zscalerBudgetKey(credentials), 1);+ } catch (rejection) {+ if (rejection instanceof RateLimiterRes) {+ throw new ZscalerError(+ "quota",+ "Zscaler urlLookup hourly budget is exhausted",+ undefined,+ rejection.msBeforeNext,+ );+ }+ throw rejection;+ }+}++// =========================================+// Requester side+// =========================================++/**+ * Queue a URL for batched classification and wait for its verdict. Throws a+ * {@link ZscalerError} on quota exhaustion, queue overflow, wait timeout, or+ * a failed provider call — the caller resolves all of those through the+ * org's failurePolicy.+ */+export async function enqueueZscalerLookup(+ orgId: string,+ lookupUrl: string,+ signal?: AbortSignal,+): Promise<ZscalerUrlLookupResult> {+ // Fail fast on a queue that cannot drain before the wait timeout. The+ // LLEN/RPUSH pair is deliberately non-atomic: this is a fail-fast+ // heuristic, and a brief overshoot from concurrent enqueuers is bounded+ // by their count and harmless.+ const depth = await redisRateLimitClient.llen(queueKey(orgId));+ if (depth >= config.ZSCALER_LOOKUP_MAX_QUEUE_DEPTH) {+ throw new ZscalerError(+ "quota",+ `Zscaler lookup queue is over capacity (${depth} URLs waiting)`,+ );+ }++ // Fail fast when the hourly budget is already spent — enqueueing would+ // only park the scrape until the timeout. Budget state is keyed by the+ // ZIA tenant, which needs the org's credentials (60s-cached read).+ const credentials = await getOrgZscalerCredentials(orgId);+ if (credentials) {+ const budgetState = await getHourlyBudget()+ .get(zscalerBudgetKey(credentials))+ .catch(() => null);+ if (+ budgetState &&+ budgetState.consumedPoints >= config.ZSCALER_LOOKUP_HOURLY_BUDGET+ ) {+ throw new ZscalerError(+ "quota",+ "Zscaler urlLookup hourly budget is exhausted",+ undefined,+ budgetState.msBeforeNext,+ );+ }+ }++ const entry: QueueEntry = {+ id: randomUUID(),+ url: lookupUrl,+ at: Date.now(),+ };+ await redisRateLimitClient.rpush(queueKey(orgId), JSON.stringify(entry));+ ensureQueueDrainer(orgId);++ let lastDrainerCheck = Date.now();+ for (;;) {+ if (signal?.aborted) {+ throw new ZscalerError(+ "api",+ "Timed out waiting for the batched Zscaler verdict",+ );+ }++ const raw = await redisRateLimitClient.get(replyKey(entry.id));+ if (raw !== null) {+ redisRateLimitClient.del(replyKey(entry.id)).catch(() => {});+ let payload: ReplyPayload;+ try {+ payload = JSON.parse(raw) as ReplyPayload;+ } catch {+ throw new ZscalerError("api", "Malformed Zscaler lookup reply");+ }+ if (payload.status === "ok") {+ // A parseable but malformed reply must become a provider failure,+ // not invalid category data inside policy evaluation.+ if (+ !Array.isArray(payload.urlClassifications) ||+ !Array.isArray(payload.urlClassificationsWithSecurityAlert)+ ) {+ throw new ZscalerError("api", "Malformed Zscaler lookup reply");+ }+ return {+ url: lookupUrl,+ urlClassifications: payload.urlClassifications.filter(+ (value): value is string => typeof value === "string",+ ),+ urlClassificationsWithSecurityAlert:+ payload.urlClassificationsWithSecurityAlert.filter(+ (value): value is string => typeof value === "string",+ ),+ };+ }+ if (payload.status === "quota") {+ throw new ZscalerError(+ "quota",+ "Zscaler urlLookup hourly budget is exhausted",+ );+ }+ throw new ZscalerError("api", payload.message);+ }++ // The drainer may have died with our entry unclaimed (its lock expires+ // within DRAIN_LOCK_TTL_MS); periodically try to become / revive it.+ if (Date.now() - lastDrainerCheck >= DRAINER_REENSURE_INTERVAL_MS) {+ lastDrainerCheck = Date.now();+ ensureQueueDrainer(orgId);+ }++ await sleep(REPLY_POLL_INTERVAL_MS);+ }+}++// =========================================+// Drainer side+// =========================================++const drainersInFlight = new Set<string>();++/** Fire-and-forget: try to become the org's queue drainer. */+function ensureQueueDrainer(orgId: string): void {+ if (drainersInFlight.has(orgId)) return;+ drainersInFlight.add(orgId);+ drainQueue(orgId)+ .catch(error => {+ logger.warn("Zscaler lookup queue drainer failed", { error, orgId });+ })+ .finally(() => {+ drainersInFlight.delete(orgId);+ });+}++async function popBatch(orgId: string): Promise<QueueEntry[]> {+ const raw = await redisRateLimitClient.lpop(queueKey(orgId), BATCH_SIZE);+ if (!raw) return [];+ const entries: QueueEntry[] = [];+ for (const item of raw) {+ try {+ const parsed = JSON.parse(item) as QueueEntry;+ if (typeof parsed.id === "string" && typeof parsed.url === "string") {+ entries.push({+ ...parsed,+ // Entries from a pre-`at` process (deploy rollout window) are of+ // unknown age; treat them as fresh so their requesters still get a+ // verdict instead of an immediate expiry error.+ at: typeof parsed.at === "number" ? parsed.at : Date.now(),+ });+ }+ } catch {+ // Skip malformed entries; their requesters time out into failurePolicy.+ }+ }+ return entries;+}++// Entries whose requester has certainly stopped polling (its wait timeout+// plus a grace period has passed). Spending budget on them would classify+// URLs nobody is waiting for.+const ENTRY_EXPIRY_GRACE_MS = 2000;++function isExpired(entry: QueueEntry): boolean {+ return (+ Date.now() - entry.at >+ config.ZSCALER_LOOKUP_TIMEOUT_MS + ENTRY_EXPIRY_GRACE_MS+ );+}++async function replyAll(+ entries: QueueEntry[],+ payloadFor: (entry: QueueEntry) => ReplyPayload,+): Promise<void> {+ const pipeline = redisRateLimitClient.pipeline();+ for (const entry of entries) {+ pipeline.set(+ replyKey(entry.id),+ JSON.stringify(payloadFor(entry)),+ "EX",+ REPLY_TTL_SECONDS,+ );+ }+ await pipeline.exec();+}++async function drainQueue(orgId: string): Promise<void> {+ let lock;+ try {+ lock = await redlock.acquire([drainLockKey(orgId)], DRAIN_LOCK_TTL_MS, {+ retryCount: 0,+ });+ } catch {+ // Another process is draining this org.+ return;+ }++ try {+ const credentials = await getOrgZscalerCredentials(orgId);++ for (;;) {+ let entries = await popBatch(orgId);+ if (entries.length === 0) {+ // A URL enqueued a moment after our pop should not wait for the next+ // re-ensure cycle; check once more before exiting.+ await sleep(DRAIN_IDLE_LINGER_MS);+ entries = await popBatch(orgId);+ if (entries.length === 0) return;+ }++ // Abandoned entries (requester timed out) get a free error reply+ // instead of a budgeted classification.+ const expired = entries.filter(isExpired);+ if (expired.length > 0) {+ await replyAll(expired, () => ({+ status: "error",+ message: "The lookup request expired before the batch was sent",+ })).catch(() => {});+ entries = entries.filter(entry => !isExpired(entry));+ if (entries.length === 0) continue;+ }++ // From here on the popped entries are this drainer's responsibility:+ // an unexpected throw (Redis blip, expired lock) must not orphan them+ // into their full wait timeout — reply an error first, then unwind.+ try {+ lock = await lock.extend(DRAIN_LOCK_TTL_MS);++ if (!credentials) {+ await replyAll(entries, () => ({+ status: "error",+ message: "No Zscaler credentials are configured for this org",+ }));+ continue;+ }++ // Hourly budget: one point per urlLookup CALL, keyed by the ZIA+ // tenant. When it is gone, this loop keeps popping and answers+ // everything "quota" without touching the tenant, so waiting scrapes+ // resolve immediately via failurePolicy.+ const budgetKey = zscalerBudgetKey(credentials);+ try {+ await getHourlyBudget().consume(budgetKey, 1);+ } catch (rejection) {+ if (rejection instanceof RateLimiterRes) {+ await replyAll(entries, () => ({ status: "quota" }));+ continue;+ }+ throw rejection;+ }++ // 1 call/second pacer.+ for (;;) {+ try {+ await getPerSecondPacer().consume(budgetKey, 1);+ break;+ } catch (rejection) {+ if (rejection instanceof RateLimiterRes) {+ await sleep(Math.max(rejection.msBeforeNext, 50));+ continue;+ }+ throw rejection;+ }+ }+ } catch (error) {+ await replyAll(entries, () => ({+ status: "error",+ message: "Zscaler lookup drainer failed before the batch was sent",+ })).catch(() => {});+ throw error;+ }++ const uniqueUrls = [...new Set(entries.map(entry => entry.url))];+ try {+ const results = await zscalerLookupUrls(credentials, uniqueUrls, {+ signal: AbortSignal.timeout(LOOKUP_CALL_TIMEOUT_MS),+ });+ const byUrl = new Map(results.map(result => [result.url, result]));+ await replyAll(entries, entry => {+ const result = byUrl.get(entry.url);+ if (!result) {+ return {+ status: "error",+ message: "Zscaler returned no verdict for this URL",+ };+ }+ return {+ status: "ok",+ urlClassifications: result.urlClassifications,+ urlClassificationsWithSecurityAlert:+ result.urlClassificationsWithSecurityAlert,+ };+ });+ } catch (error) {+ const zscalerError =+ error instanceof ZscalerError+ ? error+ : new ZscalerError(+ "api",+ error instanceof Error ? error.message : String(error),+ );+ logger.warn("Batched Zscaler urlLookup call failed", {+ error,+ orgId,+ kind: zscalerError.kind,+ urls: uniqueUrls.length,+ });+ if (+ zscalerError.kind === "rate-limit" &&+ zscalerError.retryAfterMs !== undefined+ ) {+ // Tenant-side 429 (something else shares the tenant's budget):+ // pause the drainer so we do not compound it. The current batch+ // still fails into failurePolicy — no retries that could stall+ // every waiting scrape behind one bad call.+ await sleep(Math.min(zscalerError.retryAfterMs, 10_000));+ }+ await replyAll(entries, () => ({+ status: "error",+ message: zscalerError.message,+ }));+ }+ }+ } finally {+ await lock.release().catch(() => {});+ }+}diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.ts b/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.tsnew file mode 100644index 000000000..9a0ae4aea--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.ts@@ -0,0 +1,165 @@+import type { ZscalerUrlCategory } from "./client";+import { materializeCategories, matchSyncedRules } from "./sync";++// Fixture payload following the GET /urlCategories response shape from the+// official Zscaler SDK services (urls / dbCategorizedUrls / keywords /+// keywordsRetainingParentCategory), not hand-invented fields.+const CATEGORIES: ZscalerUrlCategory[] = [+ {+ id: "CUSTOM_01",+ configuredName: "Blocked Vendors",+ customCategory: true,+ urls: ["vendor-blocked.example.com", "partner.example.com/private"],+ keywords: ["forbiddenword"],+ },+ {+ id: "CUSTOM_02",+ configuredName: "Tracked Tools",+ customCategory: true,+ dbCategorizedUrls: ["tool.example.com"],+ keywordsRetainingParentCategory: ["retainedword"],+ },+ {+ // Customer addition to a Zscaler-defined category ("URLs retaining+ // parent category" on a predefined category).+ id: "GAMBLING",+ dbCategorizedUrls: ["sweepstakes.example.com"],+ },+ {+ id: "CUSTOM_03",+ configuredName: "Port-Scoped",+ customCategory: true,+ urls: ["internal.example.com:8443"],+ },+ {+ // Pure taxonomy entry: no custom content, so no local rule.+ id: "ENTERTAINMENT",+ },+];++describe("materializeCategories", () => {+ const { taxonomy, rules } = materializeCategories(CATEGORIES);++ it("keeps every category in the taxonomy for the picker", () => {+ expect(taxonomy).toHaveLength(5);+ expect(taxonomy.map(entry => entry.id).sort()).toEqual([+ "CUSTOM_01",+ "CUSTOM_02",+ "CUSTOM_03",+ "ENTERTAINMENT",+ "GAMBLING",+ ]);+ });++ it("uses the configured name and falls back to the ID", () => {+ const byId = new Map(taxonomy.map(entry => [entry.id, entry]));+ expect(byId.get("CUSTOM_01")).toMatchObject({+ name: "Blocked Vendors",+ custom: true,+ });+ expect(byId.get("GAMBLING")).toMatchObject({+ name: "GAMBLING",+ custom: false,+ });+ });++ it("materializes rules only for categories with local content", () => {+ expect(rules.map(rule => rule.id).sort()).toEqual([+ "CUSTOM_01",+ "CUSTOM_02",+ "CUSTOM_03",+ "GAMBLING",+ ]);+ });++ it("separates reclassifying and retaining entries", () => {+ const custom01 = rules.find(rule => rule.id === "CUSTOM_01")!;+ expect(custom01.urls).toContain("vendor-blocked.example.com");+ expect(custom01.retainingUrls).toEqual([]);+ expect(custom01.keywords).toEqual(["forbiddenword"]);++ const custom02 = rules.find(rule => rule.id === "CUSTOM_02")!;+ expect(custom02.urls).toEqual([]);+ expect(custom02.retainingUrls).toEqual(["tool.example.com"]);+ expect(custom02.retainingKeywords).toEqual(["retainedword"]);+ });+});++describe("matchSyncedRules", () => {+ const { rules } = materializeCategories(CATEGORIES);++ function match(url: string) {+ return matchSyncedRules(url, rules).map(entry => ({+ id: entry.rule.id,+ reclassifies: entry.reclassifies,+ }));+ }++ it("matches a URL-list entry on the host and its subdomains", () => {+ expect(match("https://vendor-blocked.example.com/page")).toEqual([+ { id: "CUSTOM_01", reclassifies: true },+ ]);+ expect(match("https://deep.vendor-blocked.example.com/")).toEqual([+ { id: "CUSTOM_01", reclassifies: true },+ ]);+ expect(match("https://not-vendor-blocked.example.com/")).toEqual([]);+ });++ it("restricts path-scoped entries to that path", () => {+ expect(match("https://partner.example.com/private/report")).toEqual([+ { id: "CUSTOM_01", reclassifies: true },+ ]);+ expect(match("https://partner.example.com/public")).toEqual([]);+ });++ it("matches keywords anywhere in the schemeless URL, case-insensitively", () => {+ expect(match("https://anything.example.org/ForbiddenWord-page")).toEqual([+ { id: "CUSTOM_01", reclassifies: true },+ ]);+ expect(match("https://anything.example.org/?q=retainedword")).toEqual([+ { id: "CUSTOM_02", reclassifies: false },+ ]);+ });++ it("ignores ports on both the entry and the checked URL", () => {+ expect(match("https://internal.example.com:8443/console")).toEqual([+ { id: "CUSTOM_03", reclassifies: true },+ ]);+ expect(match("https://internal.example.com/console")).toEqual([+ { id: "CUSTOM_03", reclassifies: true },+ ]);+ });++ it("marks retaining-parent-category entries as non-reclassifying", () => {+ expect(match("https://tool.example.com/")).toEqual([+ { id: "CUSTOM_02", reclassifies: false },+ ]);+ expect(match("https://sweepstakes.example.com/")).toEqual([+ { id: "GAMBLING", reclassifies: false },+ ]);+ });++ it("flags keyword matches as inexact so they can never allow on their own", () => {+ const matches = matchSyncedRules(+ "https://anything.example.org/ForbiddenWord-page",+ rules,+ );+ expect(matches).toHaveLength(1);+ expect(matches[0]).toMatchObject({ reclassifies: true, exact: false });++ const exactMatches = matchSyncedRules(+ "https://vendor-blocked.example.com/",+ rules,+ );+ expect(exactMatches[0]).toMatchObject({ reclassifies: true, exact: true });+ });++ it("collects every matching category", () => {+ expect(match("https://tool.example.com/forbiddenword")).toEqual(+ expect.arrayContaining([+ { id: "CUSTOM_01", reclassifies: true },+ { id: "CUSTOM_02", reclassifies: false },+ ]),+ );+ });+});diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/sync.ts b/apps/api/src/lib/threat-protection/providers/zscaler/sync.tsnew file mode 100644index 000000000..32a4a5c4b--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/sync.ts@@ -0,0 +1,585 @@+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";+import { z } from "zod";+import { config } from "../../../../config";+import { logger as _logger } from "../../../logger";+import { redisRateLimitClient } from "../../../../services/rate-limiter";+import { redlock } from "../../../../services/redlock";+import { getOrgZscalerCredentials } from "../../store";+import type {+ ThreatDecision,+ ThreatProtectionPolicy,+ ZscalerPolicySettings,+} from "../../types";+import { canonicalizeHost, splitUrl } from "../web-risk/canonicalize";+import {+ ZscalerError,+ zscalerGetUrlCategories,+ type ZscalerCredentials,+ type ZscalerUrlCategory,+} from "./client";+import { zscalerBudgetKey } from "./lookup-queue";++// Synchronization plane for the Zscaler ZIA provider.+//+// POST /urlLookup never returns custom URL classifications ("Custom URL+// classification is not returned by this request" — ZIA API docs), so the+// tenant's custom categories, custom-category keywords, and customer+// additions to Zscaler-defined categories (dbCategorizedUrls) are synced via+// GET /urlCategories and evaluated locally against every checked URL.+//+// The synced document holds the customer's own policy configuration — never+// scrape-derived data — so persisting it in Redis is compatible with+// zero-data-retention requirements (same reasoning as the org-config cache+// in ../../store.ts).+//+// Sync runs lazily: any process that notices the document is older than the+// org's sync interval kicks a re-sync in the background, guarded by a+// distributed lock so only one process calls the tenant per interval. There+// is no dedicated sync worker to deploy or babysit. Each sync replaces the+// whole document, so entries removed upstream disappear on the next pass.+//+// Keyword matching note: locally re-implementing Zscaler's keyword semantics+// exactly is not possible from the published docs. We apply the documented+// best-effort approximation (case-insensitive substring over the schemeless+// URL); URL-list entries match exactly.++const logger = _logger.child({ module: "threat-protection-zscaler-sync" });++const SYNC_DOC_VERSION = 1;+const syncDocKey = (orgId: string) => `threat-protection:zscaler:sync:${orgId}`;+const syncLockKey = (orgId: string) =>+ `threat-protection:zscaler:sync-lock:${orgId}`;++// Process-local cache of the parsed sync document so the enforcement hot+// path does not deserialize a potentially large Redis value per URL.+const DOC_CACHE_TTL_MS = 60_000;+const DOC_CACHE_MAX_ENTRIES = 100;+const docCache = new Map<+ string,+ { expiresAt: number; doc: ZscalerSyncDocument | null }+>();++/** One synced category's locally-evaluable content. */+export interface ZscalerSyncedRule {+ /** Category ID ("CUSTOM_01", or a Zscaler-defined ID with customer additions). */+ id: string;+ name: string;+ custom: boolean;+ /** URL-list entries that reclassify the URL into this category. */+ urls: string[];+ /** "Retaining parent category" URL entries — the URL keeps its Zscaler-defined categories too. */+ retainingUrls: string[];+ /** Keywords that reclassify (best-effort substring match). */+ keywords: string[];+ /** "Retaining parent category" keywords. */+ retainingKeywords: string[];+}++export interface ZscalerTaxonomyEntry {+ id: string;+ name: string;+ custom: boolean;+}++export interface ZscalerSyncDocument {+ version: number;+ status: "ok" | "error";+ /** Last successful sync; null when no sync has ever succeeded. */+ syncedAt: string | null;+ attemptedAt: string;+ error: string | null;+ taxonomy: ZscalerTaxonomyEntry[];+ rules: ZscalerSyncedRule[];+}++const syncedRuleSchema = z.object({+ id: z.string(),+ name: z.string(),+ custom: z.boolean(),+ urls: z.array(z.string()),+ retainingUrls: z.array(z.string()),+ keywords: z.array(z.string()),+ retainingKeywords: z.array(z.string()),+});++const syncDocumentSchema = z.object({+ version: z.number(),+ status: z.enum(["ok", "error"]),+ syncedAt: z.string().nullable(),+ attemptedAt: z.string(),+ error: z.string().nullable(),+ taxonomy: z.array(+ z.object({ id: z.string(), name: z.string(), custom: z.boolean() }),+ ),+ rules: z.array(syncedRuleSchema),+});++// =========================================+// Document storage+// =========================================++export async function getZscalerSyncDocument(+ orgId: string,+): Promise<ZscalerSyncDocument | null> {+ const cached = docCache.get(orgId);+ if (cached && cached.expiresAt > Date.now()) return cached.doc;++ let doc: ZscalerSyncDocument | null = null;+ try {+ const raw = await redisRateLimitClient.get(syncDocKey(orgId));+ if (raw !== null) {+ doc = syncDocumentSchema.parse(JSON.parse(raw));+ }+ } catch (error) {+ logger.warn("Failed to read the Zscaler sync document", { error, orgId });+ // Treat as missing: the caller decides via failure semantics.+ doc = null;+ }++ const now = Date.now();+ for (const [key, entry] of docCache) {+ if (entry.expiresAt <= now) docCache.delete(key);+ }+ while (docCache.size >= DOC_CACHE_MAX_ENTRIES) {+ const oldest = docCache.keys().next().value;+ if (oldest === undefined) break;+ docCache.delete(oldest);+ }+ docCache.set(orgId, { expiresAt: now + DOC_CACHE_TTL_MS, doc });+ return doc;+}++async function putZscalerSyncDocument(+ orgId: string,+ doc: ZscalerSyncDocument,+): Promise<void> {+ await redisRateLimitClient.set(syncDocKey(orgId), JSON.stringify(doc));+ docCache.set(orgId, { expiresAt: Date.now() + DOC_CACHE_TTL_MS, doc });+}++/** Removes all synced state; used when the org's Zscaler connection is removed. */+export async function clearZscalerSyncState(orgId: string): Promise<void> {+ docCache.delete(orgId);+ await redisRateLimitClient.del(syncDocKey(orgId));+}++// =========================================+// Sync+// =========================================++// GET /urlCategories budget (ZIA: 1,000/hour per tenant). Shared between+// interval syncs, dashboard "sync now", and connection tests.+let categoriesBudget: RateLimiterRedis | undefined;+function getCategoriesBudget(): RateLimiterRedis {+ if (!categoriesBudget) {+ categoriesBudget = new RateLimiterRedis({+ storeClient: redisRateLimitClient,+ keyPrefix: "threat-protection:zscaler:categories-budget",+ points: config.ZSCALER_CATEGORIES_HOURLY_BUDGET,+ duration: 3600,+ });+ }+ return categoriesBudget;+}++/**+ * Spend one point of the tenant's urlCategories budget (interval syncs,+ * "Sync now", and the connection test's taxonomy read all share it). Throws+ * a quota ZscalerError when exhausted.+ */+export async function consumeZscalerCategoriesBudget(+ credentials: ZscalerCredentials,+): Promise<void> {+ try {+ await getCategoriesBudget().consume(zscalerBudgetKey(credentials), 1);+ } catch (rejection) {+ if (rejection instanceof RateLimiterRes) {+ throw new ZscalerError(+ "quota",+ "Zscaler urlCategories hourly budget is exhausted",+ undefined,+ rejection.msBeforeNext,+ );+ }+ throw rejection;+ }+}++const MAX_LIST_ENTRY_LENGTH = 2048;++function cleanEntries(values: string[] | undefined): string[] {+ if (!Array.isArray(values)) return [];+ const cleaned = new Set<string>();+ for (const value of values) {+ if (typeof value !== "string") continue;+ const trimmed = value.trim().toLowerCase();+ if (trimmed.length === 0 || trimmed.length > MAX_LIST_ENTRY_LENGTH) {+ continue;+ }+ cleaned.add(trimmed);+ }+ return [...cleaned];+}++/**+ * Turns the GET /urlCategories payload into the synced document's taxonomy+ * and locally-evaluable rules. Exported for tests.+ */+export function materializeCategories(categories: ZscalerUrlCategory[]): {+ taxonomy: ZscalerTaxonomyEntry[];+ rules: ZscalerSyncedRule[];+} {+ const taxonomy: ZscalerTaxonomyEntry[] = [];+ const rules: ZscalerSyncedRule[] = [];++ for (const category of categories) {+ const name =+ typeof category.configuredName === "string" &&+ category.configuredName.trim().length > 0+ ? category.configuredName.trim()+ : category.id;+ const custom = category.customCategory === true;+ taxonomy.push({ id: category.id, name, custom });++ const rule: ZscalerSyncedRule = {+ id: category.id,+ name,+ custom,+ urls: cleanEntries(category.urls),+ retainingUrls: cleanEntries(category.dbCategorizedUrls),+ keywords: cleanEntries(category.keywords),+ retainingKeywords: cleanEntries(category.keywordsRetainingParentCategory),+ };+ if (+ rule.urls.length > 0 ||+ rule.retainingUrls.length > 0 ||+ rule.keywords.length > 0 ||+ rule.retainingKeywords.length > 0+ ) {+ rules.push(rule);+ }+ }++ taxonomy.sort((a, b) => a.name.localeCompare(b.name));+ return { taxonomy, rules };+}++/**+ * Fetches the tenant taxonomy and replaces the org's synced document.+ * Distributed-lock guarded: when another process is already syncing, this+ * returns the current document without calling the tenant. Throws only when+ * there is no usable document at all (first sync failing).+ */+export async function syncOrgZscalerRules(+ orgId: string,+): Promise<ZscalerSyncDocument | null> {+ let lock;+ try {+ lock = await redlock.acquire([syncLockKey(orgId)], 120_000, {+ retryCount: 0,+ });+ } catch (error) {+ // Redlock reports contention and infrastructure failure with the same+ // error; the lock key existing is what distinguishes "someone else is+ // syncing" (return their in-progress state) from a Redis outage (which+ // must surface, not masquerade as a successful no-op).+ let lockHeld = false;+ try {+ lockHeld = (await redisRateLimitClient.exists(syncLockKey(orgId))) === 1;+ } catch {+ // Redis is down; fall through to rethrow the original error.+ }+ if (lockHeld) {+ return await getZscalerSyncDocument(orgId);+ }+ throw error;+ }++ try {+ const previous = await getZscalerSyncDocument(orgId);+ const attemptedAt = new Date().toISOString();++ const credentials = await getOrgZscalerCredentials(orgId);+ if (!credentials) {+ logger.warn("Zscaler sync requested for an org without credentials", {+ orgId,+ });+ return previous;+ }++ try {+ await consumeZscalerCategoriesBudget(credentials);+ const categories = await zscalerGetUrlCategories(credentials, {+ signal: AbortSignal.timeout(60_000),+ });+ const { taxonomy, rules } = materializeCategories(categories);+ const doc: ZscalerSyncDocument = {+ version: SYNC_DOC_VERSION,+ status: "ok",+ syncedAt: attemptedAt,+ attemptedAt,+ error: null,+ taxonomy,+ rules,+ };+ await putZscalerSyncDocument(orgId, doc);+ logger.info("Zscaler category sync succeeded", {+ orgId,+ categories: taxonomy.length,+ rules: rules.length,+ });+ return doc;+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ logger.warn("Zscaler category sync failed", { error, orgId });+ // Keep the last good rules — stale custom lists beat no custom lists —+ // but surface the failure and its time for the dashboard.+ const doc: ZscalerSyncDocument = {+ version: SYNC_DOC_VERSION,+ status: "error",+ syncedAt: previous?.syncedAt ?? null,+ attemptedAt,+ error: message,+ taxonomy: previous?.taxonomy ?? [],+ rules: previous?.rules ?? [],+ };+ await putZscalerSyncDocument(orgId, doc);+ return doc;+ }+ } finally {+ await lock.release().catch(() => {});+ }+}++const syncsInFlight = new Set<string>();++/**+ * Fire-and-forget staleness check used by the enforcement hot path: when the+ * synced document is missing or older than the org's sync interval, kick a+ * background re-sync (at most one per process at a time per org).+ */+function ensureZscalerSyncFresh(+ orgId: string,+ syncIntervalMinutes: number,+ doc: ZscalerSyncDocument | null,+): void {+ const intervalMs = syncIntervalMinutes * 60_000;+ if (+ doc &&+ doc.attemptedAt &&+ Date.now() - Date.parse(doc.attemptedAt) < intervalMs+ ) {+ return;+ }+ if (syncsInFlight.has(orgId)) return;+ syncsInFlight.add(orgId);+ syncOrgZscalerRules(orgId)+ .catch(error => {+ logger.warn("Background Zscaler sync failed", { error, orgId });+ })+ .finally(() => {+ syncsInFlight.delete(orgId);+ });+}++// =========================================+// Local rule evaluation+// =========================================++interface UrlParts {+ host: string;+ /** Path + optional query, always starting with "/". */+ pathAndQuery: string;+}++function parseUrlParts(url: string): UrlParts {+ const { host, path, query } = splitUrl(url.trim().replace(/[\t\r\n]/g, ""));+ return {+ host: canonicalizeHost(host),+ pathAndQuery:+ `${path || "/"}${query !== null ? `?${query}` : ""}`.toLowerCase(),+ };+}++/**+ * Whether a URL-list entry ("example.com", "sub.example.com/path",+ * "internal.example.com:8443") matches. Host matching follows Zscaler list+ * semantics: an entry matches the host itself and its subdomains. An entry+ * with a path component additionally requires the URL path to sit under it.+ * Ports are ignored on both sides — the checked host comes from splitUrl+ * (port already stripped), so a port kept in the entry could never match+ * and the rule would silently not enforce.+ */+function matchesZscalerUrlEntry(parts: UrlParts, entry: string): boolean {+ const slash = entry.indexOf("/");+ let entryHostRaw = slash === -1 ? entry : entry.slice(0, slash);+ const portMatch = entryHostRaw.match(/^(.*):(\d+)$/);+ if (portMatch) entryHostRaw = portMatch[1];+ const entryHost = canonicalizeHost(entryHostRaw);+ if (entryHost.length === 0) return false;+ if (parts.host !== entryHost && !parts.host.endsWith(`.${entryHost}`)) {+ return false;+ }+ if (slash === -1) return true;+ let entryPath = entry.slice(slash).toLowerCase();+ if (!entryPath.startsWith("/")) entryPath = `/${entryPath}`;+ return parts.pathAndQuery.startsWith(entryPath);+}++/**+ * Best-effort keyword match: case-insensitive substring over the schemeless+ * URL ("host/path?query"), which is how ZIA keyword categories are commonly+ * understood to behave. Documented as an approximation.+ */+function matchesKeyword(parts: UrlParts, keyword: string): boolean {+ return `${parts.host}${parts.pathAndQuery}`.includes(keyword);+}++interface ZscalerSyncedMatch {+ rule: ZscalerSyncedRule;+ /** True when the matching entry reclassifies (non-"retaining") the URL. */+ reclassifies: boolean;+ /**+ * True when the match came from a URL-list entry (exact semantics). A+ * keyword match is a best-effort approximation and must never be the+ * reason a URL SKIPS the provider lookup — only exact matches may allow.+ */+ exact: boolean;+}++/** All synced categories the URL falls into. Exported for tests. */+export function matchSyncedRules(+ url: string,+ rules: ZscalerSyncedRule[],+): ZscalerSyncedMatch[] {+ const parts = parseUrlParts(url);+ const matches: ZscalerSyncedMatch[] = [];+ for (const rule of rules) {+ const exactReclassify = rule.urls.some(entry =>+ matchesZscalerUrlEntry(parts, entry),+ );+ const keywordReclassify =+ !exactReclassify &&+ rule.keywords.some(keyword => matchesKeyword(parts, keyword));+ const reclassifies = exactReclassify || keywordReclassify;+ const exactRetain =+ !reclassifies &&+ rule.retainingUrls.some(entry => matchesZscalerUrlEntry(parts, entry));+ const keywordRetain =+ !reclassifies &&+ !exactRetain &&+ rule.retainingKeywords.some(keyword => matchesKeyword(parts, keyword));+ if (reclassifies || exactRetain || keywordRetain) {+ matches.push({+ rule,+ reclassifies,+ exact: exactReclassify || exactRetain,+ });+ }+ }+ return matches;+}++// Custom category IDs follow this shape; Zscaler-defined categories never do.+const CUSTOM_CATEGORY_ID_REGEX = /^CUSTOM_\d+$/i;++/**+ * Evaluates the org's synced Zscaler rules for a URL.+ *+ * - A match in a denied category → deny ("denied-category"), no provider call.+ * - An exact (URL-list) reclassifying match in a non-denied category → allow+ * ("custom-category"), no provider call: ZIA reclassifies such URLs out of+ * their Zscaler-defined categories, so a lookup verdict would not apply.+ * Keyword matches are approximate and never allow on their own — the+ * provider lookup still runs.+ * - Only "retaining parent category" matches in non-denied categories →+ * null: the URL also keeps its Zscaler-defined categories, so the provider+ * lookup still decides.+ * - No match → null (provider lookup decides).+ *+ * When the org denies custom categories but no sync has ever succeeded, the+ * custom rules cannot be evaluated AND the lookup cannot see them (urlLookup+ * never returns custom classifications) — that resolves through the org's+ * failurePolicy instead of silently allowing. A previously synced (stale)+ * document keeps enforcing.+ *+ * Never throws on storage failure; also kicks the lazy background re-sync.+ */+export async function evaluateZscalerSyncedRules(+ url: string,+ zscaler: ZscalerPolicySettings,+ failurePolicy: ThreatProtectionPolicy["failurePolicy"],+ base: {+ url: string;+ domain: string;+ mode: ThreatProtectionPolicy["mode"];+ },+): Promise<ThreatDecision | null> {+ let doc: ZscalerSyncDocument | null = null;+ try {+ doc = await getZscalerSyncDocument(zscaler.orgId);+ } catch (error) {+ logger.warn("Failed to load synced Zscaler rules", {+ error,+ orgId: zscaler.orgId,+ });+ }+ ensureZscalerSyncFresh(zscaler.orgId, zscaler.syncIntervalMinutes, doc);++ if (!doc || doc.syncedAt === null) {+ const deniesCustomCategories = zscaler.deniedCategories.some(id =>+ CUSTOM_CATEGORY_ID_REGEX.test(id),+ );+ if (deniesCustomCategories) {+ return {+ allowed: failurePolicy === "open",+ rule: "provider-failure",+ providerConsulted: false,+ verdict: null,+ ...base,+ };+ }+ return null;+ }+ if (doc.rules.length === 0) return null;++ const matches = matchSyncedRules(url, doc.rules);+ if (matches.length === 0) return null;++ const denied = matches.filter(match =>+ zscaler.deniedCategories.includes(match.rule.id),+ );+ if (denied.length > 0) {+ return {+ allowed: false,+ rule: "denied-category",+ providerConsulted: false,+ verdict: {+ provider: "zscaler-zia",+ riskScore: null,+ categories: denied.map(match => match.rule.id),+ fromCache: false,+ raw: {+ source: "synced-rules",+ matched: denied.map(match => ({+ category: match.rule.id,+ reclassifies: match.reclassifies,+ })),+ },+ },+ ...base,+ };+ }++ if (matches.some(match => match.reclassifies && match.exact)) {+ return {+ allowed: true,+ rule: "custom-category",+ providerConsulted: false,+ verdict: null,+ ...base,+ };+ }++ return null;+}diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/testing.ts b/apps/api/src/lib/threat-protection/providers/zscaler/testing.tsnew file mode 100644index 000000000..090006aa4--- /dev/null+++ b/apps/api/src/lib/threat-protection/providers/zscaler/testing.ts@@ -0,0 +1,163 @@+import type http from "http";+import type { ZscalerUrlCategory } from "./client";++// Mock ZIA server for tests: the Zidentity token endpoint plus the two ZIA+// endpoints the provider uses. Response shapes follow the official Zscaler+// SDK services (zscaler-sdk-go/py `urlcategories`), not hand-invented+// payloads. Point the API at it with:+//+// ZSCALER_TOKEN_URL_OVERRIDE=http://localhost:<port>/oauth2/v1/token+// ZSCALER_API_URL_OVERRIDE=http://localhost:<port>+//+// The urlLookup path then resolves to /zia/api/v1/urlLookup on this server.++export interface ZscalerMockDatabase {+ /** Credentials the token endpoint accepts. */+ clientId: string;+ clientSecret: string;+ /** Simulate a role without urlLookup access (View Only probe): 403. */+ denyLookupScope?: boolean;+ /** Simulate tenant-side throttling: every urlLookup call returns 429. */+ rateLimitLookups?: boolean;+ /** GET /urlCategories payload. */+ categories: ZscalerUrlCategory[];+ /**+ * POST /urlLookup classifications, keyed by the exact looked-up URL string+ * (schemeless). Unknown URLs return MISCELLANEOUS_OR_UNKNOWN, like ZIA.+ */+ classifications: Record<+ string,+ {+ urlClassifications: string[];+ urlClassificationsWithSecurityAlert: string[];+ }+ >;+}++interface ZscalerMockCounters {+ tokenCalls: number;+ categoriesCalls: number;+ lookupCalls: number;+ lookupUrls: number;+}++export function createZscalerMockCounters(): ZscalerMockCounters {+ return { tokenCalls: 0, categoriesCalls: 0, lookupCalls: 0, lookupUrls: 0 };+}++const MOCK_TOKEN = "mock-zscaler-access-token";++function json(res: http.ServerResponse, status: number, body: unknown): void {+ res.statusCode = status;+ res.setHeader("Content-Type", "application/json");+ res.end(JSON.stringify(body));+}++function readBody(req: http.IncomingMessage): Promise<string> {+ return new Promise((resolve, reject) => {+ const chunks: Buffer[] = [];+ req.on("data", chunk => chunks.push(chunk));+ req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));+ req.on("error", reject);+ });+}++export function createZscalerMockHandler(+ db: ZscalerMockDatabase,+ counters: ZscalerMockCounters,+): (req: http.IncomingMessage, res: http.ServerResponse) => void {+ return (req, res) => {+ void (async () => {+ const url = new URL(req.url ?? "/", "http://localhost");++ if (req.method === "POST" && url.pathname === "/oauth2/v1/token") {+ counters.tokenCalls++;+ const body = new URLSearchParams(await readBody(req));+ if (+ body.get("grant_type") !== "client_credentials" ||+ body.get("client_id") !== db.clientId ||+ body.get("client_secret") !== db.clientSecret+ ) {+ json(res, 401, { error: "invalid_client" });+ return;+ }+ json(res, 200, {+ access_token: MOCK_TOKEN,+ token_type: "Bearer",+ expires_in: 3600,+ });+ return;+ }++ if (req.headers.authorization !== `Bearer ${MOCK_TOKEN}`) {+ json(res, 401, { message: "Invalid or missing bearer token" });+ return;+ }++ if (+ req.method === "GET" &&+ url.pathname === "/zia/api/v1/urlCategories"+ ) {+ counters.categoriesCalls++;+ json(res, 200, db.categories);+ return;+ }++ if (req.method === "POST" && url.pathname === "/zia/api/v1/urlLookup") {+ counters.lookupCalls++;+ if (db.denyLookupScope) {+ json(res, 403, { message: "Access to this resource is denied" });+ return;+ }+ if (db.rateLimitLookups) {+ res.setHeader("Retry-After", "1");+ json(res, 429, { message: "Rate limit exceeded" });+ return;+ }+ let urls: unknown;+ try {+ urls = JSON.parse(await readBody(req));+ } catch {+ json(res, 400, { message: "Invalid JSON body" });+ return;+ }+ if (+ !Array.isArray(urls) ||+ urls.length > 100 ||+ urls.some(entry => typeof entry !== "string" || entry.length > 1024)+ ) {+ json(res, 400, {+ message:+ "Expected an array of at most 100 URL strings of at most 1024 characters",+ });+ return;+ }+ counters.lookupUrls += urls.length;+ json(+ res,+ 200,+ urls.map(lookupUrl => {+ const entry =+ typeof lookupUrl === "string"+ ? db.classifications[lookupUrl]+ : undefined;+ return {+ url: lookupUrl,+ urlClassifications: entry?.urlClassifications ?? [+ "MISCELLANEOUS_OR_UNKNOWN",+ ],+ urlClassificationsWithSecurityAlert:+ entry?.urlClassificationsWithSecurityAlert ?? [],+ };+ }),+ );+ return;+ }++ json(res, 404, { message: `No mock for ${req.method} ${url.pathname}` });+ })().catch(() => {+ res.statusCode = 500;+ res.end();+ });+ };+}diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.tsindex ababe76e8..b2d8faa75 100644--- a/apps/api/src/lib/threat-protection/request.ts+++ b/apps/api/src/lib/threat-protection/request.ts@@ -31,6 +31,9 @@ export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE ="Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";+const THREAT_PROTECTION_ZSCALER_NOT_CONFIGURED_MESSAGE =+ 'Threat protection mode "zscaler" requires a configured Zscaler connection. Configure one via PUT /v2/team/threat-protection first.';+/*** The deprecated v0 endpoints never resolve a threat protection policy, so* teams whose flag is "forced" must not be able to fetch content through@@ -105,6 +108,19 @@ export async function resolveThreatProtection(args: {}const policy = resolveEffectivePolicy(orgConfig, args.override);++ // "zscaler" mode without a connection can only happen via a per-request+ // override on an org that never saved credentials (the config API refuses+ // to store that combination). Reject clearly instead of failing every+ // lookup into the failurePolicy.+ if (policy.mode === "zscaler" && !policy.zscaler) {+ return {+ error: THREAT_PROTECTION_ZSCALER_NOT_CONFIGURED_MESSAGE,+ policy: null,+ orgConfig,+ };+ }+return {policy: policy.mode === "off" ? null : policy,orgConfig,@@ -150,9 +166,18 @@ export async function checkUrlsAgainstThreatPolicy(dedup: ctx.dedup ?? new Map(),};+ // "zscaler" mode checks resolve through a shared per-org lookup queue+ // whose budget is spent per CALL (≤100 URLs each) — checking only 16 at a+ // time would cap batch fill at ~16 URLs per budgeted call. Run a full+ // batch's worth concurrently so the queue drainer can pack real batches.+ const concurrency =+ policy.mode === "zscaler" && !ctx.localRulesOnly+ ? 100+ : URL_CHECK_CONCURRENCY;+const decisionsByUrl = new Map<string, ThreatDecision>();- for (let i = 0; i < uniqueUrls.length; i += URL_CHECK_CONCURRENCY) {- const batch = uniqueUrls.slice(i, i + URL_CHECK_CONCURRENCY);+ for (let i = 0; i < uniqueUrls.length; i += concurrency) {+ const batch = uniqueUrls.slice(i, i + concurrency);const decisions = await Promise.all(batch.map(url => checkUrl(url, policy, dedupCtx)),);diff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.tsindex d7987607f..a3e0aea50 100644--- a/apps/api/src/lib/threat-protection/store.test.ts+++ b/apps/api/src/lib/threat-protection/store.test.ts@@ -24,6 +24,7 @@ function makeOrgConfig(orgId: "00000000-0000-0000-0000-000000000000",policy: { ...orgPolicy },allowRequestOverrides: true,+ zscaler: null,createdAt: null,updatedAt: null,...overrides,@@ -71,11 +72,54 @@ describe("rowToConfig", () => {failurePolicy: "open",},allowRequestOverrides: false,+ zscaler: null,createdAt: "2026-01-01T00:00:00.000Z",updatedAt: "2026-01-02T00:00:00.000Z",});});+ it("maps a stored Zscaler block into settings and policy", () => {+ const config = rowToConfig(+ makeRow("zscaler", {+ failurePolicy: "closed",+ zscaler: {+ clientId: "client-1",+ secretCiphertext: "gcm:iv:tag:ct",+ vanityDomain: "tenant",+ cloud: null,+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ syncIntervalMinutes: 30,+ },+ }),+ );++ expect(config.policy.mode).toBe("zscaler");+ expect(config.policy.zscaler).toEqual({+ orgId: ORG_ID,+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ syncIntervalMinutes: 30,+ });+ expect(config.zscaler).toEqual({+ clientId: "client-1",+ secretCiphertext: "gcm:iv:tag:ct",+ vanityDomain: "tenant",+ cloud: null,+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ syncIntervalMinutes: 30,+ });+ });++ it("degrades a malformed Zscaler block to no connection instead of throwing", () => {+ const config = rowToConfig(+ makeRow("zscaler", {+ zscaler: { clientId: 42 },+ }),+ );+ expect(config.policy.mode).toBe("zscaler");+ expect(config.zscaler).toBeNull();+ expect(config.policy.zscaler).toBeUndefined();+ });+it("falls back to defaults for an empty config document", () => {const config = rowToConfig(makeRow("normal", {}));expect(config.policy).toEqual({diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.tsindex e7bde17b8..3d00ce019 100644--- a/apps/api/src/lib/threat-protection/store.ts+++ b/apps/api/src/lib/threat-protection/store.ts@@ -1,9 +1,12 @@import { eq } from "drizzle-orm";import { z } from "zod";+import { config as appConfig } from "../../config";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { deleteKey, getValue, setValue } from "../../services/redis";import { logger as _logger } from "../logger";+import { decryptOrgSecret, encryptOrgSecret } from "../org-secret-crypto";+import type { ZscalerCredentials } from "./providers/zscaler/client";import {THREAT_PROTECTION_POLICY_DEFAULTS,type ThreatProtectionPolicy,@@ -41,10 +44,26 @@ function isMissingTableError(error: unknown): boolean {return false;}+/**+ * Stored (non-secret-decrypted) Zscaler settings. `secretCiphertext` is the+ * AES-256-GCM output of the write-only client secret — decrypted only by+ * {@link getOrgZscalerCredentials} at call time, never serialized to the API+ * or into job payloads.+ */+export interface OrgZscalerSettings {+ clientId: string;+ secretCiphertext: string;+ vanityDomain: string;+ cloud: string | null;+ deniedCategories: string[];+ syncIntervalMinutes: number;+}+export interface OrgThreatProtectionConfig {orgId: string;policy: ThreatProtectionPolicy;allowRequestOverrides: boolean;+ zscaler: OrgZscalerSettings | null;createdAt: string | null;updatedAt: string | null;}@@ -60,6 +79,20 @@ type ThreatProtectionConfigRow =* field defaults instead of throwing — a bad row must never take down the* enforcement hot path.*/+const storedZscalerSchema = z+ .object({+ clientId: z.string().min(1),+ secretCiphertext: z.string().min(1),+ vanityDomain: z.string().min(1),+ cloud: z.string().nullable().catch(null),+ deniedCategories: z.array(z.string()).catch([]),+ syncIntervalMinutes: z.number().int().positive().catch(60),+ })+ // A malformed block degrades to "no Zscaler connection" (lookups then fail+ // per the org's failurePolicy) instead of throwing on the hot path.+ .nullable()+ .catch(null);+const storedConfigDocumentSchema = z.object({riskScoreThreshold: z@@ -75,10 +108,12 @@ const storedConfigDocumentSchema = z.enum(["open", "closed"]).catch(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),allowRequestOverrides: z.boolean().catch(true),+ zscaler: storedZscalerSchema.optional().catch(null),}).catch({...THREAT_PROTECTION_POLICY_DEFAULTS,allowRequestOverrides: true,+ zscaler: null,});/**@@ -91,17 +126,33 @@ export function rowToConfig(row: ThreatProtectionConfigRow,): OrgThreatProtectionConfig {const doc = storedConfigDocumentSchema.parse(row.config ?? {});+ const zscaler = doc.zscaler ?? null;return {orgId: row.org_id,policy: {- mode: row.mode === "normal" ? "normal" : "off",+ mode:+ row.mode === "normal"+ ? "normal"+ : row.mode === "zscaler"+ ? "zscaler"+ : "off",riskScoreThreshold: doc.riskScoreThreshold,blacklist: doc.blacklist,whitelist: doc.whitelist,blockedTlds: doc.blockedTlds,failurePolicy: doc.failurePolicy,+ ...(zscaler+ ? {+ zscaler: {+ orgId: row.org_id,+ deniedCategories: zscaler.deniedCategories,+ syncIntervalMinutes: zscaler.syncIntervalMinutes,+ },+ }+ : {}),},allowRequestOverrides: doc.allowRequestOverrides,+ zscaler,createdAt: row.created_at,updatedAt: row.updated_at,};@@ -163,14 +214,64 @@ export async function getOrgThreatProtectionConfig(return config;}+/**+ * The PUT config document needs a client secret we do not have: thrown when a+ * Zscaler block is first configured without one, or when the client ID+ * changes without a matching new secret. Mapped to a 400 by the controller.+ */+export class ZscalerSecretRequiredError extends Error {+ constructor() {+ super(+ "zscaler.clientSecret is required when configuring a Zscaler connection or changing its client ID",+ );+ this.name = "ZscalerSecretRequiredError";+ }+}+/*** Full-document upsert of the org's threat protection config. Invalidates* the read cache on success.+ *+ * The Zscaler client secret is write-only: when the input block omits it,+ * the stored ciphertext is kept — but only while the client ID is unchanged+ * (pairing a new client with an old secret is always a mistake, so that+ * throws {@link ZscalerSecretRequiredError} instead).*/export async function upsertOrgThreatProtectionConfig(orgId: string,config: ThreatProtectionConfigInput,): Promise<OrgThreatProtectionConfig> {+ let zscaler: z.infer<typeof storedZscalerSchema> = null;+ if (config.zscaler) {+ let secretCiphertext: string;+ if (config.zscaler.clientSecret) {+ secretCiphertext = encryptOrgSecret(config.zscaler.clientSecret, orgId);+ } else {+ // Primary read: the upsert below writes to the primary, and a stale+ // replica must not make us discard a just-saved secret.+ const existingRows = await db+ .select({ config: schema.threat_protection_config.config })+ .from(schema.threat_protection_config)+ .where(eq(schema.threat_protection_config.org_id, orgId))+ .limit(1);+ const existing = existingRows[0]+ ? storedConfigDocumentSchema.parse(existingRows[0].config ?? {}).zscaler+ : null;+ if (!existing || existing.clientId !== config.zscaler.clientId) {+ throw new ZscalerSecretRequiredError();+ }+ secretCiphertext = existing.secretCiphertext;+ }+ zscaler = {+ clientId: config.zscaler.clientId,+ secretCiphertext,+ vanityDomain: config.zscaler.vanityDomain,+ cloud: config.zscaler.cloud,+ deniedCategories: config.zscaler.deniedCategories,+ syncIntervalMinutes: config.zscaler.syncIntervalMinutes,+ };+ }+const values = {org_id: orgId,mode: config.mode,@@ -182,6 +283,7 @@ export async function upsertOrgThreatProtectionConfig(blockedTlds: config.blockedTlds,failurePolicy: config.failurePolicy,allowRequestOverrides: config.allowRequestOverrides,+ zscaler,},};@@ -233,7 +335,9 @@ export async function getOrgIdForTeam(teamId: string): Promise<string | null> {*/export function resolveEffectivePolicy(orgConfig: OrgThreatProtectionConfig | null,- requestOverride?: Partial<ThreatProtectionPolicy>,+ // Omit is deliberate: the Zscaler connection is org-owned and never+ // request-settable — passing one is a compile error, not an ignored field.+ requestOverride?: Partial<Omit<ThreatProtectionPolicy, "zscaler">>,): ThreatProtectionPolicy {const base: ThreatProtectionPolicy = orgConfig? { ...orgConfig.policy }@@ -246,6 +350,10 @@ export function resolveEffectivePolicy(for (const key of Object.keys(requestOverride) as Array<keyof ThreatProtectionPolicy>) {+ // The Zscaler connection settings come from the org config only — the+ // override schema has no such field, and this guard keeps it that way+ // even if a wider Partial ever reaches here.+ if (key === "zscaler") continue;const value = requestOverride[key];if (value !== undefined) {(base as unknown as Record<string, unknown>)[key] = value;@@ -254,3 +362,100 @@ export function resolveEffectivePolicy(return base;}++/**+ * Decrypted Zscaler API credentials for an org, or null when the org has no+ * (valid) Zscaler connection. Backed by the 60s org-config cache, so a saved+ * secret rotation is picked up within a minute — the plaintext itself lives+ * only in this return value, never in a cache or a job payload.+ */+export async function getOrgZscalerCredentials(+ orgId: string,+): Promise<ZscalerCredentials | null> {+ const orgConfig = await getOrgThreatProtectionConfig(orgId);+ if (!orgConfig?.zscaler) return null;+ try {+ return {+ clientId: orgConfig.zscaler.clientId,+ clientSecret: decryptOrgSecret(orgConfig.zscaler.secretCiphertext, orgId),+ vanityDomain: orgConfig.zscaler.vanityDomain,+ cloud: orgConfig.zscaler.cloud,+ };+ } catch (error) {+ logger.error("Failed to decrypt the stored Zscaler client secret", {+ error,+ orgId,+ });+ return null;+ }+}++const concurrencyCapCacheKey = (teamId: string) =>+ `threat-protection-concurrency-cap:${teamId}`;++/**+ * Scrape-concurrency cap imposed by the team's threat protection mode, or+ * null when none applies. "zscaler" mode caps concurrency so URL demand+ * stays near the tenant's classification budget (POST /urlLookup allows 1+ * request/second × 100 URLs, 400 requests/hour) — without the cap, a large+ * crawl outruns the budget, every lookup times out, and the failurePolicy+ * fires for everything.+ *+ * The budget is org-scoped while concurrency admission is team-scoped, so+ * the org-level cap is split across the org's teams — otherwise an org with+ * N teams would run at N× the intended concurrency.+ *+ * Cached in Redis for 60s per team; fails open (null) so a Redis or DB+ * hiccup never blocks job admission.+ */+export async function getThreatProtectionConcurrencyCap(+ teamId: string,+): Promise<number | null> {+ // Self-hosted deployments have no org config database (and no enterprise+ // threat protection) — skip the lookup instead of warning on every job.+ if (!appConfig.USE_DB_AUTHENTICATION) return null;++ const key = concurrencyCapCacheKey(teamId);+ try {+ const cached = await getValue(key);+ if (cached === "none") return null;+ if (cached !== null) {+ const parsed = Number(cached);+ // A malformed cache entry must not become a NaN/zero cap that breaks+ // job admission — fall through and resolve from the source instead.+ if (Number.isFinite(parsed) && parsed > 0) return parsed;+ }+ } catch (error) {+ logger.warn("Failed to read the concurrency cap cache", { error, teamId });+ }++ let cap: number | null = null;+ try {+ const orgId = await getOrgIdForTeam(teamId);+ const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;+ if (orgId && orgConfig?.policy.mode === "zscaler") {+ const teamRows = await dbRr+ .select({ id: schema.teams.id })+ .from(schema.teams)+ .where(eq(schema.teams.org_id, orgId));+ const teamCount = Math.max(teamRows.length, 1);+ cap = Math.max(+ 1,+ Math.floor(appConfig.ZSCALER_MODE_CONCURRENCY_CAP / teamCount),+ );+ }+ } catch (error) {+ logger.warn("Failed to resolve the threat protection concurrency cap", {+ error,+ teamId,+ });+ return null;+ }++ try {+ await setValue(key, cap === null ? "none" : String(cap), 60);+ } catch (error) {+ logger.warn("Failed to write the concurrency cap cache", { error, teamId });+ }+ return cap;+}diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.tsindex a15e7c9a5..c045ddc20 100644--- a/apps/api/src/lib/threat-protection/types.ts+++ b/apps/api/src/lib/threat-protection/types.ts@@ -1,7 +1,25 @@// Shared type contract for the threat protection feature.// NOTE: concurrent in-flight branches create this exact file — do not modify without coordinating.-export type ThreatProtectionMode = "off" | "normal";+export type ThreatProtectionMode = "off" | "normal" | "zscaler";++/**+ * The non-secret slice of an org's Zscaler settings that policy evaluation+ * needs. Threaded through {@link ThreatProtectionPolicy} — and therefore+ * serialized into scrape job payloads — so it must NEVER carry credentials.+ * The provider loads credentials fresh from the org config store by `orgId`+ * at lookup time (so a secret rotation applies within the config cache TTL).+ */+export interface ZscalerPolicySettings {+ orgId: string;+ /**+ * Category IDs (Zscaler-defined and custom) the org blocks. Matched against+ * the normalized verdict categories and against synced custom-list rules.+ */+ deniedCategories: string[];+ /** How often the synced taxonomy/custom lists are considered fresh. */+ syncIntervalMinutes: number;+}export interface ThreatProtectionPolicy {mode: ThreatProtectionMode;@@ -15,6 +33,11 @@ export interface ThreatProtectionPolicy {blockedTlds: string[];/** Behavior when the provider is unavailable: "closed" blocks, "open" allows. */failurePolicy: "open" | "closed";+ /**+ * Present when the org has a Zscaler connection configured. Attached from+ * the org config by resolveEffectivePolicy — never settable per-request.+ */+ zscaler?: ZscalerPolicySettings;}export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<@@ -28,11 +51,11 @@ export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<failurePolicy: "closed",};-// Only Google Web Risk today. The union (and the provider seam around it —-// separate provider modules under ./providers, the mode→provider dispatch in-// ./index.ts, and the provider-agnostic verdict engine in ./verdict.ts) is-// deliberately kept so future partner classifiers slot in as new members.-export type ThreatProvider = "google-web-risk";+// The union (and the provider seam around it — separate provider modules+// under ./providers, the mode→provider dispatch in ./index.ts, and the+// provider-agnostic verdict engine in ./verdict.ts) is deliberately kept so+// further partner classifiers slot in as new members.+export type ThreatProvider = "google-web-risk" | "zscaler-zia";export interface RawVerdict {provider: ThreatProvider;@@ -40,6 +63,12 @@ export interface RawVerdict {riskScore: number | null;/** Provider threat categories (Web Risk threat types map through here). */categories: string[];+ /**+ * True when the provider flagged the URL with a security-alert+ * classification (Zscaler `urlClassificationsWithSecurityAlert`).+ * Undefined for providers without the concept.+ */+ securityAlert?: boolean;/*** Always false: verdicts are never persisted (ZDR) — repeated checks within* one request share the same in-flight decision via the request-scoped@@ -66,6 +95,13 @@ export type ThreatDecisionRule =| "blacklist"| "blocked-tld"| "risk-score"+ /** A verdict category is on the org's denied-categories list (Zscaler mode). */+ | "denied-category"+ /**+ * The URL matched a synced custom-list entry that reclassifies it into a+ * non-denied custom category, so no provider lookup ran (Zscaler mode).+ */+ | "custom-category"| "provider-failure"| "default-allow";diff --git a/apps/api/src/lib/threat-protection/verdict.test.ts b/apps/api/src/lib/threat-protection/verdict.test.tsindex 95d7497b2..9d08498de 100644--- a/apps/api/src/lib/threat-protection/verdict.test.ts+++ b/apps/api/src/lib/threat-protection/verdict.test.ts@@ -425,3 +425,119 @@ describe("normalizeDomain", () => {).toMatchObject({ allowed: false, rule: "blacklist" });});});++describe("evaluatePolicy — zscaler denied categories", () => {+ function zscalerPolicy(+ overrides: Partial<ThreatProtectionPolicy> = {},+ ): ThreatProtectionPolicy {+ return policy({+ mode: "zscaler",+ zscaler: {+ orgId: "00000000-0000-0000-0000-000000000000",+ deniedCategories: ["GAMBLING", "CUSTOM_01"],+ syncIntervalMinutes: 60,+ },+ ...overrides,+ });+ }++ function zscalerVerdict(overrides: Partial<RawVerdict> = {}): RawVerdict {+ return verdict({+ provider: "zscaler-zia",+ riskScore: null,+ ...overrides,+ });+ }++ test("blocks when a verdict category is on the denied list", () => {+ const decision = evaluatePolicy(+ "https://casino.example.com/",+ zscalerVerdict({ categories: ["GAMBLING", "ENTERTAINMENT"] }),+ zscalerPolicy(),+ );+ expect(decision).toMatchObject({+ allowed: false,+ rule: "denied-category",+ providerConsulted: true,+ });+ });++ test("blocks on a denied custom category ID in the verdict", () => {+ const decision = evaluatePolicy(+ "https://vendor.example.com/",+ zscalerVerdict({ categories: ["CUSTOM_01"] }),+ zscalerPolicy(),+ );+ expect(decision).toMatchObject({ allowed: false, rule: "denied-category" });+ });++ test("allows when no verdict category is denied", () => {+ const decision = evaluatePolicy(+ "https://news.example.com/",+ zscalerVerdict({ categories: ["NEWS_AND_MEDIA"] }),+ zscalerPolicy(),+ );+ expect(decision).toMatchObject({ allowed: true, rule: "default-allow" });+ });++ test("uncategorized URLs block only when the org denies the unknown bucket", () => {+ const uncategorized = zscalerVerdict({+ categories: ["MISCELLANEOUS_OR_UNKNOWN"],+ });+ expect(+ evaluatePolicy(+ "https://new.example.com/",+ uncategorized,+ zscalerPolicy(),+ ),+ ).toMatchObject({ allowed: true, rule: "default-allow" });+ expect(+ evaluatePolicy(+ "https://new.example.com/",+ uncategorized,+ zscalerPolicy({+ zscaler: {+ orgId: "00000000-0000-0000-0000-000000000000",+ deniedCategories: ["MISCELLANEOUS_OR_UNKNOWN"],+ syncIntervalMinutes: 60,+ },+ }),+ ),+ ).toMatchObject({ allowed: false, rule: "denied-category" });+ });++ test("whitelist wins over a denied category", () => {+ const decision = evaluatePolicy(+ "https://casino.example.com/",+ zscalerVerdict({ categories: ["GAMBLING"] }),+ zscalerPolicy({ whitelist: ["casino.example.com"] }),+ );+ expect(decision).toMatchObject({ allowed: true, rule: "whitelist" });+ });++ test("a null verdict resolves through the failurePolicy", () => {+ expect(+ evaluatePolicy(+ "https://example.com/",+ null,+ zscalerPolicy({ failurePolicy: "closed" }),+ ),+ ).toMatchObject({ allowed: false, rule: "provider-failure" });+ expect(+ evaluatePolicy(+ "https://example.com/",+ null,+ zscalerPolicy({ failurePolicy: "open" }),+ ),+ ).toMatchObject({ allowed: true, rule: "provider-failure" });+ });++ test("the null risk score never trips the score threshold", () => {+ const decision = evaluatePolicy(+ "https://example.com/",+ zscalerVerdict({ categories: [], riskScore: null }),+ zscalerPolicy({ riskScoreThreshold: 0 }),+ );+ expect(decision).toMatchObject({ allowed: true, rule: "default-allow" });+ });+});diff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.tsindex d65022305..938245439 100644--- a/apps/api/src/lib/threat-protection/verdict.ts+++ b/apps/api/src/lib/threat-protection/verdict.ts@@ -167,6 +167,18 @@ export function evaluatePolicy(}if (verdict !== null) {+ // Zscaler mode blocks on the org's denied-categories selection. The+ // verdict's categories are the normalized union of urlClassifications+ // and urlClassificationsWithSecurityAlert.+ if (policy.mode === "zscaler" && policy.zscaler) {+ const denied = verdict.categories.filter(category =>+ policy.zscaler!.deniedCategories.includes(category),+ );+ if (denied.length > 0) {+ return { allowed: false, rule: "denied-category", ...base };+ }+ }+if (verdict.riskScore !== null &&verdict.riskScore >= policy.riskScoreThresholddiff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex a2767c5f2..da6361cf9 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -68,7 +68,10 @@ import {import { activityController } from "../controllers/v1/activity";import {getTeamThreatProtectionController,+ getTeamZscalerCategoriesController,putTeamThreatProtectionController,+ syncTeamZscalerController,+ testTeamZscalerConnectionController,} from "../controllers/v2/team-threat-protection";import {getTeamSiemLoggingController,@@ -444,6 +447,24 @@ v2Router.put(wrap(putTeamThreatProtectionController),);+v2Router.post(+ "/team/threat-protection/zscaler/test-connection",+ authMiddleware(RateLimiterMode.Account),+ wrap(testTeamZscalerConnectionController),+);++v2Router.get(+ "/team/threat-protection/zscaler/categories",+ authMiddleware(RateLimiterMode.Account),+ wrap(getTeamZscalerCategoriesController),+);++v2Router.post(+ "/team/threat-protection/zscaler/sync",+ authMiddleware(RateLimiterMode.Account),+ wrap(syncTeamZscalerController),+);+v2Router.get("/team/siem",authMiddleware(RateLimiterMode.Account),