Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(threat-protection): Zscaler ZIA provider (zscaler mode) (#4161)

Gergő Móriczf18fbe46ffc2
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.ts
new file mode 100644
index 000000000..8eccbde21
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/threat-protection-zscaler.test.ts
@@ -0,0 +1,364 @@
+import http from "http";
+import request from "supertest";
+import {
+ describeIf,
+ idmux,
+ Identity,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+ scrapeTimeout,
+} from "../lib";
+import { scrape, scrapeRaw } from "./lib";
+import {
+ createZscalerMockCounters,
+ createZscalerMockHandler,
+ type ZscalerMockDatabase,
+} from "../../../lib/threat-protection/providers/zscaler/testing";
+
+// =========================================
+// Threat protection "zscaler" mode
+//
+// Config-API tests need a team with the threatProtection flag (idmux) and
+// are gated on TEST_PRODUCTION; the ones that store a client secret also
+// need SIEM_LOGGING_ENCRYPTION_KEY (shared org-secret encryption key).
+//
+// Enforcement tests additionally need the mock ZIA server below. The API
+// must be started with the base-URL overrides pointing at it, e.g.:
+//
+// SIEM_LOGGING_ENCRYPTION_KEY=<32-byte hex> \
+// ZSCALER_TOKEN_URL_OVERRIDE=http://localhost:4519/oauth2/v1/token \
+// ZSCALER_API_URL_OVERRIDE=http://localhost:4519 \
+// pnpm harness pnpm exec vitest run src/__tests__/snips/v2/threat-protection-zscaler.test.ts
+//
+// Those tests self-skip when the override is not set to a local address.
+// =========================================
+
+const HAS_ENCRYPTION_KEY = !!process.env.SIEM_LOGGING_ENCRYPTION_KEY;
+
+const mockApiUrl = process.env.ZSCALER_API_URL_OVERRIDE ?? "";
+const HAS_MOCK_PROVIDER =
+ /^http:\/\/(localhost|127\.0\.0\.1):\d+$/.test(mockApiUrl) &&
+ HAS_ENCRYPTION_KEY;
+
+// Fixture domains. *.example.com is reserved (RFC 2606) — blocked scrapes
+// fail before any outbound request, so these never need to exist.
+const CASINO_DOMAIN = "threat-zscaler-casino.example.com";
+const VENDOR_DOMAIN = "threat-zscaler-vendor.example.com";
+const UNKNOWN_DOMAIN = "threat-zscaler-unknown.example.com";
+const CLEAN_URL = TEST_SUITE_WEBSITE;
+
+const MOCK_CLIENT_ID = "firecrawl-test-client";
+const MOCK_CLIENT_SECRET = "firecrawl-test-secret";
+
+const zscalerDb: ZscalerMockDatabase = {
+ clientId: MOCK_CLIENT_ID,
+ clientSecret: MOCK_CLIENT_SECRET,
+ categories: [
+ { id: "GAMBLING" },
+ { id: "NEWS_AND_MEDIA" },
+ {
+ id: "CUSTOM_01",
+ configuredName: "Blocked Vendors",
+ customCategory: true,
+ urls: [VENDOR_DOMAIN],
+ },
+ ],
+ classifications: {
+ [CASINO_DOMAIN]: {
+ urlClassifications: ["GAMBLING"],
+ urlClassificationsWithSecurityAlert: [],
+ },
+ },
+};
+const zscalerCounters = createZscalerMockCounters();
+
+let mockServer: http.Server | null = null;
+
+function startMockProvider(): Promise<void> {
+ const port = Number(new URL(mockApiUrl).port);
+ mockServer = http.createServer(
+ createZscalerMockHandler(zscalerDb, zscalerCounters),
+ );
+ return new Promise((resolve, reject) => {
+ mockServer!.once("error", reject);
+ mockServer!.listen(port, () => resolve());
+ });
+}
+
+async function getConfigRaw(identity: Identity) {
+ return await request(TEST_API_URL)
+ .get("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+async function putConfigRaw(body: unknown, identity: Identity) {
+ return await request(TEST_API_URL)
+ .put("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body as object);
+}
+
+async function postRaw(path: string, body: unknown, identity: Identity) {
+ return await request(TEST_API_URL)
+ .post(path)
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send((body ?? {}) as object);
+}
+
+function zscalerConfigDoc(overrides: Record<string, unknown> = {}) {
+ return {
+ mode: "zscaler",
+ failurePolicy: "closed",
+ zscaler: {
+ clientId: MOCK_CLIENT_ID,
+ clientSecret: MOCK_CLIENT_SECRET,
+ vanityDomain: "testtenant",
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ ...overrides,
+ },
+ };
+}
+
+describeIf(TEST_PRODUCTION)("Threat protection zscaler config API", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-zscaler/config",
+ flags: { threatProtection: "allowed" },
+ });
+ });
+
+ it('PUT rejects mode "zscaler" without a connection block', async () => {
+ const res = await putConfigRaw({ mode: "zscaler" }, identity);
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("zscaler");
+ });
+
+ it("test-connection without stored or inline credentials returns 400", async () => {
+ const res = await postRaw(
+ "/v2/team/threat-protection/zscaler/test-connection",
+ {},
+ identity,
+ );
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ });
+
+ it("categories endpoint without a connection returns 400", async () => {
+ const res = await request(TEST_API_URL)
+ .get("/v2/team/threat-protection/zscaler/categories")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ });
+
+ describeIf(HAS_ENCRYPTION_KEY)("with the encryption key", () => {
+ it("stores the connection, never serves the secret back", async () => {
+ const put = await putConfigRaw(zscalerConfigDoc(), identity);
+ expect(put.statusCode).toBe(200);
+ expect(put.body.success).toBe(true);
+ expect(put.body.data.mode).toBe("zscaler");
+ expect(put.body.data.zscaler).toMatchObject({
+ clientId: MOCK_CLIENT_ID,
+ hasClientSecret: true,
+ vanityDomain: "testtenant",
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ syncIntervalMinutes: 60,
+ });
+ expect(JSON.stringify(put.body)).not.toContain(MOCK_CLIENT_SECRET);
+
+ const get = await getConfigRaw(identity);
+ expect(get.statusCode).toBe(200);
+ expect(get.body.data.zscaler).toMatchObject({
+ clientId: MOCK_CLIENT_ID,
+ hasClientSecret: true,
+ });
+ expect(JSON.stringify(get.body)).not.toContain(MOCK_CLIENT_SECRET);
+ });
+
+ it("keeps the stored secret when the update omits it", async () => {
+ const doc = zscalerConfigDoc({ deniedCategories: ["GAMBLING"] });
+ delete (doc.zscaler as Record<string, unknown>).clientSecret;
+ const put = await putConfigRaw(doc, identity);
+ expect(put.statusCode).toBe(200);
+ expect(put.body.data.zscaler).toMatchObject({
+ hasClientSecret: true,
+ deniedCategories: ["GAMBLING"],
+ });
+ });
+
+ it("requires a new secret when the client ID changes", async () => {
+ const doc = zscalerConfigDoc({ clientId: "different-client" });
+ delete (doc.zscaler as Record<string, unknown>).clientSecret;
+ const put = await putConfigRaw(doc, identity);
+ expect(put.statusCode).toBe(400);
+ expect(put.body.success).toBe(false);
+ expect(put.body.error).toContain("clientSecret");
+ });
+
+ it("clears the connection when the block is removed", async () => {
+ const put = await putConfigRaw({ mode: "off" }, identity);
+ expect(put.statusCode).toBe(200);
+ expect(put.body.data.zscaler).toBeNull();
+
+ // Re-configuring after a clear needs the secret again.
+ const doc = zscalerConfigDoc();
+ delete (doc.zscaler as Record<string, unknown>).clientSecret;
+ const rePut = await putConfigRaw(doc, identity);
+ expect(rePut.statusCode).toBe(400);
+ });
+ });
+});
+
+describeIf(TEST_PRODUCTION && HAS_MOCK_PROVIDER)(
+ "Threat protection zscaler enforcement (mock ZIA)",
+ () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ await startMockProvider();
+ identity = await idmux({
+ name: "threat-protection-zscaler/enforcement",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+
+ const put = await putConfigRaw(zscalerConfigDoc(), identity);
+ expect(put.statusCode).toBe(200);
+
+ // Deterministic setup: force the custom-list sync before enforcement.
+ const sync = await postRaw(
+ "/v2/team/threat-protection/zscaler/sync",
+ {},
+ identity,
+ );
+ expect(sync.statusCode).toBe(200);
+ expect(sync.body.data.syncStatus.lastSyncedAt).not.toBeNull();
+ expect(sync.body.data.syncStatus.ruleCount).toBeGreaterThanOrEqual(1);
+ }, 60_000);
+
+ afterAll(async () => {
+ if (mockServer) {
+ await new Promise(resolve => mockServer!.close(resolve));
+ mockServer = null;
+ }
+ });
+
+ beforeEach(() => {
+ zscalerDb.rateLimitLookups = false;
+ });
+
+ it("connection test passes against the mock tenant", async () => {
+ const res = await postRaw(
+ "/v2/team/threat-protection/zscaler/test-connection",
+ {},
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+ expect(res.body.data).toMatchObject({
+ ok: true,
+ tokenOk: true,
+ categoriesOk: true,
+ lookupOk: true,
+ });
+ });
+
+ it("connection test distinguishes bad credentials", async () => {
+ const res = await postRaw(
+ "/v2/team/threat-protection/zscaler/test-connection",
+ {
+ zscaler: {
+ clientId: "wrong-client",
+ clientSecret: "wrong-secret",
+ vanityDomain: "testtenant",
+ },
+ },
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+ expect(res.body.data).toMatchObject({
+ ok: false,
+ tokenOk: false,
+ error: { kind: "auth" },
+ });
+ });
+
+ it("serves the synced taxonomy for the category picker", async () => {
+ const res = await request(TEST_API_URL)
+ .get("/v2/team/threat-protection/zscaler/categories")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+ expect(res.statusCode).toBe(200);
+ const ids = res.body.data.categories.map(
+ (category: { id: string }) => category.id,
+ );
+ expect(ids).toContain("GAMBLING");
+ expect(ids).toContain("CUSTOM_01");
+ const custom = res.body.data.categories.find(
+ (category: { id: string }) => category.id === "CUSTOM_01",
+ );
+ expect(custom).toMatchObject({ name: "Blocked Vendors", custom: true });
+ });
+
+ it(
+ "scrape: a denied Zscaler-defined category blocks via urlLookup",
+ async () => {
+ const res = await scrapeRaw(
+ { url: `https://${CASINO_DOMAIN}/` } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ expect(res.body.error).toContain(CASINO_DOMAIN);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: a denied custom-list domain blocks from synced rules, no lookup",
+ async () => {
+ const lookupCallsBefore = zscalerCounters.lookupCalls;
+ const res = await scrapeRaw(
+ { url: `https://${VENDOR_DOMAIN}/any/page` } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ expect(res.body.error).toContain(VENDOR_DOMAIN);
+ // Custom-list decisions never spend the tenant's lookup budget.
+ expect(zscalerCounters.lookupCalls).toBe(lookupCallsBefore);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: a clean domain passes with the lookup consulted",
+ async () => {
+ const lookupUrlsBefore = zscalerCounters.lookupUrls;
+ const doc = await scrape({ url: CLEAN_URL } as any, identity);
+ expect(doc.metadata.statusCode).toBe(200);
+ expect(zscalerCounters.lookupUrls).toBeGreaterThan(lookupUrlsBefore);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "scrape: provider failure resolves through failurePolicy closed",
+ async () => {
+ zscalerDb.rateLimitLookups = true;
+ const res = await scrapeRaw(
+ { url: `https://${UNKNOWN_DOMAIN}/` } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+ },
+);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 827e8bf06..15080288d 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -81,6 +81,35 @@ const configSchema = z.object({
.positive()
.default(6 * 60 * 60),
+ // Zscaler ZIA provider ("zscaler" threat protection mode). Base-URL
+ // overrides exist for tests (mock ZIA server); production always uses the
+ // real endpoints derived from the org's vanity domain and cloud name.
+ ZSCALER_TOKEN_URL_OVERRIDE: z.string().url().optional(),
+ ZSCALER_API_URL_OVERRIDE: z.string().url().optional(),
+ // Per-org hourly budget for POST /urlLookup calls. ZIA allows 400/hour;
+ // the default keeps headroom for connection tests.
+ ZSCALER_LOOKUP_HOURLY_BUDGET: z.coerce.number().int().positive().default(380),
+ // Per-org hourly budget for GET /urlCategories calls (ZIA allows 1,000/hour).
+ ZSCALER_CATEGORIES_HOURLY_BUDGET: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(900),
+ // How long one queued urlLookup may wait for its batched verdict before the
+ // org's failurePolicy decides.
+ ZSCALER_LOOKUP_TIMEOUT_MS: z.coerce.number().int().positive().default(15000),
+ // Queued-URL depth per org above which new lookups fail fast into the
+ // failurePolicy instead of waiting in a line they cannot exit.
+ ZSCALER_LOOKUP_MAX_QUEUE_DEPTH: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(500),
+ // Effective scrape concurrency cap for teams in "zscaler" mode, keeping URL
+ // demand near the classification budget (≤100 URLs/s burst, ~11 URLs/s
+ // sustained at 400 lookups/hour).
+ ZSCALER_MODE_CONCURRENCY_CAP: z.coerce.number().int().positive().default(15),
+
// Organization SIEM logging delivery. The encryption key must decode to
// exactly 32 bytes; validation happens when a secret is encrypted/decrypted
// so self-hosted deployments that do not use this feature need no key.
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index b9ec77ecd..2b46f79ac 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -490,12 +490,20 @@ export async function mapController(
// Threat protection: remove blocked links from the returned URL list
// entirely. Checks are URL-level; scan fees bill +2 per unique scanned
// URL (see calculateThreatScanCredits).
+ //
+ // "zscaler" mode evaluates map results against local rules only, same as
+ // the v2 map controller: one map can return thousands of URLs, and inline
+ // classification would burn the tenant's 400/hour urlLookup budget on
+ // links that may never be fetched.
let threatScanCredits = 0;
if (threatProtection.policy && result.links.length > 0) {
const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
result.links,
threatProtection.policy,
- { teamId: req.auth.team_id },
+ {
+ teamId: req.auth.team_id,
+ localRulesOnly: threatProtection.policy.mode === "zscaler",
+ },
);
threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());
result.links = result.links.filter(x => {
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index eb02d2ae9..e6a2165b5 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -210,12 +210,21 @@ export async function mapController(
// Threat protection: remove blocked links from the returned URL list
// entirely. Checks are URL-level; scan fees bill +2 per unique scanned
// URL (see calculateThreatScanCredits).
+ //
+ // "zscaler" mode evaluates map results against local rules only (org
+ // lists + synced custom categories): one map can return thousands of
+ // URLs, and inline classification would burn the tenant's 400/hour
+ // urlLookup budget on links that may never be fetched. Every URL still
+ // gets the full provider check when a scrape of it starts.
let threatScanCredits = 0;
if (threatProtection.policy && result.mapResults.length > 0) {
const { decisionsByUrl } = await checkUrlsAgainstThreatPolicy(
result.mapResults.map(x => x.url),
threatProtection.policy,
- { teamId: req.auth.team_id },
+ {
+ teamId: req.auth.team_id,
+ localRulesOnly: threatProtection.policy.mode === "zscaler",
+ },
);
threatScanCredits = calculateThreatScanCredits(decisionsByUrl.values());
result.mapResults = result.mapResults.filter(x => {
diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.ts
index db2f7012c..6378ea3ed 100644
--- a/apps/api/src/controllers/v2/team-threat-protection.ts
+++ b/apps/api/src/controllers/v2/team-threat-protection.ts
@@ -3,15 +3,29 @@ import { z } from "zod";
import { logger as _logger } from "../../lib/logger";
import { RequestWithAuth } from "./types";
import { getThreatProtection } from "../../lib/zdr-helpers";
-import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";
+import {
+ threatProtectionConfigSchema,
+ zscalerConfigSchema,
+} from "../../lib/threat-protection/config";
import { THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE } from "../../lib/threat-protection/request";
import {
getOrgIdForTeam,
getOrgThreatProtectionConfig,
+ getOrgZscalerCredentials,
resolveEffectivePolicy,
upsertOrgThreatProtectionConfig,
+ ZscalerSecretRequiredError,
type OrgThreatProtectionConfig,
} from "../../lib/threat-protection/store";
+import { zscalerTestConnection } from "../../lib/threat-protection/providers/zscaler/client";
+import { consumeZscalerLookupBudget } from "../../lib/threat-protection/providers/zscaler/lookup-queue";
+import {
+ clearZscalerSyncState,
+ consumeZscalerCategoriesBudget,
+ getZscalerSyncDocument,
+ syncOrgZscalerRules,
+ type ZscalerSyncDocument,
+} from "../../lib/threat-protection/providers/zscaler/sync";
const logger = _logger.child({ module: "team-threat-protection" });
@@ -50,16 +64,46 @@ async function resolveOrgId(
return orgId;
}
+/**
+ * Dashboard view of the sync plane: last success, last attempt, staleness,
+ * and the last failure. Derived from the synced document — no tenant call.
+ */
+function serializeSyncStatus(doc: ZscalerSyncDocument | null) {
+ return {
+ lastSyncedAt: doc?.syncedAt ?? null,
+ lastAttemptedAt: doc?.attemptedAt ?? null,
+ error: doc?.status === "error" ? doc.error : null,
+ ruleCount: doc?.rules.length ?? 0,
+ categoryCount: doc?.taxonomy.length ?? 0,
+ };
+}
+
/**
* Effective config document served by GET and PUT. Always includes every
* policy field (defaults applied), so the dashboard can render the form
- * without knowing the defaults.
+ * without knowing the defaults. The Zscaler client secret is write-only and
+ * never serialized; `hasClientSecret` tells the dashboard one is stored.
*/
-function serializeConfig(orgConfig: OrgThreatProtectionConfig | null) {
- const policy = resolveEffectivePolicy(orgConfig);
+function serializeConfig(
+ orgConfig: OrgThreatProtectionConfig | null,
+ syncDoc: ZscalerSyncDocument | null = null,
+) {
+ const { zscaler: _zscalerPolicy, ...policy } =
+ resolveEffectivePolicy(orgConfig);
return {
...policy,
allowRequestOverrides: orgConfig?.allowRequestOverrides ?? true,
+ zscaler: orgConfig?.zscaler
+ ? {
+ clientId: orgConfig.zscaler.clientId,
+ hasClientSecret: true,
+ vanityDomain: orgConfig.zscaler.vanityDomain,
+ cloud: orgConfig.zscaler.cloud,
+ deniedCategories: orgConfig.zscaler.deniedCategories,
+ syncIntervalMinutes: orgConfig.zscaler.syncIntervalMinutes,
+ syncStatus: serializeSyncStatus(syncDoc),
+ }
+ : null,
configured: orgConfig !== null,
updatedAt: orgConfig?.updatedAt ?? null,
};
@@ -89,10 +133,13 @@ export async function getTeamThreatProtectionController(
if (!orgId) return;
const orgConfig = await getOrgThreatProtectionConfig(orgId);
+ const syncDoc = orgConfig?.zscaler
+ ? await getZscalerSyncDocument(orgId)
+ : null;
res.status(200).json({
success: true,
- data: serializeConfig(orgConfig),
+ data: serializeConfig(orgConfig, syncDoc),
});
}
@@ -117,23 +164,248 @@ export async function putTeamThreatProtectionController(
return;
}
+ if (input.mode === "zscaler" && !input.zscaler) {
+ res.status(400).json({
+ success: false,
+ error:
+ 'Threat protection mode "zscaler" requires the zscaler connection block.',
+ });
+ return;
+ }
+
const orgId = await resolveOrgId(req, res);
if (!orgId) return;
const previous = await getOrgThreatProtectionConfig(orgId);
- const updated = await upsertOrgThreatProtectionConfig(orgId, input);
+ let updated: OrgThreatProtectionConfig;
+ try {
+ updated = await upsertOrgThreatProtectionConfig(orgId, input);
+ } catch (error) {
+ if (error instanceof ZscalerSecretRequiredError) {
+ res.status(400).json({ success: false, error: error.message });
+ return;
+ }
+ throw error;
+ }
+
+ // Keep the sync plane consistent with the connection: a removed connection
+ // drops the synced rules, and so does a change of connection identity — the
+ // old tenant's custom rules must not keep enforcing while the new tenant's
+ // first sync runs. A (re)configured connection syncs in the background so
+ // custom-list rules and the category picker are ready shortly after save.
+ const connectionIdentityChanged =
+ previous?.zscaler &&
+ updated.zscaler &&
+ (previous.zscaler.clientId !== updated.zscaler.clientId ||
+ previous.zscaler.vanityDomain !== updated.zscaler.vanityDomain ||
+ previous.zscaler.cloud !== updated.zscaler.cloud);
+ if (previous?.zscaler && (!updated.zscaler || connectionIdentityChanged)) {
+ await clearZscalerSyncState(orgId).catch(error => {
+ logger.warn("Failed to clear Zscaler sync state", { error, orgId });
+ });
+ }
+ if (updated.zscaler) {
+ syncOrgZscalerRules(orgId).catch(error => {
+ logger.warn("Background Zscaler sync after config save failed", {
+ error,
+ orgId,
+ });
+ });
+ }
- // Audit log — org-level security configuration change.
+ // Audit log — org-level security configuration change. The serialized view
+ // never carries the secret, so a rotation gets an explicit marker.
+ const auditChangedFields = changedFields(previous, updated);
+ if (input.zscaler?.clientSecret) {
+ auditChangedFields.push("zscaler.clientSecret");
+ }
logger.info("Threat protection config updated", {
teamId: req.auth.team_id,
orgId,
mode: updated.policy.mode,
- changedFields: changedFields(previous, updated),
+ changedFields: auditChangedFields,
+ });
+
+ res.status(200).json({
+ success: true,
+ data: serializeConfig(
+ updated,
+ updated.zscaler ? await getZscalerSyncDocument(orgId) : null,
+ ),
+ });
+}
+
+// Connection test accepts either inline credentials (pre-save validation
+// from the dashboard) or none (test the stored connection). The inline shape
+// requires the secret — testing brand-new credentials without one is
+// meaningless — except when the client ID matches the stored connection, in
+// which case the stored secret fills in (dashboards round-trip the config
+// without the write-only secret).
+const testConnectionBodySchema = z
+ .strictObject({
+ zscaler: zscalerConfigSchema
+ .pick({
+ clientId: true,
+ clientSecret: true,
+ vanityDomain: true,
+ cloud: true,
+ })
+ .optional(),
+ })
+ .optional();
+
+export async function testTeamZscalerConnectionController(
+ req: RequestWithAuth<{}, any, unknown>,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ // Primitive JSON bodies (0, false, "x") must not silently become "test the
+ // stored connection" — only an object body (or none) is valid.
+ if (
+ req.body !== undefined &&
+ (typeof req.body !== "object" ||
+ req.body === null ||
+ Array.isArray(req.body))
+ ) {
+ res.status(400).json({
+ success: false,
+ error: "The request body must be a JSON object.",
+ });
+ return;
+ }
+ const body = testConnectionBodySchema.parse(
+ req.body && Object.keys(req.body as object).length > 0
+ ? req.body
+ : undefined,
+ );
+
+ const stored = await getOrgZscalerCredentials(orgId);
+ let credentials = stored;
+ if (body?.zscaler) {
+ const inline = body.zscaler;
+ let clientSecret = inline.clientSecret ?? null;
+ if (!clientSecret && stored && stored.clientId === inline.clientId) {
+ clientSecret = stored.clientSecret;
+ }
+ if (!clientSecret) {
+ res.status(400).json({
+ success: false,
+ error:
+ "zscaler.clientSecret is required to test credentials that are not stored yet",
+ });
+ return;
+ }
+ credentials = {
+ clientId: inline.clientId,
+ clientSecret,
+ vanityDomain: inline.vanityDomain,
+ cloud: inline.cloud,
+ };
+ }
+
+ if (!credentials) {
+ res.status(400).json({
+ success: false,
+ error:
+ "No Zscaler connection is configured; provide credentials in the request body to test them.",
+ });
+ return;
+ }
+
+ // The probe spends one urlCategories call and one urlLookup call of the
+ // tenant's quota. Each stage charges the shared budget right before its
+ // own API call, so repeated tests can never starve real scrapes — and a
+ // test that dies earlier (bad credentials) spends nothing.
+ const result = await zscalerTestConnection(credentials, {
+ signal: AbortSignal.timeout(30_000),
+ beforeCategories: () => consumeZscalerCategoriesBudget(credentials),
+ beforeLookup: () => consumeZscalerLookupBudget(credentials),
+ });
+
+ logger.info("Zscaler connection test", {
+ teamId: req.auth.team_id,
+ orgId,
+ ok: result.ok,
+ tokenOk: result.tokenOk,
+ categoriesOk: result.categoriesOk,
+ lookupOk: result.lookupOk,
+ errorKind: result.error?.kind ?? null,
+ });
+
+ res.status(200).json({ success: true, data: result });
+}
+
+/**
+ * Category taxonomy for the dashboard's denied-categories picker: Zscaler-
+ * defined and custom categories, from the synced document. Syncs inline
+ * when no document exists yet (first visit after connecting).
+ */
+export async function getTeamZscalerCategoriesController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ const orgConfig = await getOrgThreatProtectionConfig(orgId);
+ if (!orgConfig?.zscaler) {
+ res.status(400).json({
+ success: false,
+ error: "No Zscaler connection is configured for this organization.",
+ });
+ return;
+ }
+
+ let doc = await getZscalerSyncDocument(orgId);
+ if (!doc || doc.taxonomy.length === 0) {
+ doc = await syncOrgZscalerRules(orgId);
+ }
+
+ res.status(200).json({
+ success: true,
+ data: {
+ categories: doc?.taxonomy ?? [],
+ syncStatus: serializeSyncStatus(doc),
+ },
+ });
+}
+
+/** Explicit "Sync now" — bypasses the interval, still lock- and budget-guarded. */
+export async function syncTeamZscalerController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ if (rejectWithoutFlag(req, res)) return;
+
+ const orgId = await resolveOrgId(req, res);
+ if (!orgId) return;
+
+ const orgConfig = await getOrgThreatProtectionConfig(orgId);
+ if (!orgConfig?.zscaler) {
+ res.status(400).json({
+ success: false,
+ error: "No Zscaler connection is configured for this organization.",
+ });
+ return;
+ }
+
+ const doc = await syncOrgZscalerRules(orgId);
+
+ logger.info("Zscaler manual sync", {
+ teamId: req.auth.team_id,
+ orgId,
+ status: doc?.status ?? "unknown",
});
res.status(200).json({
success: true,
- data: serializeConfig(updated),
+ data: { syncStatus: serializeSyncStatus(doc) },
});
}
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 246e376f0..6b8ae5c73 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -19,6 +19,7 @@ import {
removeConcurrencyLimitActiveJob,
} from "./concurrency-redis";
import { autumnService } from "../services/autumn/autumn.service";
+import { getThreatProtectionConcurrencyCap } from "./threat-protection/store";
// Fallback when Autumn can't give us a concurrency value.
const DEFAULT_CONCURRENCY_LIMIT = 2;
@@ -28,13 +29,22 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;
* balance. Autumn is authoritative; when the entity is missing we fall back to
* the low default of 2. When Autumn errors, getConcurrencyLimit already returns
* a high fail-open value, so that carries through here.
+ *
+ * Threat protection can clamp the result: "zscaler" mode caps concurrency so
+ * URL demand stays near the tenant's classification budget — otherwise a
+ * large crawl outruns the 400-lookups/hour budget and every scrape resolves
+ * through the failurePolicy instead of a verdict.
*/
export async function getEffectiveConcurrencyLimit(
teamId: string,
orgId?: string | null,
): Promise<number> {
const autumnValue = await autumnService.getConcurrencyLimit(teamId, orgId);
- return autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;
+ const limit = autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;
+ const threatProtectionCap = await getThreatProtectionConcurrencyCap(teamId);
+ return threatProtectionCap !== null
+ ? Math.min(limit, threatProtectionCap)
+ : limit;
}
const constructKey = constructConcurrencyLimitKey;
diff --git a/apps/api/src/lib/siem-logging/crypto.test.ts b/apps/api/src/lib/org-secret-crypto.test.ts
similarity index 60%
rename from apps/api/src/lib/siem-logging/crypto.test.ts
rename to apps/api/src/lib/org-secret-crypto.test.ts
index ea0c98a14..e478f992f 100644
--- a/apps/api/src/lib/siem-logging/crypto.test.ts
+++ b/apps/api/src/lib/org-secret-crypto.test.ts
@@ -1,7 +1,7 @@
import crypto from "crypto";
import { afterEach, describe, expect, it } from "vitest";
-import { config } from "../../config";
-import { decryptSiemSecret, encryptSiemSecret } from "./crypto";
+import { config } from "../config";
+import { decryptOrgSecret, encryptOrgSecret } from "./org-secret-crypto";
const originalKey = config.SIEM_LOGGING_ENCRYPTION_KEY;
@@ -9,25 +9,25 @@ afterEach(() => {
config.SIEM_LOGGING_ENCRYPTION_KEY = originalKey;
});
-describe("SIEM secret encryption", () => {
+describe("Org secret encryption", () => {
it("round-trips with AES-256-GCM without storing plaintext", () => {
config.SIEM_LOGGING_ENCRYPTION_KEY = crypto.randomBytes(32).toString("hex");
- const encrypted = encryptSiemSecret("client-secret-value", "org-one");
+ const encrypted = encryptOrgSecret("client-secret-value", "org-one");
expect(encrypted).toMatch(/^gcm:/);
expect(encrypted).not.toContain("client-secret-value");
- expect(decryptSiemSecret(encrypted, "org-one")).toBe("client-secret-value");
- expect(() => decryptSiemSecret(encrypted, "org-two")).toThrow();
+ expect(decryptOrgSecret(encrypted, "org-one")).toBe("client-secret-value");
+ expect(() => decryptOrgSecret(encrypted, "org-two")).toThrow();
});
it("refuses to store a secret without a valid external key", () => {
config.SIEM_LOGGING_ENCRYPTION_KEY = undefined;
- expect(() => encryptSiemSecret("secret", "org-one")).toThrow(
+ expect(() => encryptOrgSecret("secret", "org-one")).toThrow(
"SIEM_LOGGING_ENCRYPTION_KEY is not configured",
);
config.SIEM_LOGGING_ENCRYPTION_KEY = "too-short";
- expect(() => encryptSiemSecret("secret", "org-one")).toThrow(
+ expect(() => encryptOrgSecret("secret", "org-one")).toThrow(
"must be a 32-byte",
);
});
diff --git a/apps/api/src/lib/siem-logging/crypto.ts b/apps/api/src/lib/org-secret-crypto.ts
similarity index 72%
rename from apps/api/src/lib/siem-logging/crypto.ts
rename to apps/api/src/lib/org-secret-crypto.ts
index 04cd3fc4e..39d0dc7a0 100644
--- a/apps/api/src/lib/siem-logging/crypto.ts
+++ b/apps/api/src/lib/org-secret-crypto.ts
@@ -1,5 +1,10 @@
import crypto from "crypto";
-import { config } from "../../config";
+import { config } from "../config";
+
+// AES-256-GCM encryption for org-scoped partner credentials at rest (SIEM
+// destination secrets, Zscaler OAuth client secrets). The key lives outside
+// the database (env), and the org ID is bound in as AAD so a ciphertext
+// pasted onto another org's row fails authentication instead of decrypting.
const GCM_PREFIX = "gcm:";
@@ -25,7 +30,7 @@ function getEncryptionKey(): Buffer {
return key;
}
-export function encryptSiemSecret(secret: string, orgId: string): string {
+export function encryptOrgSecret(secret: string, orgId: string): string {
const key = getEncryptionKey();
const iv = crypto.randomBytes(12);
const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);
@@ -38,16 +43,16 @@ export function encryptSiemSecret(secret: string, orgId: string): string {
return `${GCM_PREFIX}${iv.toString("base64")}:${tag.toString("base64")}:${ciphertext.toString("base64")}`;
}
-export function decryptSiemSecret(stored: string, orgId: string): string {
+export function decryptOrgSecret(stored: string, orgId: string): string {
if (!stored.startsWith(GCM_PREFIX)) {
- throw new Error("SIEM client secret is not encrypted with AES-256-GCM");
+ throw new Error("Stored secret is not encrypted with AES-256-GCM");
}
const [ivB64, tagB64, ciphertextB64] = stored
.slice(GCM_PREFIX.length)
.split(":");
if (!ivB64 || !tagB64 || !ciphertextB64) {
- throw new Error("Malformed encrypted SIEM client secret");
+ throw new Error("Malformed encrypted secret");
}
const decipher = crypto.createDecipheriv(
diff --git a/apps/api/src/lib/siem-logging/event.ts b/apps/api/src/lib/siem-logging/event.ts
index 33984e953..e87f78800 100644
--- a/apps/api/src/lib/siem-logging/event.ts
+++ b/apps/api/src/lib/siem-logging/event.ts
@@ -96,6 +96,9 @@ function threatForDecisions(
rule: decision.rule,
provider: decision.verdict?.provider ?? null,
categories: [...new Set(decision.verdict?.categories ?? [])],
+ ...(typeof decision.verdict?.securityAlert === "boolean"
+ ? { security_alert: decision.verdict.securityAlert }
+ : {}),
};
}
diff --git a/apps/api/src/lib/siem-logging/store.ts b/apps/api/src/lib/siem-logging/store.ts
index 4c69885f3..35a59ca09 100644
--- a/apps/api/src/lib/siem-logging/store.ts
+++ b/apps/api/src/lib/siem-logging/store.ts
@@ -4,7 +4,7 @@ import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { getRedisConnection } from "../../services/queue-service";
import { logger as _logger } from "../logger";
-import { decryptSiemSecret, encryptSiemSecret } from "./crypto";
+import { decryptOrgSecret, encryptOrgSecret } from "../org-secret-crypto";
import type { OrgSiemLoggingConfig, SiemLoggingConfigInput } from "./types";
const CACHE_TTL_MS = 60_000;
@@ -47,7 +47,7 @@ function rowToConfig(row: SiemConfigRow): OrgSiemLoggingConfig {
enabled: row.enabled,
destination: {
...destination,
- clientSecret: decryptSiemSecret(row.secret_ciphertext, row.org_id),
+ clientSecret: decryptOrgSecret(row.secret_ciphertext, row.org_id),
},
createdAt: row.created_at,
updatedAt: row.updated_at,
@@ -164,7 +164,7 @@ export async function upsertOrgSiemLoggingConfig(
}
const secretCiphertext = input.destination.clientSecret
- ? encryptSiemSecret(input.destination.clientSecret, orgId)
+ ? encryptOrgSecret(input.destination.clientSecret, orgId)
: existingCiphertext!;
const values = {
org_id: orgId,
diff --git a/apps/api/src/lib/siem-logging/types.ts b/apps/api/src/lib/siem-logging/types.ts
index 688f57e0a..7bbd9f38f 100644
--- a/apps/api/src/lib/siem-logging/types.ts
+++ b/apps/api/src/lib/siem-logging/types.ts
@@ -87,6 +87,13 @@ export interface ScrapeActivityThreat {
rule: ThreatDecisionRule;
provider: ThreatProvider | null;
categories: string[];
+ /**
+ * Present only for providers with the concept (Zscaler): true when the
+ * URL carried a security-alert classification
+ * (`urlClassificationsWithSecurityAlert`). Only this normalized flag is
+ * exported — never the provider's raw response payload.
+ */
+ security_alert?: boolean;
}
export interface ScrapeActivityEvent {
diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.ts
index 34d439546..9891ad963 100644
--- a/apps/api/src/lib/threat-protection/config.ts
+++ b/apps/api/src/lib/threat-protection/config.ts
@@ -34,6 +34,62 @@ const tldSchema = z
`Invalid TLD ${JSON.stringify(iss.input)}: must be a lowercase alphanumeric TLD without the leading dot, e.g. "zip"`,
});
+// A single DNS label, as used for the Zidentity vanity domain
+// (<vanityDomain>.zslogin.net) and the optional OneAPI cloud name
+// (api.<cloud>.zsapi.net).
+const dnsLabelSchema = z
+ .string()
+ .trim()
+ .toLowerCase()
+ .refine(value => new RegExp(`^${DOMAIN_LABEL}$`).test(value), {
+ error: iss =>
+ `Invalid value ${JSON.stringify(iss.input)}: must be a single DNS label (letters, digits, inner hyphens)`,
+ });
+
+/**
+ * Zscaler category IDs: predefined ("GAMBLING", "OTHER_ADULT_MATERIAL") and
+ * custom ("CUSTOM_01") IDs are uppercase with digits/underscores.
+ */
+const zscalerCategoryIdSchema = z
+ .string()
+ .trim()
+ .refine(value => /^[A-Z0-9_]{1,128}$/.test(value), {
+ error: iss =>
+ `Invalid category ID ${JSON.stringify(iss.input)}: must be an uppercase Zscaler category ID like "GAMBLING" or "CUSTOM_01"`,
+ });
+
+/**
+ * Zscaler connection + policy block, accepted by
+ * `PUT /v2/team/threat-protection` when the org uses mode "zscaler".
+ *
+ * `clientSecret` is write-only: it is encrypted at rest, never serialized
+ * back, and may be omitted on update to keep the stored secret (only when
+ * `clientId` is unchanged — pairing a new client ID with an old secret is
+ * always a mistake).
+ */
+export const zscalerConfigSchema = z.strictObject({
+ clientId: z.string().trim().min(1).max(256),
+ // Not trimmed (the secret is stored verbatim), but a whitespace-only value
+ // must not be persisted as a "configured" secret that can never mint tokens.
+ clientSecret: z
+ .string()
+ .min(1)
+ .max(1024)
+ .refine(value => value.trim().length > 0, {
+ error: "clientSecret must not be blank",
+ })
+ .optional(),
+ /** Zidentity vanity domain: tokens come from https://<vanityDomain>.zslogin.net */
+ vanityDomain: dnsLabelSchema,
+ /** Optional OneAPI cloud name (api.<cloud>.zsapi.net); null = default cloud. */
+ cloud: dnsLabelSchema.nullable().prefault(null),
+ deniedCategories: z.array(zscalerCategoryIdSchema).max(512).prefault([]),
+ /** How often the category taxonomy + custom lists re-sync. */
+ syncIntervalMinutes: z.number().int().min(5).max(1440).prefault(60),
+});
+
+type ZscalerConfigInput = z.infer<typeof zscalerConfigSchema>;
+
// =========================================
// Policy + org config schemas
// =========================================
@@ -43,7 +99,7 @@ const tldSchema = z
* except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.
*/
export const threatProtectionPolicySchema = z.strictObject({
- mode: z.enum(["off", "normal"]),
+ mode: z.enum(["off", "normal", "zscaler"]),
riskScoreThreshold: z
.number()
.int()
@@ -81,7 +137,7 @@ void _policyContractCheck;
* provides replace the org policy's values (see `resolveEffectivePolicy`).
*/
export const threatProtectionOverrideSchema = z.strictObject({
- mode: z.enum(["off", "normal"]).optional(),
+ mode: z.enum(["off", "normal", "zscaler"]).optional(),
riskScoreThreshold: z.number().int().min(0).max(100).optional(),
blacklist: z.array(domainGlobSchema).max(1000).optional(),
whitelist: z.array(domainGlobSchema).max(1000).optional(),
@@ -91,11 +147,14 @@ export const threatProtectionOverrideSchema = z.strictObject({
type ThreatProtectionOverride = z.infer<typeof threatProtectionOverrideSchema>;
-// Compile-time assertion that the override shape stays assignable to
-// Partial<ThreatProtectionPolicy> (what `resolveEffectivePolicy` consumes).
+// Compile-time assertion that the override shape stays assignable to what
+// `resolveEffectivePolicy` consumes. The Omit is deliberate: the Zscaler
+// connection is org-owned and never request-settable, and typing it out of
+// the override surface makes passing one a compile error instead of a
+// silently ignored field.
const _overrideContractCheck = (
x: ThreatProtectionOverride,
-): Partial<ThreatProtectionPolicy> => x;
+): Partial<Omit<ThreatProtectionPolicy, "zscaler">> => x;
void _overrideContractCheck;
/**
@@ -105,6 +164,13 @@ void _overrideContractCheck;
export const threatProtectionConfigSchema = threatProtectionPolicySchema.extend(
{
allowRequestOverrides: z.boolean().prefault(true),
+ /**
+ * Zscaler connection block. Absent = no Zscaler connection (clears any
+ * stored one, consistent with the full-document PUT semantics). Required
+ * when mode is "zscaler" — enforced in the controller, where the stored
+ * secret is available for the omitted-secret update path.
+ */
+ zscaler: zscalerConfigSchema.optional(),
},
);
diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.ts
index f38a87ff2..74b5e38b5 100644
--- a/apps/api/src/lib/threat-protection/index.ts
+++ b/apps/api/src/lib/threat-protection/index.ts
@@ -1,6 +1,9 @@
+import { config } from "../../config";
import { logger } from "../logger";
import { fetchGoogleWebRiskVerdict } from "./providers/google-web-risk";
import { canonicalizeUrl } from "./providers/web-risk/canonicalize";
+import { fetchZscalerVerdict } from "./providers/zscaler";
+import { evaluateZscalerSyncedRules } from "./providers/zscaler/sync";
import type {
RawVerdict,
ThreatCheckDedup,
@@ -27,6 +30,15 @@ export interface ThreatCheckContext {
* request-scoped — never persisted, never shared across requests (ZDR).
*/
dedup?: ThreatCheckDedup;
+ /**
+ * Evaluate local rules only (whitelist/blacklist/blocked-TLD and, in
+ * "zscaler" mode, the synced custom-list rules) and default-allow instead
+ * of consulting the provider. Used for map results in "zscaler" mode: one
+ * map can return thousands of URLs, and classifying them inline would
+ * burn the tenant's 400-lookups/hour budget on links that may never be
+ * scraped — each URL still gets its full check when a scrape of it starts.
+ */
+ localRulesOnly?: boolean;
}
// Public entry point for the threat protection core library (enterprise
@@ -36,13 +48,18 @@ export interface ThreatCheckContext {
// this request/job reuses the same in-flight decision
// 3. local-only rules (whitelist/blacklist/blocked-tld, evaluated against
// the URL's host) → decide without a provider call (no billing)
-// 4. provider ("normal" = Google Web Risk local hash-prefix database),
-// with a per-attempt timeout and one retry. Lookups are URL-level:
-// host-suffix × path-prefix expressions per the Safe Browsing spec, so
-// a listing that flags only a specific page is caught even when its
-// domain is otherwise clean.
-// 5. evaluate the policy against the verdict; provider failure → the org's
-// failurePolicy decides (fail-open allows, fail-closed blocks)
+// 4. "zscaler" mode only: the tenant's synced custom-list rules (custom
+// categories, keywords, customer category additions — urlLookup never
+// returns those) → a decisive match skips the provider call
+// 5. provider ("normal" = Google Web Risk local hash-prefix database,
+// "zscaler" = batched ZIA urlLookup through the shared per-org queue),
+// with a per-attempt timeout; "normal" gets one retry. Web Risk lookups
+// are URL-level: host-suffix × path-prefix expressions per the Safe
+// Browsing spec, so a listing that flags only a specific page is caught
+// even when its domain is otherwise clean.
+// 6. evaluate the policy against the verdict (zscaler: the org's denied
+// categories); provider failure → the org's failurePolicy decides
+// (fail-open allows, fail-closed blocks)
// Any decision backed by a verdict sets providerConsulted, which drives
// billing in the enforcement layer (+2 credits per unique scanned URL per
// billing scope — see calculateThreatScanCredits). This module performs no
@@ -71,20 +88,37 @@ const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry
* Single mode→provider dispatch point. Every provider is a separate module
* under ./providers exporting `fetch<X>Verdict(url) → RawVerdict`; this
* switch is the only place that knows which mode maps to which classifier.
- * Deliberately kept as a dispatch even with one live branch — future partner
- * classifiers add a mode + a case here and nothing else changes.
+ *
+ * Per-mode transport behavior: "normal" (Web Risk) is a local hash-prefix
+ * check, so it gets a short timeout and one retry. "zscaler" waits on a
+ * batched remote lookup whose pacing the queue already owns — a longer
+ * timeout and NO retry (a retry would spend a second slice of the tenant's
+ * 400/hour budget on the same URL).
*/
async function fetchProviderVerdict(
url: string,
mode: Exclude<ThreatProtectionMode, "off">,
+ policy: ThreatProtectionPolicy,
): Promise<RawVerdict> {
+ const attempts = mode === "zscaler" ? 1 : PROVIDER_ATTEMPTS;
+ const timeoutMs =
+ mode === "zscaler" ? config.ZSCALER_LOOKUP_TIMEOUT_MS : PROVIDER_TIMEOUT_MS;
+
let lastError: unknown;
- for (let attempt = 1; attempt <= PROVIDER_ATTEMPTS; attempt++) {
+ for (let attempt = 1; attempt <= attempts; attempt++) {
try {
- const signal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS);
+ const signal = AbortSignal.timeout(timeoutMs);
switch (mode) {
case "normal":
return await fetchGoogleWebRiskVerdict(url, { signal });
+ case "zscaler": {
+ if (!policy.zscaler) {
+ throw new Error(
+ "Threat protection mode is zscaler but the org has no Zscaler connection configured",
+ );
+ }
+ return await fetchZscalerVerdict(url, policy.zscaler, { signal });
+ }
}
} catch (error) {
lastError = error;
@@ -112,11 +146,43 @@ async function checkUrlFresh(
return local;
}
+ // Zscaler mode: the tenant's synced custom-list rules next. urlLookup
+ // never returns custom classifications, so these can only be decided
+ // here — and a decisive match skips the budgeted provider call entirely.
+ if (policy.mode === "zscaler" && policy.zscaler) {
+ const synced = await evaluateZscalerSyncedRules(
+ canonicalUrl,
+ policy.zscaler,
+ policy.failurePolicy,
+ {
+ url: canonicalUrl,
+ domain: normalizeDomain(canonicalUrl),
+ mode: policy.mode,
+ },
+ );
+ if (synced !== null) {
+ return synced;
+ }
+ }
+
+ if (ctx.localRulesOnly) {
+ return {
+ allowed: true,
+ rule: "default-allow",
+ url: canonicalUrl,
+ domain: normalizeDomain(canonicalUrl),
+ providerConsulted: false,
+ verdict: null,
+ mode: policy.mode,
+ };
+ }
+
let verdict: RawVerdict | null = null;
try {
verdict = await fetchProviderVerdict(
canonicalUrl,
policy.mode as Exclude<ThreatProtectionMode, "off">,
+ policy,
);
} catch {
// Already logged per-attempt; a null verdict routes the decision
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/client.test.ts b/apps/api/src/lib/threat-protection/providers/zscaler/client.test.ts
new file mode 100644
index 000000000..be383451e
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/client.test.ts
@@ -0,0 +1,201 @@
+import http from "http";
+import { config } from "../../../../config";
+import {
+ toLookupUrl,
+ ZscalerError,
+ zscalerGetUrlCategories,
+ zscalerLookupUrls,
+ zscalerTestConnection,
+ type ZscalerCredentials,
+} from "./client";
+import {
+ createZscalerMockCounters,
+ createZscalerMockHandler,
+ type ZscalerMockDatabase,
+} from "./testing";
+
+const counters = createZscalerMockCounters();
+const db: ZscalerMockDatabase = {
+ clientId: "test-client",
+ clientSecret: "test-secret",
+ categories: [
+ { id: "GAMBLING" },
+ {
+ id: "CUSTOM_01",
+ configuredName: "Blocked Vendors",
+ customCategory: true,
+ urls: ["vendor.example.com"],
+ },
+ ],
+ classifications: {
+ "casino.example.com": {
+ urlClassifications: ["GAMBLING"],
+ urlClassificationsWithSecurityAlert: [],
+ },
+ "malware.example.com": {
+ urlClassifications: [],
+ urlClassificationsWithSecurityAlert: ["MALWARE_SITE"],
+ },
+ },
+};
+
+let server: http.Server;
+let originalTokenOverride: string | undefined;
+let originalApiOverride: string | undefined;
+
+function credentials(
+ overrides: Partial<ZscalerCredentials> = {},
+): ZscalerCredentials {
+ return {
+ clientId: "test-client",
+ clientSecret: "test-secret",
+ vanityDomain: "testtenant",
+ cloud: null,
+ ...overrides,
+ };
+}
+
+beforeAll(async () => {
+ server = http.createServer(createZscalerMockHandler(db, counters));
+ await new Promise<void>((resolve, reject) => {
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", resolve);
+ });
+ const address = server.address();
+ if (address === null || typeof address !== "object") {
+ throw new Error("Mock server did not bind");
+ }
+ const base = `http://127.0.0.1:${address.port}`;
+ originalTokenOverride = config.ZSCALER_TOKEN_URL_OVERRIDE;
+ originalApiOverride = config.ZSCALER_API_URL_OVERRIDE;
+ config.ZSCALER_TOKEN_URL_OVERRIDE = `${base}/oauth2/v1/token`;
+ config.ZSCALER_API_URL_OVERRIDE = base;
+});
+
+afterAll(async () => {
+ config.ZSCALER_TOKEN_URL_OVERRIDE = originalTokenOverride;
+ config.ZSCALER_API_URL_OVERRIDE = originalApiOverride;
+ await new Promise(resolve => server.close(resolve));
+});
+
+beforeEach(() => {
+ db.denyLookupScope = false;
+ db.rateLimitLookups = false;
+});
+
+describe("zscalerLookupUrls", () => {
+ it("classifies a batch and normalizes the response", async () => {
+ const results = await zscalerLookupUrls(credentials(), [
+ "casino.example.com",
+ "malware.example.com",
+ "unknown.example.com",
+ ]);
+ expect(results).toEqual([
+ {
+ url: "casino.example.com",
+ urlClassifications: ["GAMBLING"],
+ urlClassificationsWithSecurityAlert: [],
+ },
+ {
+ url: "malware.example.com",
+ urlClassifications: [],
+ urlClassificationsWithSecurityAlert: ["MALWARE_SITE"],
+ },
+ {
+ url: "unknown.example.com",
+ urlClassifications: ["MISCELLANEOUS_OR_UNKNOWN"],
+ urlClassificationsWithSecurityAlert: [],
+ },
+ ]);
+ });
+
+ it("reuses the cached token across calls", async () => {
+ // Prime the cache explicitly instead of relying on test order.
+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);
+ const tokenCallsBefore = counters.tokenCalls;
+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);
+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);
+ expect(counters.tokenCalls).toBe(tokenCallsBefore);
+ });
+
+ it("refuses more than 100 URLs per call", async () => {
+ const urls = Array.from({ length: 101 }, (_, i) => `u${i}.example.com`);
+ await expect(zscalerLookupUrls(credentials(), urls)).rejects.toMatchObject({
+ kind: "api",
+ });
+ });
+
+ it("maps a tenant-side 429 to a rate-limit error with Retry-After", async () => {
+ db.rateLimitLookups = true;
+ let caught: unknown;
+ try {
+ await zscalerLookupUrls(credentials(), ["casino.example.com"]);
+ } catch (error) {
+ caught = error;
+ }
+ expect(caught).toBeInstanceOf(ZscalerError);
+ expect(caught).toMatchObject({ kind: "rate-limit", status: 429 });
+ expect((caught as ZscalerError).retryAfterMs).toBeGreaterThan(0);
+ });
+});
+
+describe("authentication errors", () => {
+ it("maps rejected credentials to an auth error", async () => {
+ await expect(
+ zscalerGetUrlCategories(credentials({ clientSecret: "wrong-secret" })),
+ ).rejects.toMatchObject({ kind: "auth" });
+ });
+});
+
+describe("zscalerTestConnection", () => {
+ it("passes all three stages with a full-access client", async () => {
+ const result = await zscalerTestConnection(credentials());
+ expect(result).toMatchObject({
+ ok: true,
+ tokenOk: true,
+ categoriesOk: true,
+ lookupOk: true,
+ categoryCount: 2,
+ error: null,
+ });
+ });
+
+ it("distinguishes bad credentials from a scope problem", async () => {
+ const badCredentials = await zscalerTestConnection(
+ credentials({ clientSecret: "wrong-secret" }),
+ );
+ expect(badCredentials).toMatchObject({
+ ok: false,
+ tokenOk: false,
+ error: { kind: "auth" },
+ });
+
+ // View Only-style role: taxonomy reads fine, the lookup probe is denied.
+ db.denyLookupScope = true;
+ const scopeProblem = await zscalerTestConnection(credentials());
+ expect(scopeProblem).toMatchObject({
+ ok: false,
+ tokenOk: true,
+ categoriesOk: true,
+ lookupOk: false,
+ error: { kind: "scope" },
+ });
+ });
+});
+
+describe("toLookupUrl", () => {
+ test.each([
+ ["https://Example.com/Path?q=1#frag", "example.com/Path?q=1"],
+ ["http://example.com/", "example.com"],
+ ["example.com", "example.com"],
+ ["https://example.com:8080/x", "example.com:8080/x"],
+ ])("shapes %s into %s", (input, expected) => {
+ expect(toLookupUrl(input)).toBe(expected);
+ });
+
+ it("rejects entries over the ZIA limit instead of truncating them", () => {
+ const long = `example.com/${"a".repeat(2000)}`;
+ // Truncating would classify a different URL; the failurePolicy decides.
+ expect(() => toLookupUrl(long)).toThrow(ZscalerError);
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/client.ts b/apps/api/src/lib/threat-protection/providers/zscaler/client.ts
new file mode 100644
index 000000000..74257d5e7
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/client.ts
@@ -0,0 +1,488 @@
+import { createHash } from "crypto";
+import { ProxyAgent, type Dispatcher } from "undici";
+import { config } from "../../../../config";
+
+// Minimal Zscaler OneAPI client for the ZIA endpoints the "zscaler" threat
+// protection mode needs: OAuth2 client-credentials tokens from Zidentity,
+// GET /urlCategories (taxonomy + custom-list sync), and POST /urlLookup
+// (inline classification). All traffic goes through the partner egress proxy
+// (PARTNER_EGRESS_PROXY_URL) when configured, so the customer only has to
+// allowlist our published static IP.
+//
+// Tokens are cached in memory only — never in Redis or Postgres. The lookup
+// batcher and the sync path are the only sustained callers, and both run
+// behind per-org distributed locks, so at any moment only a handful of
+// processes hold a token. Zscaler supports two active client secrets, which
+// makes rotation zero-downtime: verdicts keep flowing on the old secret's
+// tokens until the new secret is saved here.
+
+export interface ZscalerCredentials {
+ clientId: string;
+ clientSecret: string;
+ /** Zidentity vanity domain: tokens come from https://<vanityDomain>.zslogin.net */
+ vanityDomain: string;
+ /** OneAPI cloud name (api.<cloud>.zsapi.net); null = default (api.zsapi.net). */
+ cloud: string | null;
+}
+
+type ZscalerErrorKind =
+ /** Token endpoint rejected the client (bad ID/secret, revoked client). */
+ | "auth"
+ /** Authenticated, but the role/scope does not allow the resource (403). */
+ | "scope"
+ /** 429 from the API — the tenant-side rate limit, not our local budget. */
+ | "rate-limit"
+ /** Our own per-org hourly budget is exhausted (no API call was made). */
+ | "quota"
+ /** Anything else: transport failures, 5xx, malformed responses. */
+ | "api";
+
+export class ZscalerError extends Error {
+ constructor(
+ public readonly kind: ZscalerErrorKind,
+ message: string,
+ public readonly status?: number,
+ public readonly retryAfterMs?: number,
+ ) {
+ super(message);
+ this.name = "ZscalerError";
+ }
+}
+
+export interface ZscalerUrlLookupResult {
+ url: string;
+ urlClassifications: string[];
+ urlClassificationsWithSecurityAlert: string[];
+}
+
+/**
+ * One entry of GET /urlCategories. Field names follow the ZIA API (see the
+ * official zscaler-sdk-go/py `urlcategories` services): custom categories
+ * carry `urls`/`keywords`, and `dbCategorizedUrls` /
+ * `keywordsRetainingParentCategory` hold the "retaining parent category"
+ * variants (the URL/keyword keeps its Zscaler-defined classification AND
+ * gains the custom category).
+ */
+export interface ZscalerUrlCategory {
+ id: string;
+ configuredName?: string;
+ customCategory?: boolean;
+ urls?: string[];
+ dbCategorizedUrls?: string[];
+ keywords?: string[];
+ keywordsRetainingParentCategory?: string[];
+}
+
+const TOKEN_EXPIRY_SKEW_MS = 60_000;
+const MAX_TOKEN_CACHE_ENTRIES = 100;
+
+interface CachedToken {
+ token: string;
+ expiresAt: number;
+}
+
+const tokenCache = new Map<string, CachedToken>();
+let proxyDispatcher: ProxyAgent | undefined;
+let proxyDispatcherUrl: string | undefined;
+
+function getDispatcher(): Dispatcher | undefined {
+ if (!config.PARTNER_EGRESS_PROXY_URL) return undefined;
+ if (
+ !proxyDispatcher ||
+ proxyDispatcherUrl !== config.PARTNER_EGRESS_PROXY_URL
+ ) {
+ proxyDispatcher = new ProxyAgent(config.PARTNER_EGRESS_PROXY_URL);
+ proxyDispatcherUrl = config.PARTNER_EGRESS_PROXY_URL;
+ }
+ return proxyDispatcher;
+}
+
+function tokenUrl(credentials: ZscalerCredentials): string {
+ return (
+ config.ZSCALER_TOKEN_URL_OVERRIDE ??
+ `https://${credentials.vanityDomain}.zslogin.net/oauth2/v1/token`
+ );
+}
+
+function apiBaseUrl(credentials: ZscalerCredentials): string {
+ const base =
+ config.ZSCALER_API_URL_OVERRIDE ??
+ (credentials.cloud
+ ? `https://api.${credentials.cloud}.zsapi.net`
+ : "https://api.zsapi.net");
+ return `${base.replace(/\/+$/, "")}/zia/api/v1`;
+}
+
+function tokenCacheKey(credentials: ZscalerCredentials): string {
+ const secretDigest = createHash("sha256")
+ .update(credentials.clientSecret)
+ .digest("base64url");
+ return `${credentials.vanityDomain}:${credentials.clientId}:${secretDigest}`;
+}
+
+function cacheToken(cacheKey: string, token: CachedToken): void {
+ const now = Date.now();
+ for (const [key, cached] of tokenCache) {
+ if (cached.expiresAt <= now) tokenCache.delete(key);
+ }
+ tokenCache.delete(cacheKey);
+ while (tokenCache.size >= MAX_TOKEN_CACHE_ENTRIES) {
+ const oldestKey = tokenCache.keys().next().value;
+ if (oldestKey === undefined) break;
+ tokenCache.delete(oldestKey);
+ }
+ tokenCache.set(cacheKey, token);
+}
+
+function retryAfterMs(response: Response): number | undefined {
+ const value = response.headers.get("retry-after");
+ if (!value) return undefined;
+ const seconds = Number(value);
+ if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
+ const date = Date.parse(value);
+ if (Number.isNaN(date)) return undefined;
+ return Math.max(0, date - Date.now());
+}
+
+// Error responses must have their bodies consumed, or undici cannot reuse
+// the connection and sockets pile up under repeated failures.
+function drainBody(response: Response): void {
+ response.body?.cancel().catch(() => {});
+}
+
+async function fetchAccessToken(
+ credentials: ZscalerCredentials,
+ signal: AbortSignal | undefined,
+): Promise<string> {
+ const cacheKey = tokenCacheKey(credentials);
+ const cached = tokenCache.get(cacheKey);
+ if (cached && cached.expiresAt > Date.now()) {
+ return cached.token;
+ }
+
+ const body = new URLSearchParams({
+ grant_type: "client_credentials",
+ client_id: credentials.clientId,
+ client_secret: credentials.clientSecret,
+ audience: "https://api.zscaler.com",
+ });
+
+ let response: Response;
+ try {
+ response = await fetch(tokenUrl(credentials), {
+ method: "POST",
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
+ body: body.toString(),
+ signal,
+ // undici extension: route through the partner egress proxy.
+ dispatcher: getDispatcher(),
+ } as RequestInit);
+ } catch (error) {
+ throw new ZscalerError(
+ "api",
+ `Failed to reach the Zscaler token endpoint: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+
+ if (response.status === 400 || response.status === 401) {
+ drainBody(response);
+ throw new ZscalerError(
+ "auth",
+ "Zscaler rejected the OAuth client credentials",
+ response.status,
+ );
+ }
+ if (response.status === 429) {
+ drainBody(response);
+ throw new ZscalerError(
+ "rate-limit",
+ "Zscaler token endpoint rate limit hit",
+ response.status,
+ retryAfterMs(response),
+ );
+ }
+ if (!response.ok) {
+ drainBody(response);
+ throw new ZscalerError(
+ "api",
+ `Zscaler token endpoint returned status ${response.status}`,
+ response.status,
+ );
+ }
+
+ let payload: { access_token?: string; expires_in?: number };
+ try {
+ payload = (await response.json()) as typeof payload;
+ } catch {
+ throw new ZscalerError("api", "Zscaler token response was not JSON");
+ }
+ if (!payload.access_token) {
+ throw new ZscalerError("api", "Zscaler token response had no access_token");
+ }
+
+ const expiresInMs =
+ (typeof payload.expires_in === "number" && payload.expires_in > 0
+ ? payload.expires_in * 1000
+ : 3600_000) - TOKEN_EXPIRY_SKEW_MS;
+ cacheToken(cacheKey, {
+ token: payload.access_token,
+ expiresAt: Date.now() + Math.max(expiresInMs, 10_000),
+ });
+ return payload.access_token;
+}
+
+async function apiRequest<T>(
+ credentials: ZscalerCredentials,
+ method: "GET" | "POST",
+ path: string,
+ body: unknown | undefined,
+ signal: AbortSignal | undefined,
+): Promise<T> {
+ let token = await fetchAccessToken(credentials, signal);
+
+ for (let attempt = 1; ; attempt++) {
+ let response: Response;
+ try {
+ response = await fetch(`${apiBaseUrl(credentials)}${path}`, {
+ method,
+ headers: {
+ Authorization: `Bearer ${token}`,
+ ...(body !== undefined ? { "Content-Type": "application/json" } : {}),
+ },
+ body: body !== undefined ? JSON.stringify(body) : undefined,
+ signal,
+ dispatcher: getDispatcher(),
+ } as RequestInit);
+ } catch (error) {
+ throw new ZscalerError(
+ "api",
+ `Failed to reach the Zscaler API: ${error instanceof Error ? error.message : String(error)}`,
+ );
+ }
+
+ if (response.status === 401 && attempt === 1) {
+ // Token expired or was revoked mid-flight: drop it and retry once with
+ // a fresh one. A second 401 is a real auth problem.
+ drainBody(response);
+ tokenCache.delete(tokenCacheKey(credentials));
+ token = await fetchAccessToken(credentials, signal);
+ continue;
+ }
+ if (response.status === 401) {
+ drainBody(response);
+ throw new ZscalerError(
+ "auth",
+ "Zscaler API rejected the access token",
+ response.status,
+ );
+ }
+ if (response.status === 403) {
+ drainBody(response);
+ throw new ZscalerError(
+ "scope",
+ `Zscaler API denied access to ${path} — the API client's role does not cover it`,
+ response.status,
+ );
+ }
+ if (response.status === 429) {
+ drainBody(response);
+ throw new ZscalerError(
+ "rate-limit",
+ `Zscaler API rate limit hit on ${path}`,
+ response.status,
+ retryAfterMs(response),
+ );
+ }
+ if (!response.ok) {
+ drainBody(response);
+ throw new ZscalerError(
+ "api",
+ `Zscaler API returned status ${response.status} for ${path}`,
+ response.status,
+ );
+ }
+
+ try {
+ return (await response.json()) as T;
+ } catch {
+ throw new ZscalerError(
+ "api",
+ `Zscaler API response for ${path} was not JSON`,
+ );
+ }
+ }
+}
+
+/** ZIA rejects lookup entries longer than this. */
+const MAX_LOOKUP_URL_LENGTH = 1024;
+
+/**
+ * Shape a URL the way POST /urlLookup expects: no scheme, no fragment,
+ * lowercased host. Accepts anything checkUrl's canonicalizer produces.
+ * URLs over the ZIA length limit throw (a truncated URL would classify a
+ * DIFFERENT URL); the caller resolves that through the failurePolicy.
+ */
+export function toLookupUrl(url: string): string {
+ let value = url.trim();
+ value = value.replace(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//, "");
+ const hash = value.indexOf("#");
+ if (hash !== -1) value = value.slice(0, hash);
+ const slash = value.indexOf("/");
+ const host = (slash === -1 ? value : value.slice(0, slash)).toLowerCase();
+ const rest = slash === -1 ? "" : value.slice(slash);
+ // A bare host needs no trailing slash; strip one so "example.com/" and
+ // "example.com" share a verdict key.
+ const combined = `${host}${rest === "/" ? "" : rest}`;
+ if (combined.length > MAX_LOOKUP_URL_LENGTH) {
+ throw new ZscalerError(
+ "api",
+ `URL exceeds the ${MAX_LOOKUP_URL_LENGTH}-character ZIA lookup limit`,
+ );
+ }
+ return combined;
+}
+
+/**
+ * Classify up to 100 URLs. Returns one entry per distinct URL; entries the
+ * tenant has not categorized come back with `MISCELLANEOUS_OR_UNKNOWN`.
+ * NOTE (ZIA behavior): custom URL classifications are NOT returned by this
+ * endpoint — custom-list decisions come from the synced local rules instead.
+ */
+export async function zscalerLookupUrls(
+ credentials: ZscalerCredentials,
+ urls: string[],
+ options?: { signal?: AbortSignal },
+): Promise<ZscalerUrlLookupResult[]> {
+ if (urls.length === 0) return [];
+ if (urls.length > 100) {
+ throw new ZscalerError("api", "urlLookup accepts at most 100 URLs");
+ }
+ const results = await apiRequest<
+ Array<{
+ url?: string;
+ urlClassifications?: string[];
+ urlClassificationsWithSecurityAlert?: string[];
+ }>
+ >(credentials, "POST", "/urlLookup", urls, options?.signal);
+ if (!Array.isArray(results)) {
+ throw new ZscalerError("api", "urlLookup response was not an array");
+ }
+ return results.map(entry => ({
+ url: entry.url ?? "",
+ urlClassifications: Array.isArray(entry.urlClassifications)
+ ? entry.urlClassifications.filter(c => typeof c === "string")
+ : [],
+ urlClassificationsWithSecurityAlert: Array.isArray(
+ entry.urlClassificationsWithSecurityAlert,
+ )
+ ? entry.urlClassificationsWithSecurityAlert.filter(
+ c => typeof c === "string",
+ )
+ : [],
+ }));
+}
+
+/**
+ * Fetch the tenant's full category taxonomy: Zscaler-defined categories,
+ * custom categories with their URL lists and keywords, and customer
+ * additions to Zscaler-defined categories (`dbCategorizedUrls`).
+ */
+export async function zscalerGetUrlCategories(
+ credentials: ZscalerCredentials,
+ options?: { signal?: AbortSignal },
+): Promise<ZscalerUrlCategory[]> {
+ const results = await apiRequest<ZscalerUrlCategory[]>(
+ credentials,
+ "GET",
+ "/urlCategories",
+ undefined,
+ options?.signal,
+ );
+ if (!Array.isArray(results)) {
+ throw new ZscalerError("api", "urlCategories response was not an array");
+ }
+ return results.filter(
+ entry => entry && typeof entry === "object" && typeof entry.id === "string",
+ );
+}
+
+interface ZscalerConnectionTestResult {
+ ok: boolean;
+ /** Token endpoint accepted the client credentials. */
+ tokenOk: boolean;
+ /** GET /urlCategories succeeded (taxonomy readable). */
+ categoriesOk: boolean;
+ /** POST /urlLookup succeeded (inline classification permitted). */
+ lookupOk: boolean;
+ categoryCount: number | null;
+ /** Set when ok is false: which stage failed and why. */
+ error: { kind: ZscalerErrorKind; message: string } | null;
+}
+
+/**
+ * Three-stage connection test: token (distinguishes bad credentials),
+ * taxonomy read, and a lookup probe (a View Only role can read the taxonomy
+ * but may be refused the read-semantic POST /urlLookup — surfacing that here
+ * beats surfacing it on the first scrape).
+ *
+ * The optional stage hooks run immediately before their API call and abort
+ * the stage by throwing — the caller charges its shared tenant budgets there,
+ * so a test that dies at an earlier stage never spends points for calls that
+ * were never made.
+ */
+export async function zscalerTestConnection(
+ credentials: ZscalerCredentials,
+ options?: {
+ signal?: AbortSignal;
+ beforeCategories?: () => Promise<void>;
+ beforeLookup?: () => Promise<void>;
+ },
+): Promise<ZscalerConnectionTestResult> {
+ const result: ZscalerConnectionTestResult = {
+ ok: false,
+ tokenOk: false,
+ categoriesOk: false,
+ lookupOk: false,
+ categoryCount: null,
+ error: null,
+ };
+
+ const fail = (error: unknown): ZscalerConnectionTestResult => {
+ const zscalerError =
+ error instanceof ZscalerError
+ ? error
+ : new ZscalerError(
+ "api",
+ error instanceof Error ? error.message : String(error),
+ );
+ result.error = { kind: zscalerError.kind, message: zscalerError.message };
+ return result;
+ };
+
+ try {
+ await fetchAccessToken(credentials, options?.signal);
+ result.tokenOk = true;
+ } catch (error) {
+ return fail(error);
+ }
+
+ try {
+ await options?.beforeCategories?.();
+ const categories = await zscalerGetUrlCategories(credentials, options);
+ result.categoriesOk = true;
+ result.categoryCount = categories.length;
+ } catch (error) {
+ return fail(error);
+ }
+
+ try {
+ await options?.beforeLookup?.();
+ await zscalerLookupUrls(credentials, ["example.com"], options);
+ result.lookupOk = true;
+ } catch (error) {
+ return fail(error);
+ }
+
+ result.ok = true;
+ return result;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/index.ts b/apps/api/src/lib/threat-protection/providers/zscaler/index.ts
new file mode 100644
index 000000000..b1a68bafb
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/index.ts
@@ -0,0 +1,56 @@
+import type { RawVerdict, ZscalerPolicySettings } from "../../types";
+import { toLookupUrl } from "./client";
+import { enqueueZscalerLookup } from "./lookup-queue";
+
+// Zscaler ZIA provider ("zscaler" threat protection mode): inline
+// classification via batched POST /urlLookup calls (see ./lookup-queue.ts).
+// Custom categories and customer additions never come back from urlLookup —
+// those are evaluated locally from the synced rules BEFORE this provider is
+// consulted (see ./sync.ts and checkUrlFresh in ../../index.ts).
+//
+// Verdict normalization into the provider-agnostic contract:
+// - categories = urlClassifications ∪ urlClassificationsWithSecurityAlert;
+// the org's denied-categories selection decides in evaluatePolicy.
+// - securityAlert = the URL carries a security-alert classification.
+// - riskScore stays null: ZIA is categorical, so the score threshold rule
+// never fires for this provider.
+// - raw carries the split lists for server logs only — never for SIEM
+// export (the SIEM event builder reads only the normalized fields).
+
+/**
+ * Classify a URL through the org's Zscaler tenant. Throws on quota
+ * exhaustion, queue overflow, timeout, and API failures — the caller
+ * resolves those through the org's failurePolicy.
+ */
+export async function fetchZscalerVerdict(
+ url: string,
+ zscaler: ZscalerPolicySettings,
+ options?: { signal?: AbortSignal },
+): Promise<RawVerdict> {
+ const lookupUrl = toLookupUrl(url);
+ const result = await enqueueZscalerLookup(
+ zscaler.orgId,
+ lookupUrl,
+ options?.signal,
+ );
+
+ const categories = [
+ ...new Set([
+ ...result.urlClassifications,
+ ...result.urlClassificationsWithSecurityAlert,
+ ]),
+ ];
+
+ return {
+ provider: "zscaler-zia",
+ riskScore: null,
+ categories,
+ securityAlert: result.urlClassificationsWithSecurityAlert.length > 0,
+ fromCache: false,
+ raw: {
+ urlClassifications: result.urlClassifications,
+ urlClassificationsWithSecurityAlert:
+ result.urlClassificationsWithSecurityAlert,
+ },
+ };
+}
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
new file mode 100644
index 000000000..7cb0ae4df
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
@@ -0,0 +1,465 @@
+import { randomUUID } from "crypto";
+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
+import { config } from "../../../../config";
+import { logger as _logger } from "../../../logger";
+import { redisRateLimitClient } from "../../../../services/rate-limiter";
+import { redlock } from "../../../../services/redlock";
+import { getOrgZscalerCredentials } from "../../store";
+import {
+ ZscalerError,
+ zscalerLookupUrls,
+ type ZscalerCredentials,
+ type ZscalerUrlLookupResult,
+} from "./client";
+
+// Inline classification plane for the Zscaler ZIA provider.
+//
+// POST /urlLookup accepts up to 100 URLs per call, but the tenant budget is
+// 1 call/second and 400 calls/hour — and the budget counts CALLS, not URLs.
+// With scrape workers running on hundreds of pods, per-pod batching would
+// fragment the batches and waste the scarce budget, so lookups are funneled
+// through one shared per-org queue in Redis with a single drainer:
+//
+// 1. A waiting scrape RPUSHes {id, url} onto the org's queue and polls its
+// reply key.
+// 2. Whichever process notices work tries to become the org's drainer via
+// a non-blocking distributed lock; losers just keep polling.
+// 3. The drainer pops up to 100 entries, spends 1 point of the org's
+// hourly budget and 1 point of the 1/second pacer (both enforced in
+// Redis, so they hold across all pods and drainer failovers), calls
+// urlLookup once, and writes each entry's verdict to its reply key.
+//
+// Fail-fast beats waiting: when the hourly budget is exhausted or the queue
+// is too deep to drain before the lookup timeout, the caller gets an
+// immediate error and the org's failurePolicy decides — a scrape must never
+// sit in a line it cannot exit. Verdicts live only in the short-TTL reply
+// keys consumed moments later; there is no verdict cache (ZDR).
+//
+// Capacity note: the sustained ceiling is ~11 URLs/second per org (400
+// calls/hour × 100 URLs). The "zscaler"-mode concurrency cap (see
+// getThreatProtectionConcurrencyCap) keeps URL demand near this budget;
+// this queue only absorbs bursts.
+
+const logger = _logger.child({ module: "threat-protection-zscaler-lookup" });
+
+const queueKey = (orgId: string) =>
+ `threat-protection:zscaler:lookup-queue:${orgId}`;
+const replyKey = (id: string) => `threat-protection:zscaler:lookup-reply:${id}`;
+const drainLockKey = (orgId: string) =>
+ `threat-protection:zscaler:drain-lock:${orgId}`;
+
+const BATCH_SIZE = 100;
+const REPLY_TTL_SECONDS = 120;
+const REPLY_POLL_INTERVAL_MS = 150;
+/** Re-attempt to become the drainer this often while waiting (drainer death recovery). */
+const DRAINER_REENSURE_INTERVAL_MS = 2000;
+/** One idle re-check before the drainer exits, so a straggler doesn't wait a full re-ensure cycle. */
+const DRAIN_IDLE_LINGER_MS = 200;
+const LOOKUP_CALL_TIMEOUT_MS = 30_000;
+// Must comfortably exceed one full iteration (pacer wait + the lookup call
+// timeout), or the lock expires mid-iteration and a second drainer starts
+// while the first is still replying.
+const DRAIN_LOCK_TTL_MS = LOOKUP_CALL_TIMEOUT_MS + 15_000;
+
+interface QueueEntry {
+ id: string;
+ url: string;
+ /** Enqueue time (epoch ms): the drainer drops entries whose requester has given up. */
+ at: number;
+}
+
+/**
+ * Budget keys are tenant-scoped, not org-scoped: ZIA rate limits apply to the
+ * customer's tenant, and several Firecrawl orgs may share one.
+ */
+export function zscalerBudgetKey(credentials: ZscalerCredentials): string {
+ return `${credentials.vanityDomain}:${credentials.cloud ?? "default"}`;
+}
+
+type ReplyPayload =
+ | {
+ status: "ok";
+ urlClassifications: string[];
+ urlClassificationsWithSecurityAlert: string[];
+ }
+ | { status: "quota" }
+ | { status: "error"; message: string };
+
+let hourlyBudget: RateLimiterRedis | undefined;
+function getHourlyBudget(): RateLimiterRedis {
+ if (!hourlyBudget) {
+ hourlyBudget = new RateLimiterRedis({
+ storeClient: redisRateLimitClient,
+ keyPrefix: "threat-protection:zscaler:lookup-budget",
+ points: config.ZSCALER_LOOKUP_HOURLY_BUDGET,
+ duration: 3600,
+ });
+ }
+ return hourlyBudget;
+}
+
+let perSecondPacer: RateLimiterRedis | undefined;
+function getPerSecondPacer(): RateLimiterRedis {
+ if (!perSecondPacer) {
+ perSecondPacer = new RateLimiterRedis({
+ storeClient: redisRateLimitClient,
+ keyPrefix: "threat-protection:zscaler:lookup-pacer",
+ points: 1,
+ duration: 1,
+ });
+ }
+ return perSecondPacer;
+}
+
+function sleep(ms: number): Promise<void> {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Spend one point of the tenant's urlLookup budget outside the queue (the
+ * connection test's lookup probe). Throws a quota ZscalerError when the
+ * budget is exhausted, so ad-hoc tests can never starve real scrapes.
+ */
+export async function consumeZscalerLookupBudget(
+ credentials: ZscalerCredentials,
+): Promise<void> {
+ try {
+ await getHourlyBudget().consume(zscalerBudgetKey(credentials), 1);
+ } catch (rejection) {
+ if (rejection instanceof RateLimiterRes) {
+ throw new ZscalerError(
+ "quota",
+ "Zscaler urlLookup hourly budget is exhausted",
+ undefined,
+ rejection.msBeforeNext,
+ );
+ }
+ throw rejection;
+ }
+}
+
+// =========================================
+// Requester side
+// =========================================
+
+/**
+ * Queue a URL for batched classification and wait for its verdict. Throws a
+ * {@link ZscalerError} on quota exhaustion, queue overflow, wait timeout, or
+ * a failed provider call — the caller resolves all of those through the
+ * org's failurePolicy.
+ */
+export async function enqueueZscalerLookup(
+ orgId: string,
+ lookupUrl: string,
+ signal?: AbortSignal,
+): Promise<ZscalerUrlLookupResult> {
+ // Fail fast on a queue that cannot drain before the wait timeout. The
+ // LLEN/RPUSH pair is deliberately non-atomic: this is a fail-fast
+ // heuristic, and a brief overshoot from concurrent enqueuers is bounded
+ // by their count and harmless.
+ const depth = await redisRateLimitClient.llen(queueKey(orgId));
+ if (depth >= config.ZSCALER_LOOKUP_MAX_QUEUE_DEPTH) {
+ throw new ZscalerError(
+ "quota",
+ `Zscaler lookup queue is over capacity (${depth} URLs waiting)`,
+ );
+ }
+
+ // Fail fast when the hourly budget is already spent — enqueueing would
+ // only park the scrape until the timeout. Budget state is keyed by the
+ // ZIA tenant, which needs the org's credentials (60s-cached read).
+ const credentials = await getOrgZscalerCredentials(orgId);
+ if (credentials) {
+ const budgetState = await getHourlyBudget()
+ .get(zscalerBudgetKey(credentials))
+ .catch(() => null);
+ if (
+ budgetState &&
+ budgetState.consumedPoints >= config.ZSCALER_LOOKUP_HOURLY_BUDGET
+ ) {
+ throw new ZscalerError(
+ "quota",
+ "Zscaler urlLookup hourly budget is exhausted",
+ undefined,
+ budgetState.msBeforeNext,
+ );
+ }
+ }
+
+ const entry: QueueEntry = {
+ id: randomUUID(),
+ url: lookupUrl,
+ at: Date.now(),
+ };
+ await redisRateLimitClient.rpush(queueKey(orgId), JSON.stringify(entry));
+ ensureQueueDrainer(orgId);
+
+ let lastDrainerCheck = Date.now();
+ for (;;) {
+ if (signal?.aborted) {
+ throw new ZscalerError(
+ "api",
+ "Timed out waiting for the batched Zscaler verdict",
+ );
+ }
+
+ const raw = await redisRateLimitClient.get(replyKey(entry.id));
+ if (raw !== null) {
+ redisRateLimitClient.del(replyKey(entry.id)).catch(() => {});
+ let payload: ReplyPayload;
+ try {
+ payload = JSON.parse(raw) as ReplyPayload;
+ } catch {
+ throw new ZscalerError("api", "Malformed Zscaler lookup reply");
+ }
+ if (payload.status === "ok") {
+ // A parseable but malformed reply must become a provider failure,
+ // not invalid category data inside policy evaluation.
+ if (
+ !Array.isArray(payload.urlClassifications) ||
+ !Array.isArray(payload.urlClassificationsWithSecurityAlert)
+ ) {
+ throw new ZscalerError("api", "Malformed Zscaler lookup reply");
+ }
+ return {
+ url: lookupUrl,
+ urlClassifications: payload.urlClassifications.filter(
+ (value): value is string => typeof value === "string",
+ ),
+ urlClassificationsWithSecurityAlert:
+ payload.urlClassificationsWithSecurityAlert.filter(
+ (value): value is string => typeof value === "string",
+ ),
+ };
+ }
+ if (payload.status === "quota") {
+ throw new ZscalerError(
+ "quota",
+ "Zscaler urlLookup hourly budget is exhausted",
+ );
+ }
+ throw new ZscalerError("api", payload.message);
+ }
+
+ // The drainer may have died with our entry unclaimed (its lock expires
+ // within DRAIN_LOCK_TTL_MS); periodically try to become / revive it.
+ if (Date.now() - lastDrainerCheck >= DRAINER_REENSURE_INTERVAL_MS) {
+ lastDrainerCheck = Date.now();
+ ensureQueueDrainer(orgId);
+ }
+
+ await sleep(REPLY_POLL_INTERVAL_MS);
+ }
+}
+
+// =========================================
+// Drainer side
+// =========================================
+
+const drainersInFlight = new Set<string>();
+
+/** Fire-and-forget: try to become the org's queue drainer. */
+function ensureQueueDrainer(orgId: string): void {
+ if (drainersInFlight.has(orgId)) return;
+ drainersInFlight.add(orgId);
+ drainQueue(orgId)
+ .catch(error => {
+ logger.warn("Zscaler lookup queue drainer failed", { error, orgId });
+ })
+ .finally(() => {
+ drainersInFlight.delete(orgId);
+ });
+}
+
+async function popBatch(orgId: string): Promise<QueueEntry[]> {
+ const raw = await redisRateLimitClient.lpop(queueKey(orgId), BATCH_SIZE);
+ if (!raw) return [];
+ const entries: QueueEntry[] = [];
+ for (const item of raw) {
+ try {
+ const parsed = JSON.parse(item) as QueueEntry;
+ if (typeof parsed.id === "string" && typeof parsed.url === "string") {
+ entries.push({
+ ...parsed,
+ // Entries from a pre-`at` process (deploy rollout window) are of
+ // unknown age; treat them as fresh so their requesters still get a
+ // verdict instead of an immediate expiry error.
+ at: typeof parsed.at === "number" ? parsed.at : Date.now(),
+ });
+ }
+ } catch {
+ // Skip malformed entries; their requesters time out into failurePolicy.
+ }
+ }
+ return entries;
+}
+
+// Entries whose requester has certainly stopped polling (its wait timeout
+// plus a grace period has passed). Spending budget on them would classify
+// URLs nobody is waiting for.
+const ENTRY_EXPIRY_GRACE_MS = 2000;
+
+function isExpired(entry: QueueEntry): boolean {
+ return (
+ Date.now() - entry.at >
+ config.ZSCALER_LOOKUP_TIMEOUT_MS + ENTRY_EXPIRY_GRACE_MS
+ );
+}
+
+async function replyAll(
+ entries: QueueEntry[],
+ payloadFor: (entry: QueueEntry) => ReplyPayload,
+): Promise<void> {
+ const pipeline = redisRateLimitClient.pipeline();
+ for (const entry of entries) {
+ pipeline.set(
+ replyKey(entry.id),
+ JSON.stringify(payloadFor(entry)),
+ "EX",
+ REPLY_TTL_SECONDS,
+ );
+ }
+ await pipeline.exec();
+}
+
+async function drainQueue(orgId: string): Promise<void> {
+ let lock;
+ try {
+ lock = await redlock.acquire([drainLockKey(orgId)], DRAIN_LOCK_TTL_MS, {
+ retryCount: 0,
+ });
+ } catch {
+ // Another process is draining this org.
+ return;
+ }
+
+ try {
+ const credentials = await getOrgZscalerCredentials(orgId);
+
+ for (;;) {
+ let entries = await popBatch(orgId);
+ if (entries.length === 0) {
+ // A URL enqueued a moment after our pop should not wait for the next
+ // re-ensure cycle; check once more before exiting.
+ await sleep(DRAIN_IDLE_LINGER_MS);
+ entries = await popBatch(orgId);
+ if (entries.length === 0) return;
+ }
+
+ // Abandoned entries (requester timed out) get a free error reply
+ // instead of a budgeted classification.
+ const expired = entries.filter(isExpired);
+ if (expired.length > 0) {
+ await replyAll(expired, () => ({
+ status: "error",
+ message: "The lookup request expired before the batch was sent",
+ })).catch(() => {});
+ entries = entries.filter(entry => !isExpired(entry));
+ if (entries.length === 0) continue;
+ }
+
+ // From here on the popped entries are this drainer's responsibility:
+ // an unexpected throw (Redis blip, expired lock) must not orphan them
+ // into their full wait timeout — reply an error first, then unwind.
+ try {
+ lock = await lock.extend(DRAIN_LOCK_TTL_MS);
+
+ if (!credentials) {
+ await replyAll(entries, () => ({
+ status: "error",
+ message: "No Zscaler credentials are configured for this org",
+ }));
+ continue;
+ }
+
+ // Hourly budget: one point per urlLookup CALL, keyed by the ZIA
+ // tenant. When it is gone, this loop keeps popping and answers
+ // everything "quota" without touching the tenant, so waiting scrapes
+ // resolve immediately via failurePolicy.
+ const budgetKey = zscalerBudgetKey(credentials);
+ try {
+ await getHourlyBudget().consume(budgetKey, 1);
+ } catch (rejection) {
+ if (rejection instanceof RateLimiterRes) {
+ await replyAll(entries, () => ({ status: "quota" }));
+ continue;
+ }
+ throw rejection;
+ }
+
+ // 1 call/second pacer.
+ for (;;) {
+ try {
+ await getPerSecondPacer().consume(budgetKey, 1);
+ break;
+ } catch (rejection) {
+ if (rejection instanceof RateLimiterRes) {
+ await sleep(Math.max(rejection.msBeforeNext, 50));
+ continue;
+ }
+ throw rejection;
+ }
+ }
+ } catch (error) {
+ await replyAll(entries, () => ({
+ status: "error",
+ message: "Zscaler lookup drainer failed before the batch was sent",
+ })).catch(() => {});
+ throw error;
+ }
+
+ const uniqueUrls = [...new Set(entries.map(entry => entry.url))];
+ try {
+ const results = await zscalerLookupUrls(credentials, uniqueUrls, {
+ signal: AbortSignal.timeout(LOOKUP_CALL_TIMEOUT_MS),
+ });
+ const byUrl = new Map(results.map(result => [result.url, result]));
+ await replyAll(entries, entry => {
+ const result = byUrl.get(entry.url);
+ if (!result) {
+ return {
+ status: "error",
+ message: "Zscaler returned no verdict for this URL",
+ };
+ }
+ return {
+ status: "ok",
+ urlClassifications: result.urlClassifications,
+ urlClassificationsWithSecurityAlert:
+ result.urlClassificationsWithSecurityAlert,
+ };
+ });
+ } catch (error) {
+ const zscalerError =
+ error instanceof ZscalerError
+ ? error
+ : new ZscalerError(
+ "api",
+ error instanceof Error ? error.message : String(error),
+ );
+ logger.warn("Batched Zscaler urlLookup call failed", {
+ error,
+ orgId,
+ kind: zscalerError.kind,
+ urls: uniqueUrls.length,
+ });
+ if (
+ zscalerError.kind === "rate-limit" &&
+ zscalerError.retryAfterMs !== undefined
+ ) {
+ // Tenant-side 429 (something else shares the tenant's budget):
+ // pause the drainer so we do not compound it. The current batch
+ // still fails into failurePolicy — no retries that could stall
+ // every waiting scrape behind one bad call.
+ await sleep(Math.min(zscalerError.retryAfterMs, 10_000));
+ }
+ await replyAll(entries, () => ({
+ status: "error",
+ message: zscalerError.message,
+ }));
+ }
+ }
+ } finally {
+ await lock.release().catch(() => {});
+ }
+}
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.ts b/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.ts
new file mode 100644
index 000000000..9a0ae4aea
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/sync.test.ts
@@ -0,0 +1,165 @@
+import type { ZscalerUrlCategory } from "./client";
+import { materializeCategories, matchSyncedRules } from "./sync";
+
+// Fixture payload following the GET /urlCategories response shape from the
+// official Zscaler SDK services (urls / dbCategorizedUrls / keywords /
+// keywordsRetainingParentCategory), not hand-invented fields.
+const CATEGORIES: ZscalerUrlCategory[] = [
+ {
+ id: "CUSTOM_01",
+ configuredName: "Blocked Vendors",
+ customCategory: true,
+ urls: ["vendor-blocked.example.com", "partner.example.com/private"],
+ keywords: ["forbiddenword"],
+ },
+ {
+ id: "CUSTOM_02",
+ configuredName: "Tracked Tools",
+ customCategory: true,
+ dbCategorizedUrls: ["tool.example.com"],
+ keywordsRetainingParentCategory: ["retainedword"],
+ },
+ {
+ // Customer addition to a Zscaler-defined category ("URLs retaining
+ // parent category" on a predefined category).
+ id: "GAMBLING",
+ dbCategorizedUrls: ["sweepstakes.example.com"],
+ },
+ {
+ id: "CUSTOM_03",
+ configuredName: "Port-Scoped",
+ customCategory: true,
+ urls: ["internal.example.com:8443"],
+ },
+ {
+ // Pure taxonomy entry: no custom content, so no local rule.
+ id: "ENTERTAINMENT",
+ },
+];
+
+describe("materializeCategories", () => {
+ const { taxonomy, rules } = materializeCategories(CATEGORIES);
+
+ it("keeps every category in the taxonomy for the picker", () => {
+ expect(taxonomy).toHaveLength(5);
+ expect(taxonomy.map(entry => entry.id).sort()).toEqual([
+ "CUSTOM_01",
+ "CUSTOM_02",
+ "CUSTOM_03",
+ "ENTERTAINMENT",
+ "GAMBLING",
+ ]);
+ });
+
+ it("uses the configured name and falls back to the ID", () => {
+ const byId = new Map(taxonomy.map(entry => [entry.id, entry]));
+ expect(byId.get("CUSTOM_01")).toMatchObject({
+ name: "Blocked Vendors",
+ custom: true,
+ });
+ expect(byId.get("GAMBLING")).toMatchObject({
+ name: "GAMBLING",
+ custom: false,
+ });
+ });
+
+ it("materializes rules only for categories with local content", () => {
+ expect(rules.map(rule => rule.id).sort()).toEqual([
+ "CUSTOM_01",
+ "CUSTOM_02",
+ "CUSTOM_03",
+ "GAMBLING",
+ ]);
+ });
+
+ it("separates reclassifying and retaining entries", () => {
+ const custom01 = rules.find(rule => rule.id === "CUSTOM_01")!;
+ expect(custom01.urls).toContain("vendor-blocked.example.com");
+ expect(custom01.retainingUrls).toEqual([]);
+ expect(custom01.keywords).toEqual(["forbiddenword"]);
+
+ const custom02 = rules.find(rule => rule.id === "CUSTOM_02")!;
+ expect(custom02.urls).toEqual([]);
+ expect(custom02.retainingUrls).toEqual(["tool.example.com"]);
+ expect(custom02.retainingKeywords).toEqual(["retainedword"]);
+ });
+});
+
+describe("matchSyncedRules", () => {
+ const { rules } = materializeCategories(CATEGORIES);
+
+ function match(url: string) {
+ return matchSyncedRules(url, rules).map(entry => ({
+ id: entry.rule.id,
+ reclassifies: entry.reclassifies,
+ }));
+ }
+
+ it("matches a URL-list entry on the host and its subdomains", () => {
+ expect(match("https://vendor-blocked.example.com/page")).toEqual([
+ { id: "CUSTOM_01", reclassifies: true },
+ ]);
+ expect(match("https://deep.vendor-blocked.example.com/")).toEqual([
+ { id: "CUSTOM_01", reclassifies: true },
+ ]);
+ expect(match("https://not-vendor-blocked.example.com/")).toEqual([]);
+ });
+
+ it("restricts path-scoped entries to that path", () => {
+ expect(match("https://partner.example.com/private/report")).toEqual([
+ { id: "CUSTOM_01", reclassifies: true },
+ ]);
+ expect(match("https://partner.example.com/public")).toEqual([]);
+ });
+
+ it("matches keywords anywhere in the schemeless URL, case-insensitively", () => {
+ expect(match("https://anything.example.org/ForbiddenWord-page")).toEqual([
+ { id: "CUSTOM_01", reclassifies: true },
+ ]);
+ expect(match("https://anything.example.org/?q=retainedword")).toEqual([
+ { id: "CUSTOM_02", reclassifies: false },
+ ]);
+ });
+
+ it("ignores ports on both the entry and the checked URL", () => {
+ expect(match("https://internal.example.com:8443/console")).toEqual([
+ { id: "CUSTOM_03", reclassifies: true },
+ ]);
+ expect(match("https://internal.example.com/console")).toEqual([
+ { id: "CUSTOM_03", reclassifies: true },
+ ]);
+ });
+
+ it("marks retaining-parent-category entries as non-reclassifying", () => {
+ expect(match("https://tool.example.com/")).toEqual([
+ { id: "CUSTOM_02", reclassifies: false },
+ ]);
+ expect(match("https://sweepstakes.example.com/")).toEqual([
+ { id: "GAMBLING", reclassifies: false },
+ ]);
+ });
+
+ it("flags keyword matches as inexact so they can never allow on their own", () => {
+ const matches = matchSyncedRules(
+ "https://anything.example.org/ForbiddenWord-page",
+ rules,
+ );
+ expect(matches).toHaveLength(1);
+ expect(matches[0]).toMatchObject({ reclassifies: true, exact: false });
+
+ const exactMatches = matchSyncedRules(
+ "https://vendor-blocked.example.com/",
+ rules,
+ );
+ expect(exactMatches[0]).toMatchObject({ reclassifies: true, exact: true });
+ });
+
+ it("collects every matching category", () => {
+ expect(match("https://tool.example.com/forbiddenword")).toEqual(
+ expect.arrayContaining([
+ { id: "CUSTOM_01", reclassifies: true },
+ { id: "CUSTOM_02", reclassifies: false },
+ ]),
+ );
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/sync.ts b/apps/api/src/lib/threat-protection/providers/zscaler/sync.ts
new file mode 100644
index 000000000..32a4a5c4b
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/sync.ts
@@ -0,0 +1,585 @@
+import { RateLimiterRedis, RateLimiterRes } from "rate-limiter-flexible";
+import { z } from "zod";
+import { config } from "../../../../config";
+import { logger as _logger } from "../../../logger";
+import { redisRateLimitClient } from "../../../../services/rate-limiter";
+import { redlock } from "../../../../services/redlock";
+import { getOrgZscalerCredentials } from "../../store";
+import type {
+ ThreatDecision,
+ ThreatProtectionPolicy,
+ ZscalerPolicySettings,
+} from "../../types";
+import { canonicalizeHost, splitUrl } from "../web-risk/canonicalize";
+import {
+ ZscalerError,
+ zscalerGetUrlCategories,
+ type ZscalerCredentials,
+ type ZscalerUrlCategory,
+} from "./client";
+import { zscalerBudgetKey } from "./lookup-queue";
+
+// Synchronization plane for the Zscaler ZIA provider.
+//
+// POST /urlLookup never returns custom URL classifications ("Custom URL
+// classification is not returned by this request" — ZIA API docs), so the
+// tenant's custom categories, custom-category keywords, and customer
+// additions to Zscaler-defined categories (dbCategorizedUrls) are synced via
+// GET /urlCategories and evaluated locally against every checked URL.
+//
+// The synced document holds the customer's own policy configuration — never
+// scrape-derived data — so persisting it in Redis is compatible with
+// zero-data-retention requirements (same reasoning as the org-config cache
+// in ../../store.ts).
+//
+// Sync runs lazily: any process that notices the document is older than the
+// org's sync interval kicks a re-sync in the background, guarded by a
+// distributed lock so only one process calls the tenant per interval. There
+// is no dedicated sync worker to deploy or babysit. Each sync replaces the
+// whole document, so entries removed upstream disappear on the next pass.
+//
+// Keyword matching note: locally re-implementing Zscaler's keyword semantics
+// exactly is not possible from the published docs. We apply the documented
+// best-effort approximation (case-insensitive substring over the schemeless
+// URL); URL-list entries match exactly.
+
+const logger = _logger.child({ module: "threat-protection-zscaler-sync" });
+
+const SYNC_DOC_VERSION = 1;
+const syncDocKey = (orgId: string) => `threat-protection:zscaler:sync:${orgId}`;
+const syncLockKey = (orgId: string) =>
+ `threat-protection:zscaler:sync-lock:${orgId}`;
+
+// Process-local cache of the parsed sync document so the enforcement hot
+// path does not deserialize a potentially large Redis value per URL.
+const DOC_CACHE_TTL_MS = 60_000;
+const DOC_CACHE_MAX_ENTRIES = 100;
+const docCache = new Map<
+ string,
+ { expiresAt: number; doc: ZscalerSyncDocument | null }
+>();
+
+/** One synced category's locally-evaluable content. */
+export interface ZscalerSyncedRule {
+ /** Category ID ("CUSTOM_01", or a Zscaler-defined ID with customer additions). */
+ id: string;
+ name: string;
+ custom: boolean;
+ /** URL-list entries that reclassify the URL into this category. */
+ urls: string[];
+ /** "Retaining parent category" URL entries — the URL keeps its Zscaler-defined categories too. */
+ retainingUrls: string[];
+ /** Keywords that reclassify (best-effort substring match). */
+ keywords: string[];
+ /** "Retaining parent category" keywords. */
+ retainingKeywords: string[];
+}
+
+export interface ZscalerTaxonomyEntry {
+ id: string;
+ name: string;
+ custom: boolean;
+}
+
+export interface ZscalerSyncDocument {
+ version: number;
+ status: "ok" | "error";
+ /** Last successful sync; null when no sync has ever succeeded. */
+ syncedAt: string | null;
+ attemptedAt: string;
+ error: string | null;
+ taxonomy: ZscalerTaxonomyEntry[];
+ rules: ZscalerSyncedRule[];
+}
+
+const syncedRuleSchema = z.object({
+ id: z.string(),
+ name: z.string(),
+ custom: z.boolean(),
+ urls: z.array(z.string()),
+ retainingUrls: z.array(z.string()),
+ keywords: z.array(z.string()),
+ retainingKeywords: z.array(z.string()),
+});
+
+const syncDocumentSchema = z.object({
+ version: z.number(),
+ status: z.enum(["ok", "error"]),
+ syncedAt: z.string().nullable(),
+ attemptedAt: z.string(),
+ error: z.string().nullable(),
+ taxonomy: z.array(
+ z.object({ id: z.string(), name: z.string(), custom: z.boolean() }),
+ ),
+ rules: z.array(syncedRuleSchema),
+});
+
+// =========================================
+// Document storage
+// =========================================
+
+export async function getZscalerSyncDocument(
+ orgId: string,
+): Promise<ZscalerSyncDocument | null> {
+ const cached = docCache.get(orgId);
+ if (cached && cached.expiresAt > Date.now()) return cached.doc;
+
+ let doc: ZscalerSyncDocument | null = null;
+ try {
+ const raw = await redisRateLimitClient.get(syncDocKey(orgId));
+ if (raw !== null) {
+ doc = syncDocumentSchema.parse(JSON.parse(raw));
+ }
+ } catch (error) {
+ logger.warn("Failed to read the Zscaler sync document", { error, orgId });
+ // Treat as missing: the caller decides via failure semantics.
+ doc = null;
+ }
+
+ const now = Date.now();
+ for (const [key, entry] of docCache) {
+ if (entry.expiresAt <= now) docCache.delete(key);
+ }
+ while (docCache.size >= DOC_CACHE_MAX_ENTRIES) {
+ const oldest = docCache.keys().next().value;
+ if (oldest === undefined) break;
+ docCache.delete(oldest);
+ }
+ docCache.set(orgId, { expiresAt: now + DOC_CACHE_TTL_MS, doc });
+ return doc;
+}
+
+async function putZscalerSyncDocument(
+ orgId: string,
+ doc: ZscalerSyncDocument,
+): Promise<void> {
+ await redisRateLimitClient.set(syncDocKey(orgId), JSON.stringify(doc));
+ docCache.set(orgId, { expiresAt: Date.now() + DOC_CACHE_TTL_MS, doc });
+}
+
+/** Removes all synced state; used when the org's Zscaler connection is removed. */
+export async function clearZscalerSyncState(orgId: string): Promise<void> {
+ docCache.delete(orgId);
+ await redisRateLimitClient.del(syncDocKey(orgId));
+}
+
+// =========================================
+// Sync
+// =========================================
+
+// GET /urlCategories budget (ZIA: 1,000/hour per tenant). Shared between
+// interval syncs, dashboard "sync now", and connection tests.
+let categoriesBudget: RateLimiterRedis | undefined;
+function getCategoriesBudget(): RateLimiterRedis {
+ if (!categoriesBudget) {
+ categoriesBudget = new RateLimiterRedis({
+ storeClient: redisRateLimitClient,
+ keyPrefix: "threat-protection:zscaler:categories-budget",
+ points: config.ZSCALER_CATEGORIES_HOURLY_BUDGET,
+ duration: 3600,
+ });
+ }
+ return categoriesBudget;
+}
+
+/**
+ * Spend one point of the tenant's urlCategories budget (interval syncs,
+ * "Sync now", and the connection test's taxonomy read all share it). Throws
+ * a quota ZscalerError when exhausted.
+ */
+export async function consumeZscalerCategoriesBudget(
+ credentials: ZscalerCredentials,
+): Promise<void> {
+ try {
+ await getCategoriesBudget().consume(zscalerBudgetKey(credentials), 1);
+ } catch (rejection) {
+ if (rejection instanceof RateLimiterRes) {
+ throw new ZscalerError(
+ "quota",
+ "Zscaler urlCategories hourly budget is exhausted",
+ undefined,
+ rejection.msBeforeNext,
+ );
+ }
+ throw rejection;
+ }
+}
+
+const MAX_LIST_ENTRY_LENGTH = 2048;
+
+function cleanEntries(values: string[] | undefined): string[] {
+ if (!Array.isArray(values)) return [];
+ const cleaned = new Set<string>();
+ for (const value of values) {
+ if (typeof value !== "string") continue;
+ const trimmed = value.trim().toLowerCase();
+ if (trimmed.length === 0 || trimmed.length > MAX_LIST_ENTRY_LENGTH) {
+ continue;
+ }
+ cleaned.add(trimmed);
+ }
+ return [...cleaned];
+}
+
+/**
+ * Turns the GET /urlCategories payload into the synced document's taxonomy
+ * and locally-evaluable rules. Exported for tests.
+ */
+export function materializeCategories(categories: ZscalerUrlCategory[]): {
+ taxonomy: ZscalerTaxonomyEntry[];
+ rules: ZscalerSyncedRule[];
+} {
+ const taxonomy: ZscalerTaxonomyEntry[] = [];
+ const rules: ZscalerSyncedRule[] = [];
+
+ for (const category of categories) {
+ const name =
+ typeof category.configuredName === "string" &&
+ category.configuredName.trim().length > 0
+ ? category.configuredName.trim()
+ : category.id;
+ const custom = category.customCategory === true;
+ taxonomy.push({ id: category.id, name, custom });
+
+ const rule: ZscalerSyncedRule = {
+ id: category.id,
+ name,
+ custom,
+ urls: cleanEntries(category.urls),
+ retainingUrls: cleanEntries(category.dbCategorizedUrls),
+ keywords: cleanEntries(category.keywords),
+ retainingKeywords: cleanEntries(category.keywordsRetainingParentCategory),
+ };
+ if (
+ rule.urls.length > 0 ||
+ rule.retainingUrls.length > 0 ||
+ rule.keywords.length > 0 ||
+ rule.retainingKeywords.length > 0
+ ) {
+ rules.push(rule);
+ }
+ }
+
+ taxonomy.sort((a, b) => a.name.localeCompare(b.name));
+ return { taxonomy, rules };
+}
+
+/**
+ * Fetches the tenant taxonomy and replaces the org's synced document.
+ * Distributed-lock guarded: when another process is already syncing, this
+ * returns the current document without calling the tenant. Throws only when
+ * there is no usable document at all (first sync failing).
+ */
+export async function syncOrgZscalerRules(
+ orgId: string,
+): Promise<ZscalerSyncDocument | null> {
+ let lock;
+ try {
+ lock = await redlock.acquire([syncLockKey(orgId)], 120_000, {
+ retryCount: 0,
+ });
+ } catch (error) {
+ // Redlock reports contention and infrastructure failure with the same
+ // error; the lock key existing is what distinguishes "someone else is
+ // syncing" (return their in-progress state) from a Redis outage (which
+ // must surface, not masquerade as a successful no-op).
+ let lockHeld = false;
+ try {
+ lockHeld = (await redisRateLimitClient.exists(syncLockKey(orgId))) === 1;
+ } catch {
+ // Redis is down; fall through to rethrow the original error.
+ }
+ if (lockHeld) {
+ return await getZscalerSyncDocument(orgId);
+ }
+ throw error;
+ }
+
+ try {
+ const previous = await getZscalerSyncDocument(orgId);
+ const attemptedAt = new Date().toISOString();
+
+ const credentials = await getOrgZscalerCredentials(orgId);
+ if (!credentials) {
+ logger.warn("Zscaler sync requested for an org without credentials", {
+ orgId,
+ });
+ return previous;
+ }
+
+ try {
+ await consumeZscalerCategoriesBudget(credentials);
+ const categories = await zscalerGetUrlCategories(credentials, {
+ signal: AbortSignal.timeout(60_000),
+ });
+ const { taxonomy, rules } = materializeCategories(categories);
+ const doc: ZscalerSyncDocument = {
+ version: SYNC_DOC_VERSION,
+ status: "ok",
+ syncedAt: attemptedAt,
+ attemptedAt,
+ error: null,
+ taxonomy,
+ rules,
+ };
+ await putZscalerSyncDocument(orgId, doc);
+ logger.info("Zscaler category sync succeeded", {
+ orgId,
+ categories: taxonomy.length,
+ rules: rules.length,
+ });
+ return doc;
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ logger.warn("Zscaler category sync failed", { error, orgId });
+ // Keep the last good rules — stale custom lists beat no custom lists —
+ // but surface the failure and its time for the dashboard.
+ const doc: ZscalerSyncDocument = {
+ version: SYNC_DOC_VERSION,
+ status: "error",
+ syncedAt: previous?.syncedAt ?? null,
+ attemptedAt,
+ error: message,
+ taxonomy: previous?.taxonomy ?? [],
+ rules: previous?.rules ?? [],
+ };
+ await putZscalerSyncDocument(orgId, doc);
+ return doc;
+ }
+ } finally {
+ await lock.release().catch(() => {});
+ }
+}
+
+const syncsInFlight = new Set<string>();
+
+/**
+ * Fire-and-forget staleness check used by the enforcement hot path: when the
+ * synced document is missing or older than the org's sync interval, kick a
+ * background re-sync (at most one per process at a time per org).
+ */
+function ensureZscalerSyncFresh(
+ orgId: string,
+ syncIntervalMinutes: number,
+ doc: ZscalerSyncDocument | null,
+): void {
+ const intervalMs = syncIntervalMinutes * 60_000;
+ if (
+ doc &&
+ doc.attemptedAt &&
+ Date.now() - Date.parse(doc.attemptedAt) < intervalMs
+ ) {
+ return;
+ }
+ if (syncsInFlight.has(orgId)) return;
+ syncsInFlight.add(orgId);
+ syncOrgZscalerRules(orgId)
+ .catch(error => {
+ logger.warn("Background Zscaler sync failed", { error, orgId });
+ })
+ .finally(() => {
+ syncsInFlight.delete(orgId);
+ });
+}
+
+// =========================================
+// Local rule evaluation
+// =========================================
+
+interface UrlParts {
+ host: string;
+ /** Path + optional query, always starting with "/". */
+ pathAndQuery: string;
+}
+
+function parseUrlParts(url: string): UrlParts {
+ const { host, path, query } = splitUrl(url.trim().replace(/[\t\r\n]/g, ""));
+ return {
+ host: canonicalizeHost(host),
+ pathAndQuery:
+ `${path || "/"}${query !== null ? `?${query}` : ""}`.toLowerCase(),
+ };
+}
+
+/**
+ * Whether a URL-list entry ("example.com", "sub.example.com/path",
+ * "internal.example.com:8443") matches. Host matching follows Zscaler list
+ * semantics: an entry matches the host itself and its subdomains. An entry
+ * with a path component additionally requires the URL path to sit under it.
+ * Ports are ignored on both sides — the checked host comes from splitUrl
+ * (port already stripped), so a port kept in the entry could never match
+ * and the rule would silently not enforce.
+ */
+function matchesZscalerUrlEntry(parts: UrlParts, entry: string): boolean {
+ const slash = entry.indexOf("/");
+ let entryHostRaw = slash === -1 ? entry : entry.slice(0, slash);
+ const portMatch = entryHostRaw.match(/^(.*):(\d+)$/);
+ if (portMatch) entryHostRaw = portMatch[1];
+ const entryHost = canonicalizeHost(entryHostRaw);
+ if (entryHost.length === 0) return false;
+ if (parts.host !== entryHost && !parts.host.endsWith(`.${entryHost}`)) {
+ return false;
+ }
+ if (slash === -1) return true;
+ let entryPath = entry.slice(slash).toLowerCase();
+ if (!entryPath.startsWith("/")) entryPath = `/${entryPath}`;
+ return parts.pathAndQuery.startsWith(entryPath);
+}
+
+/**
+ * Best-effort keyword match: case-insensitive substring over the schemeless
+ * URL ("host/path?query"), which is how ZIA keyword categories are commonly
+ * understood to behave. Documented as an approximation.
+ */
+function matchesKeyword(parts: UrlParts, keyword: string): boolean {
+ return `${parts.host}${parts.pathAndQuery}`.includes(keyword);
+}
+
+interface ZscalerSyncedMatch {
+ rule: ZscalerSyncedRule;
+ /** True when the matching entry reclassifies (non-"retaining") the URL. */
+ reclassifies: boolean;
+ /**
+ * True when the match came from a URL-list entry (exact semantics). A
+ * keyword match is a best-effort approximation and must never be the
+ * reason a URL SKIPS the provider lookup — only exact matches may allow.
+ */
+ exact: boolean;
+}
+
+/** All synced categories the URL falls into. Exported for tests. */
+export function matchSyncedRules(
+ url: string,
+ rules: ZscalerSyncedRule[],
+): ZscalerSyncedMatch[] {
+ const parts = parseUrlParts(url);
+ const matches: ZscalerSyncedMatch[] = [];
+ for (const rule of rules) {
+ const exactReclassify = rule.urls.some(entry =>
+ matchesZscalerUrlEntry(parts, entry),
+ );
+ const keywordReclassify =
+ !exactReclassify &&
+ rule.keywords.some(keyword => matchesKeyword(parts, keyword));
+ const reclassifies = exactReclassify || keywordReclassify;
+ const exactRetain =
+ !reclassifies &&
+ rule.retainingUrls.some(entry => matchesZscalerUrlEntry(parts, entry));
+ const keywordRetain =
+ !reclassifies &&
+ !exactRetain &&
+ rule.retainingKeywords.some(keyword => matchesKeyword(parts, keyword));
+ if (reclassifies || exactRetain || keywordRetain) {
+ matches.push({
+ rule,
+ reclassifies,
+ exact: exactReclassify || exactRetain,
+ });
+ }
+ }
+ return matches;
+}
+
+// Custom category IDs follow this shape; Zscaler-defined categories never do.
+const CUSTOM_CATEGORY_ID_REGEX = /^CUSTOM_\d+$/i;
+
+/**
+ * Evaluates the org's synced Zscaler rules for a URL.
+ *
+ * - A match in a denied category → deny ("denied-category"), no provider call.
+ * - An exact (URL-list) reclassifying match in a non-denied category → allow
+ * ("custom-category"), no provider call: ZIA reclassifies such URLs out of
+ * their Zscaler-defined categories, so a lookup verdict would not apply.
+ * Keyword matches are approximate and never allow on their own — the
+ * provider lookup still runs.
+ * - Only "retaining parent category" matches in non-denied categories →
+ * null: the URL also keeps its Zscaler-defined categories, so the provider
+ * lookup still decides.
+ * - No match → null (provider lookup decides).
+ *
+ * When the org denies custom categories but no sync has ever succeeded, the
+ * custom rules cannot be evaluated AND the lookup cannot see them (urlLookup
+ * never returns custom classifications) — that resolves through the org's
+ * failurePolicy instead of silently allowing. A previously synced (stale)
+ * document keeps enforcing.
+ *
+ * Never throws on storage failure; also kicks the lazy background re-sync.
+ */
+export async function evaluateZscalerSyncedRules(
+ url: string,
+ zscaler: ZscalerPolicySettings,
+ failurePolicy: ThreatProtectionPolicy["failurePolicy"],
+ base: {
+ url: string;
+ domain: string;
+ mode: ThreatProtectionPolicy["mode"];
+ },
+): Promise<ThreatDecision | null> {
+ let doc: ZscalerSyncDocument | null = null;
+ try {
+ doc = await getZscalerSyncDocument(zscaler.orgId);
+ } catch (error) {
+ logger.warn("Failed to load synced Zscaler rules", {
+ error,
+ orgId: zscaler.orgId,
+ });
+ }
+ ensureZscalerSyncFresh(zscaler.orgId, zscaler.syncIntervalMinutes, doc);
+
+ if (!doc || doc.syncedAt === null) {
+ const deniesCustomCategories = zscaler.deniedCategories.some(id =>
+ CUSTOM_CATEGORY_ID_REGEX.test(id),
+ );
+ if (deniesCustomCategories) {
+ return {
+ allowed: failurePolicy === "open",
+ rule: "provider-failure",
+ providerConsulted: false,
+ verdict: null,
+ ...base,
+ };
+ }
+ return null;
+ }
+ if (doc.rules.length === 0) return null;
+
+ const matches = matchSyncedRules(url, doc.rules);
+ if (matches.length === 0) return null;
+
+ const denied = matches.filter(match =>
+ zscaler.deniedCategories.includes(match.rule.id),
+ );
+ if (denied.length > 0) {
+ return {
+ allowed: false,
+ rule: "denied-category",
+ providerConsulted: false,
+ verdict: {
+ provider: "zscaler-zia",
+ riskScore: null,
+ categories: denied.map(match => match.rule.id),
+ fromCache: false,
+ raw: {
+ source: "synced-rules",
+ matched: denied.map(match => ({
+ category: match.rule.id,
+ reclassifies: match.reclassifies,
+ })),
+ },
+ },
+ ...base,
+ };
+ }
+
+ if (matches.some(match => match.reclassifies && match.exact)) {
+ return {
+ allowed: true,
+ rule: "custom-category",
+ providerConsulted: false,
+ verdict: null,
+ ...base,
+ };
+ }
+
+ return null;
+}
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/testing.ts b/apps/api/src/lib/threat-protection/providers/zscaler/testing.ts
new file mode 100644
index 000000000..090006aa4
--- /dev/null
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/testing.ts
@@ -0,0 +1,163 @@
+import type http from "http";
+import type { ZscalerUrlCategory } from "./client";
+
+// Mock ZIA server for tests: the Zidentity token endpoint plus the two ZIA
+// endpoints the provider uses. Response shapes follow the official Zscaler
+// SDK services (zscaler-sdk-go/py `urlcategories`), not hand-invented
+// payloads. Point the API at it with:
+//
+// ZSCALER_TOKEN_URL_OVERRIDE=http://localhost:<port>/oauth2/v1/token
+// ZSCALER_API_URL_OVERRIDE=http://localhost:<port>
+//
+// The urlLookup path then resolves to /zia/api/v1/urlLookup on this server.
+
+export interface ZscalerMockDatabase {
+ /** Credentials the token endpoint accepts. */
+ clientId: string;
+ clientSecret: string;
+ /** Simulate a role without urlLookup access (View Only probe): 403. */
+ denyLookupScope?: boolean;
+ /** Simulate tenant-side throttling: every urlLookup call returns 429. */
+ rateLimitLookups?: boolean;
+ /** GET /urlCategories payload. */
+ categories: ZscalerUrlCategory[];
+ /**
+ * POST /urlLookup classifications, keyed by the exact looked-up URL string
+ * (schemeless). Unknown URLs return MISCELLANEOUS_OR_UNKNOWN, like ZIA.
+ */
+ classifications: Record<
+ string,
+ {
+ urlClassifications: string[];
+ urlClassificationsWithSecurityAlert: string[];
+ }
+ >;
+}
+
+interface ZscalerMockCounters {
+ tokenCalls: number;
+ categoriesCalls: number;
+ lookupCalls: number;
+ lookupUrls: number;
+}
+
+export function createZscalerMockCounters(): ZscalerMockCounters {
+ return { tokenCalls: 0, categoriesCalls: 0, lookupCalls: 0, lookupUrls: 0 };
+}
+
+const MOCK_TOKEN = "mock-zscaler-access-token";
+
+function json(res: http.ServerResponse, status: number, body: unknown): void {
+ res.statusCode = status;
+ res.setHeader("Content-Type", "application/json");
+ res.end(JSON.stringify(body));
+}
+
+function readBody(req: http.IncomingMessage): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const chunks: Buffer[] = [];
+ req.on("data", chunk => chunks.push(chunk));
+ req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+ req.on("error", reject);
+ });
+}
+
+export function createZscalerMockHandler(
+ db: ZscalerMockDatabase,
+ counters: ZscalerMockCounters,
+): (req: http.IncomingMessage, res: http.ServerResponse) => void {
+ return (req, res) => {
+ void (async () => {
+ const url = new URL(req.url ?? "/", "http://localhost");
+
+ if (req.method === "POST" && url.pathname === "/oauth2/v1/token") {
+ counters.tokenCalls++;
+ const body = new URLSearchParams(await readBody(req));
+ if (
+ body.get("grant_type") !== "client_credentials" ||
+ body.get("client_id") !== db.clientId ||
+ body.get("client_secret") !== db.clientSecret
+ ) {
+ json(res, 401, { error: "invalid_client" });
+ return;
+ }
+ json(res, 200, {
+ access_token: MOCK_TOKEN,
+ token_type: "Bearer",
+ expires_in: 3600,
+ });
+ return;
+ }
+
+ if (req.headers.authorization !== `Bearer ${MOCK_TOKEN}`) {
+ json(res, 401, { message: "Invalid or missing bearer token" });
+ return;
+ }
+
+ if (
+ req.method === "GET" &&
+ url.pathname === "/zia/api/v1/urlCategories"
+ ) {
+ counters.categoriesCalls++;
+ json(res, 200, db.categories);
+ return;
+ }
+
+ if (req.method === "POST" && url.pathname === "/zia/api/v1/urlLookup") {
+ counters.lookupCalls++;
+ if (db.denyLookupScope) {
+ json(res, 403, { message: "Access to this resource is denied" });
+ return;
+ }
+ if (db.rateLimitLookups) {
+ res.setHeader("Retry-After", "1");
+ json(res, 429, { message: "Rate limit exceeded" });
+ return;
+ }
+ let urls: unknown;
+ try {
+ urls = JSON.parse(await readBody(req));
+ } catch {
+ json(res, 400, { message: "Invalid JSON body" });
+ return;
+ }
+ if (
+ !Array.isArray(urls) ||
+ urls.length > 100 ||
+ urls.some(entry => typeof entry !== "string" || entry.length > 1024)
+ ) {
+ json(res, 400, {
+ message:
+ "Expected an array of at most 100 URL strings of at most 1024 characters",
+ });
+ return;
+ }
+ counters.lookupUrls += urls.length;
+ json(
+ res,
+ 200,
+ urls.map(lookupUrl => {
+ const entry =
+ typeof lookupUrl === "string"
+ ? db.classifications[lookupUrl]
+ : undefined;
+ return {
+ url: lookupUrl,
+ urlClassifications: entry?.urlClassifications ?? [
+ "MISCELLANEOUS_OR_UNKNOWN",
+ ],
+ urlClassificationsWithSecurityAlert:
+ entry?.urlClassificationsWithSecurityAlert ?? [],
+ };
+ }),
+ );
+ return;
+ }
+
+ json(res, 404, { message: `No mock for ${req.method} ${url.pathname}` });
+ })().catch(() => {
+ res.statusCode = 500;
+ res.end();
+ });
+ };
+}
diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.ts
index ababe76e8..b2d8faa75 100644
--- a/apps/api/src/lib/threat-protection/request.ts
+++ b/apps/api/src/lib/threat-protection/request.ts
@@ -31,6 +31,9 @@ export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =
export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE =
"Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";
+const THREAT_PROTECTION_ZSCALER_NOT_CONFIGURED_MESSAGE =
+ 'Threat protection mode "zscaler" requires a configured Zscaler connection. Configure one via PUT /v2/team/threat-protection first.';
+
/**
* The deprecated v0 endpoints never resolve a threat protection policy, so
* teams whose flag is "forced" must not be able to fetch content through
@@ -105,6 +108,19 @@ export async function resolveThreatProtection(args: {
}
const policy = resolveEffectivePolicy(orgConfig, args.override);
+
+ // "zscaler" mode without a connection can only happen via a per-request
+ // override on an org that never saved credentials (the config API refuses
+ // to store that combination). Reject clearly instead of failing every
+ // lookup into the failurePolicy.
+ if (policy.mode === "zscaler" && !policy.zscaler) {
+ return {
+ error: THREAT_PROTECTION_ZSCALER_NOT_CONFIGURED_MESSAGE,
+ policy: null,
+ orgConfig,
+ };
+ }
+
return {
policy: policy.mode === "off" ? null : policy,
orgConfig,
@@ -150,9 +166,18 @@ export async function checkUrlsAgainstThreatPolicy(
dedup: ctx.dedup ?? new Map(),
};
+ // "zscaler" mode checks resolve through a shared per-org lookup queue
+ // whose budget is spent per CALL (≤100 URLs each) — checking only 16 at a
+ // time would cap batch fill at ~16 URLs per budgeted call. Run a full
+ // batch's worth concurrently so the queue drainer can pack real batches.
+ const concurrency =
+ policy.mode === "zscaler" && !ctx.localRulesOnly
+ ? 100
+ : URL_CHECK_CONCURRENCY;
+
const decisionsByUrl = new Map<string, ThreatDecision>();
- for (let i = 0; i < uniqueUrls.length; i += URL_CHECK_CONCURRENCY) {
- const batch = uniqueUrls.slice(i, i + URL_CHECK_CONCURRENCY);
+ for (let i = 0; i < uniqueUrls.length; i += concurrency) {
+ const batch = uniqueUrls.slice(i, i + concurrency);
const decisions = await Promise.all(
batch.map(url => checkUrl(url, policy, dedupCtx)),
);
diff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.ts
index d7987607f..a3e0aea50 100644
--- a/apps/api/src/lib/threat-protection/store.test.ts
+++ b/apps/api/src/lib/threat-protection/store.test.ts
@@ -24,6 +24,7 @@ function makeOrgConfig(
orgId: "00000000-0000-0000-0000-000000000000",
policy: { ...orgPolicy },
allowRequestOverrides: true,
+ zscaler: null,
createdAt: null,
updatedAt: null,
...overrides,
@@ -71,11 +72,54 @@ describe("rowToConfig", () => {
failurePolicy: "open",
},
allowRequestOverrides: false,
+ zscaler: null,
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-02T00:00:00.000Z",
});
});
+ it("maps a stored Zscaler block into settings and policy", () => {
+ const config = rowToConfig(
+ makeRow("zscaler", {
+ failurePolicy: "closed",
+ zscaler: {
+ clientId: "client-1",
+ secretCiphertext: "gcm:iv:tag:ct",
+ vanityDomain: "tenant",
+ cloud: null,
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ syncIntervalMinutes: 30,
+ },
+ }),
+ );
+
+ expect(config.policy.mode).toBe("zscaler");
+ expect(config.policy.zscaler).toEqual({
+ orgId: ORG_ID,
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ syncIntervalMinutes: 30,
+ });
+ expect(config.zscaler).toEqual({
+ clientId: "client-1",
+ secretCiphertext: "gcm:iv:tag:ct",
+ vanityDomain: "tenant",
+ cloud: null,
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ syncIntervalMinutes: 30,
+ });
+ });
+
+ it("degrades a malformed Zscaler block to no connection instead of throwing", () => {
+ const config = rowToConfig(
+ makeRow("zscaler", {
+ zscaler: { clientId: 42 },
+ }),
+ );
+ expect(config.policy.mode).toBe("zscaler");
+ expect(config.zscaler).toBeNull();
+ expect(config.policy.zscaler).toBeUndefined();
+ });
+
it("falls back to defaults for an empty config document", () => {
const config = rowToConfig(makeRow("normal", {}));
expect(config.policy).toEqual({
diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.ts
index e7bde17b8..3d00ce019 100644
--- a/apps/api/src/lib/threat-protection/store.ts
+++ b/apps/api/src/lib/threat-protection/store.ts
@@ -1,9 +1,12 @@
import { eq } from "drizzle-orm";
import { z } from "zod";
+import { config as appConfig } from "../../config";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { deleteKey, getValue, setValue } from "../../services/redis";
import { logger as _logger } from "../logger";
+import { decryptOrgSecret, encryptOrgSecret } from "../org-secret-crypto";
+import type { ZscalerCredentials } from "./providers/zscaler/client";
import {
THREAT_PROTECTION_POLICY_DEFAULTS,
type ThreatProtectionPolicy,
@@ -41,10 +44,26 @@ function isMissingTableError(error: unknown): boolean {
return false;
}
+/**
+ * Stored (non-secret-decrypted) Zscaler settings. `secretCiphertext` is the
+ * AES-256-GCM output of the write-only client secret — decrypted only by
+ * {@link getOrgZscalerCredentials} at call time, never serialized to the API
+ * or into job payloads.
+ */
+export interface OrgZscalerSettings {
+ clientId: string;
+ secretCiphertext: string;
+ vanityDomain: string;
+ cloud: string | null;
+ deniedCategories: string[];
+ syncIntervalMinutes: number;
+}
+
export interface OrgThreatProtectionConfig {
orgId: string;
policy: ThreatProtectionPolicy;
allowRequestOverrides: boolean;
+ zscaler: OrgZscalerSettings | null;
createdAt: string | null;
updatedAt: string | null;
}
@@ -60,6 +79,20 @@ type ThreatProtectionConfigRow =
* field defaults instead of throwing — a bad row must never take down the
* enforcement hot path.
*/
+const storedZscalerSchema = z
+ .object({
+ clientId: z.string().min(1),
+ secretCiphertext: z.string().min(1),
+ vanityDomain: z.string().min(1),
+ cloud: z.string().nullable().catch(null),
+ deniedCategories: z.array(z.string()).catch([]),
+ syncIntervalMinutes: z.number().int().positive().catch(60),
+ })
+ // A malformed block degrades to "no Zscaler connection" (lookups then fail
+ // per the org's failurePolicy) instead of throwing on the hot path.
+ .nullable()
+ .catch(null);
+
const storedConfigDocumentSchema = z
.object({
riskScoreThreshold: z
@@ -75,10 +108,12 @@ const storedConfigDocumentSchema = z
.enum(["open", "closed"])
.catch(THREAT_PROTECTION_POLICY_DEFAULTS.failurePolicy),
allowRequestOverrides: z.boolean().catch(true),
+ zscaler: storedZscalerSchema.optional().catch(null),
})
.catch({
...THREAT_PROTECTION_POLICY_DEFAULTS,
allowRequestOverrides: true,
+ zscaler: null,
});
/**
@@ -91,17 +126,33 @@ export function rowToConfig(
row: ThreatProtectionConfigRow,
): OrgThreatProtectionConfig {
const doc = storedConfigDocumentSchema.parse(row.config ?? {});
+ const zscaler = doc.zscaler ?? null;
return {
orgId: row.org_id,
policy: {
- mode: row.mode === "normal" ? "normal" : "off",
+ mode:
+ row.mode === "normal"
+ ? "normal"
+ : row.mode === "zscaler"
+ ? "zscaler"
+ : "off",
riskScoreThreshold: doc.riskScoreThreshold,
blacklist: doc.blacklist,
whitelist: doc.whitelist,
blockedTlds: doc.blockedTlds,
failurePolicy: doc.failurePolicy,
+ ...(zscaler
+ ? {
+ zscaler: {
+ orgId: row.org_id,
+ deniedCategories: zscaler.deniedCategories,
+ syncIntervalMinutes: zscaler.syncIntervalMinutes,
+ },
+ }
+ : {}),
},
allowRequestOverrides: doc.allowRequestOverrides,
+ zscaler,
createdAt: row.created_at,
updatedAt: row.updated_at,
};
@@ -163,14 +214,64 @@ export async function getOrgThreatProtectionConfig(
return config;
}
+/**
+ * The PUT config document needs a client secret we do not have: thrown when a
+ * Zscaler block is first configured without one, or when the client ID
+ * changes without a matching new secret. Mapped to a 400 by the controller.
+ */
+export class ZscalerSecretRequiredError extends Error {
+ constructor() {
+ super(
+ "zscaler.clientSecret is required when configuring a Zscaler connection or changing its client ID",
+ );
+ this.name = "ZscalerSecretRequiredError";
+ }
+}
+
/**
* Full-document upsert of the org's threat protection config. Invalidates
* the read cache on success.
+ *
+ * The Zscaler client secret is write-only: when the input block omits it,
+ * the stored ciphertext is kept — but only while the client ID is unchanged
+ * (pairing a new client with an old secret is always a mistake, so that
+ * throws {@link ZscalerSecretRequiredError} instead).
*/
export async function upsertOrgThreatProtectionConfig(
orgId: string,
config: ThreatProtectionConfigInput,
): Promise<OrgThreatProtectionConfig> {
+ let zscaler: z.infer<typeof storedZscalerSchema> = null;
+ if (config.zscaler) {
+ let secretCiphertext: string;
+ if (config.zscaler.clientSecret) {
+ secretCiphertext = encryptOrgSecret(config.zscaler.clientSecret, orgId);
+ } else {
+ // Primary read: the upsert below writes to the primary, and a stale
+ // replica must not make us discard a just-saved secret.
+ const existingRows = await db
+ .select({ config: schema.threat_protection_config.config })
+ .from(schema.threat_protection_config)
+ .where(eq(schema.threat_protection_config.org_id, orgId))
+ .limit(1);
+ const existing = existingRows[0]
+ ? storedConfigDocumentSchema.parse(existingRows[0].config ?? {}).zscaler
+ : null;
+ if (!existing || existing.clientId !== config.zscaler.clientId) {
+ throw new ZscalerSecretRequiredError();
+ }
+ secretCiphertext = existing.secretCiphertext;
+ }
+ zscaler = {
+ clientId: config.zscaler.clientId,
+ secretCiphertext,
+ vanityDomain: config.zscaler.vanityDomain,
+ cloud: config.zscaler.cloud,
+ deniedCategories: config.zscaler.deniedCategories,
+ syncIntervalMinutes: config.zscaler.syncIntervalMinutes,
+ };
+ }
+
const values = {
org_id: orgId,
mode: config.mode,
@@ -182,6 +283,7 @@ export async function upsertOrgThreatProtectionConfig(
blockedTlds: config.blockedTlds,
failurePolicy: config.failurePolicy,
allowRequestOverrides: config.allowRequestOverrides,
+ zscaler,
},
};
@@ -233,7 +335,9 @@ export async function getOrgIdForTeam(teamId: string): Promise<string | null> {
*/
export function resolveEffectivePolicy(
orgConfig: OrgThreatProtectionConfig | null,
- requestOverride?: Partial<ThreatProtectionPolicy>,
+ // Omit is deliberate: the Zscaler connection is org-owned and never
+ // request-settable — passing one is a compile error, not an ignored field.
+ requestOverride?: Partial<Omit<ThreatProtectionPolicy, "zscaler">>,
): ThreatProtectionPolicy {
const base: ThreatProtectionPolicy = orgConfig
? { ...orgConfig.policy }
@@ -246,6 +350,10 @@ export function resolveEffectivePolicy(
for (const key of Object.keys(requestOverride) as Array<
keyof ThreatProtectionPolicy
>) {
+ // The Zscaler connection settings come from the org config only — the
+ // override schema has no such field, and this guard keeps it that way
+ // even if a wider Partial ever reaches here.
+ if (key === "zscaler") continue;
const value = requestOverride[key];
if (value !== undefined) {
(base as unknown as Record<string, unknown>)[key] = value;
@@ -254,3 +362,100 @@ export function resolveEffectivePolicy(
return base;
}
+
+/**
+ * Decrypted Zscaler API credentials for an org, or null when the org has no
+ * (valid) Zscaler connection. Backed by the 60s org-config cache, so a saved
+ * secret rotation is picked up within a minute — the plaintext itself lives
+ * only in this return value, never in a cache or a job payload.
+ */
+export async function getOrgZscalerCredentials(
+ orgId: string,
+): Promise<ZscalerCredentials | null> {
+ const orgConfig = await getOrgThreatProtectionConfig(orgId);
+ if (!orgConfig?.zscaler) return null;
+ try {
+ return {
+ clientId: orgConfig.zscaler.clientId,
+ clientSecret: decryptOrgSecret(orgConfig.zscaler.secretCiphertext, orgId),
+ vanityDomain: orgConfig.zscaler.vanityDomain,
+ cloud: orgConfig.zscaler.cloud,
+ };
+ } catch (error) {
+ logger.error("Failed to decrypt the stored Zscaler client secret", {
+ error,
+ orgId,
+ });
+ return null;
+ }
+}
+
+const concurrencyCapCacheKey = (teamId: string) =>
+ `threat-protection-concurrency-cap:${teamId}`;
+
+/**
+ * Scrape-concurrency cap imposed by the team's threat protection mode, or
+ * null when none applies. "zscaler" mode caps concurrency so URL demand
+ * stays near the tenant's classification budget (POST /urlLookup allows 1
+ * request/second × 100 URLs, 400 requests/hour) — without the cap, a large
+ * crawl outruns the budget, every lookup times out, and the failurePolicy
+ * fires for everything.
+ *
+ * The budget is org-scoped while concurrency admission is team-scoped, so
+ * the org-level cap is split across the org's teams — otherwise an org with
+ * N teams would run at N× the intended concurrency.
+ *
+ * Cached in Redis for 60s per team; fails open (null) so a Redis or DB
+ * hiccup never blocks job admission.
+ */
+export async function getThreatProtectionConcurrencyCap(
+ teamId: string,
+): Promise<number | null> {
+ // Self-hosted deployments have no org config database (and no enterprise
+ // threat protection) — skip the lookup instead of warning on every job.
+ if (!appConfig.USE_DB_AUTHENTICATION) return null;
+
+ const key = concurrencyCapCacheKey(teamId);
+ try {
+ const cached = await getValue(key);
+ if (cached === "none") return null;
+ if (cached !== null) {
+ const parsed = Number(cached);
+ // A malformed cache entry must not become a NaN/zero cap that breaks
+ // job admission — fall through and resolve from the source instead.
+ if (Number.isFinite(parsed) && parsed > 0) return parsed;
+ }
+ } catch (error) {
+ logger.warn("Failed to read the concurrency cap cache", { error, teamId });
+ }
+
+ let cap: number | null = null;
+ try {
+ const orgId = await getOrgIdForTeam(teamId);
+ const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;
+ if (orgId && orgConfig?.policy.mode === "zscaler") {
+ const teamRows = await dbRr
+ .select({ id: schema.teams.id })
+ .from(schema.teams)
+ .where(eq(schema.teams.org_id, orgId));
+ const teamCount = Math.max(teamRows.length, 1);
+ cap = Math.max(
+ 1,
+ Math.floor(appConfig.ZSCALER_MODE_CONCURRENCY_CAP / teamCount),
+ );
+ }
+ } catch (error) {
+ logger.warn("Failed to resolve the threat protection concurrency cap", {
+ error,
+ teamId,
+ });
+ return null;
+ }
+
+ try {
+ await setValue(key, cap === null ? "none" : String(cap), 60);
+ } catch (error) {
+ logger.warn("Failed to write the concurrency cap cache", { error, teamId });
+ }
+ return cap;
+}
diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.ts
index a15e7c9a5..c045ddc20 100644
--- a/apps/api/src/lib/threat-protection/types.ts
+++ b/apps/api/src/lib/threat-protection/types.ts
@@ -1,7 +1,25 @@
// Shared type contract for the threat protection feature.
// NOTE: concurrent in-flight branches create this exact file — do not modify without coordinating.
-export type ThreatProtectionMode = "off" | "normal";
+export type ThreatProtectionMode = "off" | "normal" | "zscaler";
+
+/**
+ * The non-secret slice of an org's Zscaler settings that policy evaluation
+ * needs. Threaded through {@link ThreatProtectionPolicy} — and therefore
+ * serialized into scrape job payloads — so it must NEVER carry credentials.
+ * The provider loads credentials fresh from the org config store by `orgId`
+ * at lookup time (so a secret rotation applies within the config cache TTL).
+ */
+export interface ZscalerPolicySettings {
+ orgId: string;
+ /**
+ * Category IDs (Zscaler-defined and custom) the org blocks. Matched against
+ * the normalized verdict categories and against synced custom-list rules.
+ */
+ deniedCategories: string[];
+ /** How often the synced taxonomy/custom lists are considered fresh. */
+ syncIntervalMinutes: number;
+}
export interface ThreatProtectionPolicy {
mode: ThreatProtectionMode;
@@ -15,6 +33,11 @@ export interface ThreatProtectionPolicy {
blockedTlds: string[];
/** Behavior when the provider is unavailable: "closed" blocks, "open" allows. */
failurePolicy: "open" | "closed";
+ /**
+ * Present when the org has a Zscaler connection configured. Attached from
+ * the org config by resolveEffectivePolicy — never settable per-request.
+ */
+ zscaler?: ZscalerPolicySettings;
}
export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<
@@ -28,11 +51,11 @@ export const THREAT_PROTECTION_POLICY_DEFAULTS: Omit<
failurePolicy: "closed",
};
-// Only Google Web Risk today. The union (and the provider seam around it —
-// separate provider modules under ./providers, the mode→provider dispatch in
-// ./index.ts, and the provider-agnostic verdict engine in ./verdict.ts) is
-// deliberately kept so future partner classifiers slot in as new members.
-export type ThreatProvider = "google-web-risk";
+// The union (and the provider seam around it — separate provider modules
+// under ./providers, the mode→provider dispatch in ./index.ts, and the
+// provider-agnostic verdict engine in ./verdict.ts) is deliberately kept so
+// further partner classifiers slot in as new members.
+export type ThreatProvider = "google-web-risk" | "zscaler-zia";
export interface RawVerdict {
provider: ThreatProvider;
@@ -40,6 +63,12 @@ export interface RawVerdict {
riskScore: number | null;
/** Provider threat categories (Web Risk threat types map through here). */
categories: string[];
+ /**
+ * True when the provider flagged the URL with a security-alert
+ * classification (Zscaler `urlClassificationsWithSecurityAlert`).
+ * Undefined for providers without the concept.
+ */
+ securityAlert?: boolean;
/**
* Always false: verdicts are never persisted (ZDR) — repeated checks within
* one request share the same in-flight decision via the request-scoped
@@ -66,6 +95,13 @@ export type ThreatDecisionRule =
| "blacklist"
| "blocked-tld"
| "risk-score"
+ /** A verdict category is on the org's denied-categories list (Zscaler mode). */
+ | "denied-category"
+ /**
+ * The URL matched a synced custom-list entry that reclassifies it into a
+ * non-denied custom category, so no provider lookup ran (Zscaler mode).
+ */
+ | "custom-category"
| "provider-failure"
| "default-allow";
diff --git a/apps/api/src/lib/threat-protection/verdict.test.ts b/apps/api/src/lib/threat-protection/verdict.test.ts
index 95d7497b2..9d08498de 100644
--- a/apps/api/src/lib/threat-protection/verdict.test.ts
+++ b/apps/api/src/lib/threat-protection/verdict.test.ts
@@ -425,3 +425,119 @@ describe("normalizeDomain", () => {
).toMatchObject({ allowed: false, rule: "blacklist" });
});
});
+
+describe("evaluatePolicy — zscaler denied categories", () => {
+ function zscalerPolicy(
+ overrides: Partial<ThreatProtectionPolicy> = {},
+ ): ThreatProtectionPolicy {
+ return policy({
+ mode: "zscaler",
+ zscaler: {
+ orgId: "00000000-0000-0000-0000-000000000000",
+ deniedCategories: ["GAMBLING", "CUSTOM_01"],
+ syncIntervalMinutes: 60,
+ },
+ ...overrides,
+ });
+ }
+
+ function zscalerVerdict(overrides: Partial<RawVerdict> = {}): RawVerdict {
+ return verdict({
+ provider: "zscaler-zia",
+ riskScore: null,
+ ...overrides,
+ });
+ }
+
+ test("blocks when a verdict category is on the denied list", () => {
+ const decision = evaluatePolicy(
+ "https://casino.example.com/",
+ zscalerVerdict({ categories: ["GAMBLING", "ENTERTAINMENT"] }),
+ zscalerPolicy(),
+ );
+ expect(decision).toMatchObject({
+ allowed: false,
+ rule: "denied-category",
+ providerConsulted: true,
+ });
+ });
+
+ test("blocks on a denied custom category ID in the verdict", () => {
+ const decision = evaluatePolicy(
+ "https://vendor.example.com/",
+ zscalerVerdict({ categories: ["CUSTOM_01"] }),
+ zscalerPolicy(),
+ );
+ expect(decision).toMatchObject({ allowed: false, rule: "denied-category" });
+ });
+
+ test("allows when no verdict category is denied", () => {
+ const decision = evaluatePolicy(
+ "https://news.example.com/",
+ zscalerVerdict({ categories: ["NEWS_AND_MEDIA"] }),
+ zscalerPolicy(),
+ );
+ expect(decision).toMatchObject({ allowed: true, rule: "default-allow" });
+ });
+
+ test("uncategorized URLs block only when the org denies the unknown bucket", () => {
+ const uncategorized = zscalerVerdict({
+ categories: ["MISCELLANEOUS_OR_UNKNOWN"],
+ });
+ expect(
+ evaluatePolicy(
+ "https://new.example.com/",
+ uncategorized,
+ zscalerPolicy(),
+ ),
+ ).toMatchObject({ allowed: true, rule: "default-allow" });
+ expect(
+ evaluatePolicy(
+ "https://new.example.com/",
+ uncategorized,
+ zscalerPolicy({
+ zscaler: {
+ orgId: "00000000-0000-0000-0000-000000000000",
+ deniedCategories: ["MISCELLANEOUS_OR_UNKNOWN"],
+ syncIntervalMinutes: 60,
+ },
+ }),
+ ),
+ ).toMatchObject({ allowed: false, rule: "denied-category" });
+ });
+
+ test("whitelist wins over a denied category", () => {
+ const decision = evaluatePolicy(
+ "https://casino.example.com/",
+ zscalerVerdict({ categories: ["GAMBLING"] }),
+ zscalerPolicy({ whitelist: ["casino.example.com"] }),
+ );
+ expect(decision).toMatchObject({ allowed: true, rule: "whitelist" });
+ });
+
+ test("a null verdict resolves through the failurePolicy", () => {
+ expect(
+ evaluatePolicy(
+ "https://example.com/",
+ null,
+ zscalerPolicy({ failurePolicy: "closed" }),
+ ),
+ ).toMatchObject({ allowed: false, rule: "provider-failure" });
+ expect(
+ evaluatePolicy(
+ "https://example.com/",
+ null,
+ zscalerPolicy({ failurePolicy: "open" }),
+ ),
+ ).toMatchObject({ allowed: true, rule: "provider-failure" });
+ });
+
+ test("the null risk score never trips the score threshold", () => {
+ const decision = evaluatePolicy(
+ "https://example.com/",
+ zscalerVerdict({ categories: [], riskScore: null }),
+ zscalerPolicy({ riskScoreThreshold: 0 }),
+ );
+ expect(decision).toMatchObject({ allowed: true, rule: "default-allow" });
+ });
+});
diff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.ts
index d65022305..938245439 100644
--- a/apps/api/src/lib/threat-protection/verdict.ts
+++ b/apps/api/src/lib/threat-protection/verdict.ts
@@ -167,6 +167,18 @@ export function evaluatePolicy(
}
if (verdict !== null) {
+ // Zscaler mode blocks on the org's denied-categories selection. The
+ // verdict's categories are the normalized union of urlClassifications
+ // and urlClassificationsWithSecurityAlert.
+ if (policy.mode === "zscaler" && policy.zscaler) {
+ const denied = verdict.categories.filter(category =>
+ policy.zscaler!.deniedCategories.includes(category),
+ );
+ if (denied.length > 0) {
+ return { allowed: false, rule: "denied-category", ...base };
+ }
+ }
+
if (
verdict.riskScore !== null &&
verdict.riskScore >= policy.riskScoreThreshold
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index a2767c5f2..da6361cf9 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -68,7 +68,10 @@ import {
import { activityController } from "../controllers/v1/activity";
import {
getTeamThreatProtectionController,
+ getTeamZscalerCategoriesController,
putTeamThreatProtectionController,
+ syncTeamZscalerController,
+ testTeamZscalerConnectionController,
} from "../controllers/v2/team-threat-protection";
import {
getTeamSiemLoggingController,
@@ -444,6 +447,24 @@ v2Router.put(
wrap(putTeamThreatProtectionController),
);
+v2Router.post(
+ "/team/threat-protection/zscaler/test-connection",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(testTeamZscalerConnectionController),
+);
+
+v2Router.get(
+ "/team/threat-protection/zscaler/categories",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(getTeamZscalerCategoriesController),
+);
+
+v2Router.post(
+ "/team/threat-protection/zscaler/sync",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(syncTeamZscalerController),
+);
+
v2Router.get(
"/team/siem",
authMiddleware(RateLimiterMode.Account),