Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(threat-protection): remove the zscaler-mode concurrency cap (#4163)

Gergő Móricz5ad8dc051104
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 15080288d..9f80645bd 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -105,10 +105,6 @@ const configSchema = z.object({
.int()
.positive()
.default(500),
- // Effective scrape concurrency cap for teams in "zscaler" mode, keeping URL
- // demand near the classification budget (≤100 URLs/s burst, ~11 URLs/s
- // sustained at 400 lookups/hour).
- ZSCALER_MODE_CONCURRENCY_CAP: z.coerce.number().int().positive().default(15),
// Organization SIEM logging delivery. The encryption key must decode to
// exactly 32 bytes; validation happens when a secret is encrypted/decrypted
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 6b8ae5c73..246e376f0 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -19,7 +19,6 @@ import {
removeConcurrencyLimitActiveJob,
} from "./concurrency-redis";
import { autumnService } from "../services/autumn/autumn.service";
-import { getThreatProtectionConcurrencyCap } from "./threat-protection/store";
// Fallback when Autumn can't give us a concurrency value.
const DEFAULT_CONCURRENCY_LIMIT = 2;
@@ -29,22 +28,13 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;
* balance. Autumn is authoritative; when the entity is missing we fall back to
* the low default of 2. When Autumn errors, getConcurrencyLimit already returns
* a high fail-open value, so that carries through here.
- *
- * Threat protection can clamp the result: "zscaler" mode caps concurrency so
- * URL demand stays near the tenant's classification budget — otherwise a
- * large crawl outruns the 400-lookups/hour budget and every scrape resolves
- * through the failurePolicy instead of a verdict.
*/
export async function getEffectiveConcurrencyLimit(
teamId: string,
orgId?: string | null,
): Promise<number> {
const autumnValue = await autumnService.getConcurrencyLimit(teamId, orgId);
- const limit = autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;
- const threatProtectionCap = await getThreatProtectionConcurrencyCap(teamId);
- return threatProtectionCap !== null
- ? Math.min(limit, threatProtectionCap)
- : limit;
+ return autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;
}
const constructKey = constructConcurrencyLimitKey;
diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
index 7cb0ae4df..1a5b4f62b 100644
--- a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts
@@ -35,10 +35,10 @@ import {
// sit in a line it cannot exit. Verdicts live only in the short-TTL reply
// keys consumed moments later; there is no verdict cache (ZDR).
//
-// Capacity note: the sustained ceiling is ~11 URLs/second per org (400
-// calls/hour × 100 URLs). The "zscaler"-mode concurrency cap (see
-// getThreatProtectionConcurrencyCap) keeps URL demand near this budget;
-// this queue only absorbs bursts.
+// Capacity note: the sustained ceiling is ~11 URLs/second per tenant (400
+// calls/hour × 100 URLs). Demand above that resolves through the fail-fast
+// paths below (queue depth cap, exhausted hourly budget) into the org's
+// failurePolicy — scrape concurrency is deliberately NOT throttled.
const logger = _logger.child({ module: "threat-protection-zscaler-lookup" });
diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.ts
index 3d00ce019..3f744f141 100644
--- a/apps/api/src/lib/threat-protection/store.ts
+++ b/apps/api/src/lib/threat-protection/store.ts
@@ -1,6 +1,5 @@
import { eq } from "drizzle-orm";
import { z } from "zod";
-import { config as appConfig } from "../../config";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { deleteKey, getValue, setValue } from "../../services/redis";
@@ -389,73 +388,3 @@ export async function getOrgZscalerCredentials(
return null;
}
}
-
-const concurrencyCapCacheKey = (teamId: string) =>
- `threat-protection-concurrency-cap:${teamId}`;
-
-/**
- * Scrape-concurrency cap imposed by the team's threat protection mode, or
- * null when none applies. "zscaler" mode caps concurrency so URL demand
- * stays near the tenant's classification budget (POST /urlLookup allows 1
- * request/second × 100 URLs, 400 requests/hour) — without the cap, a large
- * crawl outruns the budget, every lookup times out, and the failurePolicy
- * fires for everything.
- *
- * The budget is org-scoped while concurrency admission is team-scoped, so
- * the org-level cap is split across the org's teams — otherwise an org with
- * N teams would run at N× the intended concurrency.
- *
- * Cached in Redis for 60s per team; fails open (null) so a Redis or DB
- * hiccup never blocks job admission.
- */
-export async function getThreatProtectionConcurrencyCap(
- teamId: string,
-): Promise<number | null> {
- // Self-hosted deployments have no org config database (and no enterprise
- // threat protection) — skip the lookup instead of warning on every job.
- if (!appConfig.USE_DB_AUTHENTICATION) return null;
-
- const key = concurrencyCapCacheKey(teamId);
- try {
- const cached = await getValue(key);
- if (cached === "none") return null;
- if (cached !== null) {
- const parsed = Number(cached);
- // A malformed cache entry must not become a NaN/zero cap that breaks
- // job admission — fall through and resolve from the source instead.
- if (Number.isFinite(parsed) && parsed > 0) return parsed;
- }
- } catch (error) {
- logger.warn("Failed to read the concurrency cap cache", { error, teamId });
- }
-
- let cap: number | null = null;
- try {
- const orgId = await getOrgIdForTeam(teamId);
- const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;
- if (orgId && orgConfig?.policy.mode === "zscaler") {
- const teamRows = await dbRr
- .select({ id: schema.teams.id })
- .from(schema.teams)
- .where(eq(schema.teams.org_id, orgId));
- const teamCount = Math.max(teamRows.length, 1);
- cap = Math.max(
- 1,
- Math.floor(appConfig.ZSCALER_MODE_CONCURRENCY_CAP / teamCount),
- );
- }
- } catch (error) {
- logger.warn("Failed to resolve the threat protection concurrency cap", {
- error,
- teamId,
- });
- return null;
- }
-
- try {
- await setValue(key, cap === null ? "none" : String(cap), 60);
- } catch (error) {
- logger.warn("Failed to write the concurrency cap cache", { error, teamId });
- }
- return cap;
-}