Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(threat-protection): remove the zscaler-mode concurrency cap (#4163)
Gergő Móricz5ad8dc051104
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 15080288d..9f80645bd 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -105,10 +105,6 @@ const configSchema = z.object({.int().positive().default(500),- // Effective scrape concurrency cap for teams in "zscaler" mode, keeping URL- // demand near the classification budget (≤100 URLs/s burst, ~11 URLs/s- // sustained at 400 lookups/hour).- ZSCALER_MODE_CONCURRENCY_CAP: z.coerce.number().int().positive().default(15),// Organization SIEM logging delivery. The encryption key must decode to// exactly 32 bytes; validation happens when a secret is encrypted/decrypteddiff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 6b8ae5c73..246e376f0 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -19,7 +19,6 @@ import {removeConcurrencyLimitActiveJob,} from "./concurrency-redis";import { autumnService } from "../services/autumn/autumn.service";-import { getThreatProtectionConcurrencyCap } from "./threat-protection/store";// Fallback when Autumn can't give us a concurrency value.const DEFAULT_CONCURRENCY_LIMIT = 2;@@ -29,22 +28,13 @@ const DEFAULT_CONCURRENCY_LIMIT = 2;* balance. Autumn is authoritative; when the entity is missing we fall back to* the low default of 2. When Autumn errors, getConcurrencyLimit already returns* a high fail-open value, so that carries through here.- *- * Threat protection can clamp the result: "zscaler" mode caps concurrency so- * URL demand stays near the tenant's classification budget — otherwise a- * large crawl outruns the 400-lookups/hour budget and every scrape resolves- * through the failurePolicy instead of a verdict.*/export async function getEffectiveConcurrencyLimit(teamId: string,orgId?: string | null,): Promise<number> {const autumnValue = await autumnService.getConcurrencyLimit(teamId, orgId);- const limit = autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;- const threatProtectionCap = await getThreatProtectionConcurrencyCap(teamId);- return threatProtectionCap !== null- ? Math.min(limit, threatProtectionCap)- : limit;+ return autumnValue ?? DEFAULT_CONCURRENCY_LIMIT;}const constructKey = constructConcurrencyLimitKey;diff --git a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.tsindex 7cb0ae4df..1a5b4f62b 100644--- a/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts+++ b/apps/api/src/lib/threat-protection/providers/zscaler/lookup-queue.ts@@ -35,10 +35,10 @@ import {// sit in a line it cannot exit. Verdicts live only in the short-TTL reply// keys consumed moments later; there is no verdict cache (ZDR).//-// Capacity note: the sustained ceiling is ~11 URLs/second per org (400-// calls/hour × 100 URLs). The "zscaler"-mode concurrency cap (see-// getThreatProtectionConcurrencyCap) keeps URL demand near this budget;-// this queue only absorbs bursts.+// Capacity note: the sustained ceiling is ~11 URLs/second per tenant (400+// calls/hour × 100 URLs). Demand above that resolves through the fail-fast+// paths below (queue depth cap, exhausted hourly budget) into the org's+// failurePolicy — scrape concurrency is deliberately NOT throttled.const logger = _logger.child({ module: "threat-protection-zscaler-lookup" });diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.tsindex 3d00ce019..3f744f141 100644--- a/apps/api/src/lib/threat-protection/store.ts+++ b/apps/api/src/lib/threat-protection/store.ts@@ -1,6 +1,5 @@import { eq } from "drizzle-orm";import { z } from "zod";-import { config as appConfig } from "../../config";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { deleteKey, getValue, setValue } from "../../services/redis";@@ -389,73 +388,3 @@ export async function getOrgZscalerCredentials(return null;}}--const concurrencyCapCacheKey = (teamId: string) =>- `threat-protection-concurrency-cap:${teamId}`;--/**- * Scrape-concurrency cap imposed by the team's threat protection mode, or- * null when none applies. "zscaler" mode caps concurrency so URL demand- * stays near the tenant's classification budget (POST /urlLookup allows 1- * request/second × 100 URLs, 400 requests/hour) — without the cap, a large- * crawl outruns the budget, every lookup times out, and the failurePolicy- * fires for everything.- *- * The budget is org-scoped while concurrency admission is team-scoped, so- * the org-level cap is split across the org's teams — otherwise an org with- * N teams would run at N× the intended concurrency.- *- * Cached in Redis for 60s per team; fails open (null) so a Redis or DB- * hiccup never blocks job admission.- */-export async function getThreatProtectionConcurrencyCap(- teamId: string,-): Promise<number | null> {- // Self-hosted deployments have no org config database (and no enterprise- // threat protection) — skip the lookup instead of warning on every job.- if (!appConfig.USE_DB_AUTHENTICATION) return null;-- const key = concurrencyCapCacheKey(teamId);- try {- const cached = await getValue(key);- if (cached === "none") return null;- if (cached !== null) {- const parsed = Number(cached);- // A malformed cache entry must not become a NaN/zero cap that breaks- // job admission — fall through and resolve from the source instead.- if (Number.isFinite(parsed) && parsed > 0) return parsed;- }- } catch (error) {- logger.warn("Failed to read the concurrency cap cache", { error, teamId });- }-- let cap: number | null = null;- try {- const orgId = await getOrgIdForTeam(teamId);- const orgConfig = orgId ? await getOrgThreatProtectionConfig(orgId) : null;- if (orgId && orgConfig?.policy.mode === "zscaler") {- const teamRows = await dbRr- .select({ id: schema.teams.id })- .from(schema.teams)- .where(eq(schema.teams.org_id, orgId));- const teamCount = Math.max(teamRows.length, 1);- cap = Math.max(- 1,- Math.floor(appConfig.ZSCALER_MODE_CONCURRENCY_CAP / teamCount),- );- }- } catch (error) {- logger.warn("Failed to resolve the threat protection concurrency cap", {- error,- teamId,- });- return null;- }-- try {- await setValue(key, cap === null ? "none" : String(cap), 60);- } catch (error) {- logger.warn("Failed to write the concurrency cap cache", { error, teamId });- }- return cap;-}