Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): enforce forced threat protection on v0 endpoints and team config (#3965)
Gergő Móricz93c2bf71c7fd
diff --git a/apps/api/src/__tests__/snips/v0/threat-protection.test.ts b/apps/api/src/__tests__/snips/v0/threat-protection.test.tsnew file mode 100644index 000000000..3b75daf53--- /dev/null+++ b/apps/api/src/__tests__/snips/v0/threat-protection.test.ts@@ -0,0 +1,72 @@+import request from "supertest";+import {+ describeIf,+ idmux,+ Identity,+ TEST_API_URL,+ TEST_PRODUCTION,+} from "../lib";+import { scrapeRaw } from "./lib";++// =========================================+// v0 + forced threat protection+//+// The deprecated v0 endpoints never resolve a threat protection policy, so+// teams whose flag is "forced" must be rejected on the content-fetching v0+// endpoints (scrape, crawl, search) with 403. The rejection happens right+// after auth — before any credit check or scraping — so no provider or+// scrape target is needed. crawl-status/crawl-cancel intentionally stay+// open so existing jobs can drain.+// =========================================++async function crawlRaw(body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .post("/v0/crawl")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++async function searchRaw(body: unknown, identity: Identity) {+ return await request(TEST_API_URL)+ .post("/v0/search")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body as object);+}++// Requires idmux-provisioned team flags, which only exist in the production+// test configuration.+describeIf(TEST_PRODUCTION)("V0 threat protection (forced flag)", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "v0-threat-protection/forced",+ flags: {+ threatProtection: "forced",+ },+ });+ }, 10000);++ it.concurrent("rejects v0 scrape with 403", async () => {+ const res = await scrapeRaw({ url: "https://firecrawl.dev" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toContain("Threat protection");+ expect(res.body.error).toContain("v0");+ });++ it.concurrent("rejects v0 crawl with 403", async () => {+ const res = await crawlRaw({ url: "https://firecrawl.dev" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toContain("Threat protection");+ expect(res.body.error).toContain("v0");+ });++ it.concurrent("rejects v0 search with 403", async () => {+ const res = await searchRaw({ query: "firecrawl" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toContain("Threat protection");+ expect(res.body.error).toContain("v0");+ });+});diff --git a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.tsindex d34552388..03eedb8bb 100644--- a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts+++ b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts@@ -164,4 +164,35 @@ describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {}});});++ describe("with the forced team flag", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "team-threat-protection/forced",+ flags: {+ threatProtection: "forced",+ },+ });+ });++ it('PUT rejects mode "off" with 403', async () => {+ const res = await putConfigRaw({ mode: "off" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("enforced");+ expect(res.body.error).toContain('"off"');+ });++ it('PUT accepts mode "normal" with 200 (forced teams may tighten config)', async () => {+ const res = await putConfigRaw({ mode: "normal" }, identity);+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.body.data).toMatchObject({+ mode: "normal",+ configured: true,+ });+ });+ });});diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex ccfdb3c19..49ac2e42d 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -39,6 +39,10 @@ import {} from "../../services/worker/nuq-router";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ isThreatProtectionForced,+ THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/threat-protection/request";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export async function crawlController(req: Request, res: Response) {@@ -58,6 +62,12 @@ export async function crawlController(req: Request, res: Response) {});}+ if (isThreatProtectionForced(chunk?.flags)) {+ return res.status(403).json({+ error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+ });+ }+const id = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex eee86b19e..dc601a9b3 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -26,6 +26,10 @@ import { scrapeQueue } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ isThreatProtectionForced,+ THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/threat-protection/request";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function scrapeHelper(@@ -200,6 +204,12 @@ export async function scrapeController(req: Request, res: Response) {});}+ if (isThreatProtectionForced(chunk?.flags)) {+ return res.status(403).json({+ error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+ });+ }+const jobId = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 5a4def2af..c2a1aa1fc 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -23,6 +23,10 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq-router";import { defaultOrigin } from "../../lib/default-values";import { getSearchZDR } from "../../lib/zdr-helpers";+import {+ isThreatProtectionForced,+ THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/threat-protection/request";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function searchHelper(@@ -192,6 +196,12 @@ export async function searchController(req: Request, res: Response) {});}+ if (isThreatProtectionForced(chunk?.flags)) {+ return res.status(403).json({+ error: THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,+ });+ }+const jobId = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.tsindex ea2528203..db2f7012c 100644--- a/apps/api/src/controllers/v2/team-threat-protection.ts+++ b/apps/api/src/controllers/v2/team-threat-protection.ts@@ -4,6 +4,7 @@ import { logger as _logger } from "../../lib/logger";import { RequestWithAuth } from "./types";import { getThreatProtection } from "../../lib/zdr-helpers";import { threatProtectionConfigSchema } from "../../lib/threat-protection/config";+import { THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE } from "../../lib/threat-protection/request";import {getOrgIdForTeam,getOrgThreatProtectionConfig,@@ -103,6 +104,19 @@ export async function putTeamThreatProtectionController(const input = threatProtectionConfigSchema.parse(req.body);+ // "forced" guarantees enforcement: the org config may be tightened, but its+ // mode may never be turned off through the API.+ if (+ getThreatProtection(req.acuc?.flags) === "forced" &&+ input.mode === "off"+ ) {+ res.status(403).json({+ success: false,+ error: THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE,+ });+ return;+ }+const orgId = await resolveOrgId(req, res);if (!orgId) return;diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.tsindex efb0abadd..ababe76e8 100644--- a/apps/api/src/lib/threat-protection/request.ts+++ b/apps/api/src/lib/threat-protection/request.ts@@ -25,6 +25,25 @@ export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE =export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE ='Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';+export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =+ 'Threat protection is enforced for your team and its mode cannot be set to "off". Contact your organization administrator.';++export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE =+ "Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";++/**+ * The deprecated v0 endpoints never resolve a threat protection policy, so+ * teams whose flag is "forced" must not be able to fetch content through+ * them. Content-fetching v0 controllers (scrape, crawl, search) call this+ * right after auth and reject with 403 when it returns true; crawl-status and+ * crawl-cancel intentionally do not, so existing jobs can drain.+ */+export function isThreatProtectionForced(+ flags: TeamFlags | undefined,+): boolean {+ return getThreatProtection(flags) === "forced";+}+interface ResolvedThreatProtection {/** Set when the request must be rejected with a 403. */error?: string;