Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): safe mode (#4612)
Rafael Millerf62a24c5d0bd
diff --git a/apps/api/src/__tests__/snips/v2/safe-mode.test.ts b/apps/api/src/__tests__/snips/v2/safe-mode.test.tsnew file mode 100644index 000000000..fd1174d7e--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/safe-mode.test.ts@@ -0,0 +1,493 @@+import request from "supertest";+import {+ ALLOW_TEST_SUITE_WEBSITE,+ concurrentIf,+ createTestIdUrl,+ describeIf,+ idmux,+ Identity,+ scrapeTimeout,+ TEST_API_URL,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+} from "../lib";+import { scrape, scrapeRaw, crawlStart } from "./lib";++const TEST_SUITE_HOST = new URL(TEST_SUITE_WEBSITE).hostname;++async function expectSafeModeBlocked(+ body: Parameters<typeof scrapeRaw>[0],+ identity: Identity,+) {+ const res = await scrapeRaw(body, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");+ return res.body;+}++describe("Safe Mode (v2 scrape, request-time)", () => {+ describe("org flag off", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({ name: "safe-mode/unflagged" });+ }, 10000);++ it.concurrent(+ "rejects safeMode: true",+ async () => {+ const body = await expectSafeModeBlocked(+ { url: createTestIdUrl(), safeMode: true },+ identity,+ );+ expect(body.error).toMatch(/not enabled/i);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "v1 accepts + honors the safeMode param (rejects safeMode: true)",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ url: createTestIdUrl(), safeMode: true });+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");+ },+ scrapeTimeout,+ );++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "treats safeMode: false as a no-op",+ async () => {+ const doc = await scrape(+ { url: createTestIdUrl(), safeMode: false },+ identity,+ );+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );+ });++ describeIf(TEST_PRODUCTION)("org flag on (strict defaults)", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/flagged",+ flags: { safeMode: true },+ });+ }, 10000);++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "plain scrapes still work",+ async () => {+ const doc = await scrape({ url: createTestIdUrl() }, identity);+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects stealth and enhanced proxies",+ async () => {+ for (const proxy of ["stealth", "enhanced"] as const) {+ const body = await expectSafeModeBlocked(+ { url: createTestIdUrl(), proxy },+ identity,+ );+ expect(body.error).toMatch(/prox/i);+ }+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects browser profiles",+ async () => {+ await expectSafeModeBlocked(+ { url: createTestIdUrl(), profile: { name: "test-profile" } },+ identity,+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects login-capable actions",+ async () => {+ const body = await expectSafeModeBlocked(+ {+ url: createTestIdUrl(),+ actions: [{ type: "write", text: "hunter2" }],+ },+ identity,+ );+ expect(body.error).toMatch(/write/);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects credential-bearing headers, case-insensitively",+ async () => {+ for (const header of ["Authorization", "cookie"]) {+ const body = await expectSafeModeBlocked(+ { url: createTestIdUrl(), headers: { [header]: "secret" } },+ identity,+ );+ expect(body.error.toLowerCase()).toContain(header.toLowerCase());+ }+ },+ scrapeTimeout,+ );++ it.concurrent(+ "reports basic proxy for auto requests (no stealth escalation)",+ async () => {+ const doc = await scrape(+ { url: createTestIdUrl(), proxy: "auto" },+ identity,+ );+ expect(doc.metadata?.proxyUsed).toBe("basic");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "allows benign actions",+ async () => {+ const doc = await scrape(+ { url: createTestIdUrl(), actions: [{ type: "scroll" }] },+ identity,+ );+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "allows benign headers",+ async () => {+ const doc = await scrape(+ {+ url: createTestIdUrl(),+ headers: { "Accept-Language": "en-US" },+ },+ identity,+ );+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects a bypass when the org has not allowed it",+ async () => {+ const body = await expectSafeModeBlocked(+ { url: createTestIdUrl(), safeMode: false },+ identity,+ );+ expect(body.error).toMatch(/disable Safe Mode/i);+ },+ scrapeTimeout,+ );++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "treats safeMode: true as a redundant affirmation",+ async () => {+ const doc = await scrape(+ { url: createTestIdUrl(), safeMode: true },+ identity,+ );+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );+ });++ describeIf(TEST_PRODUCTION)("org flag on with allowBypassSafeMode", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/bypassable",+ flags: {+ safeMode: true,+ safeModeConfig: { allowBypassSafeMode: true },+ },+ });+ }, 10000);++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "honors safeMode: false β otherwise-blocked params pass",+ async () => {+ const doc = await scrape(+ {+ url: createTestIdUrl(),+ safeMode: false,+ headers: { Authorization: "Bearer not-a-real-token" },+ },+ identity,+ );+ expect(doc.markdown).toBeDefined();+ },+ scrapeTimeout,+ );++ it.concurrent(+ "still enforces when no bypass is requested",+ async () => {+ await expectSafeModeBlocked(+ { url: createTestIdUrl(), headers: { Authorization: "x" } },+ identity,+ );+ },+ scrapeTimeout,+ );+ });++ describeIf(TEST_PRODUCTION)(+ "org config lockdown: true (lockdown supersedes)",+ () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/lockdown",+ flags: { safeMode: true, safeModeConfig: { lockdown: true } },+ });+ }, 10000);++ it.concurrent(+ "uncached URLs return the lockdown cache-miss error",+ async () => {+ const res = await scrapeRaw({ url: createTestIdUrl() }, identity);+ expect(res.statusCode).toBe(404);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("SCRAPE_LOCKDOWN_CACHE_MISS");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "accepts and ignores params the other controls would reject",+ async () => {+ const res = await scrapeRaw(+ {+ url: createTestIdUrl(),+ proxy: "stealth",+ profile: { name: "ignored" },+ headers: { Cookie: "ignored" },+ },+ identity,+ );+ expect(res.body.code).toBe("SCRAPE_LOCKDOWN_CACHE_MISS");+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");+ },+ scrapeTimeout,+ );+ },+ );++ describeIf(TEST_PRODUCTION)("domainControls forces threat protection", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/domain-controls",+ flags: { safeMode: true, threatProtection: "allowed" },+ });+ const res = await request(TEST_API_URL)+ .put("/v2/team/threat-protection")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ mode: "off", blacklist: ["blocked.example.com"] });+ expect(res.statusCode).toBe(200);+ }, 15000);++ it.concurrent(+ "blocks blacklisted domains even though the TP config mode is off",+ async () => {+ const res = await scrapeRaw(+ { url: "https://blocked.example.com/page" },+ identity,+ );+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects a request-level threat protection opt-out",+ async () => {+ const res = await scrapeRaw(+ { url: createTestIdUrl(), threatProtection: { mode: "off" } },+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toMatch(/threat protection/i);+ },+ scrapeTimeout,+ );+ });++ // Each capability is individually relaxable from the org's safeModeConfig:+ // flipping its control off restores the normal behavior while Safe Mode+ // stays on for everything else.+ describeIf(TEST_PRODUCTION)("org config relaxations", () => {+ it.concurrent(+ "disableStealthProxy: false lets a request use a stealth proxy",+ async () => {+ const identity = await idmux({+ name: "safe-mode/relax-stealth",+ flags: {+ safeMode: true,+ safeModeConfig: { disableStealthProxy: false },+ },+ });+ const res = await scrapeRaw(+ { url: createTestIdUrl(), proxy: "stealth" },+ identity,+ );+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");+ expect(res.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "disableAuthentication: false lets a request send credential headers",+ async () => {+ const identity = await idmux({+ name: "safe-mode/relax-auth",+ flags: {+ safeMode: true,+ safeModeConfig: { disableAuthentication: false },+ },+ });+ const res = await scrapeRaw(+ {+ url: createTestIdUrl(),+ headers: { Authorization: "Bearer not-a-real-token" },+ },+ identity,+ );+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");+ expect(res.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "still blocks the capabilities left at their strict default",+ async () => {+ const identity = await idmux({+ name: "safe-mode/relax-partial",+ flags: {+ safeMode: true,+ safeModeConfig: { disableStealthProxy: false },+ },+ });+ // stealth relaxed above, but the profile (auth-path) restriction stays+ const res = await scrapeRaw(+ { url: createTestIdUrl(), profile: { name: "test-profile" } },+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");+ },+ scrapeTimeout,+ );+ });++ // An allowlisted URL relaxes *how* it is scraped (evaluated per-URL), while a+ // non-matching URL under the same org stays fully strict.+ describeIf(TEST_PRODUCTION)("allowlist", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/allowlist",+ flags: {+ safeMode: true,+ safeModeConfig: { allowlist: [TEST_SUITE_HOST] },+ },+ });+ }, 10000);++ it.concurrent(+ "relaxes a stealth proxy request for an allowlisted URL",+ async () => {+ const res = await scrapeRaw(+ { url: createTestIdUrl(), proxy: "stealth" },+ identity,+ );+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");+ expect(res.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "keeps a non-allowlisted URL strict",+ async () => {+ const body = await expectSafeModeBlocked(+ { url: "https://not-in-allowlist.example.org/", proxy: "stealth" },+ identity,+ );+ expect(body.error).toMatch(/prox/i);+ },+ scrapeTimeout,+ );+ });++ // Ticket 08: enforcement reaches beyond /v2/scrape.+ describeIf(TEST_PRODUCTION)("endpoint coverage", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "safe-mode/endpoints",+ flags: { safeMode: true },+ });+ }, 10000);++ it.concurrent(+ "crawl rejects a stealth proxy nested under scrapeOptions",+ async () => {+ const res = await crawlStart(+ {+ url: createTestIdUrl(),+ scrapeOptions: { formats: ["markdown"], proxy: "stealth" },+ },+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");+ expect(res.body.error).toMatch(/prox/i);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "v0 content endpoints reject a Safe Mode org",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v0/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ url: createTestIdUrl() });+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toMatch(/v0 API/i);+ },+ scrapeTimeout,+ );+ });+});diff --git a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.tsindex 03eedb8bb..ce25eb8f3 100644--- a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts+++ b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts@@ -185,6 +185,13 @@ describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {expect(res.body.error).toContain('"off"');});+ it('PUT rejects mode "manual-only" with 403', async () => {+ const res = await putConfigRaw({ mode: "manual-only" }, identity);+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("enforced");+ });+it('PUT accepts mode "normal" with 200 (forced teams may tighten config)', async () => {const res = await putConfigRaw({ mode: "normal" }, identity);expect(res.statusCode).toBe(200);diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.tsindex c9fece440..f43c7ab42 100644--- a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts@@ -9,7 +9,15 @@ import {TEST_SUITE_WEBSITE,scrapeTimeout,} from "../lib";-import { crawl, crawlStart, map, scrape, scrapeRaw, search } from "./lib";+import {+ crawl,+ crawlStart,+ creditUsage,+ map,+ scrape,+ scrapeRaw,+ search,+} from "./lib";import {createWebRiskMockCounters,createWebRiskMockHandler,@@ -46,6 +54,11 @@ const RISKY_DOMAIN = "threat-risky.example.com";// The scrape target for happy-path tests.const CLEAN_URL = TEST_SUITE_WEBSITE;++const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms));+// Scrape credits are billed in batches; wait for the batch to land before+// reading a credit delta.+const sleepForBatchBilling = () => sleep(40000);const CLEAN_DOMAIN = new URL(TEST_SUITE_WEBSITE).hostname;// A stable cross-hostname redirect: google.com 301s to www.google.com.@@ -427,6 +440,70 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {);});+ describe("manual-only mode (lists only, no scans)", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "threat-protection-enforcement/manual-only",+ flags: { threatProtection: "allowed" },+ credits: 1_000_000,+ });+ });++ it(+ "blocks a blacklisted domain",+ async () => {+ const res = await scrapeRaw(+ {+ url: `https://${BLACKLISTED_DOMAIN}/`,+ threatProtection: {+ mode: "manual-only",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ },+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("unsafe_domain_blocked");+ },+ scrapeTimeout,+ );++ it(+ "allows an unlisted URL without a scan fee",+ async () => {+ // A fresh team's credit grant can land a moment after idmux returns;+ // poll until the baseline reflects the full grant so the delta is exact.+ let before = 0;+ for (let i = 0; i < 30 && before < 100_000; i++) {+ before = (await creditUsage(identity)).remainingCredits;+ if (before < 100_000) await sleep(1000);+ }+ expect(before).toBeGreaterThanOrEqual(100_000);++ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: {+ mode: "manual-only",+ blacklist: [BLACKLISTED_DOMAIN],+ },+ },+ identity,+ );+ expect(res.statusCode).toBe(200);++ await sleepForBatchBilling();+ const after = (await creditUsage(identity)).remainingCredits;+ // One credit for the scrape itself; no +2 threat-scan surcharge.+ expect(before - after).toBe(1);+ },+ scrapeTimeout + 90000,+ );+ });+describe("provider verdicts (mock Google Web Risk)", () => {let identity: Identity;@@ -543,6 +620,23 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {scrapeTimeout,);+ it(+ "rejects a per-request override that lowers the mode to manual-only",+ async () => {+ const res = await scrapeRaw(+ {+ url: CLEAN_URL,+ threatProtection: { mode: "manual-only" },+ } as any,+ identity,+ );+ expect(res.statusCode).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("cannot be disabled");+ },+ scrapeTimeout,+ );+it("still allows narrowing overrides (mode stays on)",async () => {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex da77be26b..5227beabd 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -43,6 +43,10 @@ import {isThreatProtectionForced,THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,} from "../../lib/threat-protection/request";+import {+ getSafeMode,+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export async function crawlController(req: Request, res: Response) {@@ -68,6 +72,12 @@ export async function crawlController(req: Request, res: Response) {});}+ if (getSafeMode(chunk?.flags)) {+ return res.status(403).json({+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+ });+ }+const id = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 9e65a6ff7..f0be7166f 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -30,6 +30,10 @@ import {isThreatProtectionForced,THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,} from "../../lib/threat-protection/request";+import {+ getSafeMode,+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function scrapeHelper(@@ -213,6 +217,12 @@ export async function scrapeController(req: Request, res: Response) {});}+ if (getSafeMode(chunk?.flags)) {+ return res.status(403).json({+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+ });+ }+const jobId = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex a8be55b69..c924b1d6a 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -27,6 +27,10 @@ import {isThreatProtectionForced,THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,} from "../../lib/threat-protection/request";+import {+ getSafeMode,+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function searchHelper(@@ -208,6 +212,12 @@ export async function searchController(req: Request, res: Response) {});}+ if (getSafeMode(chunk?.flags)) {+ return res.status(403).json({+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,+ });+ }+const jobId = uuidv7();await logRequest({diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 04533e3eb..9f4dbfe3d 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -26,6 +26,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import { resolveSafeMode } from "../../lib/safe-mode";import {checkUrlsAgainstThreatPolicy,resolveThreatProtection,@@ -59,11 +60,21 @@ export async function batchScrapeController(req.body = batchScrapeRequestSchema.parse(req.body);}+ const safeMode = resolveSafeMode(req.acuc?.flags, req.body.safeMode);+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ } as any);+ }+const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -74,10 +85,12 @@ export async function batchScrapeController(const permissions = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -95,8 +108,11 @@ export async function batchScrapeController(});}+ // Safe Mode lockdown is cache-only, which implies zero data retention.const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body.zeroDataRetention ||+ (safeMode.safeMode?.lockdown ?? false);const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({@@ -308,6 +324,11 @@ export async function batchScrapeController(req.body.timeout,req.auth.team_id,);+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default the+ // scrape backstop applies β keep only a maxAge the request actually sent.+ if (safeMode.safeMode?.lockdown && preNormalizedBody.maxAge === undefined) {+ scrapeOptions.maxAge = undefined;+ }const sc: StoredCrawl = req.body.appendToId? ((await getCrawl(req.body.appendToId)) as StoredCrawl)@@ -325,6 +346,9 @@ export async function batchScrapeController(zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.+ teamFlags: req.acuc?.flags ?? undefined,+ safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),@@ -340,6 +364,13 @@ export async function batchScrapeController(error: "Job not found",});}+ if (req.body.appendToId && sc?.internalOptions) {+ // Refresh Safe Mode + threat-protection so appended jobs enforce the team's+ // current policy, not whatever was stored when the batch was created.+ sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;+ sc.internalOptions.threatProtection = threatProtection.policy ?? undefined;+ sc.internalOptions.safeModeBypassed = safeMode.bypassed === true;+ }if (!req.body.appendToId) {sc.queueBackend = await resolveNewGroupBackend(sc.team_id);diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex f5c0f3870..a494aaecb 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -18,6 +18,7 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import { resolveSafeMode } from "../../lib/safe-mode";import {actionTypesOf,checkKeyFormatRestriction,@@ -45,14 +46,32 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);const id = uuidv7();++ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.scrapeOptions?.safeMode,+ req.body.url,+ );+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ } as any);+ }++ // Safe Mode lockdown is cache-only, which implies zero data retention.const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body.zeroDataRetention ||+ (safeMode.safeMode?.lockdown ?? false);const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.scrapeOptions?.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -61,14 +80,27 @@ export async function crawlController(});}+ // Scrape params live under scrapeOptions; checkPermissions reads them+ // top-level, so spread scrapeOptions while keeping the top-level fields it+ // also reads (zeroDataRetention, crawlerOptions.ignoreRobotsTxt) and the+ // nested scrapeOptions (location / threatProtection).const permissions = checkPermissions(- { ...req.body, crawlerOptions: req.body },+ {+ ...req.body.scrapeOptions,+ zeroDataRetention: req.body.zeroDataRetention,+ crawlerOptions: req.body,+ scrapeOptions: req.body.scrapeOptions,+ },req.acuc?.flags,- { threatProtectionOrgConfig: threatProtection.orgConfig },+ {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,+ },);if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -184,6 +216,14 @@ export async function crawlController(bodyScrapeOptions.timeout,req.auth.team_id,);+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default the+ // scrape backstop applies β keep only a maxAge the request actually sent.+ if (+ safeMode.safeMode?.lockdown &&+ preNormalizedBody.scrapeOptions?.maxAge === undefined+ ) {+ scrapeOptions.maxAge = undefined;+ }// TODO: @rafa, is this right? copied from v0if (Array.isArray(crawlerOptions.includePaths)) {@@ -227,6 +267,9 @@ export async function crawlController(zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.+ teamFlags: req.acuc?.flags ?? undefined,+ safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 11cf401db..d526dcfa6 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -31,6 +31,7 @@ import {} from "../../services/index";import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";+import { resolveSafeMode } from "../../lib/safe-mode";import { getScrapeZDR } from "../../lib/zdr-helpers";import {checkUrlsAgainstThreatPolicy,@@ -136,7 +137,14 @@ export async function getMapResults({let links: string[] = [url];let mapResults: MapDocument[] = [];- const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;+ // Safe Mode lockdown: serve links from the index only β skip the live+ // fireEngineMap search and any sitemap fetch (both are live discovery).+ // Derived from the team flags so every caller honors it.+ const indexOnly =+ resolveSafeMode(flags, undefined, url).safeMode?.lockdown === true;++ // Lockdown (index-only) is cache-only, which implies zero data retention.+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || indexOnly;const sc: StoredCrawl = {originUrl: url,@@ -210,7 +218,10 @@ export async function getMapResults({let allResults: any[] = [];let pagePromises: Promise<any>[] = [];- if (cachedResult) {+ if (indexOnly) {+ // Lockdown: no live search discovery, serve from the index only.+ allResults = [];+ } else if (cachedResult) {allResults = JSON.parse(cachedResult);} else {const fetchPage = async (page: number) => {@@ -251,7 +262,7 @@ export async function getMapResults({// If sitemap is not ignored, fetch sitemap// This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days- if (!ignoreSitemap) {+ if (!ignoreSitemap && !indexOnly) {try {await crawler.tryGetSitemap(urls => {@@ -387,11 +398,37 @@ export async function mapController(});}+ // Safe Mode: force domainControls so discovered links are filtered, and+ // serve index-only under lockdown (no sitemap/robots fetch to the target).+ const safeMode = resolveSafeMode(req.acuc?.flags, undefined, req.body.url);+ const lockdownIndexOnly = safeMode.safeMode?.lockdown === true;+ if (lockdownIndexOnly) {+ // Lockdown: serve links from the index only β no live search / sitemap /+ // robots discovery. A sitemapOnly request keeps its contract and simply+ // yields no links (the sitemap can't be fetched under lockdown).+ req.body.useIndex = true;+ }+ // Map's ignoreRobotsTxt is top-level, so checkPermissions (which reads it+ // under crawlerOptions) can't see it β enforce robots here for Safe Mode.+ if (+ safeMode.safeMode?.enforceRobots &&+ !lockdownIndexOnly &&+ req.body.ignoreRobotsTxt+ ) {+ return res.status(403).json({+ success: false,+ code: "SAFE_MODE_BLOCKED",+ error:+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",+ });+ }+const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -402,10 +439,12 @@ export async function mapController(const permissions = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -430,7 +469,7 @@ export async function mapController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.url,- zeroDataRetention: false, // not supported for map+ zeroDataRetention: lockdownIndexOnly,api_key_id: req.acuc?.api_key_id ?? null,});@@ -550,7 +589,7 @@ export async function mapController(},results: result.links,credits_cost: creditsToBill,- zeroDataRetention: false, // not supported+ zeroDataRetention: lockdownIndexOnly,}).catch(error => {logger.error("Failed to log map", { error, mapId });});diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex a35e0fbdf..7b4864ed6 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -14,6 +14,11 @@ import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import {+ resolveSafeMode,+ applySafeMode,+ isLockdownZeroDataRetention,+} from "../../lib/safe-mode";import {actionTypesOf,checkKeyFormatRestriction,@@ -51,10 +56,13 @@ export async function scrapeController(res: Response<ScrapeResponse>,) {// Resolved before the root span starts so the whole request trace stays- // unrecorded for zero-data-retention requests (see otel-tracer).+ // unrecorded for zero-data-retention requests (see otel-tracer). Safe Mode+ // lockdown implies ZDR, so fold it in here too β otherwise child spans could+ // export target URLs before the inner handler applies lockdown.const zeroDataRetentionTrace =getScrapeZDR(req.acuc?.flags) === "forced" ||- req.body?.zeroDataRetention === true;+ req.body?.zeroDataRetention === true ||+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);return withSpan("api.scrape.request",@@ -84,11 +92,26 @@ async function scrapeControllerInner(const preNormalizedBody = { ...req.body };req.body = scrapeRequestSchema.parse(req.body);+ // Honor a per-request Safe Mode opt-out/affirmation, consistent with v2.+ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.safeMode,+ req.body.url,+ );+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ } as any);+ }+const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -99,10 +122,12 @@ async function scrapeControllerInner(const permissions = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -121,7 +146,9 @@ async function scrapeControllerInner(}const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body.zeroDataRetention ||+ (safeMode.safeMode?.lockdown ?? false);const logger = _logger.child({method: "scrapeController",@@ -172,6 +199,12 @@ async function scrapeControllerInner(req.body.timeout,req.auth.team_id,);+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default β+ // keep only a maxAge the request actually sent under forced lockdown.+ if (safeMode.safeMode?.lockdown && preNormalizedBody.maxAge === undefined) {+ scrapeOptions.maxAge = undefined;+ }+ applySafeMode(safeMode.safeMode, scrapeOptions);const projectedKeylessCredits = !isDirectToBullMQ? projectScrapeCredits(scrapeOptions,@@ -267,6 +300,8 @@ async function scrapeControllerInner(orgId: req.acuc?.org_id ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ safeMode: safeMode.allowlisted ? undefined : safeMode.safeMode,+ safeModeBypassed: safeMode.bypassed === true,},skipNuq: true,origin,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex c5f7040aa..66b0c81b6 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -23,6 +23,11 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { z } from "zod";import { executeSearch } from "../../search/execute";import { resolveThreatProtection } from "../../lib/threat-protection/request";+import {+ resolveSafeMode,+ getEffectiveSearchForcedKind,+} from "../../lib/safe-mode";+import { checkPermissions } from "../../lib/permissions";import {DocumentWithCostTracking,scrapeSearchResults,@@ -32,7 +37,6 @@ import {filterDocumentsWithContent,} from "../../search/transform";import { fromV1ScrapeOptions } from "../v2/types";-import { getSearchForcedKind } from "../../lib/zdr-helpers";import {adjustKeylessCredits,keylessLimitBody,@@ -102,7 +106,12 @@ export async function searchController(const controllerStartTime = new Date().getTime();const jobId = uuidv7();- const teamForcedKind = getSearchForcedKind(req.acuc?.flags);+ // Safe Mode lockdown forces the "zdr" kind like the searchZDR flag does+ // (see getEffectiveSearchForcedKind).+ const teamForcedKind = getEffectiveSearchForcedKind(+ req.acuc?.flags,+ req.body.scrapeOptions?.safeMode,+ );const zeroDataRetention = teamForcedKind !== null;const teamEnterprise = teamForcedKind ? [teamForcedKind] : undefined;let logger = _logger.child({@@ -154,6 +163,21 @@ export async function searchController(origin: req.body.origin,});+ // Safe Mode: validate the per-request param up front and reject scrape+ // options it forbids (the worker backstop would otherwise strip them+ // silently). Domain controls force threat protection over the results.+ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.scrapeOptions?.safeMode,+ );+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ });+ }+// Threat protection: resolve the effective policy. Blocked domains are// removed from search results entirely.const threatProtection = await resolveThreatProtection({@@ -162,6 +186,7 @@ export async function searchController(flags: req.acuc?.flags ?? null,override:req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -170,6 +195,30 @@ export async function searchController(});}+ // Search only scrapes (and only honors scrapeOptions) when formats are+ // requested, so the scrape-option checks apply only then.+ if (+ safeMode.safeMode &&+ requestedFormats.length > 0 &&+ req.body.scrapeOptions+ ) {+ const permissions = checkPermissions(+ req.body.scrapeOptions,+ req.acuc?.flags,+ {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode,+ },+ );+ if (permissions.error) {+ return res.status(403).json({+ success: false,+ code: permissions.code,+ error: permissions.error,+ });+ }+ }+await logRequest({id: jobId,kind: "search",@@ -250,6 +299,7 @@ export async function searchController(agentIndexOnly: (req as any).agentIndexOnly ?? false,keylessReserved: reservedKeylessCredits > 0,threatProtectionPolicy: threatProtection.policy,+ safeModeBypassed: safeMode.bypassed === true,},logger,);diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 78281cdf2..1954b9ce4 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -551,6 +551,10 @@ const baseScrapeOptions = z.strictObject({.gte(0).prefault(1 * 24 * 60 * 60 * 1000),storeInCache: z.boolean().prefault(true),+ // Enterprise: per-request Safe Mode opt-out/affirmation. Tri-state, so NOT+ // prefaulted β an absent value must stay undefined (a default of false would+ // read as an explicit bypass attempt). Honored only if the org allows it.+ safeMode: z.boolean().optional(),// Enterprise: per-request field-level override of the org's threat// protection policy. Gated on the team flag + org config (checkPermissions).threatProtection: threatProtectionOverrideSchema.optional(),@@ -1334,6 +1338,22 @@ export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";+ safeMode?: boolean;+ safeModeConfig?: {+ allowBypassSafeMode?: boolean;+ lockdown?: boolean;+ domainControls?: boolean;+ enforceRobots?: boolean;+ disableStealthProxy?: boolean;+ disableAuthentication?: boolean;+ disableSiteHandling?: boolean;+ exposeWebdriver?: boolean;+ useHeadlessUserAgent?: boolean;+ disablePlatformSelection?: boolean;+ disableCountrySelection?: boolean;+ disableAutomaticReferrer?: boolean;+ allowlist?: string[];+ };siemLogging?: boolean;unblockedDomains?: string[];forceZDR?: boolean;diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 280d62407..8aeedcdda 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -26,6 +26,7 @@ import { logger as _logger } from "../../lib/logger";import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";+import { resolveSafeMode } from "../../lib/safe-mode";import {actionTypesOf,checkKeyFormatRestriction,@@ -61,11 +62,23 @@ export async function batchScrapeController(req.body = batchScrapeRequestSchema.parse(req.body);}+ // Batch has many URLs; per-URL allowlist is applied at the scrapeURL backstop.+ // Resolve org-level here only for the bypass/enablement decision.+ const safeMode = resolveSafeMode(req.acuc?.flags, req.body.safeMode);+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ });+ }+const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -76,10 +89,12 @@ export async function batchScrapeController(const permissions = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -97,9 +112,11 @@ export async function batchScrapeController(});}+ // Safe Mode lockdown is cache-only, which implies zero data retention.const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" ||- (req.body.zeroDataRetention ?? false);+ (req.body.zeroDataRetention ?? false) ||+ (safeMode.safeMode?.lockdown ?? false);if (req.body.__agentInterop &&@@ -347,6 +364,10 @@ export async function batchScrapeController(bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ // Safe Mode rides the batch payload so each URL resolves it per-URL+ // at the scrapeURL backstop.+ teamFlags: req.acuc?.flags ?? undefined,+ safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),@@ -364,6 +385,12 @@ export async function batchScrapeController(error: "Job not found",});}+ // Refresh Safe Mode + threat-protection context so appended jobs enforce+ // the team's current policy, not whatever was stored when the batch was+ // first created.+ sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;+ sc.internalOptions.safeModeBypassed = safeMode.bypassed === true;+ sc.internalOptions.threatProtection = threatProtection.policy ?? undefined;}if (!req.body.appendToId) {diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex a8856de58..ae4e7ccfe 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -37,6 +37,10 @@ import {import { autumnService } from "../../services/autumn/autumn.service";import { orgIdForTeam } from "../../lib/team-org";import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import {+ getSafeMode,+ SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,+} from "../../lib/safe-mode";// ---------------------------------------------------------------------------// Zod schemas@@ -226,6 +230,13 @@ export async function browserCreateController(req.body = browserCreateRequestSchema.parse(req.body);+ if (getSafeMode(req.acuc?.flags)) {+ return res.status(403).json({+ success: false,+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,+ });+ }+if (req.body.__agentInterop &&config.AGENT_INTEROP_SECRET &&@@ -471,6 +482,13 @@ export async function browserExecuteController(req.body = browserExecuteRequestSchema.parse(req.body);+ if (getSafeMode(req.acuc?.flags)) {+ return res.status(403).json({+ success: false,+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,+ });+ }+const id = req.params.sessionId;const { code, language, timeout, origin } = req.body;diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex d2e50069a..4c051a8d0 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -19,6 +19,7 @@ import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";+import { resolveSafeMode } from "../../lib/safe-mode";import {actionTypesOf,checkKeyFormatRestriction,@@ -48,14 +49,32 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);const id = uuidv7();++ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.scrapeOptions?.safeMode,+ req.body.url,+ );+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ });+ }++ // Safe Mode lockdown is cache-only, which implies zero data retention.const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body.zeroDataRetention ||+ (safeMode.safeMode?.lockdown ?? false);const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.scrapeOptions?.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -64,14 +83,27 @@ export async function crawlController(});}+ // Scrape params live under scrapeOptions; checkPermissions reads them+ // top-level, so spread scrapeOptions while keeping the top-level fields it+ // also reads (zeroDataRetention, crawlerOptions.ignoreRobotsTxt) and the+ // nested scrapeOptions (location / threatProtection).const permissions = checkPermissions(- { ...req.body, crawlerOptions: req.body },+ {+ ...req.body.scrapeOptions,+ zeroDataRetention: req.body.zeroDataRetention,+ crawlerOptions: req.body,+ scrapeOptions: req.body.scrapeOptions,+ },req.acuc?.flags,- { threatProtectionOrgConfig: threatProtection.orgConfig },+ {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,+ },);if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -286,6 +318,10 @@ export async function crawlController(zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ // Safe Mode rides the crawl payload so every child scrape resolves it+ // per-URL at the scrapeURL backstop (allowlist applies per child).+ teamFlags: req.acuc?.flags ?? undefined,+ safeModeBypassed: safeMode.bypassed === true,},team_id: req.auth.team_id,createdAt: Date.now(),diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 6167c6a47..7e05d3d22 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -12,6 +12,10 @@ import { externalRequestId } from "../../lib/external-request-id";import { logger as _logger } from "../../lib/logger";import { MapTimeoutError, MapFailedError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";+import {+ resolveSafeMode,+ isLockdownZeroDataRetention,+} from "../../lib/safe-mode";import { getMapResults, MapResult } from "../../lib/map-utils";import { v7 as uuidv7 } from "uuid";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";@@ -29,12 +33,16 @@ export async function mapController(req: RequestWithAuth<{}, MapResponse, MapRequest>,res: Response<MapResponse>,) {+ // Safe Mode lockdown is cache-only, which implies zero data retention.+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ isLockdownZeroDataRetention(req.acuc?.flags, undefined);const logger = _logger.child({jobId: uuidv7(),teamId: req.auth.team_id,module: "api/v2",method: "mapController",- zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",+ zeroDataRetention,});// Get timing data from middleware (includes all middleware processing time)const middlewareStartTime =@@ -44,11 +52,36 @@ export async function mapController(const originalRequest = req.body;req.body = mapRequestSchema.parse(req.body);+ // Map is link discovery, not content scraping, but it is outbound to the+ // target. Under Safe Mode: force domainControls so discovered links are+ // filtered, and under lockdown serve from the index only (no sitemap/robots+ // fetch to the target).+ const safeMode = resolveSafeMode(req.acuc?.flags, undefined, req.body.url);+ const lockdownIndexOnly = safeMode.safeMode?.lockdown === true;+ if (lockdownIndexOnly) {+ req.body.useIndex = true;+ }+ // Map's ignoreRobotsTxt is top-level, so checkPermissions (which reads it+ // under crawlerOptions) can't see it β enforce robots here for Safe Mode.+ if (+ safeMode.safeMode?.enforceRobots &&+ !lockdownIndexOnly &&+ req.body.ignoreRobotsTxt+ ) {+ return res.status(403).json({+ success: false,+ code: "SAFE_MODE_BLOCKED",+ error:+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",+ });+ }+const threatProtection = await resolveThreatProtection({teamId: req.auth.team_id,orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -59,10 +92,12 @@ export async function mapController(const permissions = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});if (permissions.error) {return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -87,17 +122,17 @@ export async function mapController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.url,- zeroDataRetention: false, // not supported for map+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});- // Short-circuit: if the URL matches avgrab's resolve pattern, delegate entirely+ // Short-circuit: if the URL matches avgrab's resolve pattern, delegate+ // entirely. Skipped under Safe Mode so results still flow through the+ // standard path (domain-controls filtering + lockdown index-only).try {- const avgrabResults = await resolveViaAvgrab(- req.body.url,- req.body.limit,- logger,- );+ const avgrabResults = safeMode.safeMode+ ? null+ : await resolveViaAvgrab(req.body.url, req.body.limit, logger);if (avgrabResults !== null) {const creditsCost = avgrabResults.length;@@ -280,7 +315,7 @@ export async function mapController(},results: result.mapResults,credits_cost: creditsToBill,- zeroDataRetention: false, // not supported+ zeroDataRetention,}).catch(error => {logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);});diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 5a47b46e6..f54a0ebfc 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -360,6 +360,7 @@ export async function parseController(});return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 685e21b4b..cf8d9e363 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -6,7 +6,7 @@ import { logger as rootLogger } from "../../lib/logger";import { fetchResearchUpstream } from "../../lib/research-upstream";import { chargeKeylessCredits } from "../../lib/keyless";import { billTeam } from "../../services/billing/credit_billing";-import { getSearchForcedKind } from "../../lib/zdr-helpers";+import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";import {logRequest,logResearchEndpoint,@@ -223,7 +223,7 @@ function creditsFor(if (freeResearchKinds.has(config.kind)) return 0;if (config.billAs === "scrape") return 1;- const forcedKind = getSearchForcedKind(req.acuc?.flags);+ const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);const perTen =forcedKind === "zdr"? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS@@ -286,7 +286,7 @@ function createResearchController(return async (req, res: Response) => {const authedReq = req as RequestWithAuth<any, any, any>;const zeroDataRetention =- getSearchForcedKind(authedReq.acuc?.flags) !== null;+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;const started = Date.now();const jobId = uuidv7();diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 5d35020a2..e0e5273f4 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -43,6 +43,10 @@ import {} from "../../lib/scrape-interact/browser-agent";import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ getSafeMode,+ SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,+} from "../../lib/safe-mode";import { RequestWithAuth, ScrapeOptions } from "./types";import { billTeam } from "../../services/billing/credit_billing";import {@@ -136,6 +140,13 @@ export async function scrapeInteractController() {req.body = browserExecuteRequestSchema.parse(req.body);+ if (getSafeMode(req.acuc?.flags)) {+ return res.status(403).json({+ success: false,+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,+ });+ }+const scrapeId = req.params.jobId;const { code: rawCode, prompt, language, timeout, origin } = req.body;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex c4f73c5a7..f11b2deb4 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -19,6 +19,11 @@ import {} from "../../lib/error";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import {+ applySafeMode,+ resolveSafeMode,+ isLockdownZeroDataRetention,+} from "../../lib/safe-mode";import {actionTypesOf,checkKeyFormatRestriction,@@ -43,6 +48,7 @@ import {import { projectScrapeCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";import { isAgentInteropSecretValid } from "../../lib/agent-interop";@@ -55,11 +61,14 @@ export async function scrapeController(if (req.body && "alexandria" in req.body)return providerScrapeController(req, res);// Resolved before the root span starts so the whole request trace stays- // unrecorded for zero-data-retention requests (see otel-tracer).+ // unrecorded for zero-data-retention requests (see otel-tracer). Safe Mode+ // lockdown implies ZDR, so fold it in here too β otherwise the rejection+ // paths below would export the target URL on the root span.const zeroDataRetentionTrace =getScrapeZDR(req.acuc?.flags) === "forced" ||req.body?.zeroDataRetention === true ||- req.body?.lockdown === true;+ req.body?.lockdown === true ||+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);return withSpan("api.scrape.request",@@ -89,6 +98,43 @@ export async function scrapeController(});});+ const emitSafeModeRejection = (message: string) =>+ emitRejectedScrapeActivityEvent({+ scrapeId: jobId,+ requestId: jobId,+ endpoint: "scrape",+ teamId: req.auth.team_id,+ apiKeyId: req.acuc?.api_key_id ?? null,+ url: req.body.url,+ error: new TransportableError("SAFE_MODE_BLOCKED", message),+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ zeroDataRetention: req.body.zeroDataRetention ?? false,+ });++ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.safeMode,+ req.body.url,+ );+ if (safeMode.error) {+ setSpanAttributes(span, {+ "scrape.error": safeMode.error,+ "scrape.status_code": 403,+ });+ emitSafeModeRejection(safeMode.error);+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ });+ }+ setSpanAttributes(span, {+ "scrape.safe_mode": safeMode.safeMode !== undefined,+ "scrape.safe_mode_bypassed": safeMode.bypassed === true,+ "scrape.safe_mode_allowlisted": safeMode.allowlisted === true,+ });+// Threat protection: resolve the effective policy (org config +// per-request override). No-ops (null policy, zero I/O) for teams// without the flag.@@ -97,24 +143,28 @@ export async function scrapeController(orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,override: req.body.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {setSpanAttributes(span, {"scrape.error": threatProtection.error,"scrape.status_code": 403,});+ if (safeMode.safeMode?.domainControls) {+ emitSafeModeRejection(threatProtection.error);+ }return res.status(403).json({success: false,error: threatProtection.error,});}-// Permission check spanconst permissions = await withSpan("api.scrape.check_permissions",async permSpan => {const perms = checkPermissions(req.body, req.acuc?.flags, {threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode ?? null,});setSpanAttributes(permSpan, {"permissions.success": !perms.error,@@ -129,8 +179,12 @@ export async function scrapeController("scrape.error": permissions.error,"scrape.status_code": 403,});+ if (permissions.code === "SAFE_MODE_BLOCKED") {+ emitSafeModeRejection(permissions.error);+ }return res.status(403).json({success: false,+ code: permissions.code,error: permissions.error,});}@@ -152,6 +206,8 @@ export async function scrapeController(});}+ applySafeMode(safeMode.safeMode, req.body);+const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" ||(req.body.zeroDataRetention ?? false) ||@@ -230,6 +286,12 @@ export async function scrapeController(const middlewareTime = controllerStartTime - middlewareStartTime;+ if (safeMode.bypassed) {+ logger.info("Safe Mode bypassed by request", {+ apiKeyId: req.acuc?.api_key_id,+ });+ }+logger.debug("Scrape " + jobId + " starting", {version: "v2",scrapeId: jobId,@@ -362,6 +424,10 @@ export async function scrapeController(teamConcurrency: baseConcurrency,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,+ safeMode: safeMode.allowlisted+ ? undefined+ : safeMode.safeMode,+ safeModeBypassed: safeMode.bypassed === true,},skipNuq: true,origin,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c640def0b..f4b30fbbd 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -38,6 +38,12 @@ import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";import { isToolsOnlySearch } from "../../search/alexandria";+import {+ resolveSafeMode,+ isLockdownZeroDataRetention,+ getEffectiveSearchForcedKind,+} from "../../lib/safe-mode";+import { checkPermissions } from "../../lib/permissions";import {actionTypesOf,checkKeyEndpointRestriction,@@ -79,7 +85,11 @@ export async function searchController(const zeroDataRetentionTrace =Boolean(getSearchForcedKind(req.acuc?.flags)) ||enterprise.includes("zdr") ||- enterprise.includes("anon");+ enterprise.includes("anon") ||+ isLockdownZeroDataRetention(+ req.acuc?.flags,+ req.body?.scrapeOptions?.safeMode,+ );return withSpan("api.search.request",@@ -106,7 +116,13 @@ async function searchControllerInner(const jobId = uuidv7();const searchZDRMode = getSearchZDR(req.acuc?.flags);- const teamForcedKind = getSearchForcedKind(req.acuc?.flags);+ // Safe Mode lockdown forces the "zdr" kind like the searchZDR flag does+ // (see getEffectiveSearchForcedKind).+ const flagForcedKind = getSearchForcedKind(req.acuc?.flags);+ const teamForcedKind = getEffectiveSearchForcedKind(+ req.acuc?.flags,+ req.body?.scrapeOptions?.safeMode,+ );let logger = _logger.child({jobId,teamId: req.auth.team_id,@@ -206,6 +222,21 @@ async function searchControllerInner(});}+ // Safe Mode: validate the per-request param up front and reject scrape+ // options it forbids (the worker backstop would otherwise strip them+ // silently). Domain controls force threat protection over the results.+ const safeMode = resolveSafeMode(+ req.acuc?.flags,+ req.body.scrapeOptions?.safeMode,+ );+ if (safeMode.error) {+ return res.status(403).json({+ success: false,+ code: safeMode.code,+ error: safeMode.error,+ });+ }+// Threat protection: resolve the effective policy. Blocked domains are// removed from search results entirely, and scraped results inherit the// policy through the scrape pipeline.@@ -215,6 +246,7 @@ async function searchControllerInner(flags: req.acuc?.flags ?? null,override:req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,+ force: safeMode.safeMode?.domainControls === true,});if (threatProtection.error) {return res.status(403).json({@@ -223,6 +255,30 @@ async function searchControllerInner(});}+ // Search only scrapes (and only honors scrapeOptions) when formats are+ // requested, so the scrape-option checks apply only then.+ if (+ safeMode.safeMode &&+ requestedFormats.length > 0 &&+ req.body.scrapeOptions+ ) {+ const permissions = checkPermissions(+ req.body.scrapeOptions,+ req.acuc?.flags,+ {+ threatProtectionOrgConfig: threatProtection.orgConfig,+ safeMode: safeMode.safeMode,+ },+ );+ if (permissions.error) {+ return res.status(403).json({+ success: false,+ code: permissions.code,+ error: permissions.error,+ });+ }+ }+const shouldBill = req.body.__agentInterop?.shouldBill ?? true;const agentRequestId = req.body.__agentInterop?.requestId ?? null;const billing: BillingMetadata = req.body.__agentInterop@@ -235,6 +291,13 @@ async function searchControllerInner(origin: req.body.origin,});+ // Kinds the request itself asked for, captured before the forced kind is+ // injected: the entitlement check below applies to these only.+ const requestedZDROrAnon =+ req.body.enterprise?.includes("zdr") ||+ req.body.enterprise?.includes("anon") ||+ false;+// Inject the team-forced enterprise mode so downstream billing,// upstream routing, and ZDR cleanup all see it.if (teamForcedKind) {@@ -251,7 +314,9 @@ async function searchControllerInner(logger = logger.child({ zeroDataRetention });// Verify the team has searchZDR enabled before allowing enterprise ZDR/anon- if (isZDROrAnon && !teamForcedKind) {+ // it asked for. Only the flag-forced kind exempts a team: a lockdown-forced+ // "zdr" must not let an unentitled request add "anon".+ if (requestedZDROrAnon && !flagForcedKind) {if (searchZDRMode !== "allowed") {return res.status(403).json({success: false,@@ -344,6 +409,7 @@ async function searchControllerInner(agentIndexOnly: (req as any).agentIndexOnly ?? false,keylessReserved: reservedKeylessCredits > 0,threatProtectionPolicy: threatProtection.policy,+ safeModeBypassed: safeMode.bypassed === true,},logger,);diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.tsindex 6378ea3ed..f68646c6d 100644--- a/apps/api/src/controllers/v2/team-threat-protection.ts+++ b/apps/api/src/controllers/v2/team-threat-protection.ts@@ -152,10 +152,10 @@ export async function putTeamThreatProtectionController(const input = threatProtectionConfigSchema.parse(req.body);// "forced" guarantees enforcement: the org config may be tightened, but its- // mode may never be turned off through the API.+ // mode may never be turned off β or dropped to lists-only β through the API.if (getThreatProtection(req.acuc?.flags) === "forced" &&- input.mode === "off"+ (input.mode === "off" || input.mode === "manual-only")) {res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 29e0b4b37..9cd769e9c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -813,6 +813,7 @@ const baseScrapeOptions = z.strictObject({minAge: z.int().gte(0).optional(),storeInCache: z.boolean().prefault(true),lockdown: z.boolean().prefault(false),+ safeMode: z.boolean().optional(),redactPII: redactPIISchema,// Enterprise: per-request field-level override of the org's threat// protection policy. Gated on the team flag + org config (checkPermissions).@@ -899,10 +900,7 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(}if (obj.lockdown && obj.maxAge === undefined) {- // 2 years in ms. Number.MAX_SAFE_INTEGER lands ~285,000 years which- // overflows Postgres TIMESTAMP arithmetic in the index lookup and silently- // returns no rows. 2 years covers any practical cache retention window.- result = { ...result, maxAge: 2 * 365 * 24 * 60 * 60 * 1000 };+ result = { ...result, maxAge: LOCKDOWN_DEFAULT_MAX_AGE_MS };}return result as T extends undefined ? undefined : T;@@ -1890,11 +1888,29 @@ type Account = {remainingCredits: number;};+export const LOCKDOWN_DEFAULT_MAX_AGE_MS = 2 * 365 * 24 * 60 * 60 * 1000;+export type TeamFlags = {exchangeRetrieve?: boolean;ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";+ safeMode?: boolean;+ safeModeConfig?: {+ allowBypassSafeMode?: boolean;+ lockdown?: boolean;+ domainControls?: boolean;+ enforceRobots?: boolean;+ disableStealthProxy?: boolean;+ disableAuthentication?: boolean;+ disableSiteHandling?: boolean;+ exposeWebdriver?: boolean;+ useHeadlessUserAgent?: boolean;+ disablePlatformSelection?: boolean;+ disableCountrySelection?: boolean;+ disableAutomaticReferrer?: boolean;+ allowlist?: string[];+ };siemLogging?: boolean;unblockedDomains?: string[];forceZDR?: boolean;diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 7c1421f81..ed461f539 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -920,12 +920,11 @@ async function startServices(command?: string[]): Promise<Services> {): undefined;- // tests hammer the API instantly, so we need to ensure it's running before launching testsif (command &&Array.isArray(command) &&command[0] === "pnpm" &&- command[1].startsWith("test:snips")+ (command[1].startsWith("test:snips") || command[1] === "exec")) {logger.info(`Waiting for API on localhost:${PORT}`);await waitForPort(Number(PORT), "localhost");diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 52a2633bb..1132a920c 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -8,6 +8,7 @@ import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";import type { Logger } from "winston";import { withSpan, setSpanAttributes } from "./otel-tracer";import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";+import { resolveSafeMode } from "./safe-mode";import {firstPipelineError,reportPipelineError,@@ -979,6 +980,15 @@ export function crawlToCrawler(newBase?: string,crawlerOptions?: any,): WebCrawler {+ // Resolve the crawl's Safe Mode once (honoring a stored request bypass) to+ // drive both lockdown and the robots override below.+ const crawlerSafeMode = sc.internalOptions?.safeModeBypassed+ ? undefined+ : resolveSafeMode(+ sc.internalOptions?.teamFlags ?? teamFlags,+ undefined,+ sc.originUrl ?? undefined,+ ).safeMode;const crawler = new WebCrawler({jobId: id,initialUrl: sc.originUrl!,@@ -1000,10 +1010,12 @@ export function crawlToCrawler(allowExternalContentLinks:sc.crawlerOptions?.allowExternalContentLinks ?? false,allowSubdomains: sc.crawlerOptions?.allowSubdomains ?? false,+ // Safe Mode enforceRobots overrides even a team's forced ignore-robots.ignoreRobotsTxt:- getIgnoreRobots(teamFlags) === "forced" ||- (getIgnoreRobots(teamFlags) === "allowed" &&- (sc.crawlerOptions?.ignoreRobotsTxt ?? false)),+ !crawlerSafeMode?.enforceRobots &&+ (getIgnoreRobots(teamFlags) === "forced" ||+ (getIgnoreRobots(teamFlags) === "allowed" &&+ (sc.crawlerOptions?.ignoreRobotsTxt ?? false))),regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,@@ -1012,6 +1024,8 @@ export function crawlToCrawler(location: sc.scrapeOptions?.location,headers: sc.scrapeOptions?.headers,robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,+ // Safe Mode lockdown: skip robots/sitemap discovery (outbound to target).+ lockdown: crawlerSafeMode?.lockdown ?? false,});if (sc.robots !== undefined) {diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 7e2fb120f..6033b4a25 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -27,6 +27,7 @@ import {PDFPrefetchFailed,DocumentPrefetchFailed,SiteError,+ SiteRestrictionError,SSLError,ProxySelectionError,AgentIndexOnlyError,@@ -48,6 +49,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";const errorMap: Record<ErrorCodes, any> = {// Terms responses are API-level, never transported through workers.THIRD_PARTY_DATA_TERMS_REQUIRED: null,+ SAFE_MODE_BLOCKED: null,+ SCRAPE_SITE_RESTRICTION_BLOCKED: SiteRestrictionError,SCRAPE_TIMEOUT: ScrapeJobTimeoutError,MAP_TIMEOUT: MapTimeoutError,UNKNOWN_ERROR: UnknownError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex a694ed141..09394ba0b 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -43,6 +43,8 @@ export type ErrorCodes =| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST"| "CONCURRENCY_QUEUE_TIMEOUT"+ | "SAFE_MODE_BLOCKED"+ | "SCRAPE_SITE_RESTRICTION_BLOCKED"// Threat protection (enterprise domain risk blocking). Lowercase by design:// this is the documented, user-facing error code for the feature.| "unsafe_domain_blocked"diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 65fe024a8..c015d2346 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -72,6 +72,8 @@ export async function scrapeDocument(? true: false,bypassBilling: true,+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.+ teamFlags: options.flags ?? undefined,},origin: options.origin,is_scrape: true,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 41324da1e..4c78b50c4 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -74,6 +74,8 @@ export async function scrapeDocument_F0(orgId: options.orgId ?? null,bypassBilling: true,threatProtection: options.threatProtectionPolicy ?? undefined,+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.+ teamFlags: options.flags ?? undefined,},origin: options.origin,is_scrape: true,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 43d6fb66f..55ba4c0b5 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -8,6 +8,7 @@ import {} from "../controllers/v2/types";import { crawlToCrawler, StoredCrawl } from "./crawl-redis";import { getScrapeZDR } from "./zdr-helpers";+import { resolveSafeMode } from "./safe-mode";import {checkAndUpdateURLForMap,isSameDomain,@@ -121,7 +122,15 @@ export async function getMapResults({}): Promise<MapResult> {const functionStartTime = Date.now();- const resolvedUrl = await resolveRedirects(url, abort);+ // Safe Mode lockdown: serve links from the index only β skip live search ++ // sitemap discovery. Derived here from the team flags so every caller+ // (map, extract, llmstxt, prompt enhancement) honors it.+ const indexOnly =+ resolveSafeMode(flags, undefined, url).safeMode?.lockdown === true;++ // Under lockdown (index-only) resolving redirects would HEAD/GET the target,+ // which the zero-outbound guarantee forbids β use the URL as given.+ const resolvedUrl = indexOnly ? url : await resolveRedirects(url, abort);// If the resolved URL is on a different domain, replace the hostnameif (!isSameDomain(url, resolvedUrl)) {@@ -133,7 +142,9 @@ export async function getMapResults({const id = providedId ?? uuidv7();let mapResults: MapDocument[] = [];- const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;+ // Lockdown (index-only) is cache-only, which implies zero data retention β+ // and must never write its empty live-search result into the shared cache.+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || indexOnly;const sc: StoredCrawl = {originUrl: url,@@ -224,6 +235,10 @@ export async function getMapResults({};const fetchAllPages = async (): Promise<any[]> => {+ if (indexOnly) {+ // Lockdown: no live search discovery, serve from the index only.+ return [];+ }if (cachedResult) {return JSON.parse(cachedResult);}@@ -256,7 +271,7 @@ export async function getMapResults({mapResults.push(...indexResults);}- if (crawlerOptions.sitemap === "include") {+ if (crawlerOptions.sitemap === "include" && !indexOnly) {try {await crawler.tryGetSitemap(urls => {diff --git a/apps/api/src/lib/permissions.test.ts b/apps/api/src/lib/permissions.test.tsindex d97818301..ba3971604 100644--- a/apps/api/src/lib/permissions.test.ts+++ b/apps/api/src/lib/permissions.test.ts@@ -1,4 +1,163 @@import { checkPermissions } from "./permissions";+import { ResolvedSafeMode } from "./safe-mode";++const strictSafeMode: ResolvedSafeMode = {+ lockdown: false,+ domainControls: true,+ enforceRobots: true,+ disableStealthProxy: true,+ disableAuthentication: true,+ disableSiteHandling: true,+ exposeWebdriver: true,+ useHeadlessUserAgent: true,+ disablePlatformSelection: true,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+};++describe("checkPermissions β safe mode", () => {+ it("passes untouched requests through", () => {+ expect(checkPermissions({}, null, { safeMode: strictSafeMode })).toEqual(+ {},+ );+ });++ it("does nothing when safe mode is absent", () => {+ expect(+ checkPermissions({ proxy: "stealth" }, null, { safeMode: null }),+ ).toEqual({});+ });++ it.each(["stealth", "enhanced"])(+ "rejects %s proxy when stealth proxy is disabled",+ proxy => {+ const result = checkPermissions({ proxy }, null, {+ safeMode: strictSafeMode,+ });+ expect(result.error).toMatch(/prox/i);+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ },+ );++ it("allows stealth when the org relaxes disableStealthProxy", () => {+ expect(+ checkPermissions({ proxy: "stealth" }, null, {+ safeMode: { ...strictSafeMode, disableStealthProxy: false },+ }),+ ).toEqual({});+ });++ it("rejects ignoreRobotsTxt even when the org flag would allow it", () => {+ const result = checkPermissions(+ { crawlerOptions: { ignoreRobotsTxt: true } },+ { ignoreRobots: "allowed" },+ { safeMode: strictSafeMode },+ );+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ });++ it("keeps the ignoreRobotsTxt entitlement check under lockdown", () => {+ // Lockdown skips the Safe Mode robots rejection, so the generic+ // enterprise gate must still apply to a non-entitled team.+ const result = checkPermissions(+ { crawlerOptions: { ignoreRobotsTxt: true } },+ null,+ { safeMode: { ...strictSafeMode, lockdown: true } },+ );+ expect(result.error).toMatch(/enterprise feature/i);+ expect(result.code).toBeUndefined();+ });++ it("allows ignoreRobotsTxt when the org relaxes enforceRobots", () => {+ expect(+ checkPermissions(+ { crawlerOptions: { ignoreRobotsTxt: true } },+ { ignoreRobots: "allowed" },+ { safeMode: { ...strictSafeMode, enforceRobots: false } },+ ),+ ).toEqual({});+ });++ it("rejects profile, login actions, and credential headers", () => {+ const requests: Parameters<typeof checkPermissions>[0][] = [+ { profile: { name: "p" } },+ { actions: [{ type: "press" }] },+ { headers: { AUTHORIZATION: "x" } },+ { headers: { "Proxy-Authorization": "x" } },+ ];+ for (const request of requests) {+ const result = checkPermissions(request, null, {+ safeMode: strictSafeMode,+ });+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ }+ });++ it("allows profile/login/credentials when the org relaxes disableAuthentication", () => {+ const requests: Parameters<typeof checkPermissions>[0][] = [+ { profile: { name: "p" } },+ { actions: [{ type: "press" }] },+ { headers: { AUTHORIZATION: "x" } },+ ];+ for (const request of requests) {+ expect(+ checkPermissions(request, null, {+ safeMode: { ...strictSafeMode, disableAuthentication: false },+ }),+ ).toEqual({});+ }+ });++ it("allows benign actions and headers", () => {+ expect(+ checkPermissions(+ {+ actions: [{ type: "scroll" }, { type: "screenshot" }],+ headers: { "User-Agent": "test" },+ },+ null,+ { safeMode: strictSafeMode },+ ),+ ).toEqual({});+ });++ it("treats threat protection as forced under domainControls", () => {+ const result = checkPermissions(+ { threatProtection: { mode: "off" } },+ { threatProtection: "allowed" },+ { safeMode: strictSafeMode },+ );+ expect(result.error).toMatch(/cannot be disabled|disable/i);++ const noFlag = checkPermissions(+ { threatProtection: { mode: "off" } },+ null,+ { safeMode: { ...strictSafeMode, lockdown: true } },+ );+ expect(noFlag.error).toBeDefined();++ expect(+ checkPermissions({ threatProtection: { mode: "normal" } }, null, {+ safeMode: strictSafeMode,+ }),+ ).toEqual({});+ });++ it("skips every rule under effective lockdown", () => {+ expect(+ checkPermissions(+ {+ proxy: "stealth",+ profile: { name: "p" },+ actions: [{ type: "write" }],+ headers: { Cookie: "x" },+ },+ null,+ { safeMode: { ...strictSafeMode, lockdown: true } },+ ),+ ).toEqual({});+ });+});describe("checkPermissions β threat protection", () => {const requestWithOption = { threatProtection: { mode: "normal" } };diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex bb116f47a..69e72efec 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -1,4 +1,10 @@import { TeamFlags } from "../controllers/v2/types";+import type { ErrorCodes } from "./error";+import {+ type ResolvedSafeMode,+ SAFE_MODE_CREDENTIAL_HEADERS,+ SAFE_MODE_LOGIN_ACTIONS,+} from "./safe-mode";import {getScrapeZDR,getIgnoreRobots,@@ -28,6 +34,10 @@ interface APIRequest {// Per-request threat protection policy override (field-level override of// the org config). Presence of any value gates on the team flag.threatProtection?: ThreatProtectionOption;+ proxy?: string;+ profile?: unknown;+ actions?: { type: string }[];+ headers?: Record<string, string>;}interface PermissionOptions {@@ -37,6 +47,7 @@ interface PermissionOptions {* threatProtection option is rejected.*/threatProtectionOrgConfig?: { allowRequestOverrides: boolean } | null;+ safeMode?: ResolvedSafeMode | null;}const SUPPORT_EMAIL = "[email protected]";@@ -45,7 +56,7 @@ export function checkPermissions(request: APIRequest,flags?: TeamFlags,options?: PermissionOptions,-): { error?: string } {+): { error?: string; code?: ErrorCodes } {// zdr perms β scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDRconst scrapeMode = getScrapeZDR(flags);if (@@ -63,7 +74,14 @@ export function checkPermissions(if (request.crawlerOptions?.ignoreRobotsTxt &&robotsMode !== "allowed" &&- robotsMode !== "forced"+ robotsMode !== "forced" &&+ // Under Safe Mode (outside lockdown) the robots rejection below returns+ // SAFE_MODE_BLOCKED; don't preempt it with the generic entitlement error.+ !(+ options?.safeMode &&+ !options.safeMode.lockdown &&+ options.safeMode.enforceRobots+ )) {return {error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,@@ -87,18 +105,72 @@ export function checkPermissions(const threatProtectionOption =request.threatProtection ?? request.scrapeOptions?.threatProtection;if (threatProtectionOption !== undefined) {- const threatMode = getThreatProtection(flags);- if (threatMode !== "allowed" && threatMode !== "forced") {+ const effectiveThreatMode =+ options?.safeMode?.domainControls === true+ ? "forced"+ : getThreatProtection(flags);+ if (effectiveThreatMode !== "allowed" && effectiveThreatMode !== "forced") {return { error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE };}if (options?.threatProtectionOrgConfig?.allowRequestOverrides === false) {return { error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE };}- if (threatMode === "forced" && threatProtectionOption.mode === "off") {+ if (+ effectiveThreatMode === "forced" &&+ threatProtectionOption.mode === "off"+ ) {return { error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE };}}+ const safeMode = options?.safeMode;+ if (safeMode && !safeMode.lockdown) {+ if (+ safeMode.disableStealthProxy &&+ (request.proxy === "stealth" || request.proxy === "enhanced")+ ) {+ return {+ error:+ 'Safe Mode: stealth and enhanced proxies are not allowed for your organization. Remove the proxy option or use proxy: "basic".',+ code: "SAFE_MODE_BLOCKED",+ };+ }+ if (safeMode.enforceRobots && request.crawlerOptions?.ignoreRobotsTxt) {+ return {+ error:+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",+ code: "SAFE_MODE_BLOCKED",+ };+ }+ if (safeMode.disableAuthentication) {+ if (request.profile !== undefined) {+ return {+ error:+ "Safe Mode: browser profiles are not allowed for your organization (nothing behind a login).",+ code: "SAFE_MODE_BLOCKED",+ };+ }+ const loginAction = (request.actions ?? []).find(a =>+ SAFE_MODE_LOGIN_ACTIONS.includes(a.type),+ );+ if (loginAction) {+ return {+ error: `Safe Mode: the ${loginAction.type} action is not allowed for your organization (nothing behind a login).`,+ code: "SAFE_MODE_BLOCKED",+ };+ }+ const credentialHeader = Object.keys(request.headers ?? {}).find(h =>+ SAFE_MODE_CREDENTIAL_HEADERS.includes(h.toLowerCase()),+ );+ if (credentialHeader) {+ return {+ error: `Safe Mode: the ${credentialHeader} header is not allowed for your organization (nothing behind a login).`,+ code: "SAFE_MODE_BLOCKED",+ };+ }+ }+ }+// ip whitelist permsconst needsWhitelist =request.location?.country === "us-whitelist" ||diff --git a/apps/api/src/lib/safe-mode.test.ts b/apps/api/src/lib/safe-mode.test.tsnew file mode 100644index 000000000..d9bad9744--- /dev/null+++ b/apps/api/src/lib/safe-mode.test.ts@@ -0,0 +1,325 @@+import {+ applySafeMode,+ getSafeMode,+ isLockdownZeroDataRetention,+ resolveSafeMode,+ ResolvedSafeMode,+ SafeModeConfig,+} from "./safe-mode";++// Every sub-control is polarised so that `true` == the stricter setting.+// Under the org defaults, that means every capability control is `true`+// (strictest), except `lockdown` which defaults off.+const strict: ResolvedSafeMode = {+ lockdown: false,+ domainControls: true,+ enforceRobots: true,+ disableStealthProxy: true,+ disableAuthentication: true,+ disableSiteHandling: true,+ exposeWebdriver: true,+ useHeadlessUserAgent: true,+ disablePlatformSelection: true,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+};++// The per-capability controls the allowlist relaxes (everything except the+// org-wide lockdown + domainControls gates, which the allowlist never touches).+const RELAXABLE = [+ "enforceRobots",+ "disableStealthProxy",+ "disableAuthentication",+ "disableSiteHandling",+ "exposeWebdriver",+ "useHeadlessUserAgent",+ "disablePlatformSelection",+ "disableCountrySelection",+ "disableAutomaticReferrer",+] as const;++// The allowlisted posture: every relaxable capability flipped to its lenient+// value, lockdown + domainControls untouched.+const allCapabilitiesAllowed: ResolvedSafeMode = {+ ...strict,+ ...Object.fromEntries(RELAXABLE.map(k => [k, false])),+} as ResolvedSafeMode;++describe("getSafeMode", () => {+ it("is false for absent or null flags", () => {+ expect(getSafeMode(undefined)).toBe(false);+ expect(getSafeMode(null)).toBe(false);+ expect(getSafeMode({})).toBe(false);+ });++ it("is true only when the flag is exactly true", () => {+ expect(getSafeMode({ safeMode: true })).toBe(true);+ expect(getSafeMode({ safeMode: false })).toBe(false);+ });+});++describe("isLockdownZeroDataRetention", () => {+ const lockdownOrg = { safeMode: true, safeModeConfig: { lockdown: true } };++ it("is false without org lockdown", () => {+ expect(isLockdownZeroDataRetention(null, undefined)).toBe(false);+ expect(isLockdownZeroDataRetention({ safeMode: true }, undefined)).toBe(+ false,+ );+ });++ it("is true for a lockdown org, including a rejected bypass attempt", () => {+ expect(isLockdownZeroDataRetention(lockdownOrg, undefined)).toBe(true);+ expect(isLockdownZeroDataRetention(lockdownOrg, true)).toBe(true);+ expect(isLockdownZeroDataRetention(lockdownOrg, false)).toBe(true);+ });++ it("is false only when a bypass is actually honored", () => {+ const bypassable = {+ safeMode: true,+ safeModeConfig: { lockdown: true, allowBypassSafeMode: true },+ };+ expect(isLockdownZeroDataRetention(bypassable, false)).toBe(false);+ expect(isLockdownZeroDataRetention(bypassable, undefined)).toBe(true);+ });+});++describe("resolveSafeMode β org flag off", () => {+ it("resolves to nothing when the request says nothing", () => {+ expect(resolveSafeMode(null, undefined)).toEqual({});+ expect(resolveSafeMode({}, undefined)).toEqual({});+ });++ it("treats safeMode: false as a no-op", () => {+ expect(resolveSafeMode({}, false)).toEqual({});+ });++ it("rejects safeMode: true", () => {+ const result = resolveSafeMode({}, true);+ expect(result.error).toMatch(/not enabled/i);+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ expect(result.safeMode).toBeUndefined();+ });+});++describe("resolveSafeMode β org flag on", () => {+ const flags = { safeMode: true };++ it("resolves strict defaults with lockdown off", () => {+ expect(resolveSafeMode(flags, undefined)).toEqual({ safeMode: strict });+ });++ it("every capability control defaults to its strictest (true), lockdown off", () => {+ const resolved = resolveSafeMode(flags, undefined).safeMode!;+ expect(resolved.lockdown).toBe(false);+ for (const key of [+ "domainControls",+ ...RELAXABLE,+ ] as (keyof ResolvedSafeMode)[]) {+ expect(resolved[key]).toBe(true);+ }+ });++ it("treats safeMode: true as a redundant affirmation, not an error", () => {+ expect(resolveSafeMode(flags, true).safeMode).toBeDefined();+ expect(resolveSafeMode(flags, true).error).toBeUndefined();+ });++ it("merges partial org config over the defaults", () => {+ const result = resolveSafeMode(+ {+ safeMode: true,+ safeModeConfig: { lockdown: true, disableStealthProxy: false },+ },+ undefined,+ );+ expect(result.safeMode).toEqual({+ ...strict,+ lockdown: true,+ disableStealthProxy: false,+ });+ });++ it("lets the org relax each capability independently", () => {+ for (const key of RELAXABLE) {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: { [key]: false } },+ undefined,+ );+ expect(result.safeMode).toEqual({ ...strict, [key]: false });+ }+ });++ it("lets the org opt into lockdown", () => {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: { lockdown: true } },+ undefined,+ );+ expect(result.safeMode).toEqual({ ...strict, lockdown: true });+ });++ it("rejects a bypass when allowBypassSafeMode is unset or false", () => {+ const configs: (SafeModeConfig | undefined)[] = [+ undefined,+ {},+ { allowBypassSafeMode: false },+ ];+ for (const config of configs) {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: config },+ false,+ );+ expect(result.error).toMatch(/disable Safe Mode/i);+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ expect(result.safeMode).toBeUndefined();+ expect(result.bypassed).toBeUndefined();+ }+ });++ it("enforces the bypass gate even under org-configured lockdown", () => {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: { lockdown: true } },+ false,+ );+ expect(result.error).toMatch(/disable Safe Mode/i);+ expect(result.code).toBe("SAFE_MODE_BLOCKED");+ });++ it("honors a bypass when allowBypassSafeMode is true", () => {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: { allowBypassSafeMode: true } },+ false,+ );+ expect(result).toEqual({ bypassed: true });+ });++ it("still enforces when allowBypassSafeMode is true but no bypass is requested", () => {+ const result = resolveSafeMode(+ { safeMode: true, safeModeConfig: { allowBypassSafeMode: true } },+ undefined,+ );+ expect(result.safeMode).toBeDefined();+ expect(result.bypassed).toBeUndefined();+ });+});++describe("resolveSafeMode β allowlist", () => {+ const flags = {+ safeMode: true,+ safeModeConfig: { allowlist: ["docs.example.com", "*.trusted.example"] },+ };++ it("relaxes every capability but keeps lockdown + domainControls", () => {+ const result = resolveSafeMode(+ flags,+ undefined,+ "https://docs.example.com/x",+ );+ expect(result.allowlisted).toBe(true);+ expect(result.safeMode).toEqual(allCapabilitiesAllowed);+ // spelled out: nothing enforced/disabled, but the org gates stay+ for (const key of RELAXABLE) {+ expect(result.safeMode![key]).toBe(false);+ }+ expect(result.safeMode!.domainControls).toBe(true);+ });++ it("keeps lockdown on for an allowlisted URL when the org enabled it", () => {+ const result = resolveSafeMode(+ {+ safeMode: true,+ safeModeConfig: { lockdown: true, allowlist: ["docs.example.com"] },+ },+ undefined,+ "https://docs.example.com/x",+ );+ expect(result.allowlisted).toBe(true);+ expect(result.safeMode?.lockdown).toBe(true);+ expect(result.safeMode?.disableStealthProxy).toBe(false);+ });++ it("matches globs and subdomains", () => {+ expect(+ resolveSafeMode(flags, undefined, "https://a.trusted.example/p")+ .allowlisted,+ ).toBe(true);+ expect(+ resolveSafeMode(+ { safeMode: true, safeModeConfig: { allowlist: ["example.com"] } },+ undefined,+ "https://sub.example.com/p",+ ).allowlisted,+ ).toBe(true);+ });++ it("does not exempt a non-matching URL β capabilities stay strict", () => {+ const result = resolveSafeMode(flags, undefined, "https://other.example/x");+ expect(result.allowlisted).toBeUndefined();+ expect(result.safeMode?.disableStealthProxy).toBe(true);+ });++ it("ignores the allowlist when no url is passed", () => {+ const result = resolveSafeMode(flags, undefined);+ expect(result.allowlisted).toBeUndefined();+ expect(result.safeMode?.disableAuthentication).toBe(true);+ });++ it("a request bypass still wins over the allowlist path", () => {+ const result = resolveSafeMode(+ {+ safeMode: true,+ safeModeConfig: { allowBypassSafeMode: true, allowlist: ["x.example"] },+ },+ false,+ "https://y.example/p",+ );+ expect(result.bypassed).toBe(true);+ });+});++describe("applySafeMode", () => {+ it("forces any escalated proxy tier to basic when stealth proxy is disabled", () => {+ for (const tier of ["auto", "stealth", "enhanced"] as const) {+ const options = { proxy: tier };+ applySafeMode(strict, options);+ expect(options.proxy).toBe("basic");+ }+ });++ it("leaves proxy untouched for non-auto values, allowed stealth, or lockdown", () => {+ for (const [safeMode, proxy] of [+ [strict, "basic"],+ [{ ...strict, disableStealthProxy: false }, "auto"],+ [{ ...strict, lockdown: true }, "auto"],+ [undefined, "auto"],+ ] as const) {+ const options = { proxy: proxy as "basic" | "auto" };+ applySafeMode(safeMode, options);+ expect(options.proxy).toBe(proxy);+ }+ });++ it("forces lockdown and the 2-year maxAge when unset", () => {+ const options: { lockdown?: boolean; maxAge?: number } = {};+ applySafeMode({ ...strict, lockdown: true }, options);+ expect(options.lockdown).toBe(true);+ expect(options.maxAge).toBe(2 * 365 * 24 * 60 * 60 * 1000);+ });++ it("keeps a request-supplied maxAge under forced lockdown", () => {+ const options = { maxAge: 5000 };+ applySafeMode({ ...strict, lockdown: true }, options);+ expect(options).toEqual({ lockdown: true, maxAge: 5000 });+ });++ it("no-ops lockdown when already requested or off", () => {+ const alreadyOn = { lockdown: true };+ applySafeMode({ ...strict, lockdown: true }, alreadyOn);+ expect(alreadyOn).toEqual({ lockdown: true });++ const off: { lockdown?: boolean } = {};+ applySafeMode(strict, off);+ applySafeMode(undefined, off);+ expect(off).toEqual({});+ });+});diff --git a/apps/api/src/lib/safe-mode.ts b/apps/api/src/lib/safe-mode.tsnew file mode 100644index 000000000..df7c9c0cc--- /dev/null+++ b/apps/api/src/lib/safe-mode.ts@@ -0,0 +1,248 @@+import {+ LOCKDOWN_DEFAULT_MAX_AGE_MS,+ TeamFlags,+} from "../controllers/v2/types";+import type { ErrorCodes } from "./error";+import { getSearchForcedKind } from "./zdr-helpers";+import {+ domainMatchesList,+ normalizeDomain,+} from "./threat-protection/verdict";++const SUPPORT_EMAIL = "[email protected]";++// v0 content endpoints can't enforce Safe Mode's per-request surface, so a+// Safe Mode org is rejected there outright (mirrors the threat-protection v0 gate).+export const SAFE_MODE_V0_UNSUPPORTED_MESSAGE =+ "Safe Mode is enabled for your organization, which is not supported on the v0 API. Please update your code to use the v1 or v2 API.";++// Interactive browser sessions (browser, scrape interact) run arbitrary code+// against live sites and can't enforce Safe Mode's controls, so a Safe Mode+// org is rejected there outright (mirrors the v0 gate).+export const SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE =+ "Safe Mode is enabled for your organization, which does not support interactive browser sessions.";++// Credential-bearing request headers rejected at request time and stripped at+// the worker under Safe Mode's disableAuthentication (case-insensitive match).+export const SAFE_MODE_CREDENTIAL_HEADERS = [+ "authorization",+ "cookie",+ "proxy-authorization",+];++// Actions that can enter credentials / run scripts; rejected at request time+// and stripped at the worker under Safe Mode's disableAuthentication.+export const SAFE_MODE_LOGIN_ACTIONS = ["write", "press", "executeJavascript"];++// Remove Basic Auth embedded in a URL (user:pass@host). Returns the URL+// unchanged if it has no userinfo or can't be parsed.+export function stripUrlUserinfo(url: string): string {+ try {+ const u = new URL(url);+ if (u.username || u.password) {+ u.username = "";+ u.password = "";+ return u.toString();+ }+ } catch {+ // leave non-parseable URLs as-is+ }+ return url;+}++// Remove credential-bearing headers. Used by the fire-engine builders so+// inherited scrape options (crawl children, sub-scrapes) can't carry credentials+// even when no request-time gate ran.+export function stripCredentialHeaders(+ headers: Record<string, string> | undefined,+): Record<string, string> | undefined {+ if (!headers) return headers;+ return Object.fromEntries(+ Object.entries(headers).filter(+ ([k]) => !SAFE_MODE_CREDENTIAL_HEADERS.includes(k.toLowerCase()),+ ),+ );+}++function isSafeModeAllowlisted(+ url: string,+ allowlist: string[] | undefined,+): boolean {+ if (!allowlist || allowlist.length === 0) return false;+ return domainMatchesList(normalizeDomain(url), allowlist);+}++export type SafeModeConfig = NonNullable<+ NonNullable<TeamFlags>["safeModeConfig"]+>;++export type ResolvedSafeMode = {+ lockdown: boolean;+ domainControls: boolean;+ enforceRobots: boolean;+ disableStealthProxy: boolean;+ disableAuthentication: boolean;+ disableSiteHandling: boolean;+ exposeWebdriver: boolean;+ useHeadlessUserAgent: boolean;+ disablePlatformSelection: boolean;+ disableCountrySelection: boolean;+ disableAutomaticReferrer: boolean;+};++const SAFE_MODE_DEFAULTS: ResolvedSafeMode = {+ lockdown: false,+ domainControls: true,+ enforceRobots: true,+ disableStealthProxy: true,+ disableAuthentication: true,+ disableSiteHandling: true,+ exposeWebdriver: true,+ useHeadlessUserAgent: true,+ disablePlatformSelection: true,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+};++export function getSafeMode(flags: TeamFlags | null | undefined): boolean {+ return flags?.safeMode === true;+}++export function applySafeMode(+ safeMode: ResolvedSafeMode | undefined,+ scrapeOptions: {+ proxy?: "basic" | "stealth" | "enhanced" | "auto";+ lockdown?: boolean;+ maxAge?: number;+ },+): void {+ if (!safeMode) return;++ // Force a non-stealth proxy. Request-time enforcement rejects an *explicit*+ // stealth/enhanced proxy with a 403, but inherited scrape options (crawl+ // children, search/extract sub-scrapes) reach the worker without that gate β+ // downgrade any escalated tier to basic here so the worker never uses stealth.+ if (+ !safeMode.lockdown &&+ safeMode.disableStealthProxy &&+ (scrapeOptions.proxy === "auto" ||+ scrapeOptions.proxy === "stealth" ||+ scrapeOptions.proxy === "enhanced")+ ) {+ scrapeOptions.proxy = "basic";+ }++ if (safeMode.lockdown && !scrapeOptions.lockdown) {+ scrapeOptions.lockdown = true;+ if (scrapeOptions.maxAge === undefined) {+ scrapeOptions.maxAge = LOCKDOWN_DEFAULT_MAX_AGE_MS;+ }+ }+}++// Lockdown is cache-only and implies zero data retention. Resolved from the+// org flags alone (lockdown never depends on the request param) and dropped+// only when a per-request bypass is actually HONORED β a rejected/invalid+// safeMode:false must still record as ZDR for a lockdown org. Safe to call+// before schema validation (a non-boolean request value is treated as unset).+export function isLockdownZeroDataRetention(+ flags: TeamFlags | null | undefined,+ requestSafeMode: boolean | undefined,+): boolean {+ if (resolveSafeMode(flags, requestSafeMode).bypassed === true) return false;+ return resolveSafeMode(flags, undefined).safeMode?.lockdown === true;+}++// Search's forced enterprise kind, with Safe Mode lockdown folded in: lockdown+// forces the "zdr" kind exactly like the searchZDR flag does (zero-retention+// upstream routing, the ZDR credit rate, ZDR recording). The flag still wins+// when set (it may force "anon" instead).+export function getEffectiveSearchForcedKind(+ flags: TeamFlags | null | undefined,+ requestSafeMode: boolean | undefined,+): ReturnType<typeof getSearchForcedKind> {+ return (+ getSearchForcedKind(flags ?? undefined) ??+ (isLockdownZeroDataRetention(flags, requestSafeMode) ? "zdr" : null)+ );+}++export function resolveSafeMode(+ flags: TeamFlags | null | undefined,+ requestSafeMode: boolean | undefined,+ url?: string,+): {+ safeMode?: ResolvedSafeMode;+ bypassed?: boolean;+ allowlisted?: boolean;+ error?: string;+ code?: ErrorCodes;+} {+ if (!getSafeMode(flags)) {+ if (requestSafeMode === true) {+ return {+ error: `Safe Mode is not enabled for your organization. Contact ${SUPPORT_EMAIL} to enable this feature.`,+ code: "SAFE_MODE_BLOCKED",+ };+ }+ return {};+ }++ const config = flags?.safeModeConfig;++ if (requestSafeMode === false) {+ if (config?.allowBypassSafeMode !== true) {+ return {+ error:+ "Requests are not allowed to disable Safe Mode for your organization. An organization admin can allow per-request opt-outs from the Safe Mode settings.",+ code: "SAFE_MODE_BLOCKED",+ };+ }+ return { bypassed: true };+ }++ const resolved: ResolvedSafeMode = {+ lockdown: config?.lockdown ?? SAFE_MODE_DEFAULTS.lockdown,+ domainControls: config?.domainControls ?? SAFE_MODE_DEFAULTS.domainControls,+ enforceRobots: config?.enforceRobots ?? SAFE_MODE_DEFAULTS.enforceRobots,+ disableStealthProxy:+ config?.disableStealthProxy ?? SAFE_MODE_DEFAULTS.disableStealthProxy,+ disableAuthentication:+ config?.disableAuthentication ?? SAFE_MODE_DEFAULTS.disableAuthentication,+ disableSiteHandling:+ config?.disableSiteHandling ?? SAFE_MODE_DEFAULTS.disableSiteHandling,+ exposeWebdriver:+ config?.exposeWebdriver ?? SAFE_MODE_DEFAULTS.exposeWebdriver,+ useHeadlessUserAgent:+ config?.useHeadlessUserAgent ?? SAFE_MODE_DEFAULTS.useHeadlessUserAgent,+ disablePlatformSelection:+ config?.disablePlatformSelection ??+ SAFE_MODE_DEFAULTS.disablePlatformSelection,+ disableCountrySelection:+ config?.disableCountrySelection ??+ SAFE_MODE_DEFAULTS.disableCountrySelection,+ disableAutomaticReferrer:+ config?.disableAutomaticReferrer ??+ SAFE_MODE_DEFAULTS.disableAutomaticReferrer,+ };++ if (url && isSafeModeAllowlisted(url, config?.allowlist)) {+ return {+ allowlisted: true,+ safeMode: {+ ...resolved,+ enforceRobots: false,+ disableStealthProxy: false,+ disableAuthentication: false,+ disableSiteHandling: false,+ exposeWebdriver: false,+ useHeadlessUserAgent: false,+ disablePlatformSelection: false,+ disableCountrySelection: false,+ disableAutomaticReferrer: false,+ },+ };+ }++ return { safeMode: resolved };+}diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 5f062c5af..3f43530c9 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -28,7 +28,8 @@ const redactPIIPdfPageCostBonus = 4;// scope β a scrape and its same-URL re-check share one fee, while a crawl of// N pages bills N scans (each page job is its own scope). Verdicts are never// reused across requests (no verdict cache β ZDR). Local-only decisions-// (whitelist/blacklist/blocked-tld, mode off, provider failure) never bill.+// (whitelist/blacklist/blocked-tld, modes off / manual-only, provider+// failure) never bill.const threatScanCost = 2;/**diff --git a/apps/api/src/lib/threat-protection/check-url.test.ts b/apps/api/src/lib/threat-protection/check-url.test.tsindex b699cf7ef..30027675d 100644--- a/apps/api/src/lib/threat-protection/check-url.test.ts+++ b/apps/api/src/lib/threat-protection/check-url.test.ts@@ -138,6 +138,37 @@ describe("checkUrl", () => {expect(counters.hashesSearchRequests).toBe(before);});+ it("manual-only: enforces the lists and default-allows the rest with no provider call", async () => {+ const before = counters.hashesSearchRequests;+ const manual = policy({+ mode: "manual-only",+ blacklist: ["blocked.com"],+ blockedTlds: ["zip"],+ });++ const blocked = await checkUrl("https://cdn.blocked.com/x", manual, {});+ expect(blocked).toMatchObject({+ allowed: false,+ rule: "blacklist",+ providerConsulted: false,+ mode: "manual-only",+ });++ const tld = await checkUrl("https://archive.zip/", manual, {});+ expect(tld).toMatchObject({ allowed: false, rule: "blocked-tld" });++ // A domain the provider would flag is allowed: no lookup, no billing.+ const risky = await checkUrl(`https://${RISKY_DOMAIN}/`, manual, {});+ expect(risky).toMatchObject({+ allowed: true,+ rule: "default-allow",+ providerConsulted: false,+ verdict: null,+ mode: "manual-only",+ });+ expect(counters.hashesSearchRequests).toBe(before);+ });+it("consults the provider and blocks a flagged domain (fresh scan)", async () => {const before = riskyHits();const decision = await checkUrl(RISKY_DOMAIN.toUpperCase(), policy(), {diff --git a/apps/api/src/lib/threat-protection/config.test.ts b/apps/api/src/lib/threat-protection/config.test.tsindex d43ddd820..4413dd2bf 100644--- a/apps/api/src/lib/threat-protection/config.test.ts+++ b/apps/api/src/lib/threat-protection/config.test.ts@@ -28,6 +28,18 @@ describe("threatProtectionPolicySchema", () => {expect(policy.failurePolicy).toBe("open");});+ it('accepts the "manual-only" mode in every schema', () => {+ expect(+ threatProtectionPolicySchema.parse({ mode: "manual-only" }).mode,+ ).toBe("manual-only");+ expect(+ threatProtectionOverrideSchema.parse({ mode: "manual-only" }).mode,+ ).toBe("manual-only");+ expect(+ threatProtectionConfigSchema.parse({ mode: "manual-only" }).mode,+ ).toBe("manual-only");+ });+it("rejects an invalid mode", () => {expect(() =>threatProtectionPolicySchema.parse({ mode: "paranoid" }),diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.tsindex 9891ad963..ecc892925 100644--- a/apps/api/src/lib/threat-protection/config.ts+++ b/apps/api/src/lib/threat-protection/config.ts@@ -99,7 +99,7 @@ type ZscalerConfigInput = z.infer<typeof zscalerConfigSchema>;* except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.*/export const threatProtectionPolicySchema = z.strictObject({- mode: z.enum(["off", "normal", "zscaler"]),+ mode: z.enum(["off", "manual-only", "normal", "zscaler"]),riskScoreThreshold: z.number().int()@@ -137,7 +137,7 @@ void _policyContractCheck;* provides replace the org policy's values (see `resolveEffectivePolicy`).*/export const threatProtectionOverrideSchema = z.strictObject({- mode: z.enum(["off", "normal", "zscaler"]).optional(),+ mode: z.enum(["off", "manual-only", "normal", "zscaler"]).optional(),riskScoreThreshold: z.number().int().min(0).max(100).optional(),blacklist: z.array(domainGlobSchema).max(1000).optional(),whitelist: z.array(domainGlobSchema).max(1000).optional(),diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.tsindex 138206548..2f5886c4a 100644--- a/apps/api/src/lib/threat-protection/index.ts+++ b/apps/api/src/lib/threat-protection/index.ts@@ -47,7 +47,8 @@ export interface ThreatCheckContext {// 2. request-scoped dedup (ctx.dedup) β a URL already checked within// this request/job reuses the same in-flight decision// 3. local-only rules (whitelist/blacklist/blocked-tld, evaluated against-// the URL's host) β decide without a provider call (no billing)+// the URL's host) β decide without a provider call (no billing);+// mode "manual-only" stops here and default-allows everything else// 4. "zscaler" mode only: the tenant's synced custom-list rules (custom// categories, keywords, customer category additions β urlLookup never// returns those) β a decisive match skips the provider call@@ -97,7 +98,7 @@ const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry*/async function fetchProviderVerdict(url: string,- mode: Exclude<ThreatProtectionMode, "off">,+ mode: Exclude<ThreatProtectionMode, "off" | "manual-only">,policy: ThreatProtectionPolicy,): Promise<RawVerdict> {const attempts = mode === "zscaler" ? 1 : PROVIDER_ATTEMPTS;@@ -165,7 +166,7 @@ async function checkUrlFresh(}}- if (ctx.localRulesOnly) {+ if (ctx.localRulesOnly || policy.mode === "manual-only") {return {allowed: true,rule: "default-allow",@@ -181,7 +182,7 @@ async function checkUrlFresh(try {verdict = await fetchProviderVerdict(canonicalUrl,- policy.mode as Exclude<ThreatProtectionMode, "off">,+ policy.mode as Exclude<ThreatProtectionMode, "off" | "manual-only">,policy,);} catch {diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.tsindex b2d8faa75..111102cc5 100644--- a/apps/api/src/lib/threat-protection/request.ts+++ b/apps/api/src/lib/threat-protection/request.ts@@ -7,7 +7,11 @@ import {resolveEffectivePolicy,type OrgThreatProtectionConfig,} from "./store";-import type { ThreatDecision, ThreatProtectionPolicy } from "./types";+import type {+ ThreatDecision,+ ThreatProtectionMode,+ ThreatProtectionPolicy,+} from "./types";// Controller-layer glue for threat protection enforcement. One helper// (resolveThreatProtection) is shared by every endpoint: it turns@@ -23,10 +27,19 @@ export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE ="Per-request threat protection overrides are disabled by your organization's threat protection configuration.";export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE =- 'Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';+ 'Threat protection is enforced for your team and cannot be disabled or weakened per-request (threatProtection.mode may not be "off" or lower than the mode your organization configured). Remove the mode field or contact your organization administrator.';export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =- 'Threat protection is enforced for your team and its mode cannot be set to "off". Contact your organization administrator.';+ 'Threat protection is enforced for your team and its mode cannot be set to "off" or "manual-only". Contact your organization administrator.';++// Strength order for the never-weaken rule. "normal" and "zscaler" are peers+// (both provider-backed); switching between them is allowed.+const MODE_RANK: Record<ThreatProtectionMode, number> = {+ off: 0,+ "manual-only": 1,+ normal: 2,+ zscaler: 2,+};export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE ="Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";@@ -67,16 +80,22 @@ interface ResolvedThreatProtection {* - Flag "forced": an override may never set `mode: "off"` β 403.* - Effective mode "off" resolves to `policy: null` so callers can skip all* enforcement work.+ * - `force: true` (Safe Mode domain controls): treated as flag "forced", and+ * an "off" policy is raised to "manual-only" so the org's own lists are+ * enforced without a provider scan (no scan fee). Never resolves to+ * `policy: null`; an org that chose "normal"/"zscaler" keeps that mode.*/export async function resolveThreatProtection(args: {teamId: string;orgId?: string | null;flags: TeamFlags;override?: Partial<ThreatProtectionPolicy>;+ force?: boolean;}): Promise<ResolvedThreatProtection> {- const flagMode = getThreatProtection(args.flags);+ const effectiveFlagMode =+ args.force === true ? "forced" : getThreatProtection(args.flags);- if (flagMode !== "allowed" && flagMode !== "forced") {+ if (effectiveFlagMode !== "allowed" && effectiveFlagMode !== "forced") {if (args.override !== undefined) {return {error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE,@@ -98,16 +117,30 @@ export async function resolveThreatProtection(args: {orgConfig,};}- if (flagMode === "forced" && args.override.mode === "off") {- return {- error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,- policy: null,- orgConfig,- };+ if (effectiveFlagMode === "forced" && args.override.mode !== undefined) {+ // Enforced teams may tighten per-request but never weaken: "off" is+ // always refused, and the mode may not drop below the org's effective+ // mode (Safe Mode's raise puts that floor at "manual-only").+ const orgMode = orgConfig?.policy.mode ?? "off";+ const floor: ThreatProtectionMode =+ args.force === true && orgMode === "off" ? "manual-only" : orgMode;+ if (+ args.override.mode === "off" ||+ MODE_RANK[args.override.mode] < MODE_RANK[floor]+ ) {+ return {+ error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,+ policy: null,+ orgConfig,+ };+ }}}- const policy = resolveEffectivePolicy(orgConfig, args.override);+ let policy = resolveEffectivePolicy(orgConfig, args.override);+ if (args.force === true && policy.mode === "off") {+ policy = { ...policy, mode: "manual-only" };+ }// "zscaler" mode without a connection can only happen via a per-request// override on an org that never saved credentials (the config API refusesdiff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.tsindex a3e0aea50..ec3c85acb 100644--- a/apps/api/src/lib/threat-protection/store.test.ts+++ b/apps/api/src/lib/threat-protection/store.test.ts@@ -49,6 +49,12 @@ function makeRow(mode: string, config: unknown) {}describe("rowToConfig", () => {+ it('maps a "manual-only" mode column', () => {+ expect(rowToConfig(makeRow("manual-only", {})).policy.mode).toBe(+ "manual-only",+ );+ });+it("maps a full stored document", () => {const config = rowToConfig(makeRow("normal", {diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.tsindex 3f744f141..b394cabd9 100644--- a/apps/api/src/lib/threat-protection/store.ts+++ b/apps/api/src/lib/threat-protection/store.ts@@ -134,7 +134,9 @@ export function rowToConfig(? "normal": row.mode === "zscaler"? "zscaler"- : "off",+ : row.mode === "manual-only"+ ? "manual-only"+ : "off",riskScoreThreshold: doc.riskScoreThreshold,blacklist: doc.blacklist,whitelist: doc.whitelist,diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.tsindex a85b6bba9..c56ca1a11 100644--- a/apps/api/src/lib/threat-protection/types.ts+++ b/apps/api/src/lib/threat-protection/types.ts@@ -1,7 +1,10 @@// Shared type contract for the threat protection feature.// NOTE: concurrent in-flight branches create this exact file β do not modify without coordinating.-export type ThreatProtectionMode = "off" | "normal" | "zscaler";+// "manual-only": enforce the org's own lists (whitelist / blacklist /+// blocked TLDs) and nothing else β no provider lookup, so no scan fee. This+// is also what Safe Mode's domain controls raise an "off" policy to.+export type ThreatProtectionMode = "off" | "manual-only" | "normal" | "zscaler";/*** The non-secret slice of an org's Zscaler settings that policy evaluationdiff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.tsindex 938245439..2e27d47a5 100644--- a/apps/api/src/lib/threat-protection/verdict.ts+++ b/apps/api/src/lib/threat-protection/verdict.ts@@ -51,7 +51,7 @@ function domainMatchesEntry(domain: string, entry: string): boolean {return domain === normalized || domain.endsWith(`.${normalized}`);}-function domainMatchesList(domain: string, entries: string[]): boolean {+export function domainMatchesList(domain: string, entries: string[]): boolean {return entries.some(entry => domainMatchesEntry(domain, entry));}@@ -189,9 +189,10 @@ export function evaluatePolicy(return { allowed: true, rule: "default-allow", ...base };}- // No verdict: the provider failed or was unavailable (mode "off" never- // reaches here via checkUrl). Fail open or closed per the org policy.- if (policy.mode === "off") {+ // No verdict: the provider failed or was unavailable (modes "off" and+ // "manual-only" never consult one, so they never fail). Otherwise fail open+ // or closed per the org policy.+ if (policy.mode === "off" || policy.mode === "manual-only") {return { allowed: true, rule: "default-allow", ...base };}return {diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 7e0b1f597..b212e5954 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -85,6 +85,8 @@ export class WebCrawler {private location?: ScrapeOptions["location"];private headers?: Record<string, string>;private robotsUserAgent?: string;+ // Safe Mode lockdown: no outbound discovery to the target (robots/sitemap).+ private lockdown: boolean = false;constructor({jobId,@@ -107,6 +109,7 @@ export class WebCrawler {location,headers,robotsUserAgent,+ lockdown,}: {jobId: string;initialUrl: string;@@ -128,6 +131,7 @@ export class WebCrawler {location?: ScrapeOptions["location"];headers?: Record<string, string>;robotsUserAgent?: string;+ lockdown?: boolean;}) {this.jobId = jobId;this.initialUrl = initialUrl;@@ -158,6 +162,7 @@ export class WebCrawler {this.location = location;this.headers = headers;this.robotsUserAgent = robotsUserAgent;+ this.lockdown = lockdown ?? false;}public setBaseUrl(newBase: string): void {@@ -469,6 +474,11 @@ export class WebCrawler {skipTlsVerification = false,abort?: AbortSignal,): Promise<string> {+ // Lockdown: no outbound fetch to the target. Empty robots => fail-open,+ // consistent with a failed robots fetch.+ if (this.lockdown) {+ return "";+ }try {this.logger.debug("Attempting to fetch robots.txt", {method: "getRobotsTxt",@@ -550,6 +560,13 @@ export class WebCrawler {mock?: string,maxAge: number = SITEMAP_MAX_AGE,): Promise<number> {+ // Lockdown: no outbound sitemap discovery to the target (index-only crawl).+ if (this.lockdown) {+ this.logger.debug("Skipping sitemap discovery under Safe Mode lockdown", {+ method: "tryGetSitemap",+ });+ return 0;+ }this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {method: "tryGetSitemap",});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex d200a13c3..c262c0504 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -12,6 +12,7 @@ import {DNSResolutionError,FEPageLoadFailed,ProxySelectionError,+ SiteRestrictionError,} from "../../error";import { MockState } from "../../lib/mock";import { fireEngineURL } from "./scrape";@@ -137,6 +138,7 @@ const failedSchema = z.object({processing: z.literal(false),error: z.string(),retryWithStealth: z.boolean().optional(),+ failureReason: z.literal("site_protection").optional(),});export class StillProcessingError extends Error {@@ -190,6 +192,9 @@ export async function fireEngineCheckStatus(throw new StillProcessingError(jobId);} else if (failedParse.success) {logger.debug("Scrape job failed", { status, jobId });+ if (failedParse.data.failureReason === "site_protection") {+ throw new SiteRestrictionError();+ }if (failedParse.data.retryWithStealth &&meta.options.proxy === "auto" &&diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 67b1082ed..79ea24b0f 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -7,7 +7,9 @@ import {FireEngineScrapeRequestChromeCDP,FireEngineScrapeRequestCommon,FireEngineScrapeRequestTLSClient,+ safeModeParams,} from "./scrape";+import { stripCredentialHeaders } from "../../../../lib/safe-mode";import { EngineScrapeResult } from "..";import {fireEngineCheckStatus,@@ -20,6 +22,7 @@ import {EngineError,DNSResolutionError,SiteError,+ SiteRestrictionError,SSLError,UnsupportedFileError,FEPageLoadFailed,@@ -152,6 +155,7 @@ async function performFireEngineScrape<} else if (error instanceof EngineError ||error instanceof SiteError ||+ error instanceof SiteRestrictionError ||error instanceof SSLError ||error instanceof DNSResolutionError ||error instanceof ActionError ||@@ -490,8 +494,20 @@ export async function scrapeURLWithFireEngineChromeCDP(uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,}: undefined,+ ...safeModeParams(meta.internalOptions.safeMode),};+ // Safe Mode worker-side hardening: neutralize anything that request-time+ // enforcement would reject but that inherited scrape options can still carry.+ const sm = meta.internalOptions.safeMode;+ if (sm?.disableStealthProxy) {+ request.mobileProxy = false;+ }+ if (sm?.disableAuthentication) {+ request.persistentStorage = undefined;+ request.headers = stripCredentialHeaders(request.headers);+ }+let response = await performFireEngineScrape(meta,meta.logger.child({@@ -642,8 +658,17 @@ export async function scrapeURLWithFireEngineTLSClient(!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,+ ...safeModeParams(meta.internalOptions.safeMode),};+ const sm = meta.internalOptions.safeMode;+ if (sm?.disableStealthProxy) {+ request.mobileProxy = false;+ }+ if (sm?.disableAuthentication) {+ request.headers = stripCredentialHeaders(request.headers);+ }+let response = await performFireEngineScrape(meta,meta.logger.child({diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.tsnew file mode 100644index 000000000..42988c64c--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.ts@@ -0,0 +1,87 @@+import { safeModeParams } from "./scrape";+import type { ResolvedSafeMode } from "../../../../lib/safe-mode";++const strict: ResolvedSafeMode = {+ lockdown: false,+ domainControls: true,+ enforceRobots: true,+ disableStealthProxy: true,+ disableAuthentication: true,+ disableSiteHandling: true,+ exposeWebdriver: true,+ useHeadlessUserAgent: true,+ disablePlatformSelection: true,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+};++// The safe-mode control -> engine behaviorOverride field it drives. Same+// name, same value: the mapper is a straight pass-through (no negation).+const ENGINE_FIELDS = [+ "disableSiteHandling",+ "exposeWebdriver",+ "useHeadlessUserAgent",+ "disablePlatformSelection",+ "disableCountrySelection",+ "disableAutomaticReferrer",+] as const;++describe("safeModeParams", () => {+ it("sends nothing when Safe Mode is absent", () => {+ expect(safeModeParams(undefined)).toEqual({});+ });++ it("passes the strict defaults through as all overrides on", () => {+ expect(safeModeParams(strict)).toEqual({+ behaviorOverrides: {+ disableSiteHandling: true,+ exposeWebdriver: true,+ useHeadlessUserAgent: true,+ disablePlatformSelection: true,+ disableCountrySelection: true,+ disableAutomaticReferrer: true,+ },+ });+ });++ it("only forwards the six engine fields, not the firecrawl-side controls", () => {+ expect(Object.keys(safeModeParams(strict).behaviorOverrides!)).toEqual([+ ...ENGINE_FIELDS,+ ]);+ });++ it("forwards each engine field verbatim, with no negation", () => {+ for (const field of ENGINE_FIELDS) {+ const relaxed = safeModeParams({ ...strict, [field]: false });+ expect(relaxed.behaviorOverrides![field]).toBe(false);+ // the other five stay on+ for (const other of ENGINE_FIELDS) {+ if (other !== field) {+ expect(relaxed.behaviorOverrides![other]).toBe(true);+ }+ }+ }+ });++ it("passes an allowlisted (fully relaxed) posture through as all overrides off", () => {+ const relaxed: ResolvedSafeMode = {+ ...strict,+ disableSiteHandling: false,+ exposeWebdriver: false,+ useHeadlessUserAgent: false,+ disablePlatformSelection: false,+ disableCountrySelection: false,+ disableAutomaticReferrer: false,+ };+ expect(safeModeParams(relaxed)).toEqual({+ behaviorOverrides: {+ disableSiteHandling: false,+ exposeWebdriver: false,+ useHeadlessUserAgent: false,+ disablePlatformSelection: false,+ disableCountrySelection: false,+ disableAutomaticReferrer: false,+ },+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex de0cc7cb7..339ed97fa 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -15,9 +15,11 @@ import {ProxySelectionError,SSLError,SiteError,+ SiteRestrictionError,UnsupportedFileError,} from "../../error";import { Meta } from "../..";+import type { ResolvedSafeMode } from "../../../../lib/safe-mode";import { config } from "../../../../config";@@ -57,8 +59,33 @@ export type FireEngineScrapeRequestCommon = {maxAge?: number;saveScrapeResultToGCS?: boolean;zeroDataRetention?: boolean;++ behaviorOverrides?: {+ disableSiteHandling?: boolean;+ exposeWebdriver?: boolean;+ useHeadlessUserAgent?: boolean;+ disablePlatformSelection?: boolean;+ disableCountrySelection?: boolean;+ disableAutomaticReferrer?: boolean;+ };};+export function safeModeParams(+ safeMode: ResolvedSafeMode | undefined,+): Pick<FireEngineScrapeRequestCommon, "behaviorOverrides"> {+ if (!safeMode) return {};+ return {+ behaviorOverrides: {+ disableSiteHandling: safeMode.disableSiteHandling,+ exposeWebdriver: safeMode.exposeWebdriver,+ useHeadlessUserAgent: safeMode.useHeadlessUserAgent,+ disablePlatformSelection: safeMode.disablePlatformSelection,+ disableCountrySelection: safeMode.disableCountrySelection,+ disableAutomaticReferrer: safeMode.disableAutomaticReferrer,+ },+ };+}+export type FireEngineScrapeRequestChromeCDP = {engine: "chrome-cdp";skipTlsVerification?: boolean;@@ -180,6 +207,7 @@ const processingSchema = z.object({const failedSchema = z.object({error: z.string(),retryWithStealth: z.boolean().optional(),+ failureReason: z.literal("site_protection").optional(),});export const fireEngineURL =@@ -241,6 +269,9 @@ export async function fireEngineScrape<logger.debug("Scrape job failed", {status,});+ if (failedParse.data.failureReason === "site_protection") {+ throw new SiteRestrictionError();+ }if (failedParse.data.retryWithStealth &&meta.options.proxy === "auto" &&diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 1c2ba2c34..19129fdb7 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -203,6 +203,30 @@ export class ProxySelectionError extends TransportableError {}}+export class SiteRestrictionError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_SITE_RESTRICTION_BLOCKED",+ "This site restricts automated access to the requested content. Safe Mode is enabled for your organization, so the site's restriction is returned instead of being worked around.",+ );+ }++ serialize() {+ return {+ ...super.serialize(),+ };+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new SiteRestrictionError();+ x.stack = data.stack;+ return x;+ }+}+export class ActionError extends TransportableError {constructor(public errorCode: string) {super(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 102f81368..91d9b3022 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -41,6 +41,7 @@ import {DocumentFetchProxyError,RemoveFeatureError,SiteError,+ SiteRestrictionError,UnsupportedFileError,SSLError,PDFInsufficientTimeError,@@ -113,6 +114,15 @@ import {type ThreatDecision,type ThreatProtectionPolicy,} from "../../lib/threat-protection";+import {+ type ResolvedSafeMode,+ resolveSafeMode,+ applySafeMode,+ stripCredentialHeaders,+ stripUrlUserinfo,+ SAFE_MODE_LOGIN_ACTIONS,+} from "../../lib/safe-mode";+import { resolveThreatProtection } from "../../lib/threat-protection/request";import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { canonicalizeUrl } from "../../lib/threat-protection/providers/web-risk/canonicalize";@@ -623,6 +633,12 @@ export type InternalOptions = {*/threatProtection?: ThreatProtectionPolicy;+ safeMode?: ResolvedSafeMode;+ /** Set when a request legitimately opted out of Safe Mode at the controller+ * (allowBypassSafeMode + safeMode:false). Tells the worker backstop NOT to+ * re-resolve safe mode from teamFlags β otherwise the bypass would be undone. */+ safeModeBypassed?: boolean;+v1Agent?: ScrapeOptionsV1["agent"];v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];v1JSONSystemPrompt?: string;@@ -991,6 +1007,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof AddFeatureError ||error.error instanceof RemoveFeatureError ||error.error instanceof SiteError ||+ error.error instanceof SiteRestrictionError ||error.error instanceof SSLError ||error.error instanceof DNSResolutionError ||error.error instanceof ActionError ||@@ -1231,6 +1248,66 @@ export async function scrapeURL(return withSpan("scrape.pipeline",async span => {+ // Safe Mode: the universal enforcement choke point. The controller+ // pre-resolves safe mode for a single scrape, but jobs that carry only+ // team flags (crawl children, search / extract URLs, allowlisted single+ // scrapes) resolve here, per-URL, so the allowlist applies to each+ // discovered URL. Runs before buildMetaObject so forced lockdown reaches+ // feature-flag/engine selection, and covers every endpoint that stamps+ // teamFlags onto its job payload.+ if (+ !internalOptions.safeMode &&+ !internalOptions.safeModeBypassed &&+ internalOptions.teamFlags+ ) {+ internalOptions.safeMode = resolveSafeMode(+ internalOptions.teamFlags,+ undefined,+ url,+ ).safeMode;+ }+ if (internalOptions.safeMode) {+ applySafeMode(internalOptions.safeMode, options);+ // Auth-path enforcement for every engine (not just fire-engine) and for+ // inherited options a request-time gate never saw (crawl children etc.).+ if (internalOptions.safeMode.disableAuthentication) {+ options.headers = stripCredentialHeaders(options.headers);+ if (options.actions) {+ options.actions = options.actions.filter(+ a => !SAFE_MODE_LOGIN_ACTIONS.includes(a.type),+ );+ }+ options.profile = undefined;+ // Basic Auth embedded in the URL (user:pass@host) is another way to+ // authenticate; strip the userinfo so no engine can use it, and from+ // the preserved source URL so it isn't returned/persisted in metadata.+ url = stripUrlUserinfo(url);+ if (internalOptions.unnormalizedSourceURL) {+ internalOptions.unnormalizedSourceURL = stripUrlUserinfo(+ internalOptions.unnormalizedSourceURL,+ );+ }+ }+ if (+ internalOptions.safeMode.domainControls &&+ !internalOptions.threatProtection+ ) {+ const tp = await resolveThreatProtection({+ teamId: internalOptions.teamId,+ orgId: internalOptions.orgId,+ flags: internalOptions.teamFlags ?? {},+ force: true,+ });+ // Fail closed: domainControls must never silently disable itself.+ if (tp.error) {+ throw new Error(+ `Safe Mode domain controls could not be resolved: ${tp.error}`,+ );+ }+ internalOptions.threatProtection = tp.policy ?? undefined;+ }+ }+const meta = await buildMetaObject(id,url,@@ -1737,6 +1814,11 @@ export async function scrapeURL(meta.logger.warn("scrapeURL: Site failed to load in browser", {error,});+ } else if (error instanceof SiteRestrictionError) {+ errorType = "SiteRestrictionError";+ meta.logger.warn("scrapeURL: Site restriction returned (Safe Mode)", {+ error,+ });} else if (error instanceof SSLError) {errorType = "SSLError";meta.logger.warn("scrapeURL: SSL error", { error });diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.tsnew file mode 100644index 000000000..360d9df1f--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.ts@@ -0,0 +1,46 @@+import { shouldCheckRobots } from "./shouldCheckRobots";++describe("shouldCheckRobots", () => {+ it("never checks under lockdown, even with safe mode enforcing robots", () => {+ expect(+ shouldCheckRobots(+ { lockdown: true },+ { safeMode: { enforceRobots: true } as any },+ ),+ ).toBe(false);+ });++ it("checks when the team flag opts in", () => {+ expect(+ shouldCheckRobots({}, { teamFlags: { checkRobotsOnScrape: true } }),+ ).toBe(true);+ });++ it("checks when safe mode enforces robots, regardless of the team flag", () => {+ expect(+ shouldCheckRobots({}, { safeMode: { enforceRobots: true } as any }),+ ).toBe(true);+ expect(+ shouldCheckRobots(+ {},+ {+ teamFlags: { checkRobotsOnScrape: false },+ safeMode: { enforceRobots: true } as any,+ },+ ),+ ).toBe(true);+ });++ it("does not check when safe mode relaxes robots and the flag is off", () => {+ expect(+ shouldCheckRobots({}, { safeMode: { enforceRobots: false } as any }),+ ).toBe(false);+ });++ it("does not check when neither opts in", () => {+ expect(shouldCheckRobots({}, {})).toBe(false);+ expect(+ shouldCheckRobots({}, { teamFlags: { checkRobotsOnScrape: false } }),+ ).toBe(false);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.tsindex d2bc039f3..8f68659da 100644--- a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts@@ -1,4 +1,5 @@import type { TeamFlags } from "../../controllers/v1/types";+import type { ResolvedSafeMode } from "../../lib/safe-mode";// The invariant "lockdown never fetches robots.txt" is load-bearing for the// lockdown guarantee (robots.txt is a request to the target domain). Keep this@@ -6,10 +7,13 @@ import type { TeamFlags } from "../../controllers/v1/types";// ESM-heavy module graph.export function shouldCheckRobots(options: { lockdown?: boolean },- internalOptions: { teamFlags?: TeamFlags },+ internalOptions: { teamFlags?: TeamFlags; safeMode?: ResolvedSafeMode },): boolean {if (options.lockdown) {return false;}+ if (internalOptions.safeMode?.enforceRobots) {+ return true;+ }return !!internalOptions.teamFlags?.checkRobotsOnScrape;}diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 33dda1306..d5517c431 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -65,6 +65,8 @@ interface SearchContext {keylessReserved?: boolean;/** Effective threat protection policy; blocked domains are removed from results entirely. */threatProtectionPolicy?: ThreatProtectionPolicy | null;+ /** Set when the request legitimately opted out of Safe Mode at the controller. */+ safeModeBypassed?: boolean;}interface SearchExecuteResult {@@ -305,6 +307,7 @@ export async function executeSearch(agentIndexOnly: context.agentIndexOnly,keylessReserved: context.keylessReserved,threatProtectionPolicy: threatPolicy ?? null,+ safeModeBypassed: context.safeModeBypassed ?? false,};const allDocsWithCostTracking = await scrapeSearchResults(diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex d3f06fc40..acd89fafc 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -43,6 +43,7 @@ interface ScrapeSearchOptions {agentIndexOnly?: boolean;keylessReserved?: boolean;threatProtectionPolicy?: ThreatProtectionPolicy | null;+ safeModeBypassed?: boolean;}async function scrapeSearchResultDirect(@@ -86,6 +87,7 @@ async function scrapeSearchResultDirect(teamFlags: flags,agentIndexOnly: options.agentIndexOnly ?? false,threatProtection: options.threatProtectionPolicy ?? undefined,+ safeModeBypassed: options.safeModeBypassed === true,},skipNuq: true,origin: options.origin,diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 56218a6bb..63aefb0de 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -74,6 +74,7 @@ import { verdictJsonSchema } from "./search/judge";import { computeGoalVersion } from "./search/dedupe";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { getACUCTeam } from "../../controllers/auth";+import type { TeamFlags } from "../../controllers/v1/types";import { orgIdForTeam } from "../../lib/team-org";import {reconstructKnownState,@@ -271,6 +272,7 @@ export function estimateActualCredits(doc: any, options?: any): number {// per-page β search monitors bill flat at the check level.async function scrapeSearchMonitorPage(params: {teamId: string;+ teamFlags: TeamFlags | null;checkId: string;url: string;judgePrompt: string;@@ -316,6 +318,8 @@ async function scrapeSearchMonitorPage(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,bypassBilling: true,zeroDataRetention: false,+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.+ teamFlags: params.teamFlags ?? undefined,},skipNuq: true,origin: "monitor",@@ -555,6 +559,10 @@ async function enqueueMonitorScrapeTarget(params: {throw new Error("Expected scrape target");}+ // Safe Mode applies to monitor scrapes: resolve per-URL at the backstop from+ // the team's flags. orgId stays null (monitors' separate no-blocklist rule).+ const acuc = await getACUCTeam(params.monitor.team_id);+for (const [index, url] of params.target.urls.entries()) {const scrapeId = params.targetRun.expectedJobs[index];const scrapeOptions = scrapeRequestSchema.parse({@@ -588,6 +596,7 @@ async function enqueueMonitorScrapeTarget(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,bypassBilling: true,zeroDataRetention: false,+ teamFlags: acuc?.flags ?? undefined,},origin: "monitor",integration: null,@@ -646,6 +655,10 @@ async function enqueueMonitorCrawlTarget(params: {prompt: undefined,};+ // Safe Mode applies to monitor crawls: each child resolves it per-URL at the+ // backstop from the team's flags. orgId stays null (monitors' no-blocklist rule).+ const acuc = await getACUCTeam(params.monitor.team_id);+const sc: StoredCrawl = {originUrl: body.url,crawlerOptions: toV0CrawlerOptions(crawlerOptions),@@ -658,6 +671,7 @@ async function enqueueMonitorCrawlTarget(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,bypassBilling: true,+ teamFlags: acuc?.flags ?? undefined,},team_id: params.monitor.team_id,createdAt: Date.now(),@@ -823,6 +837,7 @@ async function runMonitorSearchTarget(params: {scrapePage: ({ url, judgePrompt }) =>scrapeSearchMonitorPage({teamId: monitor.team_id,+ teamFlags,checkId: check.id,url,judgePrompt,diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 2cae892bf..6ead58ef2 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -306,8 +306,14 @@ export interface RedactPIIOptions {* fields you provide replace the team policy's values.*/export interface ThreatProtectionOptions {- /** "off" disables scanning for this request; "normal" applies the policy. */- mode?: "off" | "normal";+ /**+ * "off" disables scanning for this request; "manual-only" enforces only the+ * blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);+ * "normal" scans with Google Web Risk; "zscaler" classifies through your+ * organization's Zscaler connection. Enforced teams may raise the mode per+ * request but never lower it.+ */+ mode?: "off" | "manual-only" | "normal" | "zscaler";/** Block verdicts at or above this risk score (integer 0-100). */riskScoreThreshold?: number;/** Exact domains or globs like "*.example.com" to always block (max 1000). */diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 6eac78b3e..4ef19366e 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -843,8 +843,12 @@ class ThreatProtectionOptions(BaseModel):explicitly provide replace the team policy's values."""- # "off" disables scanning for this request; "normal" applies the policy.- mode: Optional[Literal["off", "normal"]] = None+ # "off" disables scanning for this request; "manual-only" enforces only the+ # blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);+ # "normal" scans with Google Web Risk; "zscaler" classifies through your+ # organization's Zscaler connection. Enforced teams may raise the mode per+ # request but never lower it.+ mode: Optional[Literal["off", "manual-only", "normal", "zscaler"]] = None# Block verdicts at or above this risk score (integer 0-100).risk_score_threshold: Optional[int] = Field(default=None, alias="riskScoreThreshold"