Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): safe mode (#4612)

Rafael Millerf62a24c5d0bd
diff --git a/apps/api/src/__tests__/snips/v2/safe-mode.test.ts b/apps/api/src/__tests__/snips/v2/safe-mode.test.ts
new file mode 100644
index 000000000..fd1174d7e
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/safe-mode.test.ts
@@ -0,0 +1,493 @@
+import request from "supertest";
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ concurrentIf,
+ createTestIdUrl,
+ describeIf,
+ idmux,
+ Identity,
+ scrapeTimeout,
+ TEST_API_URL,
+ TEST_PRODUCTION,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import { scrape, scrapeRaw, crawlStart } from "./lib";
+
+const TEST_SUITE_HOST = new URL(TEST_SUITE_WEBSITE).hostname;
+
+async function expectSafeModeBlocked(
+ body: Parameters<typeof scrapeRaw>[0],
+ identity: Identity,
+) {
+ const res = await scrapeRaw(body, identity);
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");
+ return res.body;
+}
+
+describe("Safe Mode (v2 scrape, request-time)", () => {
+ describe("org flag off", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({ name: "safe-mode/unflagged" });
+ }, 10000);
+
+ it.concurrent(
+ "rejects safeMode: true",
+ async () => {
+ const body = await expectSafeModeBlocked(
+ { url: createTestIdUrl(), safeMode: true },
+ identity,
+ );
+ expect(body.error).toMatch(/not enabled/i);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "v1 accepts + honors the safeMode param (rejects safeMode: true)",
+ async () => {
+ const res = await request(TEST_API_URL)
+ .post("/v1/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ url: createTestIdUrl(), safeMode: true });
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "treats safeMode: false as a no-op",
+ async () => {
+ const doc = await scrape(
+ { url: createTestIdUrl(), safeMode: false },
+ identity,
+ );
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describeIf(TEST_PRODUCTION)("org flag on (strict defaults)", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/flagged",
+ flags: { safeMode: true },
+ });
+ }, 10000);
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "plain scrapes still work",
+ async () => {
+ const doc = await scrape({ url: createTestIdUrl() }, identity);
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects stealth and enhanced proxies",
+ async () => {
+ for (const proxy of ["stealth", "enhanced"] as const) {
+ const body = await expectSafeModeBlocked(
+ { url: createTestIdUrl(), proxy },
+ identity,
+ );
+ expect(body.error).toMatch(/prox/i);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects browser profiles",
+ async () => {
+ await expectSafeModeBlocked(
+ { url: createTestIdUrl(), profile: { name: "test-profile" } },
+ identity,
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects login-capable actions",
+ async () => {
+ const body = await expectSafeModeBlocked(
+ {
+ url: createTestIdUrl(),
+ actions: [{ type: "write", text: "hunter2" }],
+ },
+ identity,
+ );
+ expect(body.error).toMatch(/write/);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects credential-bearing headers, case-insensitively",
+ async () => {
+ for (const header of ["Authorization", "cookie"]) {
+ const body = await expectSafeModeBlocked(
+ { url: createTestIdUrl(), headers: { [header]: "secret" } },
+ identity,
+ );
+ expect(body.error.toLowerCase()).toContain(header.toLowerCase());
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "reports basic proxy for auto requests (no stealth escalation)",
+ async () => {
+ const doc = await scrape(
+ { url: createTestIdUrl(), proxy: "auto" },
+ identity,
+ );
+ expect(doc.metadata?.proxyUsed).toBe("basic");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "allows benign actions",
+ async () => {
+ const doc = await scrape(
+ { url: createTestIdUrl(), actions: [{ type: "scroll" }] },
+ identity,
+ );
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "allows benign headers",
+ async () => {
+ const doc = await scrape(
+ {
+ url: createTestIdUrl(),
+ headers: { "Accept-Language": "en-US" },
+ },
+ identity,
+ );
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects a bypass when the org has not allowed it",
+ async () => {
+ const body = await expectSafeModeBlocked(
+ { url: createTestIdUrl(), safeMode: false },
+ identity,
+ );
+ expect(body.error).toMatch(/disable Safe Mode/i);
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "treats safeMode: true as a redundant affirmation",
+ async () => {
+ const doc = await scrape(
+ { url: createTestIdUrl(), safeMode: true },
+ identity,
+ );
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describeIf(TEST_PRODUCTION)("org flag on with allowBypassSafeMode", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/bypassable",
+ flags: {
+ safeMode: true,
+ safeModeConfig: { allowBypassSafeMode: true },
+ },
+ });
+ }, 10000);
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "honors safeMode: false β€” otherwise-blocked params pass",
+ async () => {
+ const doc = await scrape(
+ {
+ url: createTestIdUrl(),
+ safeMode: false,
+ headers: { Authorization: "Bearer not-a-real-token" },
+ },
+ identity,
+ );
+ expect(doc.markdown).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "still enforces when no bypass is requested",
+ async () => {
+ await expectSafeModeBlocked(
+ { url: createTestIdUrl(), headers: { Authorization: "x" } },
+ identity,
+ );
+ },
+ scrapeTimeout,
+ );
+ });
+
+ describeIf(TEST_PRODUCTION)(
+ "org config lockdown: true (lockdown supersedes)",
+ () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/lockdown",
+ flags: { safeMode: true, safeModeConfig: { lockdown: true } },
+ });
+ }, 10000);
+
+ it.concurrent(
+ "uncached URLs return the lockdown cache-miss error",
+ async () => {
+ const res = await scrapeRaw({ url: createTestIdUrl() }, identity);
+ expect(res.statusCode).toBe(404);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("SCRAPE_LOCKDOWN_CACHE_MISS");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "accepts and ignores params the other controls would reject",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: createTestIdUrl(),
+ proxy: "stealth",
+ profile: { name: "ignored" },
+ headers: { Cookie: "ignored" },
+ },
+ identity,
+ );
+ expect(res.body.code).toBe("SCRAPE_LOCKDOWN_CACHE_MISS");
+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");
+ },
+ scrapeTimeout,
+ );
+ },
+ );
+
+ describeIf(TEST_PRODUCTION)("domainControls forces threat protection", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/domain-controls",
+ flags: { safeMode: true, threatProtection: "allowed" },
+ });
+ const res = await request(TEST_API_URL)
+ .put("/v2/team/threat-protection")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ mode: "off", blacklist: ["blocked.example.com"] });
+ expect(res.statusCode).toBe(200);
+ }, 15000);
+
+ it.concurrent(
+ "blocks blacklisted domains even though the TP config mode is off",
+ async () => {
+ const res = await scrapeRaw(
+ { url: "https://blocked.example.com/page" },
+ identity,
+ );
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects a request-level threat protection opt-out",
+ async () => {
+ const res = await scrapeRaw(
+ { url: createTestIdUrl(), threatProtection: { mode: "off" } },
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toMatch(/threat protection/i);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ // Each capability is individually relaxable from the org's safeModeConfig:
+ // flipping its control off restores the normal behavior while Safe Mode
+ // stays on for everything else.
+ describeIf(TEST_PRODUCTION)("org config relaxations", () => {
+ it.concurrent(
+ "disableStealthProxy: false lets a request use a stealth proxy",
+ async () => {
+ const identity = await idmux({
+ name: "safe-mode/relax-stealth",
+ flags: {
+ safeMode: true,
+ safeModeConfig: { disableStealthProxy: false },
+ },
+ });
+ const res = await scrapeRaw(
+ { url: createTestIdUrl(), proxy: "stealth" },
+ identity,
+ );
+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");
+ expect(res.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "disableAuthentication: false lets a request send credential headers",
+ async () => {
+ const identity = await idmux({
+ name: "safe-mode/relax-auth",
+ flags: {
+ safeMode: true,
+ safeModeConfig: { disableAuthentication: false },
+ },
+ });
+ const res = await scrapeRaw(
+ {
+ url: createTestIdUrl(),
+ headers: { Authorization: "Bearer not-a-real-token" },
+ },
+ identity,
+ );
+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");
+ expect(res.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "still blocks the capabilities left at their strict default",
+ async () => {
+ const identity = await idmux({
+ name: "safe-mode/relax-partial",
+ flags: {
+ safeMode: true,
+ safeModeConfig: { disableStealthProxy: false },
+ },
+ });
+ // stealth relaxed above, but the profile (auth-path) restriction stays
+ const res = await scrapeRaw(
+ { url: createTestIdUrl(), profile: { name: "test-profile" } },
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");
+ },
+ scrapeTimeout,
+ );
+ });
+
+ // An allowlisted URL relaxes *how* it is scraped (evaluated per-URL), while a
+ // non-matching URL under the same org stays fully strict.
+ describeIf(TEST_PRODUCTION)("allowlist", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/allowlist",
+ flags: {
+ safeMode: true,
+ safeModeConfig: { allowlist: [TEST_SUITE_HOST] },
+ },
+ });
+ }, 10000);
+
+ it.concurrent(
+ "relaxes a stealth proxy request for an allowlisted URL",
+ async () => {
+ const res = await scrapeRaw(
+ { url: createTestIdUrl(), proxy: "stealth" },
+ identity,
+ );
+ expect(res.body.code).not.toBe("SAFE_MODE_BLOCKED");
+ expect(res.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "keeps a non-allowlisted URL strict",
+ async () => {
+ const body = await expectSafeModeBlocked(
+ { url: "https://not-in-allowlist.example.org/", proxy: "stealth" },
+ identity,
+ );
+ expect(body.error).toMatch(/prox/i);
+ },
+ scrapeTimeout,
+ );
+ });
+
+ // Ticket 08: enforcement reaches beyond /v2/scrape.
+ describeIf(TEST_PRODUCTION)("endpoint coverage", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "safe-mode/endpoints",
+ flags: { safeMode: true },
+ });
+ }, 10000);
+
+ it.concurrent(
+ "crawl rejects a stealth proxy nested under scrapeOptions",
+ async () => {
+ const res = await crawlStart(
+ {
+ url: createTestIdUrl(),
+ scrapeOptions: { formats: ["markdown"], proxy: "stealth" },
+ },
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.code).toBe("SAFE_MODE_BLOCKED");
+ expect(res.body.error).toMatch(/prox/i);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "v0 content endpoints reject a Safe Mode org",
+ async () => {
+ const res = await request(TEST_API_URL)
+ .post("/v0/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({ url: createTestIdUrl() });
+ expect(res.statusCode).toBe(403);
+ expect(res.body.error).toMatch(/v0 API/i);
+ },
+ scrapeTimeout,
+ );
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts
index 03eedb8bb..ce25eb8f3 100644
--- a/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts
+++ b/apps/api/src/__tests__/snips/v2/team-threat-protection.test.ts
@@ -185,6 +185,13 @@ describeIf(TEST_PRODUCTION)("Team threat protection config API", () => {
expect(res.body.error).toContain('"off"');
});
+ it('PUT rejects mode "manual-only" with 403', async () => {
+ const res = await putConfigRaw({ mode: "manual-only" }, identity);
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("enforced");
+ });
+
it('PUT accepts mode "normal" with 200 (forced teams may tighten config)', async () => {
const res = await putConfigRaw({ mode: "normal" }, identity);
expect(res.statusCode).toBe(200);
diff --git a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
index c9fece440..f43c7ab42 100644
--- a/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
+++ b/apps/api/src/__tests__/snips/v2/threat-protection-enforcement.test.ts
@@ -9,7 +9,15 @@ import {
TEST_SUITE_WEBSITE,
scrapeTimeout,
} from "../lib";
-import { crawl, crawlStart, map, scrape, scrapeRaw, search } from "./lib";
+import {
+ crawl,
+ crawlStart,
+ creditUsage,
+ map,
+ scrape,
+ scrapeRaw,
+ search,
+} from "./lib";
import {
createWebRiskMockCounters,
createWebRiskMockHandler,
@@ -46,6 +54,11 @@ const RISKY_DOMAIN = "threat-risky.example.com";
// The scrape target for happy-path tests.
const CLEAN_URL = TEST_SUITE_WEBSITE;
+
+const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms));
+// Scrape credits are billed in batches; wait for the batch to land before
+// reading a credit delta.
+const sleepForBatchBilling = () => sleep(40000);
const CLEAN_DOMAIN = new URL(TEST_SUITE_WEBSITE).hostname;
// A stable cross-hostname redirect: google.com 301s to www.google.com.
@@ -427,6 +440,70 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {
);
});
+ describe("manual-only mode (lists only, no scans)", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "threat-protection-enforcement/manual-only",
+ flags: { threatProtection: "allowed" },
+ credits: 1_000_000,
+ });
+ });
+
+ it(
+ "blocks a blacklisted domain",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: `https://${BLACKLISTED_DOMAIN}/`,
+ threatProtection: {
+ mode: "manual-only",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ },
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.code).toBe("unsafe_domain_blocked");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "allows an unlisted URL without a scan fee",
+ async () => {
+ // A fresh team's credit grant can land a moment after idmux returns;
+ // poll until the baseline reflects the full grant so the delta is exact.
+ let before = 0;
+ for (let i = 0; i < 30 && before < 100_000; i++) {
+ before = (await creditUsage(identity)).remainingCredits;
+ if (before < 100_000) await sleep(1000);
+ }
+ expect(before).toBeGreaterThanOrEqual(100_000);
+
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: {
+ mode: "manual-only",
+ blacklist: [BLACKLISTED_DOMAIN],
+ },
+ },
+ identity,
+ );
+ expect(res.statusCode).toBe(200);
+
+ await sleepForBatchBilling();
+ const after = (await creditUsage(identity)).remainingCredits;
+ // One credit for the scrape itself; no +2 threat-scan surcharge.
+ expect(before - after).toBe(1);
+ },
+ scrapeTimeout + 90000,
+ );
+ });
+
describe("provider verdicts (mock Google Web Risk)", () => {
let identity: Identity;
@@ -543,6 +620,23 @@ describeIf(TEST_PRODUCTION)("Threat protection enforcement", () => {
scrapeTimeout,
);
+ it(
+ "rejects a per-request override that lowers the mode to manual-only",
+ async () => {
+ const res = await scrapeRaw(
+ {
+ url: CLEAN_URL,
+ threatProtection: { mode: "manual-only" },
+ } as any,
+ identity,
+ );
+ expect(res.statusCode).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("cannot be disabled");
+ },
+ scrapeTimeout,
+ );
+
it(
"still allows narrowing overrides (mode stays on)",
async () => {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index da77be26b..5227beabd 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -43,6 +43,10 @@ import {
isThreatProtectionForced,
THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/threat-protection/request";
+import {
+ getSafeMode,
+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
export async function crawlController(req: Request, res: Response) {
@@ -68,6 +72,12 @@ export async function crawlController(req: Request, res: Response) {
});
}
+ if (getSafeMode(chunk?.flags)) {
+ return res.status(403).json({
+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+ });
+ }
+
const id = uuidv7();
await logRequest({
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 9e65a6ff7..f0be7166f 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -30,6 +30,10 @@ import {
isThreatProtectionForced,
THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/threat-protection/request";
+import {
+ getSafeMode,
+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
async function scrapeHelper(
@@ -213,6 +217,12 @@ export async function scrapeController(req: Request, res: Response) {
});
}
+ if (getSafeMode(chunk?.flags)) {
+ return res.status(403).json({
+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+ });
+ }
+
const jobId = uuidv7();
await logRequest({
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index a8be55b69..c924b1d6a 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -27,6 +27,10 @@ import {
isThreatProtectionForced,
THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/threat-protection/request";
+import {
+ getSafeMode,
+ SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
async function searchHelper(
@@ -208,6 +212,12 @@ export async function searchController(req: Request, res: Response) {
});
}
+ if (getSafeMode(chunk?.flags)) {
+ return res.status(403).json({
+ error: SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
+ });
+ }
+
const jobId = uuidv7();
await logRequest({
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 04533e3eb..9f4dbfe3d 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -26,6 +26,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
+import { resolveSafeMode } from "../../lib/safe-mode";
import {
checkUrlsAgainstThreatPolicy,
resolveThreatProtection,
@@ -59,11 +60,21 @@ export async function batchScrapeController(
req.body = batchScrapeRequestSchema.parse(req.body);
}
+ const safeMode = resolveSafeMode(req.acuc?.flags, req.body.safeMode);
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ } as any);
+ }
+
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -74,10 +85,12 @@ export async function batchScrapeController(
const permissions = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -95,8 +108,11 @@ export async function batchScrapeController(
});
}
+ // Safe Mode lockdown is cache-only, which implies zero data retention.
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body.zeroDataRetention ||
+ (safeMode.safeMode?.lockdown ?? false);
const id = req.body.appendToId ?? uuidv7();
const logger = _logger.child({
@@ -308,6 +324,11 @@ export async function batchScrapeController(
req.body.timeout,
req.auth.team_id,
);
+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default the
+ // scrape backstop applies β€” keep only a maxAge the request actually sent.
+ if (safeMode.safeMode?.lockdown && preNormalizedBody.maxAge === undefined) {
+ scrapeOptions.maxAge = undefined;
+ }
const sc: StoredCrawl = req.body.appendToId
? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
@@ -325,6 +346,9 @@ export async function batchScrapeController(
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
+ teamFlags: req.acuc?.flags ?? undefined,
+ safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
@@ -340,6 +364,13 @@ export async function batchScrapeController(
error: "Job not found",
});
}
+ if (req.body.appendToId && sc?.internalOptions) {
+ // Refresh Safe Mode + threat-protection so appended jobs enforce the team's
+ // current policy, not whatever was stored when the batch was created.
+ sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;
+ sc.internalOptions.threatProtection = threatProtection.policy ?? undefined;
+ sc.internalOptions.safeModeBypassed = safeMode.bypassed === true;
+ }
if (!req.body.appendToId) {
sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index f5c0f3870..a494aaecb 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -18,6 +18,7 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
+import { resolveSafeMode } from "../../lib/safe-mode";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -45,14 +46,32 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
const id = uuidv7();
+
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.safeMode,
+ req.body.url,
+ );
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ } as any);
+ }
+
+ // Safe Mode lockdown is cache-only, which implies zero data retention.
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body.zeroDataRetention ||
+ (safeMode.safeMode?.lockdown ?? false);
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.scrapeOptions?.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -61,14 +80,27 @@ export async function crawlController(
});
}
+ // Scrape params live under scrapeOptions; checkPermissions reads them
+ // top-level, so spread scrapeOptions while keeping the top-level fields it
+ // also reads (zeroDataRetention, crawlerOptions.ignoreRobotsTxt) and the
+ // nested scrapeOptions (location / threatProtection).
const permissions = checkPermissions(
- { ...req.body, crawlerOptions: req.body },
+ {
+ ...req.body.scrapeOptions,
+ zeroDataRetention: req.body.zeroDataRetention,
+ crawlerOptions: req.body,
+ scrapeOptions: req.body.scrapeOptions,
+ },
req.acuc?.flags,
- { threatProtectionOrgConfig: threatProtection.orgConfig },
+ {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
+ },
);
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -184,6 +216,14 @@ export async function crawlController(
bodyScrapeOptions.timeout,
req.auth.team_id,
);
+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default the
+ // scrape backstop applies β€” keep only a maxAge the request actually sent.
+ if (
+ safeMode.safeMode?.lockdown &&
+ preNormalizedBody.scrapeOptions?.maxAge === undefined
+ ) {
+ scrapeOptions.maxAge = undefined;
+ }
// TODO: @rafa, is this right? copied from v0
if (Array.isArray(crawlerOptions.includePaths)) {
@@ -227,6 +267,9 @@ export async function crawlController(
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
+ teamFlags: req.acuc?.flags ?? undefined,
+ safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 11cf401db..d526dcfa6 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -31,6 +31,7 @@ import {
} from "../../services/index";
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
+import { resolveSafeMode } from "../../lib/safe-mode";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import {
checkUrlsAgainstThreatPolicy,
@@ -136,7 +137,14 @@ export async function getMapResults({
let links: string[] = [url];
let mapResults: MapDocument[] = [];
- const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;
+ // Safe Mode lockdown: serve links from the index only β€” skip the live
+ // fireEngineMap search and any sitemap fetch (both are live discovery).
+ // Derived from the team flags so every caller honors it.
+ const indexOnly =
+ resolveSafeMode(flags, undefined, url).safeMode?.lockdown === true;
+
+ // Lockdown (index-only) is cache-only, which implies zero data retention.
+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || indexOnly;
const sc: StoredCrawl = {
originUrl: url,
@@ -210,7 +218,10 @@ export async function getMapResults({
let allResults: any[] = [];
let pagePromises: Promise<any>[] = [];
- if (cachedResult) {
+ if (indexOnly) {
+ // Lockdown: no live search discovery, serve from the index only.
+ allResults = [];
+ } else if (cachedResult) {
allResults = JSON.parse(cachedResult);
} else {
const fetchPage = async (page: number) => {
@@ -251,7 +262,7 @@ export async function getMapResults({
// If sitemap is not ignored, fetch sitemap
// This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days
- if (!ignoreSitemap) {
+ if (!ignoreSitemap && !indexOnly) {
try {
await crawler.tryGetSitemap(
urls => {
@@ -387,11 +398,37 @@ export async function mapController(
});
}
+ // Safe Mode: force domainControls so discovered links are filtered, and
+ // serve index-only under lockdown (no sitemap/robots fetch to the target).
+ const safeMode = resolveSafeMode(req.acuc?.flags, undefined, req.body.url);
+ const lockdownIndexOnly = safeMode.safeMode?.lockdown === true;
+ if (lockdownIndexOnly) {
+ // Lockdown: serve links from the index only β€” no live search / sitemap /
+ // robots discovery. A sitemapOnly request keeps its contract and simply
+ // yields no links (the sitemap can't be fetched under lockdown).
+ req.body.useIndex = true;
+ }
+ // Map's ignoreRobotsTxt is top-level, so checkPermissions (which reads it
+ // under crawlerOptions) can't see it β€” enforce robots here for Safe Mode.
+ if (
+ safeMode.safeMode?.enforceRobots &&
+ !lockdownIndexOnly &&
+ req.body.ignoreRobotsTxt
+ ) {
+ return res.status(403).json({
+ success: false,
+ code: "SAFE_MODE_BLOCKED",
+ error:
+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",
+ });
+ }
+
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -402,10 +439,12 @@ export async function mapController(
const permissions = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -430,7 +469,7 @@ export async function mapController(
origin: req.body.origin ?? "api",
integration: req.body.integration,
target_hint: req.body.url,
- zeroDataRetention: false, // not supported for map
+ zeroDataRetention: lockdownIndexOnly,
api_key_id: req.acuc?.api_key_id ?? null,
});
@@ -550,7 +589,7 @@ export async function mapController(
},
results: result.links,
credits_cost: creditsToBill,
- zeroDataRetention: false, // not supported
+ zeroDataRetention: lockdownIndexOnly,
}).catch(error => {
logger.error("Failed to log map", { error, mapId });
});
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index a35e0fbdf..7b4864ed6 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -14,6 +14,11 @@ import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
+import {
+ resolveSafeMode,
+ applySafeMode,
+ isLockdownZeroDataRetention,
+} from "../../lib/safe-mode";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -51,10 +56,13 @@ export async function scrapeController(
res: Response<ScrapeResponse>,
) {
// Resolved before the root span starts so the whole request trace stays
- // unrecorded for zero-data-retention requests (see otel-tracer).
+ // unrecorded for zero-data-retention requests (see otel-tracer). Safe Mode
+ // lockdown implies ZDR, so fold it in here too β€” otherwise child spans could
+ // export target URLs before the inner handler applies lockdown.
const zeroDataRetentionTrace =
getScrapeZDR(req.acuc?.flags) === "forced" ||
- req.body?.zeroDataRetention === true;
+ req.body?.zeroDataRetention === true ||
+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);
return withSpan(
"api.scrape.request",
@@ -84,11 +92,26 @@ async function scrapeControllerInner(
const preNormalizedBody = { ...req.body };
req.body = scrapeRequestSchema.parse(req.body);
+ // Honor a per-request Safe Mode opt-out/affirmation, consistent with v2.
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.safeMode,
+ req.body.url,
+ );
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ } as any);
+ }
+
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -99,10 +122,12 @@ async function scrapeControllerInner(
const permissions = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -121,7 +146,9 @@ async function scrapeControllerInner(
}
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body.zeroDataRetention ||
+ (safeMode.safeMode?.lockdown ?? false);
const logger = _logger.child({
method: "scrapeController",
@@ -172,6 +199,12 @@ async function scrapeControllerInner(
req.body.timeout,
req.auth.team_id,
);
+ // v1 prefaults maxAge (1 day), which would shadow the lockdown default β€”
+ // keep only a maxAge the request actually sent under forced lockdown.
+ if (safeMode.safeMode?.lockdown && preNormalizedBody.maxAge === undefined) {
+ scrapeOptions.maxAge = undefined;
+ }
+ applySafeMode(safeMode.safeMode, scrapeOptions);
const projectedKeylessCredits = !isDirectToBullMQ
? projectScrapeCredits(
scrapeOptions,
@@ -267,6 +300,8 @@ async function scrapeControllerInner(
orgId: req.acuc?.org_id ?? null,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ safeMode: safeMode.allowlisted ? undefined : safeMode.safeMode,
+ safeModeBypassed: safeMode.bypassed === true,
},
skipNuq: true,
origin,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index c5f7040aa..66b0c81b6 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -23,6 +23,11 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { z } from "zod";
import { executeSearch } from "../../search/execute";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import {
+ resolveSafeMode,
+ getEffectiveSearchForcedKind,
+} from "../../lib/safe-mode";
+import { checkPermissions } from "../../lib/permissions";
import {
DocumentWithCostTracking,
scrapeSearchResults,
@@ -32,7 +37,6 @@ import {
filterDocumentsWithContent,
} from "../../search/transform";
import { fromV1ScrapeOptions } from "../v2/types";
-import { getSearchForcedKind } from "../../lib/zdr-helpers";
import {
adjustKeylessCredits,
keylessLimitBody,
@@ -102,7 +106,12 @@ export async function searchController(
const controllerStartTime = new Date().getTime();
const jobId = uuidv7();
- const teamForcedKind = getSearchForcedKind(req.acuc?.flags);
+ // Safe Mode lockdown forces the "zdr" kind like the searchZDR flag does
+ // (see getEffectiveSearchForcedKind).
+ const teamForcedKind = getEffectiveSearchForcedKind(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.safeMode,
+ );
const zeroDataRetention = teamForcedKind !== null;
const teamEnterprise = teamForcedKind ? [teamForcedKind] : undefined;
let logger = _logger.child({
@@ -154,6 +163,21 @@ export async function searchController(
origin: req.body.origin,
});
+ // Safe Mode: validate the per-request param up front and reject scrape
+ // options it forbids (the worker backstop would otherwise strip them
+ // silently). Domain controls force threat protection over the results.
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.safeMode,
+ );
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ });
+ }
+
// Threat protection: resolve the effective policy. Blocked domains are
// removed from search results entirely.
const threatProtection = await resolveThreatProtection({
@@ -162,6 +186,7 @@ export async function searchController(
flags: req.acuc?.flags ?? null,
override:
req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -170,6 +195,30 @@ export async function searchController(
});
}
+ // Search only scrapes (and only honors scrapeOptions) when formats are
+ // requested, so the scrape-option checks apply only then.
+ if (
+ safeMode.safeMode &&
+ requestedFormats.length > 0 &&
+ req.body.scrapeOptions
+ ) {
+ const permissions = checkPermissions(
+ req.body.scrapeOptions,
+ req.acuc?.flags,
+ {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode,
+ },
+ );
+ if (permissions.error) {
+ return res.status(403).json({
+ success: false,
+ code: permissions.code,
+ error: permissions.error,
+ });
+ }
+ }
+
await logRequest({
id: jobId,
kind: "search",
@@ -250,6 +299,7 @@ export async function searchController(
agentIndexOnly: (req as any).agentIndexOnly ?? false,
keylessReserved: reservedKeylessCredits > 0,
threatProtectionPolicy: threatProtection.policy,
+ safeModeBypassed: safeMode.bypassed === true,
},
logger,
);
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 78281cdf2..1954b9ce4 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -551,6 +551,10 @@ const baseScrapeOptions = z.strictObject({
.gte(0)
.prefault(1 * 24 * 60 * 60 * 1000),
storeInCache: z.boolean().prefault(true),
+ // Enterprise: per-request Safe Mode opt-out/affirmation. Tri-state, so NOT
+ // prefaulted β€” an absent value must stay undefined (a default of false would
+ // read as an explicit bypass attempt). Honored only if the org allows it.
+ safeMode: z.boolean().optional(),
// Enterprise: per-request field-level override of the org's threat
// protection policy. Gated on the team flag + org config (checkPermissions).
threatProtection: threatProtectionOverrideSchema.optional(),
@@ -1334,6 +1338,22 @@ export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
+ safeMode?: boolean;
+ safeModeConfig?: {
+ allowBypassSafeMode?: boolean;
+ lockdown?: boolean;
+ domainControls?: boolean;
+ enforceRobots?: boolean;
+ disableStealthProxy?: boolean;
+ disableAuthentication?: boolean;
+ disableSiteHandling?: boolean;
+ exposeWebdriver?: boolean;
+ useHeadlessUserAgent?: boolean;
+ disablePlatformSelection?: boolean;
+ disableCountrySelection?: boolean;
+ disableAutomaticReferrer?: boolean;
+ allowlist?: string[];
+ };
siemLogging?: boolean;
unblockedDomains?: string[];
forceZDR?: boolean;
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 280d62407..8aeedcdda 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -26,6 +26,7 @@ import { logger as _logger } from "../../lib/logger";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
+import { resolveSafeMode } from "../../lib/safe-mode";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -61,11 +62,23 @@ export async function batchScrapeController(
req.body = batchScrapeRequestSchema.parse(req.body);
}
+ // Batch has many URLs; per-URL allowlist is applied at the scrapeURL backstop.
+ // Resolve org-level here only for the bypass/enablement decision.
+ const safeMode = resolveSafeMode(req.acuc?.flags, req.body.safeMode);
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ });
+ }
+
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -76,10 +89,12 @@ export async function batchScrapeController(
const permissions = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -97,9 +112,11 @@ export async function batchScrapeController(
});
}
+ // Safe Mode lockdown is cache-only, which implies zero data retention.
const zeroDataRetention =
getScrapeZDR(req.acuc?.flags) === "forced" ||
- (req.body.zeroDataRetention ?? false);
+ (req.body.zeroDataRetention ?? false) ||
+ (safeMode.safeMode?.lockdown ?? false);
if (
req.body.__agentInterop &&
@@ -347,6 +364,10 @@ export async function batchScrapeController(
bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ // Safe Mode rides the batch payload so each URL resolves it per-URL
+ // at the scrapeURL backstop.
+ teamFlags: req.acuc?.flags ?? undefined,
+ safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
createdAt: Date.now(),
@@ -364,6 +385,12 @@ export async function batchScrapeController(
error: "Job not found",
});
}
+ // Refresh Safe Mode + threat-protection context so appended jobs enforce
+ // the team's current policy, not whatever was stored when the batch was
+ // first created.
+ sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;
+ sc.internalOptions.safeModeBypassed = safeMode.bypassed === true;
+ sc.internalOptions.threatProtection = threatProtection.policy ?? undefined;
}
if (!req.body.appendToId) {
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index a8856de58..ae4e7ccfe 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -37,6 +37,10 @@ import {
import { autumnService } from "../../services/autumn/autumn.service";
import { orgIdForTeam } from "../../lib/team-org";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
+import {
+ getSafeMode,
+ SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
+} from "../../lib/safe-mode";
// ---------------------------------------------------------------------------
// Zod schemas
@@ -226,6 +230,13 @@ export async function browserCreateController(
req.body = browserCreateRequestSchema.parse(req.body);
+ if (getSafeMode(req.acuc?.flags)) {
+ return res.status(403).json({
+ success: false,
+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
+ });
+ }
+
if (
req.body.__agentInterop &&
config.AGENT_INTEROP_SECRET &&
@@ -471,6 +482,13 @@ export async function browserExecuteController(
req.body = browserExecuteRequestSchema.parse(req.body);
+ if (getSafeMode(req.acuc?.flags)) {
+ return res.status(403).json({
+ success: false,
+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
+ });
+ }
+
const id = req.params.sessionId;
const { code, language, timeout, origin } = req.body;
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index d2e50069a..4c051a8d0 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -19,6 +19,7 @@ import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
+import { resolveSafeMode } from "../../lib/safe-mode";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -48,14 +49,32 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
const id = uuidv7();
+
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.safeMode,
+ req.body.url,
+ );
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ });
+ }
+
+ // Safe Mode lockdown is cache-only, which implies zero data retention.
const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body.zeroDataRetention ||
+ (safeMode.safeMode?.lockdown ?? false);
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.scrapeOptions?.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -64,14 +83,27 @@ export async function crawlController(
});
}
+ // Scrape params live under scrapeOptions; checkPermissions reads them
+ // top-level, so spread scrapeOptions while keeping the top-level fields it
+ // also reads (zeroDataRetention, crawlerOptions.ignoreRobotsTxt) and the
+ // nested scrapeOptions (location / threatProtection).
const permissions = checkPermissions(
- { ...req.body, crawlerOptions: req.body },
+ {
+ ...req.body.scrapeOptions,
+ zeroDataRetention: req.body.zeroDataRetention,
+ crawlerOptions: req.body,
+ scrapeOptions: req.body.scrapeOptions,
+ },
req.acuc?.flags,
- { threatProtectionOrgConfig: threatProtection.orgConfig },
+ {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
+ },
);
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -286,6 +318,10 @@ export async function crawlController(
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ // Safe Mode rides the crawl payload so every child scrape resolves it
+ // per-URL at the scrapeURL backstop (allowlist applies per child).
+ teamFlags: req.acuc?.flags ?? undefined,
+ safeModeBypassed: safeMode.bypassed === true,
},
team_id: req.auth.team_id,
createdAt: Date.now(),
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 6167c6a47..7e05d3d22 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -12,6 +12,10 @@ import { externalRequestId } from "../../lib/external-request-id";
import { logger as _logger } from "../../lib/logger";
import { MapTimeoutError, MapFailedError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
+import {
+ resolveSafeMode,
+ isLockdownZeroDataRetention,
+} from "../../lib/safe-mode";
import { getMapResults, MapResult } from "../../lib/map-utils";
import { v7 as uuidv7 } from "uuid";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
@@ -29,12 +33,16 @@ export async function mapController(
req: RequestWithAuth<{}, MapResponse, MapRequest>,
res: Response<MapResponse>,
) {
+ // Safe Mode lockdown is cache-only, which implies zero data retention.
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ isLockdownZeroDataRetention(req.acuc?.flags, undefined);
const logger = _logger.child({
jobId: uuidv7(),
teamId: req.auth.team_id,
module: "api/v2",
method: "mapController",
- zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",
+ zeroDataRetention,
});
// Get timing data from middleware (includes all middleware processing time)
const middlewareStartTime =
@@ -44,11 +52,36 @@ export async function mapController(
const originalRequest = req.body;
req.body = mapRequestSchema.parse(req.body);
+ // Map is link discovery, not content scraping, but it is outbound to the
+ // target. Under Safe Mode: force domainControls so discovered links are
+ // filtered, and under lockdown serve from the index only (no sitemap/robots
+ // fetch to the target).
+ const safeMode = resolveSafeMode(req.acuc?.flags, undefined, req.body.url);
+ const lockdownIndexOnly = safeMode.safeMode?.lockdown === true;
+ if (lockdownIndexOnly) {
+ req.body.useIndex = true;
+ }
+ // Map's ignoreRobotsTxt is top-level, so checkPermissions (which reads it
+ // under crawlerOptions) can't see it β€” enforce robots here for Safe Mode.
+ if (
+ safeMode.safeMode?.enforceRobots &&
+ !lockdownIndexOnly &&
+ req.body.ignoreRobotsTxt
+ ) {
+ return res.status(403).json({
+ success: false,
+ code: "SAFE_MODE_BLOCKED",
+ error:
+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",
+ });
+ }
+
const threatProtection = await resolveThreatProtection({
teamId: req.auth.team_id,
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -59,10 +92,12 @@ export async function mapController(
const permissions = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
if (permissions.error) {
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -87,17 +122,17 @@ export async function mapController(
origin: req.body.origin ?? "api",
integration: req.body.integration,
target_hint: req.body.url,
- zeroDataRetention: false, // not supported for map
+ zeroDataRetention,
api_key_id: req.acuc?.api_key_id ?? null,
});
- // Short-circuit: if the URL matches avgrab's resolve pattern, delegate entirely
+ // Short-circuit: if the URL matches avgrab's resolve pattern, delegate
+ // entirely. Skipped under Safe Mode so results still flow through the
+ // standard path (domain-controls filtering + lockdown index-only).
try {
- const avgrabResults = await resolveViaAvgrab(
- req.body.url,
- req.body.limit,
- logger,
- );
+ const avgrabResults = safeMode.safeMode
+ ? null
+ : await resolveViaAvgrab(req.body.url, req.body.limit, logger);
if (avgrabResults !== null) {
const creditsCost = avgrabResults.length;
@@ -280,7 +315,7 @@ export async function mapController(
},
results: result.mapResults,
credits_cost: creditsToBill,
- zeroDataRetention: false, // not supported
+ zeroDataRetention,
}).catch(error => {
logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);
});
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 5a47b46e6..f54a0ebfc 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -360,6 +360,7 @@ export async function parseController(
});
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 685e21b4b..cf8d9e363 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -6,7 +6,7 @@ import { logger as rootLogger } from "../../lib/logger";
import { fetchResearchUpstream } from "../../lib/research-upstream";
import { chargeKeylessCredits } from "../../lib/keyless";
import { billTeam } from "../../services/billing/credit_billing";
-import { getSearchForcedKind } from "../../lib/zdr-helpers";
+import { getEffectiveSearchForcedKind } from "../../lib/safe-mode";
import {
logRequest,
logResearchEndpoint,
@@ -223,7 +223,7 @@ function creditsFor(
if (freeResearchKinds.has(config.kind)) return 0;
if (config.billAs === "scrape") return 1;
- const forcedKind = getSearchForcedKind(req.acuc?.flags);
+ const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);
const perTen =
forcedKind === "zdr"
? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS
@@ -286,7 +286,7 @@ function createResearchController(
return async (req, res: Response) => {
const authedReq = req as RequestWithAuth<any, any, any>;
const zeroDataRetention =
- getSearchForcedKind(authedReq.acuc?.flags) !== null;
+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;
const started = Date.now();
const jobId = uuidv7();
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 5d35020a2..e0e5273f4 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -43,6 +43,10 @@ import {
} from "../../lib/scrape-interact/browser-agent";
import { sanitizeUrlForTrace } from "../../lib/scrape-interact/langsmith";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import {
+ getSafeMode,
+ SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
+} from "../../lib/safe-mode";
import { RequestWithAuth, ScrapeOptions } from "./types";
import { billTeam } from "../../services/billing/credit_billing";
import {
@@ -136,6 +140,13 @@ export async function scrapeInteractController(
) {
req.body = browserExecuteRequestSchema.parse(req.body);
+ if (getSafeMode(req.acuc?.flags)) {
+ return res.status(403).json({
+ success: false,
+ error: SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
+ });
+ }
+
const scrapeId = req.params.jobId;
const { code: rawCode, prompt, language, timeout, origin } = req.body;
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index c4f73c5a7..f11b2deb4 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -19,6 +19,11 @@ import {
} from "../../lib/error";
import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
+import {
+ applySafeMode,
+ resolveSafeMode,
+ isLockdownZeroDataRetention,
+} from "../../lib/safe-mode";
import {
actionTypesOf,
checkKeyFormatRestriction,
@@ -43,6 +48,7 @@ import {
import { projectScrapeCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
@@ -55,11 +61,14 @@ export async function scrapeController(
if (req.body && "alexandria" in req.body)
return providerScrapeController(req, res);
// Resolved before the root span starts so the whole request trace stays
- // unrecorded for zero-data-retention requests (see otel-tracer).
+ // unrecorded for zero-data-retention requests (see otel-tracer). Safe Mode
+ // lockdown implies ZDR, so fold it in here too β€” otherwise the rejection
+ // paths below would export the target URL on the root span.
const zeroDataRetentionTrace =
getScrapeZDR(req.acuc?.flags) === "forced" ||
req.body?.zeroDataRetention === true ||
- req.body?.lockdown === true;
+ req.body?.lockdown === true ||
+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);
return withSpan(
"api.scrape.request",
@@ -89,6 +98,43 @@ export async function scrapeController(
});
});
+ const emitSafeModeRejection = (message: string) =>
+ emitRejectedScrapeActivityEvent({
+ scrapeId: jobId,
+ requestId: jobId,
+ endpoint: "scrape",
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ url: req.body.url,
+ error: new TransportableError("SAFE_MODE_BLOCKED", message),
+ origin: req.body.origin ?? "api",
+ integration: req.body.integration,
+ zeroDataRetention: req.body.zeroDataRetention ?? false,
+ });
+
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.safeMode,
+ req.body.url,
+ );
+ if (safeMode.error) {
+ setSpanAttributes(span, {
+ "scrape.error": safeMode.error,
+ "scrape.status_code": 403,
+ });
+ emitSafeModeRejection(safeMode.error);
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ });
+ }
+ setSpanAttributes(span, {
+ "scrape.safe_mode": safeMode.safeMode !== undefined,
+ "scrape.safe_mode_bypassed": safeMode.bypassed === true,
+ "scrape.safe_mode_allowlisted": safeMode.allowlisted === true,
+ });
+
// Threat protection: resolve the effective policy (org config +
// per-request override). No-ops (null policy, zero I/O) for teams
// without the flag.
@@ -97,24 +143,28 @@ export async function scrapeController(
orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
override: req.body.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
setSpanAttributes(span, {
"scrape.error": threatProtection.error,
"scrape.status_code": 403,
});
+ if (safeMode.safeMode?.domainControls) {
+ emitSafeModeRejection(threatProtection.error);
+ }
return res.status(403).json({
success: false,
error: threatProtection.error,
});
}
-
// Permission check span
const permissions = await withSpan(
"api.scrape.check_permissions",
async permSpan => {
const perms = checkPermissions(req.body, req.acuc?.flags, {
threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode ?? null,
});
setSpanAttributes(permSpan, {
"permissions.success": !perms.error,
@@ -129,8 +179,12 @@ export async function scrapeController(
"scrape.error": permissions.error,
"scrape.status_code": 403,
});
+ if (permissions.code === "SAFE_MODE_BLOCKED") {
+ emitSafeModeRejection(permissions.error);
+ }
return res.status(403).json({
success: false,
+ code: permissions.code,
error: permissions.error,
});
}
@@ -152,6 +206,8 @@ export async function scrapeController(
});
}
+ applySafeMode(safeMode.safeMode, req.body);
+
const zeroDataRetention =
getScrapeZDR(req.acuc?.flags) === "forced" ||
(req.body.zeroDataRetention ?? false) ||
@@ -230,6 +286,12 @@ export async function scrapeController(
const middlewareTime = controllerStartTime - middlewareStartTime;
+ if (safeMode.bypassed) {
+ logger.info("Safe Mode bypassed by request", {
+ apiKeyId: req.acuc?.api_key_id,
+ });
+ }
+
logger.debug("Scrape " + jobId + " starting", {
version: "v2",
scrapeId: jobId,
@@ -362,6 +424,10 @@ export async function scrapeController(
teamConcurrency: baseConcurrency,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
+ safeMode: safeMode.allowlisted
+ ? undefined
+ : safeMode.safeMode,
+ safeModeBypassed: safeMode.bypassed === true,
},
skipNuq: true,
origin,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index c640def0b..f4b30fbbd 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -38,6 +38,12 @@ import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { isToolsOnlySearch } from "../../search/alexandria";
+import {
+ resolveSafeMode,
+ isLockdownZeroDataRetention,
+ getEffectiveSearchForcedKind,
+} from "../../lib/safe-mode";
+import { checkPermissions } from "../../lib/permissions";
import {
actionTypesOf,
checkKeyEndpointRestriction,
@@ -79,7 +85,11 @@ export async function searchController(
const zeroDataRetentionTrace =
Boolean(getSearchForcedKind(req.acuc?.flags)) ||
enterprise.includes("zdr") ||
- enterprise.includes("anon");
+ enterprise.includes("anon") ||
+ isLockdownZeroDataRetention(
+ req.acuc?.flags,
+ req.body?.scrapeOptions?.safeMode,
+ );
return withSpan(
"api.search.request",
@@ -106,7 +116,13 @@ async function searchControllerInner(
const jobId = uuidv7();
const searchZDRMode = getSearchZDR(req.acuc?.flags);
- const teamForcedKind = getSearchForcedKind(req.acuc?.flags);
+ // Safe Mode lockdown forces the "zdr" kind like the searchZDR flag does
+ // (see getEffectiveSearchForcedKind).
+ const flagForcedKind = getSearchForcedKind(req.acuc?.flags);
+ const teamForcedKind = getEffectiveSearchForcedKind(
+ req.acuc?.flags,
+ req.body?.scrapeOptions?.safeMode,
+ );
let logger = _logger.child({
jobId,
teamId: req.auth.team_id,
@@ -206,6 +222,21 @@ async function searchControllerInner(
});
}
+ // Safe Mode: validate the per-request param up front and reject scrape
+ // options it forbids (the worker backstop would otherwise strip them
+ // silently). Domain controls force threat protection over the results.
+ const safeMode = resolveSafeMode(
+ req.acuc?.flags,
+ req.body.scrapeOptions?.safeMode,
+ );
+ if (safeMode.error) {
+ return res.status(403).json({
+ success: false,
+ code: safeMode.code,
+ error: safeMode.error,
+ });
+ }
+
// Threat protection: resolve the effective policy. Blocked domains are
// removed from search results entirely, and scraped results inherit the
// policy through the scrape pipeline.
@@ -215,6 +246,7 @@ async function searchControllerInner(
flags: req.acuc?.flags ?? null,
override:
req.body.threatProtection ?? req.body.scrapeOptions?.threatProtection,
+ force: safeMode.safeMode?.domainControls === true,
});
if (threatProtection.error) {
return res.status(403).json({
@@ -223,6 +255,30 @@ async function searchControllerInner(
});
}
+ // Search only scrapes (and only honors scrapeOptions) when formats are
+ // requested, so the scrape-option checks apply only then.
+ if (
+ safeMode.safeMode &&
+ requestedFormats.length > 0 &&
+ req.body.scrapeOptions
+ ) {
+ const permissions = checkPermissions(
+ req.body.scrapeOptions,
+ req.acuc?.flags,
+ {
+ threatProtectionOrgConfig: threatProtection.orgConfig,
+ safeMode: safeMode.safeMode,
+ },
+ );
+ if (permissions.error) {
+ return res.status(403).json({
+ success: false,
+ code: permissions.code,
+ error: permissions.error,
+ });
+ }
+ }
+
const shouldBill = req.body.__agentInterop?.shouldBill ?? true;
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
const billing: BillingMetadata = req.body.__agentInterop
@@ -235,6 +291,13 @@ async function searchControllerInner(
origin: req.body.origin,
});
+ // Kinds the request itself asked for, captured before the forced kind is
+ // injected: the entitlement check below applies to these only.
+ const requestedZDROrAnon =
+ req.body.enterprise?.includes("zdr") ||
+ req.body.enterprise?.includes("anon") ||
+ false;
+
// Inject the team-forced enterprise mode so downstream billing,
// upstream routing, and ZDR cleanup all see it.
if (teamForcedKind) {
@@ -251,7 +314,9 @@ async function searchControllerInner(
logger = logger.child({ zeroDataRetention });
// Verify the team has searchZDR enabled before allowing enterprise ZDR/anon
- if (isZDROrAnon && !teamForcedKind) {
+ // it asked for. Only the flag-forced kind exempts a team: a lockdown-forced
+ // "zdr" must not let an unentitled request add "anon".
+ if (requestedZDROrAnon && !flagForcedKind) {
if (searchZDRMode !== "allowed") {
return res.status(403).json({
success: false,
@@ -344,6 +409,7 @@ async function searchControllerInner(
agentIndexOnly: (req as any).agentIndexOnly ?? false,
keylessReserved: reservedKeylessCredits > 0,
threatProtectionPolicy: threatProtection.policy,
+ safeModeBypassed: safeMode.bypassed === true,
},
logger,
);
diff --git a/apps/api/src/controllers/v2/team-threat-protection.ts b/apps/api/src/controllers/v2/team-threat-protection.ts
index 6378ea3ed..f68646c6d 100644
--- a/apps/api/src/controllers/v2/team-threat-protection.ts
+++ b/apps/api/src/controllers/v2/team-threat-protection.ts
@@ -152,10 +152,10 @@ export async function putTeamThreatProtectionController(
const input = threatProtectionConfigSchema.parse(req.body);
// "forced" guarantees enforcement: the org config may be tightened, but its
- // mode may never be turned off through the API.
+ // mode may never be turned off β€” or dropped to lists-only β€” through the API.
if (
getThreatProtection(req.acuc?.flags) === "forced" &&
- input.mode === "off"
+ (input.mode === "off" || input.mode === "manual-only")
) {
res.status(403).json({
success: false,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 29e0b4b37..9cd769e9c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -813,6 +813,7 @@ const baseScrapeOptions = z.strictObject({
minAge: z.int().gte(0).optional(),
storeInCache: z.boolean().prefault(true),
lockdown: z.boolean().prefault(false),
+ safeMode: z.boolean().optional(),
redactPII: redactPIISchema,
// Enterprise: per-request field-level override of the org's threat
// protection policy. Gated on the team flag + org config (checkPermissions).
@@ -899,10 +900,7 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(
}
if (obj.lockdown && obj.maxAge === undefined) {
- // 2 years in ms. Number.MAX_SAFE_INTEGER lands ~285,000 years which
- // overflows Postgres TIMESTAMP arithmetic in the index lookup and silently
- // returns no rows. 2 years covers any practical cache retention window.
- result = { ...result, maxAge: 2 * 365 * 24 * 60 * 60 * 1000 };
+ result = { ...result, maxAge: LOCKDOWN_DEFAULT_MAX_AGE_MS };
}
return result as T extends undefined ? undefined : T;
@@ -1890,11 +1888,29 @@ type Account = {
remainingCredits: number;
};
+export const LOCKDOWN_DEFAULT_MAX_AGE_MS = 2 * 365 * 24 * 60 * 60 * 1000;
+
export type TeamFlags = {
exchangeRetrieve?: boolean;
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
+ safeMode?: boolean;
+ safeModeConfig?: {
+ allowBypassSafeMode?: boolean;
+ lockdown?: boolean;
+ domainControls?: boolean;
+ enforceRobots?: boolean;
+ disableStealthProxy?: boolean;
+ disableAuthentication?: boolean;
+ disableSiteHandling?: boolean;
+ exposeWebdriver?: boolean;
+ useHeadlessUserAgent?: boolean;
+ disablePlatformSelection?: boolean;
+ disableCountrySelection?: boolean;
+ disableAutomaticReferrer?: boolean;
+ allowlist?: string[];
+ };
siemLogging?: boolean;
unblockedDomains?: string[];
forceZDR?: boolean;
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 7c1421f81..ed461f539 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -920,12 +920,11 @@ async function startServices(command?: string[]): Promise<Services> {
)
: undefined;
- // tests hammer the API instantly, so we need to ensure it's running before launching tests
if (
command &&
Array.isArray(command) &&
command[0] === "pnpm" &&
- command[1].startsWith("test:snips")
+ (command[1].startsWith("test:snips") || command[1] === "exec")
) {
logger.info(`Waiting for API on localhost:${PORT}`);
await waitForPort(Number(PORT), "localhost");
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 52a2633bb..1132a920c 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -8,6 +8,7 @@ import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";
import type { Logger } from "winston";
import { withSpan, setSpanAttributes } from "./otel-tracer";
import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
+import { resolveSafeMode } from "./safe-mode";
import {
firstPipelineError,
reportPipelineError,
@@ -979,6 +980,15 @@ export function crawlToCrawler(
newBase?: string,
crawlerOptions?: any,
): WebCrawler {
+ // Resolve the crawl's Safe Mode once (honoring a stored request bypass) to
+ // drive both lockdown and the robots override below.
+ const crawlerSafeMode = sc.internalOptions?.safeModeBypassed
+ ? undefined
+ : resolveSafeMode(
+ sc.internalOptions?.teamFlags ?? teamFlags,
+ undefined,
+ sc.originUrl ?? undefined,
+ ).safeMode;
const crawler = new WebCrawler({
jobId: id,
initialUrl: sc.originUrl!,
@@ -1000,10 +1010,12 @@ export function crawlToCrawler(
allowExternalContentLinks:
sc.crawlerOptions?.allowExternalContentLinks ?? false,
allowSubdomains: sc.crawlerOptions?.allowSubdomains ?? false,
+ // Safe Mode enforceRobots overrides even a team's forced ignore-robots.
ignoreRobotsTxt:
- getIgnoreRobots(teamFlags) === "forced" ||
- (getIgnoreRobots(teamFlags) === "allowed" &&
- (sc.crawlerOptions?.ignoreRobotsTxt ?? false)),
+ !crawlerSafeMode?.enforceRobots &&
+ (getIgnoreRobots(teamFlags) === "forced" ||
+ (getIgnoreRobots(teamFlags) === "allowed" &&
+ (sc.crawlerOptions?.ignoreRobotsTxt ?? false))),
regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,
maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,
currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,
@@ -1012,6 +1024,8 @@ export function crawlToCrawler(
location: sc.scrapeOptions?.location,
headers: sc.scrapeOptions?.headers,
robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,
+ // Safe Mode lockdown: skip robots/sitemap discovery (outbound to target).
+ lockdown: crawlerSafeMode?.lockdown ?? false,
});
if (sc.robots !== undefined) {
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 7e2fb120f..6033b4a25 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -27,6 +27,7 @@ import {
PDFPrefetchFailed,
DocumentPrefetchFailed,
SiteError,
+ SiteRestrictionError,
SSLError,
ProxySelectionError,
AgentIndexOnlyError,
@@ -48,6 +49,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
const errorMap: Record<ErrorCodes, any> = {
// Terms responses are API-level, never transported through workers.
THIRD_PARTY_DATA_TERMS_REQUIRED: null,
+ SAFE_MODE_BLOCKED: null,
+ SCRAPE_SITE_RESTRICTION_BLOCKED: SiteRestrictionError,
SCRAPE_TIMEOUT: ScrapeJobTimeoutError,
MAP_TIMEOUT: MapTimeoutError,
UNKNOWN_ERROR: UnknownError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index a694ed141..09394ba0b 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -43,6 +43,8 @@ export type ErrorCodes =
| "BAD_REQUEST_INVALID_JSON"
| "BAD_REQUEST"
| "CONCURRENCY_QUEUE_TIMEOUT"
+ | "SAFE_MODE_BLOCKED"
+ | "SCRAPE_SITE_RESTRICTION_BLOCKED"
// Threat protection (enterprise domain risk blocking). Lowercase by design:
// this is the documented, user-facing error code for the feature.
| "unsafe_domain_blocked"
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 65fe024a8..c015d2346 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -72,6 +72,8 @@ export async function scrapeDocument(
? true
: false,
bypassBilling: true,
+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
+ teamFlags: options.flags ?? undefined,
},
origin: options.origin,
is_scrape: true,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 41324da1e..4c78b50c4 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -74,6 +74,8 @@ export async function scrapeDocument_F0(
orgId: options.orgId ?? null,
bypassBilling: true,
threatProtection: options.threatProtectionPolicy ?? undefined,
+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
+ teamFlags: options.flags ?? undefined,
},
origin: options.origin,
is_scrape: true,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 43d6fb66f..55ba4c0b5 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -8,6 +8,7 @@ import {
} from "../controllers/v2/types";
import { crawlToCrawler, StoredCrawl } from "./crawl-redis";
import { getScrapeZDR } from "./zdr-helpers";
+import { resolveSafeMode } from "./safe-mode";
import {
checkAndUpdateURLForMap,
isSameDomain,
@@ -121,7 +122,15 @@ export async function getMapResults({
}): Promise<MapResult> {
const functionStartTime = Date.now();
- const resolvedUrl = await resolveRedirects(url, abort);
+ // Safe Mode lockdown: serve links from the index only β€” skip live search +
+ // sitemap discovery. Derived here from the team flags so every caller
+ // (map, extract, llmstxt, prompt enhancement) honors it.
+ const indexOnly =
+ resolveSafeMode(flags, undefined, url).safeMode?.lockdown === true;
+
+ // Under lockdown (index-only) resolving redirects would HEAD/GET the target,
+ // which the zero-outbound guarantee forbids β€” use the URL as given.
+ const resolvedUrl = indexOnly ? url : await resolveRedirects(url, abort);
// If the resolved URL is on a different domain, replace the hostname
if (!isSameDomain(url, resolvedUrl)) {
@@ -133,7 +142,9 @@ export async function getMapResults({
const id = providedId ?? uuidv7();
let mapResults: MapDocument[] = [];
- const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;
+ // Lockdown (index-only) is cache-only, which implies zero data retention β€”
+ // and must never write its empty live-search result into the shared cache.
+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || indexOnly;
const sc: StoredCrawl = {
originUrl: url,
@@ -224,6 +235,10 @@ export async function getMapResults({
};
const fetchAllPages = async (): Promise<any[]> => {
+ if (indexOnly) {
+ // Lockdown: no live search discovery, serve from the index only.
+ return [];
+ }
if (cachedResult) {
return JSON.parse(cachedResult);
}
@@ -256,7 +271,7 @@ export async function getMapResults({
mapResults.push(...indexResults);
}
- if (crawlerOptions.sitemap === "include") {
+ if (crawlerOptions.sitemap === "include" && !indexOnly) {
try {
await crawler.tryGetSitemap(
urls => {
diff --git a/apps/api/src/lib/permissions.test.ts b/apps/api/src/lib/permissions.test.ts
index d97818301..ba3971604 100644
--- a/apps/api/src/lib/permissions.test.ts
+++ b/apps/api/src/lib/permissions.test.ts
@@ -1,4 +1,163 @@
import { checkPermissions } from "./permissions";
+import { ResolvedSafeMode } from "./safe-mode";
+
+const strictSafeMode: ResolvedSafeMode = {
+ lockdown: false,
+ domainControls: true,
+ enforceRobots: true,
+ disableStealthProxy: true,
+ disableAuthentication: true,
+ disableSiteHandling: true,
+ exposeWebdriver: true,
+ useHeadlessUserAgent: true,
+ disablePlatformSelection: true,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+};
+
+describe("checkPermissions β€” safe mode", () => {
+ it("passes untouched requests through", () => {
+ expect(checkPermissions({}, null, { safeMode: strictSafeMode })).toEqual(
+ {},
+ );
+ });
+
+ it("does nothing when safe mode is absent", () => {
+ expect(
+ checkPermissions({ proxy: "stealth" }, null, { safeMode: null }),
+ ).toEqual({});
+ });
+
+ it.each(["stealth", "enhanced"])(
+ "rejects %s proxy when stealth proxy is disabled",
+ proxy => {
+ const result = checkPermissions({ proxy }, null, {
+ safeMode: strictSafeMode,
+ });
+ expect(result.error).toMatch(/prox/i);
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ },
+ );
+
+ it("allows stealth when the org relaxes disableStealthProxy", () => {
+ expect(
+ checkPermissions({ proxy: "stealth" }, null, {
+ safeMode: { ...strictSafeMode, disableStealthProxy: false },
+ }),
+ ).toEqual({});
+ });
+
+ it("rejects ignoreRobotsTxt even when the org flag would allow it", () => {
+ const result = checkPermissions(
+ { crawlerOptions: { ignoreRobotsTxt: true } },
+ { ignoreRobots: "allowed" },
+ { safeMode: strictSafeMode },
+ );
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ });
+
+ it("keeps the ignoreRobotsTxt entitlement check under lockdown", () => {
+ // Lockdown skips the Safe Mode robots rejection, so the generic
+ // enterprise gate must still apply to a non-entitled team.
+ const result = checkPermissions(
+ { crawlerOptions: { ignoreRobotsTxt: true } },
+ null,
+ { safeMode: { ...strictSafeMode, lockdown: true } },
+ );
+ expect(result.error).toMatch(/enterprise feature/i);
+ expect(result.code).toBeUndefined();
+ });
+
+ it("allows ignoreRobotsTxt when the org relaxes enforceRobots", () => {
+ expect(
+ checkPermissions(
+ { crawlerOptions: { ignoreRobotsTxt: true } },
+ { ignoreRobots: "allowed" },
+ { safeMode: { ...strictSafeMode, enforceRobots: false } },
+ ),
+ ).toEqual({});
+ });
+
+ it("rejects profile, login actions, and credential headers", () => {
+ const requests: Parameters<typeof checkPermissions>[0][] = [
+ { profile: { name: "p" } },
+ { actions: [{ type: "press" }] },
+ { headers: { AUTHORIZATION: "x" } },
+ { headers: { "Proxy-Authorization": "x" } },
+ ];
+ for (const request of requests) {
+ const result = checkPermissions(request, null, {
+ safeMode: strictSafeMode,
+ });
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ }
+ });
+
+ it("allows profile/login/credentials when the org relaxes disableAuthentication", () => {
+ const requests: Parameters<typeof checkPermissions>[0][] = [
+ { profile: { name: "p" } },
+ { actions: [{ type: "press" }] },
+ { headers: { AUTHORIZATION: "x" } },
+ ];
+ for (const request of requests) {
+ expect(
+ checkPermissions(request, null, {
+ safeMode: { ...strictSafeMode, disableAuthentication: false },
+ }),
+ ).toEqual({});
+ }
+ });
+
+ it("allows benign actions and headers", () => {
+ expect(
+ checkPermissions(
+ {
+ actions: [{ type: "scroll" }, { type: "screenshot" }],
+ headers: { "User-Agent": "test" },
+ },
+ null,
+ { safeMode: strictSafeMode },
+ ),
+ ).toEqual({});
+ });
+
+ it("treats threat protection as forced under domainControls", () => {
+ const result = checkPermissions(
+ { threatProtection: { mode: "off" } },
+ { threatProtection: "allowed" },
+ { safeMode: strictSafeMode },
+ );
+ expect(result.error).toMatch(/cannot be disabled|disable/i);
+
+ const noFlag = checkPermissions(
+ { threatProtection: { mode: "off" } },
+ null,
+ { safeMode: { ...strictSafeMode, lockdown: true } },
+ );
+ expect(noFlag.error).toBeDefined();
+
+ expect(
+ checkPermissions({ threatProtection: { mode: "normal" } }, null, {
+ safeMode: strictSafeMode,
+ }),
+ ).toEqual({});
+ });
+
+ it("skips every rule under effective lockdown", () => {
+ expect(
+ checkPermissions(
+ {
+ proxy: "stealth",
+ profile: { name: "p" },
+ actions: [{ type: "write" }],
+ headers: { Cookie: "x" },
+ },
+ null,
+ { safeMode: { ...strictSafeMode, lockdown: true } },
+ ),
+ ).toEqual({});
+ });
+});
describe("checkPermissions β€” threat protection", () => {
const requestWithOption = { threatProtection: { mode: "normal" } };
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index bb116f47a..69e72efec 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -1,4 +1,10 @@
import { TeamFlags } from "../controllers/v2/types";
+import type { ErrorCodes } from "./error";
+import {
+ type ResolvedSafeMode,
+ SAFE_MODE_CREDENTIAL_HEADERS,
+ SAFE_MODE_LOGIN_ACTIONS,
+} from "./safe-mode";
import {
getScrapeZDR,
getIgnoreRobots,
@@ -28,6 +34,10 @@ interface APIRequest {
// Per-request threat protection policy override (field-level override of
// the org config). Presence of any value gates on the team flag.
threatProtection?: ThreatProtectionOption;
+ proxy?: string;
+ profile?: unknown;
+ actions?: { type: string }[];
+ headers?: Record<string, string>;
}
interface PermissionOptions {
@@ -37,6 +47,7 @@ interface PermissionOptions {
* threatProtection option is rejected.
*/
threatProtectionOrgConfig?: { allowRequestOverrides: boolean } | null;
+ safeMode?: ResolvedSafeMode | null;
}
const SUPPORT_EMAIL = "[email protected]";
@@ -45,7 +56,7 @@ export function checkPermissions(
request: APIRequest,
flags?: TeamFlags,
options?: PermissionOptions,
-): { error?: string } {
+): { error?: string; code?: ErrorCodes } {
// zdr perms β€” scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDR
const scrapeMode = getScrapeZDR(flags);
if (
@@ -63,7 +74,14 @@ export function checkPermissions(
if (
request.crawlerOptions?.ignoreRobotsTxt &&
robotsMode !== "allowed" &&
- robotsMode !== "forced"
+ robotsMode !== "forced" &&
+ // Under Safe Mode (outside lockdown) the robots rejection below returns
+ // SAFE_MODE_BLOCKED; don't preempt it with the generic entitlement error.
+ !(
+ options?.safeMode &&
+ !options.safeMode.lockdown &&
+ options.safeMode.enforceRobots
+ )
) {
return {
error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
@@ -87,18 +105,72 @@ export function checkPermissions(
const threatProtectionOption =
request.threatProtection ?? request.scrapeOptions?.threatProtection;
if (threatProtectionOption !== undefined) {
- const threatMode = getThreatProtection(flags);
- if (threatMode !== "allowed" && threatMode !== "forced") {
+ const effectiveThreatMode =
+ options?.safeMode?.domainControls === true
+ ? "forced"
+ : getThreatProtection(flags);
+ if (effectiveThreatMode !== "allowed" && effectiveThreatMode !== "forced") {
return { error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE };
}
if (options?.threatProtectionOrgConfig?.allowRequestOverrides === false) {
return { error: THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE };
}
- if (threatMode === "forced" && threatProtectionOption.mode === "off") {
+ if (
+ effectiveThreatMode === "forced" &&
+ threatProtectionOption.mode === "off"
+ ) {
return { error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE };
}
}
+ const safeMode = options?.safeMode;
+ if (safeMode && !safeMode.lockdown) {
+ if (
+ safeMode.disableStealthProxy &&
+ (request.proxy === "stealth" || request.proxy === "enhanced")
+ ) {
+ return {
+ error:
+ 'Safe Mode: stealth and enhanced proxies are not allowed for your organization. Remove the proxy option or use proxy: "basic".',
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ if (safeMode.enforceRobots && request.crawlerOptions?.ignoreRobotsTxt) {
+ return {
+ error:
+ "Safe Mode: robots.txt is always honored for your organization; the ignoreRobotsTxt parameter is not allowed.",
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ if (safeMode.disableAuthentication) {
+ if (request.profile !== undefined) {
+ return {
+ error:
+ "Safe Mode: browser profiles are not allowed for your organization (nothing behind a login).",
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ const loginAction = (request.actions ?? []).find(a =>
+ SAFE_MODE_LOGIN_ACTIONS.includes(a.type),
+ );
+ if (loginAction) {
+ return {
+ error: `Safe Mode: the ${loginAction.type} action is not allowed for your organization (nothing behind a login).`,
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ const credentialHeader = Object.keys(request.headers ?? {}).find(h =>
+ SAFE_MODE_CREDENTIAL_HEADERS.includes(h.toLowerCase()),
+ );
+ if (credentialHeader) {
+ return {
+ error: `Safe Mode: the ${credentialHeader} header is not allowed for your organization (nothing behind a login).`,
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ }
+ }
+
// ip whitelist perms
const needsWhitelist =
request.location?.country === "us-whitelist" ||
diff --git a/apps/api/src/lib/safe-mode.test.ts b/apps/api/src/lib/safe-mode.test.ts
new file mode 100644
index 000000000..d9bad9744
--- /dev/null
+++ b/apps/api/src/lib/safe-mode.test.ts
@@ -0,0 +1,325 @@
+import {
+ applySafeMode,
+ getSafeMode,
+ isLockdownZeroDataRetention,
+ resolveSafeMode,
+ ResolvedSafeMode,
+ SafeModeConfig,
+} from "./safe-mode";
+
+// Every sub-control is polarised so that `true` == the stricter setting.
+// Under the org defaults, that means every capability control is `true`
+// (strictest), except `lockdown` which defaults off.
+const strict: ResolvedSafeMode = {
+ lockdown: false,
+ domainControls: true,
+ enforceRobots: true,
+ disableStealthProxy: true,
+ disableAuthentication: true,
+ disableSiteHandling: true,
+ exposeWebdriver: true,
+ useHeadlessUserAgent: true,
+ disablePlatformSelection: true,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+};
+
+// The per-capability controls the allowlist relaxes (everything except the
+// org-wide lockdown + domainControls gates, which the allowlist never touches).
+const RELAXABLE = [
+ "enforceRobots",
+ "disableStealthProxy",
+ "disableAuthentication",
+ "disableSiteHandling",
+ "exposeWebdriver",
+ "useHeadlessUserAgent",
+ "disablePlatformSelection",
+ "disableCountrySelection",
+ "disableAutomaticReferrer",
+] as const;
+
+// The allowlisted posture: every relaxable capability flipped to its lenient
+// value, lockdown + domainControls untouched.
+const allCapabilitiesAllowed: ResolvedSafeMode = {
+ ...strict,
+ ...Object.fromEntries(RELAXABLE.map(k => [k, false])),
+} as ResolvedSafeMode;
+
+describe("getSafeMode", () => {
+ it("is false for absent or null flags", () => {
+ expect(getSafeMode(undefined)).toBe(false);
+ expect(getSafeMode(null)).toBe(false);
+ expect(getSafeMode({})).toBe(false);
+ });
+
+ it("is true only when the flag is exactly true", () => {
+ expect(getSafeMode({ safeMode: true })).toBe(true);
+ expect(getSafeMode({ safeMode: false })).toBe(false);
+ });
+});
+
+describe("isLockdownZeroDataRetention", () => {
+ const lockdownOrg = { safeMode: true, safeModeConfig: { lockdown: true } };
+
+ it("is false without org lockdown", () => {
+ expect(isLockdownZeroDataRetention(null, undefined)).toBe(false);
+ expect(isLockdownZeroDataRetention({ safeMode: true }, undefined)).toBe(
+ false,
+ );
+ });
+
+ it("is true for a lockdown org, including a rejected bypass attempt", () => {
+ expect(isLockdownZeroDataRetention(lockdownOrg, undefined)).toBe(true);
+ expect(isLockdownZeroDataRetention(lockdownOrg, true)).toBe(true);
+ expect(isLockdownZeroDataRetention(lockdownOrg, false)).toBe(true);
+ });
+
+ it("is false only when a bypass is actually honored", () => {
+ const bypassable = {
+ safeMode: true,
+ safeModeConfig: { lockdown: true, allowBypassSafeMode: true },
+ };
+ expect(isLockdownZeroDataRetention(bypassable, false)).toBe(false);
+ expect(isLockdownZeroDataRetention(bypassable, undefined)).toBe(true);
+ });
+});
+
+describe("resolveSafeMode β€” org flag off", () => {
+ it("resolves to nothing when the request says nothing", () => {
+ expect(resolveSafeMode(null, undefined)).toEqual({});
+ expect(resolveSafeMode({}, undefined)).toEqual({});
+ });
+
+ it("treats safeMode: false as a no-op", () => {
+ expect(resolveSafeMode({}, false)).toEqual({});
+ });
+
+ it("rejects safeMode: true", () => {
+ const result = resolveSafeMode({}, true);
+ expect(result.error).toMatch(/not enabled/i);
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ expect(result.safeMode).toBeUndefined();
+ });
+});
+
+describe("resolveSafeMode β€” org flag on", () => {
+ const flags = { safeMode: true };
+
+ it("resolves strict defaults with lockdown off", () => {
+ expect(resolveSafeMode(flags, undefined)).toEqual({ safeMode: strict });
+ });
+
+ it("every capability control defaults to its strictest (true), lockdown off", () => {
+ const resolved = resolveSafeMode(flags, undefined).safeMode!;
+ expect(resolved.lockdown).toBe(false);
+ for (const key of [
+ "domainControls",
+ ...RELAXABLE,
+ ] as (keyof ResolvedSafeMode)[]) {
+ expect(resolved[key]).toBe(true);
+ }
+ });
+
+ it("treats safeMode: true as a redundant affirmation, not an error", () => {
+ expect(resolveSafeMode(flags, true).safeMode).toBeDefined();
+ expect(resolveSafeMode(flags, true).error).toBeUndefined();
+ });
+
+ it("merges partial org config over the defaults", () => {
+ const result = resolveSafeMode(
+ {
+ safeMode: true,
+ safeModeConfig: { lockdown: true, disableStealthProxy: false },
+ },
+ undefined,
+ );
+ expect(result.safeMode).toEqual({
+ ...strict,
+ lockdown: true,
+ disableStealthProxy: false,
+ });
+ });
+
+ it("lets the org relax each capability independently", () => {
+ for (const key of RELAXABLE) {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: { [key]: false } },
+ undefined,
+ );
+ expect(result.safeMode).toEqual({ ...strict, [key]: false });
+ }
+ });
+
+ it("lets the org opt into lockdown", () => {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: { lockdown: true } },
+ undefined,
+ );
+ expect(result.safeMode).toEqual({ ...strict, lockdown: true });
+ });
+
+ it("rejects a bypass when allowBypassSafeMode is unset or false", () => {
+ const configs: (SafeModeConfig | undefined)[] = [
+ undefined,
+ {},
+ { allowBypassSafeMode: false },
+ ];
+ for (const config of configs) {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: config },
+ false,
+ );
+ expect(result.error).toMatch(/disable Safe Mode/i);
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ expect(result.safeMode).toBeUndefined();
+ expect(result.bypassed).toBeUndefined();
+ }
+ });
+
+ it("enforces the bypass gate even under org-configured lockdown", () => {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: { lockdown: true } },
+ false,
+ );
+ expect(result.error).toMatch(/disable Safe Mode/i);
+ expect(result.code).toBe("SAFE_MODE_BLOCKED");
+ });
+
+ it("honors a bypass when allowBypassSafeMode is true", () => {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: { allowBypassSafeMode: true } },
+ false,
+ );
+ expect(result).toEqual({ bypassed: true });
+ });
+
+ it("still enforces when allowBypassSafeMode is true but no bypass is requested", () => {
+ const result = resolveSafeMode(
+ { safeMode: true, safeModeConfig: { allowBypassSafeMode: true } },
+ undefined,
+ );
+ expect(result.safeMode).toBeDefined();
+ expect(result.bypassed).toBeUndefined();
+ });
+});
+
+describe("resolveSafeMode β€” allowlist", () => {
+ const flags = {
+ safeMode: true,
+ safeModeConfig: { allowlist: ["docs.example.com", "*.trusted.example"] },
+ };
+
+ it("relaxes every capability but keeps lockdown + domainControls", () => {
+ const result = resolveSafeMode(
+ flags,
+ undefined,
+ "https://docs.example.com/x",
+ );
+ expect(result.allowlisted).toBe(true);
+ expect(result.safeMode).toEqual(allCapabilitiesAllowed);
+ // spelled out: nothing enforced/disabled, but the org gates stay
+ for (const key of RELAXABLE) {
+ expect(result.safeMode![key]).toBe(false);
+ }
+ expect(result.safeMode!.domainControls).toBe(true);
+ });
+
+ it("keeps lockdown on for an allowlisted URL when the org enabled it", () => {
+ const result = resolveSafeMode(
+ {
+ safeMode: true,
+ safeModeConfig: { lockdown: true, allowlist: ["docs.example.com"] },
+ },
+ undefined,
+ "https://docs.example.com/x",
+ );
+ expect(result.allowlisted).toBe(true);
+ expect(result.safeMode?.lockdown).toBe(true);
+ expect(result.safeMode?.disableStealthProxy).toBe(false);
+ });
+
+ it("matches globs and subdomains", () => {
+ expect(
+ resolveSafeMode(flags, undefined, "https://a.trusted.example/p")
+ .allowlisted,
+ ).toBe(true);
+ expect(
+ resolveSafeMode(
+ { safeMode: true, safeModeConfig: { allowlist: ["example.com"] } },
+ undefined,
+ "https://sub.example.com/p",
+ ).allowlisted,
+ ).toBe(true);
+ });
+
+ it("does not exempt a non-matching URL β€” capabilities stay strict", () => {
+ const result = resolveSafeMode(flags, undefined, "https://other.example/x");
+ expect(result.allowlisted).toBeUndefined();
+ expect(result.safeMode?.disableStealthProxy).toBe(true);
+ });
+
+ it("ignores the allowlist when no url is passed", () => {
+ const result = resolveSafeMode(flags, undefined);
+ expect(result.allowlisted).toBeUndefined();
+ expect(result.safeMode?.disableAuthentication).toBe(true);
+ });
+
+ it("a request bypass still wins over the allowlist path", () => {
+ const result = resolveSafeMode(
+ {
+ safeMode: true,
+ safeModeConfig: { allowBypassSafeMode: true, allowlist: ["x.example"] },
+ },
+ false,
+ "https://y.example/p",
+ );
+ expect(result.bypassed).toBe(true);
+ });
+});
+
+describe("applySafeMode", () => {
+ it("forces any escalated proxy tier to basic when stealth proxy is disabled", () => {
+ for (const tier of ["auto", "stealth", "enhanced"] as const) {
+ const options = { proxy: tier };
+ applySafeMode(strict, options);
+ expect(options.proxy).toBe("basic");
+ }
+ });
+
+ it("leaves proxy untouched for non-auto values, allowed stealth, or lockdown", () => {
+ for (const [safeMode, proxy] of [
+ [strict, "basic"],
+ [{ ...strict, disableStealthProxy: false }, "auto"],
+ [{ ...strict, lockdown: true }, "auto"],
+ [undefined, "auto"],
+ ] as const) {
+ const options = { proxy: proxy as "basic" | "auto" };
+ applySafeMode(safeMode, options);
+ expect(options.proxy).toBe(proxy);
+ }
+ });
+
+ it("forces lockdown and the 2-year maxAge when unset", () => {
+ const options: { lockdown?: boolean; maxAge?: number } = {};
+ applySafeMode({ ...strict, lockdown: true }, options);
+ expect(options.lockdown).toBe(true);
+ expect(options.maxAge).toBe(2 * 365 * 24 * 60 * 60 * 1000);
+ });
+
+ it("keeps a request-supplied maxAge under forced lockdown", () => {
+ const options = { maxAge: 5000 };
+ applySafeMode({ ...strict, lockdown: true }, options);
+ expect(options).toEqual({ lockdown: true, maxAge: 5000 });
+ });
+
+ it("no-ops lockdown when already requested or off", () => {
+ const alreadyOn = { lockdown: true };
+ applySafeMode({ ...strict, lockdown: true }, alreadyOn);
+ expect(alreadyOn).toEqual({ lockdown: true });
+
+ const off: { lockdown?: boolean } = {};
+ applySafeMode(strict, off);
+ applySafeMode(undefined, off);
+ expect(off).toEqual({});
+ });
+});
diff --git a/apps/api/src/lib/safe-mode.ts b/apps/api/src/lib/safe-mode.ts
new file mode 100644
index 000000000..df7c9c0cc
--- /dev/null
+++ b/apps/api/src/lib/safe-mode.ts
@@ -0,0 +1,248 @@
+import {
+ LOCKDOWN_DEFAULT_MAX_AGE_MS,
+ TeamFlags,
+} from "../controllers/v2/types";
+import type { ErrorCodes } from "./error";
+import { getSearchForcedKind } from "./zdr-helpers";
+import {
+ domainMatchesList,
+ normalizeDomain,
+} from "./threat-protection/verdict";
+
+const SUPPORT_EMAIL = "[email protected]";
+
+// v0 content endpoints can't enforce Safe Mode's per-request surface, so a
+// Safe Mode org is rejected there outright (mirrors the threat-protection v0 gate).
+export const SAFE_MODE_V0_UNSUPPORTED_MESSAGE =
+ "Safe Mode is enabled for your organization, which is not supported on the v0 API. Please update your code to use the v1 or v2 API.";
+
+// Interactive browser sessions (browser, scrape interact) run arbitrary code
+// against live sites and can't enforce Safe Mode's controls, so a Safe Mode
+// org is rejected there outright (mirrors the v0 gate).
+export const SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE =
+ "Safe Mode is enabled for your organization, which does not support interactive browser sessions.";
+
+// Credential-bearing request headers rejected at request time and stripped at
+// the worker under Safe Mode's disableAuthentication (case-insensitive match).
+export const SAFE_MODE_CREDENTIAL_HEADERS = [
+ "authorization",
+ "cookie",
+ "proxy-authorization",
+];
+
+// Actions that can enter credentials / run scripts; rejected at request time
+// and stripped at the worker under Safe Mode's disableAuthentication.
+export const SAFE_MODE_LOGIN_ACTIONS = ["write", "press", "executeJavascript"];
+
+// Remove Basic Auth embedded in a URL (user:pass@host). Returns the URL
+// unchanged if it has no userinfo or can't be parsed.
+export function stripUrlUserinfo(url: string): string {
+ try {
+ const u = new URL(url);
+ if (u.username || u.password) {
+ u.username = "";
+ u.password = "";
+ return u.toString();
+ }
+ } catch {
+ // leave non-parseable URLs as-is
+ }
+ return url;
+}
+
+// Remove credential-bearing headers. Used by the fire-engine builders so
+// inherited scrape options (crawl children, sub-scrapes) can't carry credentials
+// even when no request-time gate ran.
+export function stripCredentialHeaders(
+ headers: Record<string, string> | undefined,
+): Record<string, string> | undefined {
+ if (!headers) return headers;
+ return Object.fromEntries(
+ Object.entries(headers).filter(
+ ([k]) => !SAFE_MODE_CREDENTIAL_HEADERS.includes(k.toLowerCase()),
+ ),
+ );
+}
+
+function isSafeModeAllowlisted(
+ url: string,
+ allowlist: string[] | undefined,
+): boolean {
+ if (!allowlist || allowlist.length === 0) return false;
+ return domainMatchesList(normalizeDomain(url), allowlist);
+}
+
+export type SafeModeConfig = NonNullable<
+ NonNullable<TeamFlags>["safeModeConfig"]
+>;
+
+export type ResolvedSafeMode = {
+ lockdown: boolean;
+ domainControls: boolean;
+ enforceRobots: boolean;
+ disableStealthProxy: boolean;
+ disableAuthentication: boolean;
+ disableSiteHandling: boolean;
+ exposeWebdriver: boolean;
+ useHeadlessUserAgent: boolean;
+ disablePlatformSelection: boolean;
+ disableCountrySelection: boolean;
+ disableAutomaticReferrer: boolean;
+};
+
+const SAFE_MODE_DEFAULTS: ResolvedSafeMode = {
+ lockdown: false,
+ domainControls: true,
+ enforceRobots: true,
+ disableStealthProxy: true,
+ disableAuthentication: true,
+ disableSiteHandling: true,
+ exposeWebdriver: true,
+ useHeadlessUserAgent: true,
+ disablePlatformSelection: true,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+};
+
+export function getSafeMode(flags: TeamFlags | null | undefined): boolean {
+ return flags?.safeMode === true;
+}
+
+export function applySafeMode(
+ safeMode: ResolvedSafeMode | undefined,
+ scrapeOptions: {
+ proxy?: "basic" | "stealth" | "enhanced" | "auto";
+ lockdown?: boolean;
+ maxAge?: number;
+ },
+): void {
+ if (!safeMode) return;
+
+ // Force a non-stealth proxy. Request-time enforcement rejects an *explicit*
+ // stealth/enhanced proxy with a 403, but inherited scrape options (crawl
+ // children, search/extract sub-scrapes) reach the worker without that gate β€”
+ // downgrade any escalated tier to basic here so the worker never uses stealth.
+ if (
+ !safeMode.lockdown &&
+ safeMode.disableStealthProxy &&
+ (scrapeOptions.proxy === "auto" ||
+ scrapeOptions.proxy === "stealth" ||
+ scrapeOptions.proxy === "enhanced")
+ ) {
+ scrapeOptions.proxy = "basic";
+ }
+
+ if (safeMode.lockdown && !scrapeOptions.lockdown) {
+ scrapeOptions.lockdown = true;
+ if (scrapeOptions.maxAge === undefined) {
+ scrapeOptions.maxAge = LOCKDOWN_DEFAULT_MAX_AGE_MS;
+ }
+ }
+}
+
+// Lockdown is cache-only and implies zero data retention. Resolved from the
+// org flags alone (lockdown never depends on the request param) and dropped
+// only when a per-request bypass is actually HONORED β€” a rejected/invalid
+// safeMode:false must still record as ZDR for a lockdown org. Safe to call
+// before schema validation (a non-boolean request value is treated as unset).
+export function isLockdownZeroDataRetention(
+ flags: TeamFlags | null | undefined,
+ requestSafeMode: boolean | undefined,
+): boolean {
+ if (resolveSafeMode(flags, requestSafeMode).bypassed === true) return false;
+ return resolveSafeMode(flags, undefined).safeMode?.lockdown === true;
+}
+
+// Search's forced enterprise kind, with Safe Mode lockdown folded in: lockdown
+// forces the "zdr" kind exactly like the searchZDR flag does (zero-retention
+// upstream routing, the ZDR credit rate, ZDR recording). The flag still wins
+// when set (it may force "anon" instead).
+export function getEffectiveSearchForcedKind(
+ flags: TeamFlags | null | undefined,
+ requestSafeMode: boolean | undefined,
+): ReturnType<typeof getSearchForcedKind> {
+ return (
+ getSearchForcedKind(flags ?? undefined) ??
+ (isLockdownZeroDataRetention(flags, requestSafeMode) ? "zdr" : null)
+ );
+}
+
+export function resolveSafeMode(
+ flags: TeamFlags | null | undefined,
+ requestSafeMode: boolean | undefined,
+ url?: string,
+): {
+ safeMode?: ResolvedSafeMode;
+ bypassed?: boolean;
+ allowlisted?: boolean;
+ error?: string;
+ code?: ErrorCodes;
+} {
+ if (!getSafeMode(flags)) {
+ if (requestSafeMode === true) {
+ return {
+ error: `Safe Mode is not enabled for your organization. Contact ${SUPPORT_EMAIL} to enable this feature.`,
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ return {};
+ }
+
+ const config = flags?.safeModeConfig;
+
+ if (requestSafeMode === false) {
+ if (config?.allowBypassSafeMode !== true) {
+ return {
+ error:
+ "Requests are not allowed to disable Safe Mode for your organization. An organization admin can allow per-request opt-outs from the Safe Mode settings.",
+ code: "SAFE_MODE_BLOCKED",
+ };
+ }
+ return { bypassed: true };
+ }
+
+ const resolved: ResolvedSafeMode = {
+ lockdown: config?.lockdown ?? SAFE_MODE_DEFAULTS.lockdown,
+ domainControls: config?.domainControls ?? SAFE_MODE_DEFAULTS.domainControls,
+ enforceRobots: config?.enforceRobots ?? SAFE_MODE_DEFAULTS.enforceRobots,
+ disableStealthProxy:
+ config?.disableStealthProxy ?? SAFE_MODE_DEFAULTS.disableStealthProxy,
+ disableAuthentication:
+ config?.disableAuthentication ?? SAFE_MODE_DEFAULTS.disableAuthentication,
+ disableSiteHandling:
+ config?.disableSiteHandling ?? SAFE_MODE_DEFAULTS.disableSiteHandling,
+ exposeWebdriver:
+ config?.exposeWebdriver ?? SAFE_MODE_DEFAULTS.exposeWebdriver,
+ useHeadlessUserAgent:
+ config?.useHeadlessUserAgent ?? SAFE_MODE_DEFAULTS.useHeadlessUserAgent,
+ disablePlatformSelection:
+ config?.disablePlatformSelection ??
+ SAFE_MODE_DEFAULTS.disablePlatformSelection,
+ disableCountrySelection:
+ config?.disableCountrySelection ??
+ SAFE_MODE_DEFAULTS.disableCountrySelection,
+ disableAutomaticReferrer:
+ config?.disableAutomaticReferrer ??
+ SAFE_MODE_DEFAULTS.disableAutomaticReferrer,
+ };
+
+ if (url && isSafeModeAllowlisted(url, config?.allowlist)) {
+ return {
+ allowlisted: true,
+ safeMode: {
+ ...resolved,
+ enforceRobots: false,
+ disableStealthProxy: false,
+ disableAuthentication: false,
+ disableSiteHandling: false,
+ exposeWebdriver: false,
+ useHeadlessUserAgent: false,
+ disablePlatformSelection: false,
+ disableCountrySelection: false,
+ disableAutomaticReferrer: false,
+ },
+ };
+ }
+
+ return { safeMode: resolved };
+}
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 5f062c5af..3f43530c9 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -28,7 +28,8 @@ const redactPIIPdfPageCostBonus = 4;
// scope β€” a scrape and its same-URL re-check share one fee, while a crawl of
// N pages bills N scans (each page job is its own scope). Verdicts are never
// reused across requests (no verdict cache β€” ZDR). Local-only decisions
-// (whitelist/blacklist/blocked-tld, mode off, provider failure) never bill.
+// (whitelist/blacklist/blocked-tld, modes off / manual-only, provider
+// failure) never bill.
const threatScanCost = 2;
/**
diff --git a/apps/api/src/lib/threat-protection/check-url.test.ts b/apps/api/src/lib/threat-protection/check-url.test.ts
index b699cf7ef..30027675d 100644
--- a/apps/api/src/lib/threat-protection/check-url.test.ts
+++ b/apps/api/src/lib/threat-protection/check-url.test.ts
@@ -138,6 +138,37 @@ describe("checkUrl", () => {
expect(counters.hashesSearchRequests).toBe(before);
});
+ it("manual-only: enforces the lists and default-allows the rest with no provider call", async () => {
+ const before = counters.hashesSearchRequests;
+ const manual = policy({
+ mode: "manual-only",
+ blacklist: ["blocked.com"],
+ blockedTlds: ["zip"],
+ });
+
+ const blocked = await checkUrl("https://cdn.blocked.com/x", manual, {});
+ expect(blocked).toMatchObject({
+ allowed: false,
+ rule: "blacklist",
+ providerConsulted: false,
+ mode: "manual-only",
+ });
+
+ const tld = await checkUrl("https://archive.zip/", manual, {});
+ expect(tld).toMatchObject({ allowed: false, rule: "blocked-tld" });
+
+ // A domain the provider would flag is allowed: no lookup, no billing.
+ const risky = await checkUrl(`https://${RISKY_DOMAIN}/`, manual, {});
+ expect(risky).toMatchObject({
+ allowed: true,
+ rule: "default-allow",
+ providerConsulted: false,
+ verdict: null,
+ mode: "manual-only",
+ });
+ expect(counters.hashesSearchRequests).toBe(before);
+ });
+
it("consults the provider and blocks a flagged domain (fresh scan)", async () => {
const before = riskyHits();
const decision = await checkUrl(RISKY_DOMAIN.toUpperCase(), policy(), {
diff --git a/apps/api/src/lib/threat-protection/config.test.ts b/apps/api/src/lib/threat-protection/config.test.ts
index d43ddd820..4413dd2bf 100644
--- a/apps/api/src/lib/threat-protection/config.test.ts
+++ b/apps/api/src/lib/threat-protection/config.test.ts
@@ -28,6 +28,18 @@ describe("threatProtectionPolicySchema", () => {
expect(policy.failurePolicy).toBe("open");
});
+ it('accepts the "manual-only" mode in every schema', () => {
+ expect(
+ threatProtectionPolicySchema.parse({ mode: "manual-only" }).mode,
+ ).toBe("manual-only");
+ expect(
+ threatProtectionOverrideSchema.parse({ mode: "manual-only" }).mode,
+ ).toBe("manual-only");
+ expect(
+ threatProtectionConfigSchema.parse({ mode: "manual-only" }).mode,
+ ).toBe("manual-only");
+ });
+
it("rejects an invalid mode", () => {
expect(() =>
threatProtectionPolicySchema.parse({ mode: "paranoid" }),
diff --git a/apps/api/src/lib/threat-protection/config.ts b/apps/api/src/lib/threat-protection/config.ts
index 9891ad963..ecc892925 100644
--- a/apps/api/src/lib/threat-protection/config.ts
+++ b/apps/api/src/lib/threat-protection/config.ts
@@ -99,7 +99,7 @@ type ZscalerConfigInput = z.infer<typeof zscalerConfigSchema>;
* except `mode` defaults to {@link THREAT_PROTECTION_POLICY_DEFAULTS}.
*/
export const threatProtectionPolicySchema = z.strictObject({
- mode: z.enum(["off", "normal", "zscaler"]),
+ mode: z.enum(["off", "manual-only", "normal", "zscaler"]),
riskScoreThreshold: z
.number()
.int()
@@ -137,7 +137,7 @@ void _policyContractCheck;
* provides replace the org policy's values (see `resolveEffectivePolicy`).
*/
export const threatProtectionOverrideSchema = z.strictObject({
- mode: z.enum(["off", "normal", "zscaler"]).optional(),
+ mode: z.enum(["off", "manual-only", "normal", "zscaler"]).optional(),
riskScoreThreshold: z.number().int().min(0).max(100).optional(),
blacklist: z.array(domainGlobSchema).max(1000).optional(),
whitelist: z.array(domainGlobSchema).max(1000).optional(),
diff --git a/apps/api/src/lib/threat-protection/index.ts b/apps/api/src/lib/threat-protection/index.ts
index 138206548..2f5886c4a 100644
--- a/apps/api/src/lib/threat-protection/index.ts
+++ b/apps/api/src/lib/threat-protection/index.ts
@@ -47,7 +47,8 @@ export interface ThreatCheckContext {
// 2. request-scoped dedup (ctx.dedup) β†’ a URL already checked within
// this request/job reuses the same in-flight decision
// 3. local-only rules (whitelist/blacklist/blocked-tld, evaluated against
-// the URL's host) β†’ decide without a provider call (no billing)
+// the URL's host) β†’ decide without a provider call (no billing);
+// mode "manual-only" stops here and default-allows everything else
// 4. "zscaler" mode only: the tenant's synced custom-list rules (custom
// categories, keywords, customer category additions β€” urlLookup never
// returns those) β†’ a decisive match skips the provider call
@@ -97,7 +98,7 @@ const PROVIDER_ATTEMPTS = 2; // 1 initial + 1 retry
*/
async function fetchProviderVerdict(
url: string,
- mode: Exclude<ThreatProtectionMode, "off">,
+ mode: Exclude<ThreatProtectionMode, "off" | "manual-only">,
policy: ThreatProtectionPolicy,
): Promise<RawVerdict> {
const attempts = mode === "zscaler" ? 1 : PROVIDER_ATTEMPTS;
@@ -165,7 +166,7 @@ async function checkUrlFresh(
}
}
- if (ctx.localRulesOnly) {
+ if (ctx.localRulesOnly || policy.mode === "manual-only") {
return {
allowed: true,
rule: "default-allow",
@@ -181,7 +182,7 @@ async function checkUrlFresh(
try {
verdict = await fetchProviderVerdict(
canonicalUrl,
- policy.mode as Exclude<ThreatProtectionMode, "off">,
+ policy.mode as Exclude<ThreatProtectionMode, "off" | "manual-only">,
policy,
);
} catch {
diff --git a/apps/api/src/lib/threat-protection/request.ts b/apps/api/src/lib/threat-protection/request.ts
index b2d8faa75..111102cc5 100644
--- a/apps/api/src/lib/threat-protection/request.ts
+++ b/apps/api/src/lib/threat-protection/request.ts
@@ -7,7 +7,11 @@ import {
resolveEffectivePolicy,
type OrgThreatProtectionConfig,
} from "./store";
-import type { ThreatDecision, ThreatProtectionPolicy } from "./types";
+import type {
+ ThreatDecision,
+ ThreatProtectionMode,
+ ThreatProtectionPolicy,
+} from "./types";
// Controller-layer glue for threat protection enforcement. One helper
// (resolveThreatProtection) is shared by every endpoint: it turns
@@ -23,10 +27,19 @@ export const THREAT_PROTECTION_OVERRIDES_DISABLED_MESSAGE =
"Per-request threat protection overrides are disabled by your organization's threat protection configuration.";
export const THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE =
- 'Threat protection is enforced for your team and cannot be disabled per-request (threatProtection.mode may not be "off"). Remove the mode field or contact your organization administrator.';
+ 'Threat protection is enforced for your team and cannot be disabled or weakened per-request (threatProtection.mode may not be "off" or lower than the mode your organization configured). Remove the mode field or contact your organization administrator.';
export const THREAT_PROTECTION_CANNOT_TURN_OFF_MESSAGE =
- 'Threat protection is enforced for your team and its mode cannot be set to "off". Contact your organization administrator.';
+ 'Threat protection is enforced for your team and its mode cannot be set to "off" or "manual-only". Contact your organization administrator.';
+
+// Strength order for the never-weaken rule. "normal" and "zscaler" are peers
+// (both provider-backed); switching between them is allowed.
+const MODE_RANK: Record<ThreatProtectionMode, number> = {
+ off: 0,
+ "manual-only": 1,
+ normal: 2,
+ zscaler: 2,
+};
export const THREAT_PROTECTION_V0_UNSUPPORTED_MESSAGE =
"Threat protection is enforced for your team and is not supported on the deprecated v0 API. Please update your code to use the v1 or v2 API.";
@@ -67,16 +80,22 @@ interface ResolvedThreatProtection {
* - Flag "forced": an override may never set `mode: "off"` β†’ 403.
* - Effective mode "off" resolves to `policy: null` so callers can skip all
* enforcement work.
+ * - `force: true` (Safe Mode domain controls): treated as flag "forced", and
+ * an "off" policy is raised to "manual-only" so the org's own lists are
+ * enforced without a provider scan (no scan fee). Never resolves to
+ * `policy: null`; an org that chose "normal"/"zscaler" keeps that mode.
*/
export async function resolveThreatProtection(args: {
teamId: string;
orgId?: string | null;
flags: TeamFlags;
override?: Partial<ThreatProtectionPolicy>;
+ force?: boolean;
}): Promise<ResolvedThreatProtection> {
- const flagMode = getThreatProtection(args.flags);
+ const effectiveFlagMode =
+ args.force === true ? "forced" : getThreatProtection(args.flags);
- if (flagMode !== "allowed" && flagMode !== "forced") {
+ if (effectiveFlagMode !== "allowed" && effectiveFlagMode !== "forced") {
if (args.override !== undefined) {
return {
error: THREAT_PROTECTION_NOT_ENABLED_MESSAGE,
@@ -98,16 +117,30 @@ export async function resolveThreatProtection(args: {
orgConfig,
};
}
- if (flagMode === "forced" && args.override.mode === "off") {
- return {
- error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,
- policy: null,
- orgConfig,
- };
+ if (effectiveFlagMode === "forced" && args.override.mode !== undefined) {
+ // Enforced teams may tighten per-request but never weaken: "off" is
+ // always refused, and the mode may not drop below the org's effective
+ // mode (Safe Mode's raise puts that floor at "manual-only").
+ const orgMode = orgConfig?.policy.mode ?? "off";
+ const floor: ThreatProtectionMode =
+ args.force === true && orgMode === "off" ? "manual-only" : orgMode;
+ if (
+ args.override.mode === "off" ||
+ MODE_RANK[args.override.mode] < MODE_RANK[floor]
+ ) {
+ return {
+ error: THREAT_PROTECTION_CANNOT_DISABLE_MESSAGE,
+ policy: null,
+ orgConfig,
+ };
+ }
}
}
- const policy = resolveEffectivePolicy(orgConfig, args.override);
+ let policy = resolveEffectivePolicy(orgConfig, args.override);
+ if (args.force === true && policy.mode === "off") {
+ policy = { ...policy, mode: "manual-only" };
+ }
// "zscaler" mode without a connection can only happen via a per-request
// override on an org that never saved credentials (the config API refuses
diff --git a/apps/api/src/lib/threat-protection/store.test.ts b/apps/api/src/lib/threat-protection/store.test.ts
index a3e0aea50..ec3c85acb 100644
--- a/apps/api/src/lib/threat-protection/store.test.ts
+++ b/apps/api/src/lib/threat-protection/store.test.ts
@@ -49,6 +49,12 @@ function makeRow(mode: string, config: unknown) {
}
describe("rowToConfig", () => {
+ it('maps a "manual-only" mode column', () => {
+ expect(rowToConfig(makeRow("manual-only", {})).policy.mode).toBe(
+ "manual-only",
+ );
+ });
+
it("maps a full stored document", () => {
const config = rowToConfig(
makeRow("normal", {
diff --git a/apps/api/src/lib/threat-protection/store.ts b/apps/api/src/lib/threat-protection/store.ts
index 3f744f141..b394cabd9 100644
--- a/apps/api/src/lib/threat-protection/store.ts
+++ b/apps/api/src/lib/threat-protection/store.ts
@@ -134,7 +134,9 @@ export function rowToConfig(
? "normal"
: row.mode === "zscaler"
? "zscaler"
- : "off",
+ : row.mode === "manual-only"
+ ? "manual-only"
+ : "off",
riskScoreThreshold: doc.riskScoreThreshold,
blacklist: doc.blacklist,
whitelist: doc.whitelist,
diff --git a/apps/api/src/lib/threat-protection/types.ts b/apps/api/src/lib/threat-protection/types.ts
index a85b6bba9..c56ca1a11 100644
--- a/apps/api/src/lib/threat-protection/types.ts
+++ b/apps/api/src/lib/threat-protection/types.ts
@@ -1,7 +1,10 @@
// Shared type contract for the threat protection feature.
// NOTE: concurrent in-flight branches create this exact file β€” do not modify without coordinating.
-export type ThreatProtectionMode = "off" | "normal" | "zscaler";
+// "manual-only": enforce the org's own lists (whitelist / blacklist /
+// blocked TLDs) and nothing else β€” no provider lookup, so no scan fee. This
+// is also what Safe Mode's domain controls raise an "off" policy to.
+export type ThreatProtectionMode = "off" | "manual-only" | "normal" | "zscaler";
/**
* The non-secret slice of an org's Zscaler settings that policy evaluation
diff --git a/apps/api/src/lib/threat-protection/verdict.ts b/apps/api/src/lib/threat-protection/verdict.ts
index 938245439..2e27d47a5 100644
--- a/apps/api/src/lib/threat-protection/verdict.ts
+++ b/apps/api/src/lib/threat-protection/verdict.ts
@@ -51,7 +51,7 @@ function domainMatchesEntry(domain: string, entry: string): boolean {
return domain === normalized || domain.endsWith(`.${normalized}`);
}
-function domainMatchesList(domain: string, entries: string[]): boolean {
+export function domainMatchesList(domain: string, entries: string[]): boolean {
return entries.some(entry => domainMatchesEntry(domain, entry));
}
@@ -189,9 +189,10 @@ export function evaluatePolicy(
return { allowed: true, rule: "default-allow", ...base };
}
- // No verdict: the provider failed or was unavailable (mode "off" never
- // reaches here via checkUrl). Fail open or closed per the org policy.
- if (policy.mode === "off") {
+ // No verdict: the provider failed or was unavailable (modes "off" and
+ // "manual-only" never consult one, so they never fail). Otherwise fail open
+ // or closed per the org policy.
+ if (policy.mode === "off" || policy.mode === "manual-only") {
return { allowed: true, rule: "default-allow", ...base };
}
return {
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 7e0b1f597..b212e5954 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -85,6 +85,8 @@ export class WebCrawler {
private location?: ScrapeOptions["location"];
private headers?: Record<string, string>;
private robotsUserAgent?: string;
+ // Safe Mode lockdown: no outbound discovery to the target (robots/sitemap).
+ private lockdown: boolean = false;
constructor({
jobId,
@@ -107,6 +109,7 @@ export class WebCrawler {
location,
headers,
robotsUserAgent,
+ lockdown,
}: {
jobId: string;
initialUrl: string;
@@ -128,6 +131,7 @@ export class WebCrawler {
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
robotsUserAgent?: string;
+ lockdown?: boolean;
}) {
this.jobId = jobId;
this.initialUrl = initialUrl;
@@ -158,6 +162,7 @@ export class WebCrawler {
this.location = location;
this.headers = headers;
this.robotsUserAgent = robotsUserAgent;
+ this.lockdown = lockdown ?? false;
}
public setBaseUrl(newBase: string): void {
@@ -469,6 +474,11 @@ export class WebCrawler {
skipTlsVerification = false,
abort?: AbortSignal,
): Promise<string> {
+ // Lockdown: no outbound fetch to the target. Empty robots => fail-open,
+ // consistent with a failed robots fetch.
+ if (this.lockdown) {
+ return "";
+ }
try {
this.logger.debug("Attempting to fetch robots.txt", {
method: "getRobotsTxt",
@@ -550,6 +560,13 @@ export class WebCrawler {
mock?: string,
maxAge: number = SITEMAP_MAX_AGE,
): Promise<number> {
+ // Lockdown: no outbound sitemap discovery to the target (index-only crawl).
+ if (this.lockdown) {
+ this.logger.debug("Skipping sitemap discovery under Safe Mode lockdown", {
+ method: "tryGetSitemap",
+ });
+ return 0;
+ }
this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {
method: "tryGetSitemap",
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index d200a13c3..c262c0504 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -12,6 +12,7 @@ import {
DNSResolutionError,
FEPageLoadFailed,
ProxySelectionError,
+ SiteRestrictionError,
} from "../../error";
import { MockState } from "../../lib/mock";
import { fireEngineURL } from "./scrape";
@@ -137,6 +138,7 @@ const failedSchema = z.object({
processing: z.literal(false),
error: z.string(),
retryWithStealth: z.boolean().optional(),
+ failureReason: z.literal("site_protection").optional(),
});
export class StillProcessingError extends Error {
@@ -190,6 +192,9 @@ export async function fireEngineCheckStatus(
throw new StillProcessingError(jobId);
} else if (failedParse.success) {
logger.debug("Scrape job failed", { status, jobId });
+ if (failedParse.data.failureReason === "site_protection") {
+ throw new SiteRestrictionError();
+ }
if (
failedParse.data.retryWithStealth &&
meta.options.proxy === "auto" &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 67b1082ed..79ea24b0f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -7,7 +7,9 @@ import {
FireEngineScrapeRequestChromeCDP,
FireEngineScrapeRequestCommon,
FireEngineScrapeRequestTLSClient,
+ safeModeParams,
} from "./scrape";
+import { stripCredentialHeaders } from "../../../../lib/safe-mode";
import { EngineScrapeResult } from "..";
import {
fireEngineCheckStatus,
@@ -20,6 +22,7 @@ import {
EngineError,
DNSResolutionError,
SiteError,
+ SiteRestrictionError,
SSLError,
UnsupportedFileError,
FEPageLoadFailed,
@@ -152,6 +155,7 @@ async function performFireEngineScrape<
} else if (
error instanceof EngineError ||
error instanceof SiteError ||
+ error instanceof SiteRestrictionError ||
error instanceof SSLError ||
error instanceof DNSResolutionError ||
error instanceof ActionError ||
@@ -490,8 +494,20 @@ export async function scrapeURLWithFireEngineChromeCDP(
uniqueId: `${createHash("sha256").update(meta.internalOptions.teamId).digest("hex").slice(0, 16)}_${meta.options.profile.name}`,
}
: undefined,
+ ...safeModeParams(meta.internalOptions.safeMode),
};
+ // Safe Mode worker-side hardening: neutralize anything that request-time
+ // enforcement would reject but that inherited scrape options can still carry.
+ const sm = meta.internalOptions.safeMode;
+ if (sm?.disableStealthProxy) {
+ request.mobileProxy = false;
+ }
+ if (sm?.disableAuthentication) {
+ request.persistentStorage = undefined;
+ request.headers = stripCredentialHeaders(request.headers);
+ }
+
let response = await performFireEngineScrape(
meta,
meta.logger.child({
@@ -642,8 +658,17 @@ export async function scrapeURLWithFireEngineTLSClient(
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ ...safeModeParams(meta.internalOptions.safeMode),
};
+ const sm = meta.internalOptions.safeMode;
+ if (sm?.disableStealthProxy) {
+ request.mobileProxy = false;
+ }
+ if (sm?.disableAuthentication) {
+ request.headers = stripCredentialHeaders(request.headers);
+ }
+
let response = await performFireEngineScrape(
meta,
meta.logger.child({
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.ts
new file mode 100644
index 000000000..42988c64c
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.test.ts
@@ -0,0 +1,87 @@
+import { safeModeParams } from "./scrape";
+import type { ResolvedSafeMode } from "../../../../lib/safe-mode";
+
+const strict: ResolvedSafeMode = {
+ lockdown: false,
+ domainControls: true,
+ enforceRobots: true,
+ disableStealthProxy: true,
+ disableAuthentication: true,
+ disableSiteHandling: true,
+ exposeWebdriver: true,
+ useHeadlessUserAgent: true,
+ disablePlatformSelection: true,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+};
+
+// The safe-mode control -> engine behaviorOverride field it drives. Same
+// name, same value: the mapper is a straight pass-through (no negation).
+const ENGINE_FIELDS = [
+ "disableSiteHandling",
+ "exposeWebdriver",
+ "useHeadlessUserAgent",
+ "disablePlatformSelection",
+ "disableCountrySelection",
+ "disableAutomaticReferrer",
+] as const;
+
+describe("safeModeParams", () => {
+ it("sends nothing when Safe Mode is absent", () => {
+ expect(safeModeParams(undefined)).toEqual({});
+ });
+
+ it("passes the strict defaults through as all overrides on", () => {
+ expect(safeModeParams(strict)).toEqual({
+ behaviorOverrides: {
+ disableSiteHandling: true,
+ exposeWebdriver: true,
+ useHeadlessUserAgent: true,
+ disablePlatformSelection: true,
+ disableCountrySelection: true,
+ disableAutomaticReferrer: true,
+ },
+ });
+ });
+
+ it("only forwards the six engine fields, not the firecrawl-side controls", () => {
+ expect(Object.keys(safeModeParams(strict).behaviorOverrides!)).toEqual([
+ ...ENGINE_FIELDS,
+ ]);
+ });
+
+ it("forwards each engine field verbatim, with no negation", () => {
+ for (const field of ENGINE_FIELDS) {
+ const relaxed = safeModeParams({ ...strict, [field]: false });
+ expect(relaxed.behaviorOverrides![field]).toBe(false);
+ // the other five stay on
+ for (const other of ENGINE_FIELDS) {
+ if (other !== field) {
+ expect(relaxed.behaviorOverrides![other]).toBe(true);
+ }
+ }
+ }
+ });
+
+ it("passes an allowlisted (fully relaxed) posture through as all overrides off", () => {
+ const relaxed: ResolvedSafeMode = {
+ ...strict,
+ disableSiteHandling: false,
+ exposeWebdriver: false,
+ useHeadlessUserAgent: false,
+ disablePlatformSelection: false,
+ disableCountrySelection: false,
+ disableAutomaticReferrer: false,
+ };
+ expect(safeModeParams(relaxed)).toEqual({
+ behaviorOverrides: {
+ disableSiteHandling: false,
+ exposeWebdriver: false,
+ useHeadlessUserAgent: false,
+ disablePlatformSelection: false,
+ disableCountrySelection: false,
+ disableAutomaticReferrer: false,
+ },
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index de0cc7cb7..339ed97fa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -15,9 +15,11 @@ import {
ProxySelectionError,
SSLError,
SiteError,
+ SiteRestrictionError,
UnsupportedFileError,
} from "../../error";
import { Meta } from "../..";
+import type { ResolvedSafeMode } from "../../../../lib/safe-mode";
import { config } from "../../../../config";
@@ -57,8 +59,33 @@ export type FireEngineScrapeRequestCommon = {
maxAge?: number;
saveScrapeResultToGCS?: boolean;
zeroDataRetention?: boolean;
+
+ behaviorOverrides?: {
+ disableSiteHandling?: boolean;
+ exposeWebdriver?: boolean;
+ useHeadlessUserAgent?: boolean;
+ disablePlatformSelection?: boolean;
+ disableCountrySelection?: boolean;
+ disableAutomaticReferrer?: boolean;
+ };
};
+export function safeModeParams(
+ safeMode: ResolvedSafeMode | undefined,
+): Pick<FireEngineScrapeRequestCommon, "behaviorOverrides"> {
+ if (!safeMode) return {};
+ return {
+ behaviorOverrides: {
+ disableSiteHandling: safeMode.disableSiteHandling,
+ exposeWebdriver: safeMode.exposeWebdriver,
+ useHeadlessUserAgent: safeMode.useHeadlessUserAgent,
+ disablePlatformSelection: safeMode.disablePlatformSelection,
+ disableCountrySelection: safeMode.disableCountrySelection,
+ disableAutomaticReferrer: safeMode.disableAutomaticReferrer,
+ },
+ };
+}
+
export type FireEngineScrapeRequestChromeCDP = {
engine: "chrome-cdp";
skipTlsVerification?: boolean;
@@ -180,6 +207,7 @@ const processingSchema = z.object({
const failedSchema = z.object({
error: z.string(),
retryWithStealth: z.boolean().optional(),
+ failureReason: z.literal("site_protection").optional(),
});
export const fireEngineURL =
@@ -241,6 +269,9 @@ export async function fireEngineScrape<
logger.debug("Scrape job failed", {
status,
});
+ if (failedParse.data.failureReason === "site_protection") {
+ throw new SiteRestrictionError();
+ }
if (
failedParse.data.retryWithStealth &&
meta.options.proxy === "auto" &&
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 1c2ba2c34..19129fdb7 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -203,6 +203,30 @@ export class ProxySelectionError extends TransportableError {
}
}
+export class SiteRestrictionError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_SITE_RESTRICTION_BLOCKED",
+ "This site restricts automated access to the requested content. Safe Mode is enabled for your organization, so the site's restriction is returned instead of being worked around.",
+ );
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ };
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new SiteRestrictionError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class ActionError extends TransportableError {
constructor(public errorCode: string) {
super(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 102f81368..91d9b3022 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -41,6 +41,7 @@ import {
DocumentFetchProxyError,
RemoveFeatureError,
SiteError,
+ SiteRestrictionError,
UnsupportedFileError,
SSLError,
PDFInsufficientTimeError,
@@ -113,6 +114,15 @@ import {
type ThreatDecision,
type ThreatProtectionPolicy,
} from "../../lib/threat-protection";
+import {
+ type ResolvedSafeMode,
+ resolveSafeMode,
+ applySafeMode,
+ stripCredentialHeaders,
+ stripUrlUserinfo,
+ SAFE_MODE_LOGIN_ACTIONS,
+} from "../../lib/safe-mode";
+import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
import { canonicalizeUrl } from "../../lib/threat-protection/providers/web-risk/canonicalize";
@@ -623,6 +633,12 @@ export type InternalOptions = {
*/
threatProtection?: ThreatProtectionPolicy;
+ safeMode?: ResolvedSafeMode;
+ /** Set when a request legitimately opted out of Safe Mode at the controller
+ * (allowBypassSafeMode + safeMode:false). Tells the worker backstop NOT to
+ * re-resolve safe mode from teamFlags β€” otherwise the bypass would be undone. */
+ safeModeBypassed?: boolean;
+
v1Agent?: ScrapeOptionsV1["agent"];
v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];
v1JSONSystemPrompt?: string;
@@ -991,6 +1007,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof AddFeatureError ||
error.error instanceof RemoveFeatureError ||
error.error instanceof SiteError ||
+ error.error instanceof SiteRestrictionError ||
error.error instanceof SSLError ||
error.error instanceof DNSResolutionError ||
error.error instanceof ActionError ||
@@ -1231,6 +1248,66 @@ export async function scrapeURL(
return withSpan(
"scrape.pipeline",
async span => {
+ // Safe Mode: the universal enforcement choke point. The controller
+ // pre-resolves safe mode for a single scrape, but jobs that carry only
+ // team flags (crawl children, search / extract URLs, allowlisted single
+ // scrapes) resolve here, per-URL, so the allowlist applies to each
+ // discovered URL. Runs before buildMetaObject so forced lockdown reaches
+ // feature-flag/engine selection, and covers every endpoint that stamps
+ // teamFlags onto its job payload.
+ if (
+ !internalOptions.safeMode &&
+ !internalOptions.safeModeBypassed &&
+ internalOptions.teamFlags
+ ) {
+ internalOptions.safeMode = resolveSafeMode(
+ internalOptions.teamFlags,
+ undefined,
+ url,
+ ).safeMode;
+ }
+ if (internalOptions.safeMode) {
+ applySafeMode(internalOptions.safeMode, options);
+ // Auth-path enforcement for every engine (not just fire-engine) and for
+ // inherited options a request-time gate never saw (crawl children etc.).
+ if (internalOptions.safeMode.disableAuthentication) {
+ options.headers = stripCredentialHeaders(options.headers);
+ if (options.actions) {
+ options.actions = options.actions.filter(
+ a => !SAFE_MODE_LOGIN_ACTIONS.includes(a.type),
+ );
+ }
+ options.profile = undefined;
+ // Basic Auth embedded in the URL (user:pass@host) is another way to
+ // authenticate; strip the userinfo so no engine can use it, and from
+ // the preserved source URL so it isn't returned/persisted in metadata.
+ url = stripUrlUserinfo(url);
+ if (internalOptions.unnormalizedSourceURL) {
+ internalOptions.unnormalizedSourceURL = stripUrlUserinfo(
+ internalOptions.unnormalizedSourceURL,
+ );
+ }
+ }
+ if (
+ internalOptions.safeMode.domainControls &&
+ !internalOptions.threatProtection
+ ) {
+ const tp = await resolveThreatProtection({
+ teamId: internalOptions.teamId,
+ orgId: internalOptions.orgId,
+ flags: internalOptions.teamFlags ?? {},
+ force: true,
+ });
+ // Fail closed: domainControls must never silently disable itself.
+ if (tp.error) {
+ throw new Error(
+ `Safe Mode domain controls could not be resolved: ${tp.error}`,
+ );
+ }
+ internalOptions.threatProtection = tp.policy ?? undefined;
+ }
+ }
+
const meta = await buildMetaObject(
id,
url,
@@ -1737,6 +1814,11 @@ export async function scrapeURL(
meta.logger.warn("scrapeURL: Site failed to load in browser", {
error,
});
+ } else if (error instanceof SiteRestrictionError) {
+ errorType = "SiteRestrictionError";
+ meta.logger.warn("scrapeURL: Site restriction returned (Safe Mode)", {
+ error,
+ });
} else if (error instanceof SSLError) {
errorType = "SSLError";
meta.logger.warn("scrapeURL: SSL error", { error });
diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.ts
new file mode 100644
index 000000000..360d9df1f
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.test.ts
@@ -0,0 +1,46 @@
+import { shouldCheckRobots } from "./shouldCheckRobots";
+
+describe("shouldCheckRobots", () => {
+ it("never checks under lockdown, even with safe mode enforcing robots", () => {
+ expect(
+ shouldCheckRobots(
+ { lockdown: true },
+ { safeMode: { enforceRobots: true } as any },
+ ),
+ ).toBe(false);
+ });
+
+ it("checks when the team flag opts in", () => {
+ expect(
+ shouldCheckRobots({}, { teamFlags: { checkRobotsOnScrape: true } }),
+ ).toBe(true);
+ });
+
+ it("checks when safe mode enforces robots, regardless of the team flag", () => {
+ expect(
+ shouldCheckRobots({}, { safeMode: { enforceRobots: true } as any }),
+ ).toBe(true);
+ expect(
+ shouldCheckRobots(
+ {},
+ {
+ teamFlags: { checkRobotsOnScrape: false },
+ safeMode: { enforceRobots: true } as any,
+ },
+ ),
+ ).toBe(true);
+ });
+
+ it("does not check when safe mode relaxes robots and the flag is off", () => {
+ expect(
+ shouldCheckRobots({}, { safeMode: { enforceRobots: false } as any }),
+ ).toBe(false);
+ });
+
+ it("does not check when neither opts in", () => {
+ expect(shouldCheckRobots({}, {})).toBe(false);
+ expect(
+ shouldCheckRobots({}, { teamFlags: { checkRobotsOnScrape: false } }),
+ ).toBe(false);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
index d2bc039f3..8f68659da 100644
--- a/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
+++ b/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
@@ -1,4 +1,5 @@
import type { TeamFlags } from "../../controllers/v1/types";
+import type { ResolvedSafeMode } from "../../lib/safe-mode";
// The invariant "lockdown never fetches robots.txt" is load-bearing for the
// lockdown guarantee (robots.txt is a request to the target domain). Keep this
@@ -6,10 +7,13 @@ import type { TeamFlags } from "../../controllers/v1/types";
// ESM-heavy module graph.
export function shouldCheckRobots(
options: { lockdown?: boolean },
- internalOptions: { teamFlags?: TeamFlags },
+ internalOptions: { teamFlags?: TeamFlags; safeMode?: ResolvedSafeMode },
): boolean {
if (options.lockdown) {
return false;
}
+ if (internalOptions.safeMode?.enforceRobots) {
+ return true;
+ }
return !!internalOptions.teamFlags?.checkRobotsOnScrape;
}
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 33dda1306..d5517c431 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -65,6 +65,8 @@ interface SearchContext {
keylessReserved?: boolean;
/** Effective threat protection policy; blocked domains are removed from results entirely. */
threatProtectionPolicy?: ThreatProtectionPolicy | null;
+ /** Set when the request legitimately opted out of Safe Mode at the controller. */
+ safeModeBypassed?: boolean;
}
interface SearchExecuteResult {
@@ -305,6 +307,7 @@ export async function executeSearch(
agentIndexOnly: context.agentIndexOnly,
keylessReserved: context.keylessReserved,
threatProtectionPolicy: threatPolicy ?? null,
+ safeModeBypassed: context.safeModeBypassed ?? false,
};
const allDocsWithCostTracking = await scrapeSearchResults(
diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.ts
index d3f06fc40..acd89fafc 100644
--- a/apps/api/src/search/scrape.ts
+++ b/apps/api/src/search/scrape.ts
@@ -43,6 +43,7 @@ interface ScrapeSearchOptions {
agentIndexOnly?: boolean;
keylessReserved?: boolean;
threatProtectionPolicy?: ThreatProtectionPolicy | null;
+ safeModeBypassed?: boolean;
}
async function scrapeSearchResultDirect(
@@ -86,6 +87,7 @@ async function scrapeSearchResultDirect(
teamFlags: flags,
agentIndexOnly: options.agentIndexOnly ?? false,
threatProtection: options.threatProtectionPolicy ?? undefined,
+ safeModeBypassed: options.safeModeBypassed === true,
},
skipNuq: true,
origin: options.origin,
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 56218a6bb..63aefb0de 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -74,6 +74,7 @@ import { verdictJsonSchema } from "./search/judge";
import { computeGoalVersion } from "./search/dedupe";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { getACUCTeam } from "../../controllers/auth";
+import type { TeamFlags } from "../../controllers/v1/types";
import { orgIdForTeam } from "../../lib/team-org";
import {
reconstructKnownState,
@@ -271,6 +272,7 @@ export function estimateActualCredits(doc: any, options?: any): number {
// per-page β€” search monitors bill flat at the check level.
async function scrapeSearchMonitorPage(params: {
teamId: string;
+ teamFlags: TeamFlags | null;
checkId: string;
url: string;
judgePrompt: string;
@@ -316,6 +318,8 @@ async function scrapeSearchMonitorPage(params: {
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
bypassBilling: true,
zeroDataRetention: false,
+ // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
+ teamFlags: params.teamFlags ?? undefined,
},
skipNuq: true,
origin: "monitor",
@@ -555,6 +559,10 @@ async function enqueueMonitorScrapeTarget(params: {
throw new Error("Expected scrape target");
}
+ // Safe Mode applies to monitor scrapes: resolve per-URL at the backstop from
+ // the team's flags. orgId stays null (monitors' separate no-blocklist rule).
+ const acuc = await getACUCTeam(params.monitor.team_id);
+
for (const [index, url] of params.target.urls.entries()) {
const scrapeId = params.targetRun.expectedJobs[index];
const scrapeOptions = scrapeRequestSchema.parse({
@@ -588,6 +596,7 @@ async function enqueueMonitorScrapeTarget(params: {
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
bypassBilling: true,
zeroDataRetention: false,
+ teamFlags: acuc?.flags ?? undefined,
},
origin: "monitor",
integration: null,
@@ -646,6 +655,10 @@ async function enqueueMonitorCrawlTarget(params: {
prompt: undefined,
};
+ // Safe Mode applies to monitor crawls: each child resolves it per-URL at the
+ // backstop from the team's flags. orgId stays null (monitors' no-blocklist rule).
+ const acuc = await getACUCTeam(params.monitor.team_id);
+
const sc: StoredCrawl = {
originUrl: body.url,
crawlerOptions: toV0CrawlerOptions(crawlerOptions),
@@ -658,6 +671,7 @@ async function enqueueMonitorCrawlTarget(params: {
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
zeroDataRetention: false,
bypassBilling: true,
+ teamFlags: acuc?.flags ?? undefined,
},
team_id: params.monitor.team_id,
createdAt: Date.now(),
@@ -823,6 +837,7 @@ async function runMonitorSearchTarget(params: {
scrapePage: ({ url, judgePrompt }) =>
scrapeSearchMonitorPage({
teamId: monitor.team_id,
+ teamFlags,
checkId: check.id,
url,
judgePrompt,
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 2cae892bf..6ead58ef2 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -306,8 +306,14 @@ export interface RedactPIIOptions {
* fields you provide replace the team policy's values.
*/
export interface ThreatProtectionOptions {
- /** "off" disables scanning for this request; "normal" applies the policy. */
- mode?: "off" | "normal";
+ /**
+ * "off" disables scanning for this request; "manual-only" enforces only the
+ * blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
+ * "normal" scans with Google Web Risk; "zscaler" classifies through your
+ * organization's Zscaler connection. Enforced teams may raise the mode per
+ * request but never lower it.
+ */
+ mode?: "off" | "manual-only" | "normal" | "zscaler";
/** Block verdicts at or above this risk score (integer 0-100). */
riskScoreThreshold?: number;
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 6eac78b3e..4ef19366e 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -843,8 +843,12 @@ class ThreatProtectionOptions(BaseModel):
explicitly provide replace the team policy's values.
"""
- # "off" disables scanning for this request; "normal" applies the policy.
- mode: Optional[Literal["off", "normal"]] = None
+ # "off" disables scanning for this request; "manual-only" enforces only the
+ # blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
+ # "normal" scans with Google Web Risk; "zscaler" classifies through your
+ # organization's Zscaler connection. Enforced teams may raise the mode per
+ # request but never lower it.
+ mode: Optional[Literal["off", "manual-only", "normal", "zscaler"]] = None
# Block verdicts at or above this risk score (integer 0-100).
risk_score_threshold: Optional[int] = Field(
default=None, alias="riskScoreThreshold"