Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add scrapeZDR/searchZDR enum support with backward compat (#3156)
firecrawl-spring[bot]ca58f5ea6e24
diff --git a/apps/api/src/__tests__/snips/v1/billing.test.ts b/apps/api/src/__tests__/snips/v1/billing.test.tsindex 325af196c..6c6c347b4 100644--- a/apps/api/src/__tests__/snips/v1/billing.test.ts+++ b/apps/api/src/__tests__/snips/v1/billing.test.ts@@ -394,7 +394,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -462,7 +462,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR batch scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -531,7 +531,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR crawl correctly",credits: 200,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -595,7 +595,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",zdrCost: 0,},});diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex 4787751d1..ab10290ca 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -24,12 +24,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/scrape`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});@@ -66,12 +61,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/crawl`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});@@ -122,12 +112,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/batch-scrape`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});diff --git a/apps/api/src/__tests__/snips/v2/billing.test.ts b/apps/api/src/__tests__/snips/v2/billing.test.tsindex 470cf7a16..71f8c3b1b 100644--- a/apps/api/src/__tests__/snips/v2/billing.test.ts+++ b/apps/api/src/__tests__/snips/v2/billing.test.ts@@ -369,7 +369,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -439,7 +439,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR batch scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -510,7 +510,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR crawl correctly",credits: 200,flags: {- allowZDR: true,+ scrapeZDR: "allowed",},});@@ -576,7 +576,7 @@ describeIf(TEST_PRODUCTION)("Billing tests", () => {name: "billing/bills ZDR scrape correctly",credits: 100,flags: {- allowZDR: true,+ scrapeZDR: "allowed",zdrCost: 0,},});diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 9ed7cce14..3b57a61e4 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -24,12 +24,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/scrape`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});@@ -66,12 +61,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/crawl`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});@@ -120,12 +110,7 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {name: `zdr/${scope}/batch-scrape`,credits: 10000,flags: {- allowZDR: true,- ...(scope === "Team-scoped"- ? {- forceZDR: true,- }- : {}),+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",},});diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex 11ea5b9f6..91fc10927 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -7,6 +7,7 @@ import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { redisEvictConnection } from "../../../src/services/redis";import { crawlGroup } from "../../services/worker/nuq";+import { getScrapeZDR } from "../../lib/zdr-helpers";configDotenv();export async function crawlCancelController(req: Request, res: Response) {@@ -18,7 +19,7 @@ export async function crawlCancelController(req: Request, res: Response) {const { team_id } = auth;- if (auth.chunk?.flags?.forceZDR) {+ if (getScrapeZDR(auth.chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 7262c4d58..3c18b3169 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -13,6 +13,7 @@ import type { DBScrape, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob } from "../../services/worker/nuq";import { includesFormat } from "../../lib/format-utils";+import { getScrapeZDR } from "../../lib/zdr-helpers";configDotenv();async function getJobs(@@ -96,7 +97,7 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(auth.status).json({ error: auth.error });}- if (auth.chunk?.flags?.forceZDR) {+ if (getScrapeZDR(auth.chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 70f8d990e..58b12d544 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -35,6 +35,7 @@ import { fromV0ScrapeOptions } from "../v2/types";import { isSelfHosted } from "../../lib/deployment";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function crawlController(req: Request, res: Response) {try {@@ -45,7 +46,7 @@ export async function crawlController(req: Request, res: Response) {const { team_id, chunk } = auth;- if (chunk?.flags?.forceZDR) {+ if (getScrapeZDR(chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.tsindex bbf636138..fd8e33127 100644--- a/apps/api/src/controllers/v0/keyAuth.ts+++ b/apps/api/src/controllers/v0/keyAuth.ts@@ -4,6 +4,7 @@ import { Request, Response } from "express";import { authenticateUser } from "../auth";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../lib/logger";+import { getScrapeZDR } from "../../lib/zdr-helpers";export const keyAuthController = async (req: Request, res: Response) => {try {@@ -13,7 +14,7 @@ export const keyAuthController = async (req: Request, res: Response) => {return res.status(auth.status).json({ error: auth.error });}- if (auth.chunk?.flags?.forceZDR) {+ if (getScrapeZDR(auth.chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 22e25054f..c89484b99 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -25,6 +25,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";import { getErrorContactMessage } from "../../lib/deployment";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";async function scrapeHelper(jobId: string,@@ -185,7 +186,7 @@ export async function scrapeController(req: Request, res: Response) {const { team_id, chunk } = auth;- if (chunk?.flags?.forceZDR) {+ if (getScrapeZDR(chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex d8dc46aae..d9955b398 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -24,6 +24,7 @@ import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";import { defaultOrigin } from "../../lib/default-values";+import { getSearchZDR } from "../../lib/zdr-helpers";async function searchHelper(jobId: string,@@ -176,7 +177,7 @@ export async function searchController(req: Request, res: Response) {}const { team_id, chunk } = auth;- if (chunk?.flags?.forceZDR) {+ if (getSearchZDR(chunk?.flags) === "forced") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 3f74fc6b3..6b9cd0b67 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -28,6 +28,7 @@ import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -49,7 +50,7 @@ export async function batchScrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 65fdb2197..55a5986a3 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -20,6 +20,7 @@ import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -37,7 +38,7 @@ export async function crawlController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const id = uuidv7();const logger = _logger.child({diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex d72a52574..f6c45b3c7 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -6,6 +6,7 @@ import * as Sentry from "@sentry/node";import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";import { z } from "zod";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";const deepResearchRequestSchema = z.object({@@ -81,7 +82,7 @@ export async function deepResearchController(req: RequestWithAuth<{}, DeepResearchResponse, DeepResearchRequest>,res: Response<DeepResearchResponse>,) {- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 1d0e947ca..f4882df0d 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -20,6 +20,7 @@ import {} from "../v2/types";import { createWebhookSender, WebhookEvent } from "../../services/webhook";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";import { config } from "../../config";async function oldExtract(@@ -105,7 +106,7 @@ export async function extractController(const originalRequest = { ...req.body };req.body = extractRequestSchema.parse(req.body);- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:@@ -138,7 +139,7 @@ export async function extractController(team_id: req.auth.team_id,subId: req.acuc?.sub_id,extractId,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});await logRequest({@@ -193,7 +194,7 @@ export async function extractController(showLLMUsage: req.body.__experimental_llmUsage,showSources: req.body.__experimental_showSources || req.body.showSources,showCostTracking: req.body.__experimental_showCostTracking,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});await addExtractJobToQueue(extractId, jobData);diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 2fe3c83bc..50210abce 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -10,6 +10,7 @@ import { getGenerateLlmsTxtQueue } from "../../services/queue-service";import * as Sentry from "@sentry/node";import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";type GenerateLLMsTextResponse =| ErrorResponse@@ -28,7 +29,7 @@ export async function generateLLMsTextController(req: RequestWithAuth<{}, GenerateLLMsTextResponse, GenerateLLMsTextRequest>,res: Response<GenerateLLMsTextResponse>,) {- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex f65481e02..5026311a1 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -30,6 +30,7 @@ import {} from "../../services/index";import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";+import { getScrapeZDR } from "../../lib/zdr-helpers";configDotenv();const redis = new Redis(config.REDIS_URL!);@@ -126,7 +127,7 @@ export async function getMapResults({let links: string[] = [url];let mapResults: MapDocument[] = [];- const zeroDataRetention = flags?.forceZDR || false;+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;const sc: StoredCrawl = {originUrl: url,@@ -369,7 +370,7 @@ export async function mapController(const originalRequest = req.body;req.body = mapRequestSchema.parse(req.body);- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 3cce861ce..9b7ebc211 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -2,6 +2,7 @@ import { Response } from "express";import { supabaseGetScrapeByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -10,10 +11,10 @@ export async function scrapeStatusController(req: any, res: any) {teamId: req.auth.team_id,jobId: req.params.jobId,scrapeId: req.params.jobId,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex e7e1d2c24..2e22d174e 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -22,6 +22,7 @@ import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManageimport { logRequest } from "../../services/logging/log_job";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -45,7 +46,7 @@ export async function scrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const logger = _logger.child({method: "scrapeController",diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex d144ea7f5..ce3313b77 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -26,6 +26,7 @@ import {filterDocumentsWithContent,} from "../../search/transform";import { fromV1ScrapeOptions } from "../v2/types";+import { getSearchZDR } from "../../lib/zdr-helpers";// Used for deep researchexport async function searchAndScrapeSearchResult(@@ -90,11 +91,11 @@ export async function searchController(teamId: req.auth.team_id,module: "search",method: "searchController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",searchQuery: req.body.query.slice(0, 100),});- if (req.acuc?.flags?.forceZDR) {+ if (getSearchZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 045c77b53..6e230c498 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1279,6 +1279,8 @@ export type TeamFlags = {unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;+ scrapeZDR?: "disabled" | "allowed" | "forced";+ searchZDR?: "disabled" | "allowed" | "forced";zdrCost?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 805b2519a..b1234257c 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -27,6 +27,7 @@ import {captureExceptionWithZdrCheck,} from "../../services/sentry";import { getJobPriority } from "../../lib/job-priority";+import { getSearchZDR } from "../../lib/zdr-helpers";interface DocumentWithCostTracking {document: Document;@@ -51,7 +52,7 @@ async function scrapeX402SearchResult(const costTracking = new CostTracking();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = getSearchZDR(flags) === "forced";applyZdrScope(zeroDataRetention);try {@@ -185,10 +186,10 @@ export async function x402SearchController(teamId: req.auth.team_id,module: "x402-search",method: "x402SearchController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",});- if (req.acuc?.flags?.forceZDR) {+ if (getSearchZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 85334ea66..34136a277 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -10,6 +10,7 @@ import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";import { config } from "../../config";import { supabase_service } from "../../services/supabase";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function agentController(req: RequestWithAuth<{}, AgentResponse, AgentRequest>,@@ -24,13 +25,13 @@ export async function agentController(team_id: req.auth.team_id,module: "api/v2",method: "agentController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});const originalRequest = { ...req.body };req.body = agentRequestSchema.parse(req.body);- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:@@ -42,7 +43,7 @@ export async function agentController(request: req.body,originalRequest,subId: req.acuc?.sub_id,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});if (!config.EXTRACT_V3_BETA_URL) {diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 00081a516..9620ca8b9 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -29,6 +29,7 @@ import { checkPermissions } from "../../lib/permissions";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";import type { BillingMetadata } from "../../services/billing/types";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -50,7 +51,7 @@ export async function batchScrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);+ getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);if (req.body.__agentInterop &&diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 2d0059b10..1dc4881a8 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -22,6 +22,7 @@ import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -39,7 +40,7 @@ export async function crawlController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;+ getScrapeZDR(req.acuc?.flags) === "forced" || req.body.zeroDataRetention;const id = uuidv7();const logger = _logger.child({diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 44b69a5b7..d7155a312 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -13,6 +13,7 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { logger as _logger } from "../../lib/logger";import { logRequest } from "../../services/logging/log_job";import { config } from "../../config";+import { getScrapeZDR } from "../../lib/zdr-helpers";/*** Extracts data from the provided URLs based on the request parameters.@@ -28,7 +29,7 @@ export async function extractController(const originalRequest = { ...req.body };req.body = extractRequestSchema.parse(req.body);- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:@@ -45,7 +46,7 @@ export async function extractController(team_id: req.auth.team_id,subId: req.acuc?.sub_id,extractId,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});if (req.body.agent?.model === "v3-beta") {@@ -100,7 +101,7 @@ export async function extractController(showLLMUsage: req.body.__experimental_llmUsage,showSources: req.body.__experimental_showSources || req.body.showSources,showCostTracking: req.body.__experimental_showCostTracking,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});await addExtractJobToQueue(extractId, {diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex ac88d6412..a46871b4d 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -14,6 +14,7 @@ import { checkPermissions } from "../../lib/permissions";import { getMapResults, MapResult } from "../../lib/map-utils";import { v7 as uuidv7 } from "uuid";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";+import { getScrapeZDR } from "../../lib/zdr-helpers";configDotenv();@@ -26,7 +27,7 @@ export async function mapController(teamId: req.auth.team_id,module: "api/v2",method: "mapController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});// Get timing data from middleware (includes all middleware processing time)const middlewareStartTime =diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.tsindex 2fa51cbfc..758f354ef 100644--- a/apps/api/src/controllers/v2/scrape-status.ts+++ b/apps/api/src/controllers/v2/scrape-status.ts@@ -1,6 +1,7 @@import { supabaseGetScrapeByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";+import { getScrapeZDR } from "../../lib/zdr-helpers";export async function scrapeStatusController(req: any, res: any) {const uuidReg =@@ -18,10 +19,10 @@ export async function scrapeStatusController(req: any, res: any) {teamId: req.auth.team_id,jobId: req.params.jobId,scrapeId: req.params.jobId,- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});- if (req.acuc?.flags?.forceZDR) {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b03f5c036..afa1af946 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -23,6 +23,7 @@ import { logRequest } from "../../services/logging/log_job";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";+import { getScrapeZDR } from "../../lib/zdr-helpers";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -83,7 +84,7 @@ export async function scrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);+ getScrapeZDR(req.acuc?.flags) === "forced" || (req.body.zeroDataRetention ?? false);const billing: BillingMetadata = req.body.__agentInterop? { endpoint: "agent" as const, jobId }: { endpoint: "scrape" as const, jobId };diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 6e557c4dc..85f4e77e6 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -19,6 +19,7 @@ import {} from "../../services/sentry";import { executeSearch } from "../../search/execute";import type { BillingMetadata } from "../../services/billing/types";+import { getSearchZDR } from "../../lib/zdr-helpers";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -34,10 +35,10 @@ export async function searchController(teamId: req.auth.team_id,module: "api/v2",method: "searchController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",});- if (req.acuc?.flags?.forceZDR) {+ if (getSearchZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 30332c56d..5fd359bf1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1287,6 +1287,8 @@ export type TeamFlags = {unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;+ scrapeZDR?: "disabled" | "allowed" | "forced";+ searchZDR?: "disabled" | "allowed" | "forced";zdrCost?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 6a1a793a4..03d43c200 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -32,6 +32,7 @@ import {applyZdrScope,captureExceptionWithZdrCheck,} from "../../services/sentry";+import { getSearchZDR } from "../../lib/zdr-helpers";interface DocumentWithCostTracking {document: Document;@@ -61,7 +62,7 @@ async function startX420ScrapeJob(): Promise<string> {const jobId = uuidv7();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = getSearchZDR(flags) === "forced";applyZdrScope(zeroDataRetention);logger.info("Adding scrape job [x402]", {@@ -210,10 +211,10 @@ export async function x402SearchController(teamId: req.auth.team_id,module: "api/v2",method: "x402SearchController",- zeroDataRetention: req.acuc?.flags?.forceZDR,+ zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",});- if (req.acuc?.flags?.forceZDR) {+ if (getSearchZDR(req.acuc?.flags) === "forced") {return res.status(400).json({success: false,error:diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 77b4f6332..0093b92c1 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -6,6 +6,7 @@ import { logger as _logger } from "./logger";import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";import type { Logger } from "winston";import { withSpan, setSpanAttributes } from "./otel-tracer";+import { getScrapeZDR } from "./zdr-helpers";export type StoredCrawl = {originUrl?: string;@@ -559,7 +560,7 @@ export function crawlToCrawler(regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,- zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false,+ zeroDataRetention: (getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,location: sc.scrapeOptions?.location,headers: sc.scrapeOptions?.headers,});diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 5a0a511e8..43bbc8b66 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -7,6 +7,7 @@ import {MAX_MAP_LIMIT,} from "../controllers/v2/types";import { crawlToCrawler, StoredCrawl } from "./crawl-redis";+import { getScrapeZDR } from "./zdr-helpers";import {checkAndUpdateURLForMap,isSameDomain,@@ -130,7 +131,7 @@ export async function getMapResults({const id = providedId ?? uuidv7();let mapResults: MapDocument[] = [];- const zeroDataRetention = flags?.forceZDR || false;+ const zeroDataRetention = getScrapeZDR(flags) === "forced" || false;const sc: StoredCrawl = {originUrl: url,diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex 3a0d6a5c1..8bca74fb1 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -1,4 +1,5 @@import { TeamFlags } from "../controllers/v2/types";+import { getScrapeZDR } from "./zdr-helpers";type LocationOptions = { country?: string };@@ -16,8 +17,9 @@ export function checkPermissions(request: APIRequest,flags?: TeamFlags,): { error?: string } {- // zdr perms- if (request.zeroDataRetention && !flags?.allowZDR) {+ // zdr perms — scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDR+ const scrapeMode = getScrapeZDR(flags);+ if (request.zeroDataRetention && scrapeMode !== "allowed" && scrapeMode !== "forced") {return {error: `Zero Data Retention (ZDR) is not enabled for your team. Contact ${SUPPORT_EMAIL} to enable this feature.`,};diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsnew file mode 100644index 000000000..1033a0266--- /dev/null+++ b/apps/api/src/lib/zdr-helpers.ts@@ -0,0 +1,29 @@+import type { TeamFlags } from "../controllers/v2/types";++type ZDRMode = "disabled" | "allowed" | "forced";++/**+ * Resolves the effective ZDR mode for scrape endpoints from team flags.+ *+ * Handles both the new enum format (scrapeZDR) and the legacy boolean+ * format (forceZDR/allowZDR) for backward compatibility during the+ * cache migration window.+ */+export function getScrapeZDR(flags: TeamFlags | undefined): ZDRMode {+ if (flags?.scrapeZDR === "forced" || flags?.forceZDR) return "forced";+ if (flags?.scrapeZDR === "allowed" || flags?.allowZDR) return "allowed";+ return "disabled";+}++/**+ * Resolves the effective ZDR mode for search endpoints from team flags.+ *+ * Handles both the new enum format (searchZDR) and the legacy boolean+ * format (forceZDR/allowZDR) for backward compatibility during the+ * cache migration window.+ */+export function getSearchZDR(flags: TeamFlags | undefined): ZDRMode {+ if (flags?.searchZDR === "forced" || flags?.forceZDR) return "forced";+ if (flags?.searchZDR === "allowed" || flags?.allowZDR) return "allowed";+ return "disabled";+}diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex 41081098b..03c9b0508 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -9,6 +9,7 @@ import { processJobInternal } from "../services/worker/scrape-worker";import { ScrapeJobData } from "../types";import { SearchV2Response } from "../lib/entities";import type { BillingMetadata } from "../services/billing/types";+import { getScrapeZDR } from "../lib/zdr-helpers";export interface DocumentWithCostTracking {document: Document;@@ -48,7 +49,7 @@ async function scrapeSearchResultDirect(): Promise<DocumentWithCostTracking> {const jobId = uuidv7();const zeroDataRetention =- flags?.forceZDR || (options.zeroDataRetention ?? false);+ getScrapeZDR(flags) === "forced" || (options.zeroDataRetention ?? false);logger.debug("Starting direct scrape for search result", {scrapeId: jobId,