Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): split searchZDR forced into forced-zdr and forced-anon
firecrawl-spring[bot]c84dd32095a0
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 83b4d5410..f0b995dbc 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -186,7 +186,8 @@ export async function searchController(req: Request, res: Response) {}const { team_id, chunk } = auth;- if (getSearchZDR(chunk?.flags) === "forced") {+ const v0SearchZDRMode = getSearchZDR(chunk?.flags);+ if (v0SearchZDRMode === "forced-zdr" || v0SearchZDRMode === "forced-anon") {return res.status(400).json({error:"Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.",diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 642e5adb7..692fd3482 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -26,7 +26,7 @@ import {filterDocumentsWithContent,} from "../../search/transform";import { fromV1ScrapeOptions } from "../v2/types";-import { getSearchZDR } from "../../lib/zdr-helpers";+import { getSearchForcedKind } from "../../lib/zdr-helpers";// Used for deep researchexport async function searchAndScrapeSearchResult(@@ -86,13 +86,16 @@ export async function searchController(const controllerStartTime = new Date().getTime();const jobId = uuidv7();- const zeroDataRetention = getSearchZDR(req.acuc?.flags) === "forced";+ const teamForcedKind = getSearchForcedKind(req.acuc?.flags);+ const zeroDataRetention = teamForcedKind !== null;+ const teamEnterprise = teamForcedKind ? [teamForcedKind] : undefined;let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "search",method: "searchController",zeroDataRetention,+ teamForcedKind,searchQuery: req.body.query.slice(0, 100),});@@ -152,6 +155,7 @@ export async function searchController(sources: [{ type: "web" }], // v1 only supports webscrapeOptions: shouldScrape ? scrapeOptions : undefined,timeout: req.body.timeout,+ enterprise: teamEnterprise,},{teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 4be7d24cf..97ab46a71 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1290,7 +1290,7 @@ export type TeamFlags = {forceZDR?: boolean;allowZDR?: boolean;scrapeZDR?: "disabled" | "allowed" | "forced";- searchZDR?: "disabled" | "allowed" | "forced";+ searchZDR?: "disabled" | "allowed" | "forced" | "forced-zdr" | "forced-anon";zdrCost?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 614bea80f..fe044a6e1 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -27,7 +27,7 @@ import {captureExceptionWithZdrCheck,} from "../../services/sentry";import { getJobPriority } from "../../lib/job-priority";-import { getSearchZDR } from "../../lib/zdr-helpers";+import { getSearchForcedKind } from "../../lib/zdr-helpers";interface DocumentWithCostTracking {document: Document;@@ -52,7 +52,7 @@ async function scrapeX402SearchResult(const costTracking = new CostTracking();- const zeroDataRetention = getSearchZDR(flags) === "forced";+ const zeroDataRetention = getSearchForcedKind(flags) !== null;applyZdrScope(zeroDataRetention);try {@@ -186,13 +186,15 @@ export async function x402SearchController(res: Response<SearchResponse & { request?: any }>,) {const jobId = uuidv7();- const zeroDataRetention = getSearchZDR(req.acuc?.flags) === "forced";+ const teamForcedKind = getSearchForcedKind(req.acuc?.flags);+ const zeroDataRetention = teamForcedKind !== null;let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "x402-search",method: "x402SearchController",zeroDataRetention,+ teamForcedKind,});let responseData: SearchResponse = {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c19ae19d1..126fd6f31 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -19,7 +19,7 @@ import {} from "../../services/sentry";import { executeSearch } from "../../search/execute";import type { BillingMetadata } from "../../services/billing/types";-import { getSearchZDR } from "../../lib/zdr-helpers";+import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -31,13 +31,14 @@ export async function searchController(const jobId = uuidv7();const searchZDRMode = getSearchZDR(req.acuc?.flags);- const teamForcedZDR = searchZDRMode === "forced";+ const teamForcedKind = getSearchForcedKind(req.acuc?.flags);let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "api/v2",method: "searchController",- zeroDataRetention: teamForcedZDR,+ zeroDataRetention: teamForcedKind !== null,+ teamForcedKind,});const middlewareTime = controllerStartTime - middlewareStartTime;@@ -45,7 +46,7 @@ export async function searchController(config.SEARCH_PREVIEW_TOKEN !== undefined &&config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;- let zeroDataRetention = teamForcedZDR;+ let zeroDataRetention = teamForcedKind !== null;try {req.body = searchRequestSchema.parse(req.body);@@ -78,15 +79,25 @@ export async function searchController(origin: req.body.origin,});+ // Inject the team-forced enterprise mode so downstream billing,+ // upstream routing, and ZDR cleanup all see it.+ if (teamForcedKind) {+ const existing = req.body.enterprise ?? [];+ if (!existing.includes(teamForcedKind)) {+ req.body.enterprise = [...existing, teamForcedKind];+ }+ }+const isZDR = req.body.enterprise?.includes("zdr");const isAnon = req.body.enterprise?.includes("anon");const isZDROrAnon = isZDR || isAnon;- zeroDataRetention = teamForcedZDR || (isZDROrAnon ?? false);+ zeroDataRetention = isZDROrAnon ?? false;+ logger = logger.child({ zeroDataRetention });applyZdrScope(zeroDataRetention);// Verify the team has searchZDR enabled before allowing enterprise ZDR/anon- if (isZDROrAnon) {- if (searchZDRMode !== "allowed" && searchZDRMode !== "forced") {+ if (isZDROrAnon && !teamForcedKind) {+ if (searchZDRMode !== "allowed") {return res.status(403).json({success: false,error:diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 3eb06ea1b..1016d4118 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1412,7 +1412,7 @@ export type TeamFlags = {forceZDR?: boolean;allowZDR?: boolean;scrapeZDR?: "disabled" | "allowed" | "forced";- searchZDR?: "disabled" | "allowed" | "forced";+ searchZDR?: "disabled" | "allowed" | "forced" | "forced-zdr" | "forced-anon";zdrCost?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 1d1d8c4ce..46e94532f 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -32,7 +32,7 @@ import {applyZdrScope,captureExceptionWithZdrCheck,} from "../../services/sentry";-import { getSearchZDR } from "../../lib/zdr-helpers";+import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";interface DocumentWithCostTracking {document: Document;@@ -62,7 +62,7 @@ async function startX420ScrapeJob(): Promise<string> {const jobId = uuidv7();- const zeroDataRetention = getSearchZDR(flags) === "forced";+ const zeroDataRetention = getSearchForcedKind(flags) !== null;applyZdrScope(zeroDataRetention);logger.info("Adding scrape job [x402]", {@@ -207,13 +207,15 @@ export async function x402SearchController() {const jobId = uuidv7();const searchZDRMode = getSearchZDR(req.acuc?.flags);- const zeroDataRetention = searchZDRMode === "forced";+ const teamForcedKind = getSearchForcedKind(req.acuc?.flags);+ let zeroDataRetention = teamForcedKind !== null;let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "api/v2",method: "x402SearchController",zeroDataRetention,+ teamForcedKind,});const startTime = new Date().getTime();@@ -237,11 +239,20 @@ export async function x402SearchController(origin: req.body.origin,});+ // Inject the team-forced enterprise mode so downstream billing,+ // upstream routing, and ZDR cleanup all see it.+ if (teamForcedKind) {+ const existing = req.body.enterprise ?? [];+ if (!existing.includes(teamForcedKind)) {+ req.body.enterprise = [...existing, teamForcedKind];+ }+ }+// Verify the team has searchZDR enabled before allowing enterprise ZDR/anonconst isZDR = req.body.enterprise?.includes("zdr");const isAnon = req.body.enterprise?.includes("anon");- if (isZDR || isAnon) {- if (searchZDRMode !== "allowed" && searchZDRMode !== "forced") {+ if ((isZDR || isAnon) && !teamForcedKind) {+ if (searchZDRMode !== "allowed") {return res.status(403).json({success: false,error:@@ -249,6 +260,10 @@ export async function x402SearchController(});}}+ zeroDataRetention =+ teamForcedKind !== null || isZDR === true || isAnon === true;+ logger = logger.child({ zeroDataRetention });+ applyZdrScope(zeroDataRetention);await logRequest({id: jobId,diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex abd42ee4f..2860d81be 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -2,7 +2,9 @@ import type { TeamFlags } from "../controllers/v2/types";type OrgFlagMode = "disabled" | "allowed" | "forced";-type ZDRMode = OrgFlagMode;+type ScrapeZDRMode = OrgFlagMode;+type SearchZDRMode = "disabled" | "allowed" | "forced-zdr" | "forced-anon";+type SearchForcedKind = "zdr" | "anon";/*** Resolves the effective ZDR mode for scrape endpoints from team flags.@@ -11,7 +13,7 @@ type ZDRMode = OrgFlagMode;* format (forceZDR/allowZDR) for backward compatibility during the* cache migration window.*/-export function getScrapeZDR(flags: TeamFlags | undefined): ZDRMode {+export function getScrapeZDR(flags: TeamFlags | undefined): ScrapeZDRMode {if (flags?.scrapeZDR === "forced" || flags?.forceZDR) return "forced";if (flags?.scrapeZDR === "allowed" || flags?.allowZDR) return "allowed";return "disabled";@@ -20,16 +22,43 @@ export function getScrapeZDR(flags: TeamFlags | undefined): ZDRMode {/*** Resolves the effective ZDR mode for search endpoints from team flags.*+ * Search has two distinct forced modes that differ in pricing and upstream+ * behavior:+ * - "forced-zdr" — ZDR enterprise mode (10 credits per 10 results)+ * - "forced-anon" — anonymous routing (default credit rate, no attribution)+ *+ * The bare "forced" value is treated as a deprecated alias for "forced-zdr"+ * for existing orgs that haven't been migrated to the explicit variant.+ ** Handles both the new enum format (searchZDR) and the legacy boolean* format (forceZDR/allowZDR) for backward compatibility during the* cache migration window.*/-export function getSearchZDR(flags: TeamFlags | undefined): ZDRMode {- if (flags?.searchZDR === "forced" || flags?.forceZDR) return "forced";+export function getSearchZDR(flags: TeamFlags | undefined): SearchZDRMode {+ if (+ flags?.searchZDR === "forced-zdr" ||+ flags?.searchZDR === "forced" ||+ flags?.forceZDR+ ) {+ return "forced-zdr";+ }+ if (flags?.searchZDR === "forced-anon") return "forced-anon";if (flags?.searchZDR === "allowed" || flags?.allowZDR) return "allowed";return "disabled";}+/**+ * Returns which enterprise mode the team is forced into, or null if not forced.+ */+export function getSearchForcedKind(+ flags: TeamFlags | undefined,+): SearchForcedKind | null {+ const mode = getSearchZDR(flags);+ if (mode === "forced-zdr") return "zdr";+ if (mode === "forced-anon") return "anon";+ return null;+}+/*** Resolves the effective ignoreRobots mode from team flags.*/