Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): allow search for teams with searchZDR forced
firecrawl-spring[bot]222373eea6a1
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 3b57a61e4..49e32f3a7 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -6,6 +6,7 @@ import {scrapeStatusRaw,zdrcleaner,idmux,+ searchRaw,} from "./lib";import { describeIf, TEST_PRODUCTION } from "../lib";import {@@ -148,4 +149,20 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {600000 + 20000,);});++ it("should allow search when searchZDR is forced", async () => {+ const identity = await idmux({+ name: "zdr/search-forced",+ credits: 10000,+ flags: { searchZDR: "forced" },+ });++ const response = await searchRaw(+ { query: "firecrawl", limit: 1 },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ }, 60000);});diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex aebc8949d..642e5adb7 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -86,23 +86,16 @@ export async function searchController(const controllerStartTime = new Date().getTime();const jobId = uuidv7();+ const zeroDataRetention = getSearchZDR(req.acuc?.flags) === "forced";let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "search",method: "searchController",- zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",+ zeroDataRetention,searchQuery: req.body.query.slice(0, 100),});- if (getSearchZDR(req.acuc?.flags) === "forced") {- return res.status(400).json({- success: false,- error:- "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature.",- });- }-let responseData: SearchResponse = {success: true,data: [],@@ -130,7 +123,7 @@ export async function searchController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.query,- zeroDataRetention: false,+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});@@ -169,7 +162,7 @@ export async function searchController(jobId,apiVersion: "v1",bypassBilling: false,- zeroDataRetention: false,+ zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,},logger,@@ -235,7 +228,7 @@ export async function searchController(scrapeOptions: undefined,},credits_cost: result.searchCredits,- zeroDataRetention: false,+ zeroDataRetention,},false,);@@ -276,7 +269,7 @@ export async function searchController(}captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: false },+ extra: { zeroDataRetention },});logger.error("Unhandled error occurred in search", {version: "v1",diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex e599a048d..614bea80f 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -186,22 +186,15 @@ export async function x402SearchController(res: Response<SearchResponse & { request?: any }>,) {const jobId = uuidv7();+ const zeroDataRetention = getSearchZDR(req.acuc?.flags) === "forced";let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "x402-search",method: "x402SearchController",- zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",+ zeroDataRetention,});- if (getSearchZDR(req.acuc?.flags) === "forced") {- return res.status(400).json({- success: false,- error:- "Your team has zero data retention enabled. This is not supported on x402/search. Please contact [email protected] to unblock this feature.",- });- }-let responseData: SearchResponse = {success: true,data: [],@@ -234,7 +227,7 @@ export async function x402SearchController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.query,- zeroDataRetention: false, // not supported for x402 search+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});@@ -351,7 +344,7 @@ export async function x402SearchController(team_id: req.auth.team_id,options: { ...req.body, scrapeOptions: undefined, query: undefined },credits_cost: responseData.data.length,- zeroDataRetention: false, // not supported+ zeroDataRetention,},false,);@@ -367,7 +360,7 @@ export async function x402SearchController(}captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: false },+ extra: { zeroDataRetention },});logger.error("Unhandled error occurred in search [x402]", { error });return res.status(500).json({diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 74160e63e..c19ae19d1 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -30,28 +30,22 @@ export async function searchController(const controllerStartTime = new Date().getTime();const jobId = uuidv7();+ const searchZDRMode = getSearchZDR(req.acuc?.flags);+ const teamForcedZDR = searchZDRMode === "forced";let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "api/v2",method: "searchController",- zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",+ zeroDataRetention: teamForcedZDR,});- if (getSearchZDR(req.acuc?.flags) === "forced") {- return res.status(400).json({- success: false,- error:- "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature.",- });- }-const middlewareTime = controllerStartTime - middlewareStartTime;const isSearchPreview =config.SEARCH_PREVIEW_TOKEN !== undefined &&config.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;- let zeroDataRetention = false;+ let zeroDataRetention = teamForcedZDR;try {req.body = searchRequestSchema.parse(req.body);@@ -87,13 +81,12 @@ export async function searchController(const isZDR = req.body.enterprise?.includes("zdr");const isAnon = req.body.enterprise?.includes("anon");const isZDROrAnon = isZDR || isAnon;- zeroDataRetention = isZDROrAnon ?? false;- applyZdrScope(isZDROrAnon ?? false);+ zeroDataRetention = teamForcedZDR || (isZDROrAnon ?? false);+ applyZdrScope(zeroDataRetention);// Verify the team has searchZDR enabled before allowing enterprise ZDR/anonif (isZDROrAnon) {- const searchMode = getSearchZDR(req.acuc?.flags);- if (searchMode !== "allowed" && searchMode !== "forced") {+ if (searchZDRMode !== "allowed" && searchZDRMode !== "forced") {return res.status(403).json({success: false,error:@@ -111,7 +104,7 @@ export async function searchController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.query,- zeroDataRetention: isZDROrAnon ?? false,+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});}@@ -142,7 +135,7 @@ export async function searchController(jobId,apiVersion: "v2",bypassBilling: !shouldBill,- zeroDataRetention: isZDROrAnon,+ zeroDataRetention,billing,agentIndexOnly: (req as any).agentIndexOnly ?? false,},@@ -180,7 +173,7 @@ export async function searchController(team_id: req.auth.team_id,options: req.body,credits_cost: shouldBill ? result.searchCredits : 0,- zeroDataRetention: isZDROrAnon ?? false,+ zeroDataRetention,},false,);diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex ba54a96bb..1d1d8c4ce 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -206,22 +206,16 @@ export async function x402SearchController(res: Response<SearchResponse>,) {const jobId = uuidv7();+ const searchZDRMode = getSearchZDR(req.acuc?.flags);+ const zeroDataRetention = searchZDRMode === "forced";let logger = _logger.child({jobId,teamId: req.auth.team_id,module: "api/v2",method: "x402SearchController",- zeroDataRetention: getSearchZDR(req.acuc?.flags) === "forced",+ zeroDataRetention,});- if (getSearchZDR(req.acuc?.flags) === "forced") {- return res.status(400).json({- success: false,- error:- "Your team has zero data retention enabled. This is not supported on search. Please contact [email protected] to unblock this feature.",- });- }-const startTime = new Date().getTime();const isSearchPreview =config.SEARCH_PREVIEW_TOKEN !== undefined &&@@ -247,8 +241,7 @@ export async function x402SearchController(const isZDR = req.body.enterprise?.includes("zdr");const isAnon = req.body.enterprise?.includes("anon");if (isZDR || isAnon) {- const searchMode = getSearchZDR(req.acuc?.flags);- if (searchMode !== "allowed" && searchMode !== "forced") {+ if (searchZDRMode !== "allowed" && searchZDRMode !== "forced") {return res.status(403).json({success: false,error:@@ -265,7 +258,7 @@ export async function x402SearchController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.query,- zeroDataRetention: false, // not supported for x402 search+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});@@ -558,7 +551,7 @@ export async function x402SearchController(scrapeOptions: undefined,},credits_cost: credits_billed,- zeroDataRetention: false,+ zeroDataRetention,},false,);@@ -643,7 +636,7 @@ export async function x402SearchController(team_id: req.auth.team_id,options: { ...req.body, scrapeOptions: undefined, query: undefined },credits_cost: credits_billed,- zeroDataRetention: false, // not supported+ zeroDataRetention,},false,);@@ -674,7 +667,7 @@ export async function x402SearchController(}captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: false },+ extra: { zeroDataRetention },});logger.error("Unhandled error occurred in search [x402]", { error });return res.status(500).json({