Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): tell the developer index when a search is zero data retention (#5281)
Karan Lokchandani99d6d501265c
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.tsindex baca9d856..6f492a11b 100644--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts@@ -128,6 +128,13 @@ describe.each([expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(expect.objectContaining({ zeroDataRetention: true }),);+ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({+ headers: expect.objectContaining({+ "firecrawl-zero-data-retention": "true",+ }),+ }),+ );});it("keeps normal retention for a team that may opt into ZDR", async () => {@@ -141,5 +148,12 @@ describe.each([expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(expect.objectContaining({ zeroDataRetention: false }),);+ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({+ headers: expect.objectContaining({+ "firecrawl-zero-data-retention": "false",+ }),+ }),+ );});});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 1b52931bf..4a226ee23 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -293,6 +293,7 @@ async function fetchForRequest(endpoint: ResearchEndpointConfig,params: Record<string, any>,queryKeys: string[],+ zeroDataRetention: boolean,) {const headers: Record<string, string> = {};for (const h of FORWARDED_REQUEST_HEADERS) {@@ -300,6 +301,7 @@ async function fetchForRequest(if (typeof v === "string") headers[h] = v;}headers["firecrawl-team-id"] = req.auth.team_id;+ headers["firecrawl-zero-data-retention"] = String(zeroDataRetention);if ("fetchUpstream" in endpoint) {return endpoint.fetchUpstream(params, headers);@@ -332,6 +334,7 @@ function createResearchController(method: endpoint.action,jobId,teamId: authedReq.auth.team_id,+ zeroDataRetention,});const source = req.method === "POST" ? (req.body ?? {}) : req.query;@@ -391,6 +394,7 @@ function createResearchController(endpoint,params,queryKeys,+ zeroDataRetention,);if (!upstream) {if (releaseGovCall) await releaseGovCall();diff --git a/apps/api/src/search/developer.test.ts b/apps/api/src/search/developer.test.tsindex b8902d0c9..766ee979b 100644--- a/apps/api/src/search/developer.test.ts+++ b/apps/api/src/search/developer.test.ts@@ -44,7 +44,13 @@ describe("searchDeveloperCategory", () => {fetchMock.mockResolvedValue(upstreamOk({ results: [] }));await searchDeveloperCategory(- { query: '"exact phrase" retries', limit: 7, teamId: "t1", timeout: 500 },+ {+ query: '"exact phrase" retries',+ limit: 7,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,);@@ -55,7 +61,29 @@ describe("searchDeveloperCategory", () => {expect([...url.searchParams.keys()]).toEqual(["query", "k"]);expect(url.searchParams.has("skills")).toBe(false);expect(firstCall()[1].method).toBe("GET");- expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });+ expect(firstCall()[1].headers).toEqual({+ "firecrawl-team-id": "t1",+ "firecrawl-zero-data-retention": "false",+ });+ });++ it("tells the upstream that a zero data retention search must not be kept", async () => {+ fetchMock.mockResolvedValue(upstreamOk({ results: [] }));++ await searchDeveloperCategory(+ {+ query: "private query",+ limit: 3,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: true,+ },+ logger,+ );++ expect(firstCall()[1].headers["firecrawl-zero-data-retention"]).toBe(+ "true",+ );});it("maps results to the exact web schema and leaks nothing upstream", async () => {@@ -78,7 +106,13 @@ describe("searchDeveloperCategory", () => {);const results = await searchDeveloperCategory(- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "retries",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,);@@ -117,7 +151,13 @@ describe("searchDeveloperCategory", () => {);const results = await searchDeveloperCategory(- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "retries",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,);@@ -137,7 +177,13 @@ describe("searchDeveloperCategory", () => {await expect(searchDeveloperCategory(- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "retries",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,),).resolves.toEqual([]);@@ -149,7 +195,13 @@ describe("searchDeveloperCategory", () => {await expect(searchDeveloperCategory(- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "retries",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,),).resolves.toEqual([]);diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.tsindex 2304ccd88..91da7067d 100644--- a/apps/api/src/search/developer.ts+++ b/apps/api/src/search/developer.ts@@ -10,7 +10,13 @@ export function wantsDeveloperCategory(categories?: CategoryOption[]): boolean {}export async function searchDeveloperCategory(- options: { query: string; limit: number; teamId: string; timeout: number },+ options: {+ query: string;+ limit: number;+ teamId: string;+ timeout: number;+ zeroDataRetention: boolean;+ },logger: Logger,): Promise<WebSearchResult[]> {try {@@ -18,7 +24,10 @@ export async function searchDeveloperCategory(path: "/v2/code/search",params: { query: options.query, k: options.limit },queryKeys: DEVELOPER_QUERY_KEYS,- headers: { "firecrawl-team-id": options.teamId },+ headers: {+ "firecrawl-team-id": options.teamId,+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),+ },timeoutMs: options.timeout,});if (upstream === null) {diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 3a1e558c2..84cef2370 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -309,6 +309,33 @@ describe("executeSearch developer category", () => {expect(result.resultCategories).toEqual({});});+ it.each([+ {+ name: "a forced team",+ context: { zeroDataRetention: true },+ enterprise: [],+ },+ { name: "an enterprise zdr request", context: {}, enterprise: ["zdr"] },+ { name: "an enterprise anon request", context: {}, enterprise: ["anon"] },+ ])(+ "marks the developer upstream call zero data retention for $name",+ async c => {+ await executeSearch(+ {+ ...options([{ type: "developer" }]),+ enterprise: c.enterprise,+ } as any,+ { ...context, ...c.context } as any,+ logger,+ );++ expect(mocks.searchDeveloperCategory).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ logger,+ );+ },+ );+it("rejects developer combined with other categories at the schema", () => {const mixed = searchRequestSchema.safeParse({query: "retries",@@ -350,7 +377,13 @@ describe("executeSearch gov category", () => {expect(mocks.search).not.toHaveBeenCalled();expect(mocks.searchDeveloperCategory).not.toHaveBeenCalled();expect(mocks.searchGovCategory).toHaveBeenCalledWith(- { query: "retries", limit: 10, teamId: "team-1", timeout: 1_000 },+ {+ query: "retries",+ limit: 10,+ teamId: "team-1",+ timeout: 1_000,+ zeroDataRetention: false,+ },logger,);expect(result.response).toEqual({ web: [govResult] });diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 903532aad..a97d74e76 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -141,7 +141,17 @@ export async function executeSearch(: null;const indexResultsPromise = indexCategorySearch? indexCategorySearch(- { query, limit, teamId, timeout: options.timeout },+ {+ query,+ limit,+ teamId,+ timeout: options.timeout,+ zeroDataRetention:+ zeroDataRetention === true ||+ options.enterprise?.some(+ mode => mode === "zdr" || mode === "anon",+ ) === true,+ },logger,): null;