Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): tell the developer index when a search is zero data retention (#5281)

Karan Lokchandani99d6d501265c
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
index baca9d856..6f492a11b 100644
--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
@@ -128,6 +128,13 @@ describe.each([
expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(
expect.objectContaining({ zeroDataRetention: true }),
);
+ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: expect.objectContaining({
+ "firecrawl-zero-data-retention": "true",
+ }),
+ }),
+ );
});
it("keeps normal retention for a team that may opt into ZDR", async () => {
@@ -141,5 +148,12 @@ describe.each([
expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(
expect.objectContaining({ zeroDataRetention: false }),
);
+ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: expect.objectContaining({
+ "firecrawl-zero-data-retention": "false",
+ }),
+ }),
+ );
});
});
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 1b52931bf..4a226ee23 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -293,6 +293,7 @@ async function fetchForRequest(
endpoint: ResearchEndpointConfig,
params: Record<string, any>,
queryKeys: string[],
+ zeroDataRetention: boolean,
) {
const headers: Record<string, string> = {};
for (const h of FORWARDED_REQUEST_HEADERS) {
@@ -300,6 +301,7 @@ async function fetchForRequest(
if (typeof v === "string") headers[h] = v;
}
headers["firecrawl-team-id"] = req.auth.team_id;
+ headers["firecrawl-zero-data-retention"] = String(zeroDataRetention);
if ("fetchUpstream" in endpoint) {
return endpoint.fetchUpstream(params, headers);
@@ -332,6 +334,7 @@ function createResearchController(
method: endpoint.action,
jobId,
teamId: authedReq.auth.team_id,
+ zeroDataRetention,
});
const source = req.method === "POST" ? (req.body ?? {}) : req.query;
@@ -391,6 +394,7 @@ function createResearchController(
endpoint,
params,
queryKeys,
+ zeroDataRetention,
);
if (!upstream) {
if (releaseGovCall) await releaseGovCall();
diff --git a/apps/api/src/search/developer.test.ts b/apps/api/src/search/developer.test.ts
index b8902d0c9..766ee979b 100644
--- a/apps/api/src/search/developer.test.ts
+++ b/apps/api/src/search/developer.test.ts
@@ -44,7 +44,13 @@ describe("searchDeveloperCategory", () => {
fetchMock.mockResolvedValue(upstreamOk({ results: [] }));
await searchDeveloperCategory(
- { query: '"exact phrase" retries', limit: 7, teamId: "t1", timeout: 500 },
+ {
+ query: '"exact phrase" retries',
+ limit: 7,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
);
@@ -55,7 +61,29 @@ describe("searchDeveloperCategory", () => {
expect([...url.searchParams.keys()]).toEqual(["query", "k"]);
expect(url.searchParams.has("skills")).toBe(false);
expect(firstCall()[1].method).toBe("GET");
- expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });
+ expect(firstCall()[1].headers).toEqual({
+ "firecrawl-team-id": "t1",
+ "firecrawl-zero-data-retention": "false",
+ });
+ });
+
+ it("tells the upstream that a zero data retention search must not be kept", async () => {
+ fetchMock.mockResolvedValue(upstreamOk({ results: [] }));
+
+ await searchDeveloperCategory(
+ {
+ query: "private query",
+ limit: 3,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: true,
+ },
+ logger,
+ );
+
+ expect(firstCall()[1].headers["firecrawl-zero-data-retention"]).toBe(
+ "true",
+ );
});
it("maps results to the exact web schema and leaks nothing upstream", async () => {
@@ -78,7 +106,13 @@ describe("searchDeveloperCategory", () => {
);
const results = await searchDeveloperCategory(
- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "retries",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
);
@@ -117,7 +151,13 @@ describe("searchDeveloperCategory", () => {
);
const results = await searchDeveloperCategory(
- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "retries",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
);
@@ -137,7 +177,13 @@ describe("searchDeveloperCategory", () => {
await expect(
searchDeveloperCategory(
- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "retries",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
),
).resolves.toEqual([]);
@@ -149,7 +195,13 @@ describe("searchDeveloperCategory", () => {
await expect(
searchDeveloperCategory(
- { query: "retries", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "retries",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
),
).resolves.toEqual([]);
diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.ts
index 2304ccd88..91da7067d 100644
--- a/apps/api/src/search/developer.ts
+++ b/apps/api/src/search/developer.ts
@@ -10,7 +10,13 @@ export function wantsDeveloperCategory(categories?: CategoryOption[]): boolean {
}
export async function searchDeveloperCategory(
- options: { query: string; limit: number; teamId: string; timeout: number },
+ options: {
+ query: string;
+ limit: number;
+ teamId: string;
+ timeout: number;
+ zeroDataRetention: boolean;
+ },
logger: Logger,
): Promise<WebSearchResult[]> {
try {
@@ -18,7 +24,10 @@ export async function searchDeveloperCategory(
path: "/v2/code/search",
params: { query: options.query, k: options.limit },
queryKeys: DEVELOPER_QUERY_KEYS,
- headers: { "firecrawl-team-id": options.teamId },
+ headers: {
+ "firecrawl-team-id": options.teamId,
+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),
+ },
timeoutMs: options.timeout,
});
if (upstream === null) {
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 3a1e558c2..84cef2370 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -309,6 +309,33 @@ describe("executeSearch developer category", () => {
expect(result.resultCategories).toEqual({});
});
+ it.each([
+ {
+ name: "a forced team",
+ context: { zeroDataRetention: true },
+ enterprise: [],
+ },
+ { name: "an enterprise zdr request", context: {}, enterprise: ["zdr"] },
+ { name: "an enterprise anon request", context: {}, enterprise: ["anon"] },
+ ])(
+ "marks the developer upstream call zero data retention for $name",
+ async c => {
+ await executeSearch(
+ {
+ ...options([{ type: "developer" }]),
+ enterprise: c.enterprise,
+ } as any,
+ { ...context, ...c.context } as any,
+ logger,
+ );
+
+ expect(mocks.searchDeveloperCategory).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ logger,
+ );
+ },
+ );
+
it("rejects developer combined with other categories at the schema", () => {
const mixed = searchRequestSchema.safeParse({
query: "retries",
@@ -350,7 +377,13 @@ describe("executeSearch gov category", () => {
expect(mocks.search).not.toHaveBeenCalled();
expect(mocks.searchDeveloperCategory).not.toHaveBeenCalled();
expect(mocks.searchGovCategory).toHaveBeenCalledWith(
- { query: "retries", limit: 10, teamId: "team-1", timeout: 1_000 },
+ {
+ query: "retries",
+ limit: 10,
+ teamId: "team-1",
+ timeout: 1_000,
+ zeroDataRetention: false,
+ },
logger,
);
expect(result.response).toEqual({ web: [govResult] });
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 903532aad..a97d74e76 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -141,7 +141,17 @@ export async function executeSearch(
: null;
const indexResultsPromise = indexCategorySearch
? indexCategorySearch(
- { query, limit, teamId, timeout: options.timeout },
+ {
+ query,
+ limit,
+ teamId,
+ timeout: options.timeout,
+ zeroDataRetention:
+ zeroDataRetention === true ||
+ options.enterprise?.some(
+ mode => mode === "zdr" || mode === "anon",
+ ) === true,
+ },
logger,
)
: null;