Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): send the ZDR header on gov category search and mark the legacy research log

PhantomInTheWiref738a9efb8c8
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
index 6f492a11b..36fd47f70 100644
--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
@@ -33,6 +33,7 @@ vi.mock("../../../lib/logger", () => ({
},
}));
+import { logger as rootLogger } from "../../../lib/logger";
import { createDeveloperRouter, createResearchRouter } from "../research-proxy";
const TEAM_ID = "11111111-1111-1111-1111-111111111111";
@@ -157,3 +158,34 @@ describe.each([
);
});
});
+
+describe("legacy research mount notice", () => {
+ function logNotice(flags: Record<string, unknown>) {
+ const req = makeReq({ query: "private research query" }, flags);
+ req.originalUrl = "/v2/research/papers?query=private+research+query";
+ createResearchRouter({ legacy: true }).stack[0].handle(
+ req,
+ makeRes(),
+ vi.fn(),
+ );
+ return (rootLogger.warn as any).mock.calls.find(
+ ([message]: any[]) => message === "Legacy research endpoint used",
+ )?.[1];
+ }
+
+ it.each([
+ { searchZDR: "forced-zdr" },
+ { searchZDR: "forced-anon" },
+ { forceZDR: true },
+ ])("marks the notice so the log filter drops it", flags => {
+ expect(logNotice(flags)).toEqual(
+ expect.objectContaining({ zeroDataRetention: true }),
+ );
+ });
+
+ it("keeps the notice for a team that may opt into ZDR", () => {
+ expect(logNotice({ searchZDR: "allowed" })).toEqual(
+ expect.objectContaining({ zeroDataRetention: false }),
+ );
+ });
+});
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 4a226ee23..4745f87e6 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -501,12 +501,16 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {
if (options.legacy) {
router.use((req, _res, next) => {
+ const authedReq = req as RequestWithAuth<any, any, any>;
rootLogger.warn("Legacy research endpoint used", {
module: "api/v2/research",
- teamId: (req as RequestWithAuth<any, any, any>).auth?.team_id,
+ teamId: authedReq.auth?.team_id,
method: req.method,
path: req.originalUrl,
requestId: req.headers["x-request-id"],
+ zeroDataRetention:
+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !==
+ null,
});
next();
});
diff --git a/apps/api/src/search/gov.test.ts b/apps/api/src/search/gov.test.ts
index 07482d81c..50ef19664 100644
--- a/apps/api/src/search/gov.test.ts
+++ b/apps/api/src/search/gov.test.ts
@@ -33,7 +33,13 @@ describe("searchGovCategory", () => {
);
await searchGovCategory(
- { query: "food labeling rules", limit: 7, teamId: "t1", timeout: 500 },
+ {
+ query: "food labeling rules",
+ limit: 7,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
);
@@ -45,6 +51,27 @@ describe("searchGovCategory", () => {
top_k: 7,
});
expect(init.headers["firecrawl-team-id"]).toBe("t1");
+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("false");
+ });
+
+ it("tells the upstream that a zero data retention search must not be kept", async () => {
+ fetchMock.mockResolvedValue(
+ upstreamOk({ success: true, data: { web: [] } }),
+ );
+
+ await searchGovCategory(
+ {
+ query: "private query",
+ limit: 3,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: true,
+ },
+ logger,
+ );
+
+ const [, init] = fetchMock.mock.calls[0] as any[];
+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("true");
});
it("maps results to the exact web schema and tags the category", async () => {
@@ -72,7 +99,13 @@ describe("searchGovCategory", () => {
);
const results = await searchGovCategory(
- { query: "food labeling", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "food labeling",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
);
@@ -104,7 +137,13 @@ describe("searchGovCategory", () => {
await expect(
searchGovCategory(
- { query: "zoning", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "zoning",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
),
).resolves.toEqual([]);
@@ -116,7 +155,13 @@ describe("searchGovCategory", () => {
await expect(
searchGovCategory(
- { query: "zoning", limit: 5, teamId: "t1", timeout: 500 },
+ {
+ query: "zoning",
+ limit: 5,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ },
logger,
),
).resolves.toEqual([]);
diff --git a/apps/api/src/search/gov.ts b/apps/api/src/search/gov.ts
index c22391792..e6cce255c 100644
--- a/apps/api/src/search/gov.ts
+++ b/apps/api/src/search/gov.ts
@@ -8,14 +8,23 @@ export function wantsGovCategory(categories?: CategoryOption[]): boolean {
}
export async function searchGovCategory(
- options: { query: string; limit: number; teamId: string; timeout: number },
+ options: {
+ query: string;
+ limit: number;
+ teamId: string;
+ timeout: number;
+ zeroDataRetention: boolean;
+ },
logger: Logger,
): Promise<WebSearchResult[]> {
try {
const upstream = await fetchGovUpstream({
query: options.query,
k: options.limit,
- headers: { "firecrawl-team-id": options.teamId },
+ headers: {
+ "firecrawl-team-id": options.teamId,
+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),
+ },
timeoutMs: options.timeout,
});
if (upstream === null) {