Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): send the ZDR header on gov category search and mark the legacy research log
PhantomInTheWiref738a9efb8c8
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.tsindex 6f492a11b..36fd47f70 100644--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts@@ -33,6 +33,7 @@ vi.mock("../../../lib/logger", () => ({},}));+import { logger as rootLogger } from "../../../lib/logger";import { createDeveloperRouter, createResearchRouter } from "../research-proxy";const TEAM_ID = "11111111-1111-1111-1111-111111111111";@@ -157,3 +158,34 @@ describe.each([);});});++describe("legacy research mount notice", () => {+ function logNotice(flags: Record<string, unknown>) {+ const req = makeReq({ query: "private research query" }, flags);+ req.originalUrl = "/v2/research/papers?query=private+research+query";+ createResearchRouter({ legacy: true }).stack[0].handle(+ req,+ makeRes(),+ vi.fn(),+ );+ return (rootLogger.warn as any).mock.calls.find(+ ([message]: any[]) => message === "Legacy research endpoint used",+ )?.[1];+ }++ it.each([+ { searchZDR: "forced-zdr" },+ { searchZDR: "forced-anon" },+ { forceZDR: true },+ ])("marks the notice so the log filter drops it", flags => {+ expect(logNotice(flags)).toEqual(+ expect.objectContaining({ zeroDataRetention: true }),+ );+ });++ it("keeps the notice for a team that may opt into ZDR", () => {+ expect(logNotice({ searchZDR: "allowed" })).toEqual(+ expect.objectContaining({ zeroDataRetention: false }),+ );+ });+});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 4a226ee23..4745f87e6 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -501,12 +501,16 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {if (options.legacy) {router.use((req, _res, next) => {+ const authedReq = req as RequestWithAuth<any, any, any>;rootLogger.warn("Legacy research endpoint used", {module: "api/v2/research",- teamId: (req as RequestWithAuth<any, any, any>).auth?.team_id,+ teamId: authedReq.auth?.team_id,method: req.method,path: req.originalUrl,requestId: req.headers["x-request-id"],+ zeroDataRetention:+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !==+ null,});next();});diff --git a/apps/api/src/search/gov.test.ts b/apps/api/src/search/gov.test.tsindex 07482d81c..50ef19664 100644--- a/apps/api/src/search/gov.test.ts+++ b/apps/api/src/search/gov.test.ts@@ -33,7 +33,13 @@ describe("searchGovCategory", () => {);await searchGovCategory(- { query: "food labeling rules", limit: 7, teamId: "t1", timeout: 500 },+ {+ query: "food labeling rules",+ limit: 7,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,);@@ -45,6 +51,27 @@ describe("searchGovCategory", () => {top_k: 7,});expect(init.headers["firecrawl-team-id"]).toBe("t1");+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("false");+ });++ it("tells the upstream that a zero data retention search must not be kept", async () => {+ fetchMock.mockResolvedValue(+ upstreamOk({ success: true, data: { web: [] } }),+ );++ await searchGovCategory(+ {+ query: "private query",+ limit: 3,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: true,+ },+ logger,+ );++ const [, init] = fetchMock.mock.calls[0] as any[];+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("true");});it("maps results to the exact web schema and tags the category", async () => {@@ -72,7 +99,13 @@ describe("searchGovCategory", () => {);const results = await searchGovCategory(- { query: "food labeling", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "food labeling",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,);@@ -104,7 +137,13 @@ describe("searchGovCategory", () => {await expect(searchGovCategory(- { query: "zoning", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "zoning",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,),).resolves.toEqual([]);@@ -116,7 +155,13 @@ describe("searchGovCategory", () => {await expect(searchGovCategory(- { query: "zoning", limit: 5, teamId: "t1", timeout: 500 },+ {+ query: "zoning",+ limit: 5,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ },logger,),).resolves.toEqual([]);diff --git a/apps/api/src/search/gov.ts b/apps/api/src/search/gov.tsindex c22391792..e6cce255c 100644--- a/apps/api/src/search/gov.ts+++ b/apps/api/src/search/gov.ts@@ -8,14 +8,23 @@ export function wantsGovCategory(categories?: CategoryOption[]): boolean {}export async function searchGovCategory(- options: { query: string; limit: number; teamId: string; timeout: number },+ options: {+ query: string;+ limit: number;+ teamId: string;+ timeout: number;+ zeroDataRetention: boolean;+ },logger: Logger,): Promise<WebSearchResult[]> {try {const upstream = await fetchGovUpstream({query: options.query,k: options.limit,- headers: { "firecrawl-team-id": options.teamId },+ headers: {+ "firecrawl-team-id": options.teamId,+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),+ },timeoutMs: options.timeout,});if (upstream === null) {