Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): refuse research, developer and gov search under zero data retention (#5280)
Gergő Móricz6ff4cf3d1dce
diff --git a/apps/api/src/__tests__/snips/v2/developer.test.ts b/apps/api/src/__tests__/snips/v2/developer.test.tsindex 2466749e9..0f30acebb 100644--- a/apps/api/src/__tests__/snips/v2/developer.test.ts+++ b/apps/api/src/__tests__/snips/v2/developer.test.ts@@ -1,5 +1,5 @@import { config } from "../../../config";-import { describeIf, TEST_PRODUCTION } from "../lib";+import { describeIf, scrapeTimeout, TEST_PRODUCTION } from "../lib";import { creditUsage, idmux, researchPostRaw, researchRaw } from "./lib";import { HAS_JOB_LOG, jobLogJson, waitForJobLogRow } from "../job-log";@@ -134,13 +134,36 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {expect(requestLog?.integration).toBe("_research_test");}, 120000);- it("redacts stored payloads for a forced-ZDR team", async () => {+ it(+ "refuses a forced-ZDR team on every serving path",+ async () => {+ const identity = await idmux({+ name: "developer/forced ZDR refusal",+ credits: 100,+ flags: { searchZDR: "forced-zdr" },+ });++ for (const path of SERVING_PATHS) {+ const res = await researchRaw(+ path,+ { query: "how do I configure retries", k: 1 },+ identity,+ );+ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("zero data retention");+ }+ },+ scrapeTimeout,+ );++ it("redacts stored payloads for a forced-anon team", async () => {if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;const identity = await idmux({- name: "developer/forced ZDR retention",+ name: "developer/forced anon retention",credits: 100,- flags: { searchZDR: "forced-zdr" },+ flags: { searchZDR: "forced-anon" },});const res = await researchRaw(diff --git a/apps/api/src/__tests__/snips/v2/gov.test.ts b/apps/api/src/__tests__/snips/v2/gov.test.tsindex f6e86479e..b8cbddd6d 100644--- a/apps/api/src/__tests__/snips/v2/gov.test.ts+++ b/apps/api/src/__tests__/snips/v2/gov.test.ts@@ -1,5 +1,5 @@import { config } from "../../../config";-import { describeIf } from "../lib";+import { describeIf, scrapeTimeout } from "../lib";import { idmux, researchRaw } from "./lib";const HAS_GOV = !!config.SEARCH_PLATFORM_URL;@@ -30,4 +30,26 @@ describeIf(HAS_GOV)("Government Index Search API", () => {expect(typeof result.position).toBe("number");}}, 120000);++ it(+ "refuses a forced-ZDR team",+ async () => {+ const identity = await idmux({+ name: "gov/forced ZDR refusal",+ credits: 100,+ flags: { searchZDR: "forced-zdr" },+ });++ const res = await researchRaw(+ PATH,+ { query: "food labeling requirements", k: 3 },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("zero data retention");+ },+ scrapeTimeout,+ );});diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex ecb18b887..65ae273c6 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -224,4 +224,58 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);}, 60000);++ it(+ "should serve the research category when searchZDR is forced",+ async () => {+ const identity = await idmux({+ name: "zdr/search-forced-research",+ credits: 10000,+ flags: { searchZDR: "forced-zdr" },+ });++ const response = await searchRaw(+ { query: "protein folding", limit: 1, categories: ["research"] },+ identity,+ );++ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ },+ scrapeTimeout,+ );++ it(+ "should refuse the developer and gov categories under ZDR",+ async () => {+ const forced = await idmux({+ name: "zdr/search-forced-index-categories",+ credits: 10000,+ flags: { searchZDR: "forced-zdr" },+ });+ const allowed = await idmux({+ name: "zdr/search-request-index-categories",+ credits: 10000,+ flags: { searchZDR: "allowed" },+ });++ const attempts = [+ { identity: forced, enterprise: undefined },+ { identity: allowed, enterprise: ["zdr" as const] },+ ];+ for (const { identity, enterprise } of attempts) {+ for (const category of ["developer", "gov"] as const) {+ const response = await searchRaw(+ { query: "retries", limit: 1, categories: [category], enterprise },+ identity,+ );++ expect(response.statusCode).toBe(400);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain("zero data retention");+ }+ }+ },+ scrapeTimeout,+ );});diff --git a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts b/apps/api/src/controllers/v2/__tests__/gov-search.test.tsindex 9cf4f69e8..d4994ebc4 100644--- a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts+++ b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts@@ -137,11 +137,32 @@ describe("/v2/search/gov", () => {},);- it("stays free under zero data retention and marks the logs", async () => {+ it("refuses a forced-ZDR team ahead of the route handler", async () => {+ const router: any = createGovRouter();+ const req = {+ ...makeReq("GET", { query: "food labeling rules" }, TEAM_ID, {+ searchZDR: "forced-zdr",+ }),+ url: "/",+ };+ const res = makeRes();+ const next = vi.fn();+ router(req, res, next);+ await flush();++ expect(res.status).toHaveBeenCalledWith(400);+ expect(next).not.toHaveBeenCalled();+ expect(mocks.fetchGovUpstream).not.toHaveBeenCalled();+ expect(billTeam).not.toHaveBeenCalled();+ expect(mocks.logRequest).not.toHaveBeenCalled();+ expect(mocks.logResearchEndpoint).not.toHaveBeenCalled();+ });++ it("stays free for a forced-anon team and marks the logs", async () => {const res = makeRes();await handler("get")(makeReq("GET", { query: "food labeling rules" }, TEAM_ID, {- searchZDR: "forced-zdr",+ searchZDR: "forced-anon",}),res,);diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.tsindex 6f492a11b..1ec17f4f0 100644--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts@@ -34,6 +34,8 @@ vi.mock("../../../lib/logger", () => ({}));import { createDeveloperRouter, createResearchRouter } from "../research-proxy";+import { billTeam } from "../../../services/billing/credit_billing";+import { logger } from "../../../lib/logger";const TEAM_ID = "11111111-1111-1111-1111-111111111111";const flush = () => new Promise(resolve => setImmediate(resolve));@@ -81,8 +83,39 @@ async function callRoute(query: Record<string, unknown>,flags: Record<string, unknown>,) {- routeHandler(router, path)(makeReq(query, flags), makeRes(), vi.fn());+ const res = makeRes();+ routeHandler(router, path)(makeReq(query, flags), res, vi.fn());await flush();+ return res;+}++async function dispatch(+ router: any,+ path: string,+ query: Record<string, unknown>,+ flags: Record<string, unknown>,+) {+ const req = { ...makeReq(query, flags), url: path };+ const res = makeRes();+ const next = vi.fn();+ router(req, res, next);+ await flush();+ return { res, next };+}++function expectRefused(res: any, next: any) {+ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ success: false,+ error: expect.stringContaining("zero data retention"),+ }),+ );+ expect(next).not.toHaveBeenCalled();+ expect(mocks.fetchResearchUpstream).not.toHaveBeenCalled();+ expect(mocks.logRequest).not.toHaveBeenCalled();+ expect(mocks.logResearchEndpoint).not.toHaveBeenCalled();+ expect(billTeam).not.toHaveBeenCalled();}beforeEach(() => {@@ -117,10 +150,23 @@ describe.each([])("standalone ZDR persistence for $name", testCase => {it.each([{ searchZDR: "forced-zdr" },- { searchZDR: "forced-anon" },+ { searchZDR: "forced" },{ forceZDR: true },- ])("marks forced team payloads for redaction and cleanup", async flags => {- await callRoute(testCase.router(), testCase.path, testCase.query, flags);+ ])("refuses a forced-ZDR team ahead of every route handler", async flags => {+ const { res, next } = await dispatch(+ testCase.router(),+ testCase.path,+ testCase.query,+ flags,+ );++ expectRefused(res, next);+ });++ it("marks forced-anon team payloads for redaction and cleanup", async () => {+ await callRoute(testCase.router(), testCase.path, testCase.query, {+ searchZDR: "forced-anon",+ });expect(mocks.logRequest).toHaveBeenCalledWith(expect.objectContaining({ zeroDataRetention: true }),@@ -157,3 +203,29 @@ describe.each([);});});++describe("forced-ZDR refusal ordering", () => {+ it("refuses on the legacy research mount before its usage log line", async () => {+ const { res, next } = await dispatch(+ createResearchRouter({ legacy: true }),+ "/papers",+ { query: "private research query" },+ { searchZDR: "forced-zdr" },+ );++ expectRefused(res, next);+ expect(logger.warn).not.toHaveBeenCalled();+ });++ it("refuses GitHub search before its deprecation notice and credit gate", async () => {+ const { res, next } = await dispatch(+ createResearchRouter(),+ "/github",+ { query: "private code query" },+ { searchZDR: "forced-zdr" },+ );++ expectRefused(res, next);+ expect(res.setHeader).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex 4c39d630b..00d1ad7bf 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -66,6 +66,7 @@ vi.mock("../../../lib/logger", () => ({}));import { searchController } from "../search";+import { billTeam } from "../../../services/billing/credit_billing";const TEAM_ID = "11111111-1111-1111-1111-111111111111";@@ -419,3 +420,62 @@ describe("tool discovery availability", () => {);});});++describe("index categories under zero data retention", () => {+ it.each(["developer", "gov"])(+ "refuses the %s category for a forced-ZDR team",+ async category => {+ const req = makeReq({ query: "http client", categories: [category] });+ req.acuc = { api_key_id: 7, flags: { searchZDR: "forced-zdr" } };+ const res = makeRes();++ await searchController(req, res);++ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ success: false,+ error: expect.stringContaining("zero data retention"),+ }),+ );+ expect(mockExecuteSearch).not.toHaveBeenCalled();+ expect(mockLogRequest).not.toHaveBeenCalled();+ expect(mockLogSearch).not.toHaveBeenCalled();+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();+ expect(billTeam).not.toHaveBeenCalled();+ },+ );++ it("refuses the developer category for a request-scoped ZDR search", async () => {+ const req = makeReq({+ query: "http client",+ categories: ["developer"],+ enterprise: ["zdr"],+ });+ req.acuc = { api_key_id: 7, flags: { searchZDR: "allowed" } };+ const res = makeRes();++ await searchController(req, res);++ expect(res.status).toHaveBeenCalledWith(400);+ expect(mockExecuteSearch).not.toHaveBeenCalled();+ expect(mockLogRequest).not.toHaveBeenCalled();+ expect(mockLogSearch).not.toHaveBeenCalled();+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();+ expect(billTeam).not.toHaveBeenCalled();+ });++ it("keeps serving the research category for a forced-ZDR team", async () => {+ const req = makeReq({ query: "protein folding", categories: ["research"] });+ req.acuc = { api_key_id: 7, flags: { searchZDR: "forced-zdr" } };+ const res = makeRes();++ await searchController(req, res);+ await flushAsync();++ expect(res.status).toHaveBeenCalledWith(200);+ expect(mockExecuteSearch).toHaveBeenCalled();+ expect(mockExecuteSearch.mock.calls[0][0].enterprise).toContain("zdr");+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 4a226ee23..93beeec10 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -1,4 +1,4 @@-import express, { Request, Response } from "express";+import express, { NextFunction, Request, Response } from "express";import { externalRequestId } from "../../lib/external-request-id";import { z } from "zod";import { v7 as uuidv7 } from "uuid";@@ -37,7 +37,10 @@ const searchCreditGate = () =>SEARCH_CREDITS_PER_TEN_RESULTS,SEARCH_CREDITS_FEATURE_ID,);-const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;+// The research, developer and government index services have no zero data+// retention mode, so forced-ZDR teams are refused before any upstream call.+const ZDR_UNSUPPORTED_MESSAGE =+ "Your team has zero data retention enabled. This is not supported on research, developer or government search. Please contact [email protected] to unblock this feature.";const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;const GOV_SEARCH_TIMEOUT_MS = 15_000;@@ -238,11 +241,7 @@ function resultCount(body: any): number {return Array.isArray(body?.data?.web) ? body.data.web.length : 0;}-function creditsFor(- config: ResearchEndpointConfig,- body: any,- req: RequestWithAuth<any, any, any>,-) {+function creditsFor(config: ResearchEndpointConfig, body: any) {// Make certain index accesses free (AI/ML & life-sciences indices and the// gov index). Research paper and gov endpoints should have no credit cost.// Explicit exceptions remain billable: GitHub search and developer/code@@ -257,12 +256,7 @@ function creditsFor(if (freeResearchKinds.has(config.kind)) return 0;if (config.billAs === "scrape") return 1;- const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);- const perTen =- forcedKind === "zdr"- ? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS- : SEARCH_CREDITS_PER_TEN_RESULTS;- return Math.ceil(resultCount(body) / 10) * perTen;+ return Math.ceil(resultCount(body) / 10) * SEARCH_CREDITS_PER_TEN_RESULTS;}function researchError(@@ -278,6 +272,16 @@ function researchError(});}+// First on every router here, so a refused query reaches no log line, credit+// check or upstream.+function refuseForcedZdr(req: Request, res: Response, next: NextFunction) {+ const flags = (req as RequestWithAuth<any, any, any>).acuc?.flags;+ if (getEffectiveSearchForcedKind(flags, undefined) === "zdr") {+ return researchError(res, 400, ZDR_UNSUPPORTED_MESSAGE);+ }+ next();+}+function isResearchTimeoutError(error: unknown): boolean {return ((error instanceof DOMException && error.name === "TimeoutError") ||@@ -417,7 +421,7 @@ function createResearchController(}if (upstream.ok) {- credits = creditsFor(endpoint, responseBody, authedReq);+ credits = creditsFor(endpoint, responseBody);if (credits > 0) {billTeam(authedReq.auth.team_id,@@ -498,6 +502,7 @@ function createResearchController(export function createResearchRouter(options: { legacy?: boolean } = {}) {const router = express.Router();+ router.use(refuseForcedZdr);if (options.legacy) {router.use((req, _res, next) => {@@ -607,6 +612,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {export function createDeveloperRouter(options: { root?: boolean } = {}) {const router = express.Router();+ router.use(refuseForcedZdr);const controller = wrap(createResearchController(@@ -638,6 +644,7 @@ export function createDeveloperRouter(options: { root?: boolean } = {}) {export function createGovRouter() {const router = express.Router();+ router.use(refuseForcedZdr);const controller = wrap(createResearchController(govSearchSchema, [], {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 141bc9fb7..aa9a86d8b 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -349,6 +349,15 @@ async function searchControllerInner(}}+ // The developer and gov indexes have no zero data retention mode upstream.+ if (isZDR && indexCategory) {+ return res.status(400).json({+ success: false,+ error:+ "The developer and gov search categories are not supported with zero data retention.",+ });+ }+// Kick off the `requests` row insert without blocking: it queues on the// Postgres pool and can take seconds under pool pressure. We only need it// committed before the child-row writes (logSearch et al. below) to keep