Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): refuse research, developer and gov search under zero data retention (#5280)

Gergő Móricz6ff4cf3d1dce
diff --git a/apps/api/src/__tests__/snips/v2/developer.test.ts b/apps/api/src/__tests__/snips/v2/developer.test.ts
index 2466749e9..0f30acebb 100644
--- a/apps/api/src/__tests__/snips/v2/developer.test.ts
+++ b/apps/api/src/__tests__/snips/v2/developer.test.ts
@@ -1,5 +1,5 @@
import { config } from "../../../config";
-import { describeIf, TEST_PRODUCTION } from "../lib";
+import { describeIf, scrapeTimeout, TEST_PRODUCTION } from "../lib";
import { creditUsage, idmux, researchPostRaw, researchRaw } from "./lib";
import { HAS_JOB_LOG, jobLogJson, waitForJobLogRow } from "../job-log";
@@ -134,13 +134,36 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {
expect(requestLog?.integration).toBe("_research_test");
}, 120000);
- it("redacts stored payloads for a forced-ZDR team", async () => {
+ it(
+ "refuses a forced-ZDR team on every serving path",
+ async () => {
+ const identity = await idmux({
+ name: "developer/forced ZDR refusal",
+ credits: 100,
+ flags: { searchZDR: "forced-zdr" },
+ });
+
+ for (const path of SERVING_PATHS) {
+ const res = await researchRaw(
+ path,
+ { query: "how do I configure retries", k: 1 },
+ identity,
+ );
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("zero data retention");
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it("redacts stored payloads for a forced-anon team", async () => {
if (!config.USE_DB_AUTHENTICATION || !HAS_JOB_LOG) return;
const identity = await idmux({
- name: "developer/forced ZDR retention",
+ name: "developer/forced anon retention",
credits: 100,
- flags: { searchZDR: "forced-zdr" },
+ flags: { searchZDR: "forced-anon" },
});
const res = await researchRaw(
diff --git a/apps/api/src/__tests__/snips/v2/gov.test.ts b/apps/api/src/__tests__/snips/v2/gov.test.ts
index f6e86479e..b8cbddd6d 100644
--- a/apps/api/src/__tests__/snips/v2/gov.test.ts
+++ b/apps/api/src/__tests__/snips/v2/gov.test.ts
@@ -1,5 +1,5 @@
import { config } from "../../../config";
-import { describeIf } from "../lib";
+import { describeIf, scrapeTimeout } from "../lib";
import { idmux, researchRaw } from "./lib";
const HAS_GOV = !!config.SEARCH_PLATFORM_URL;
@@ -30,4 +30,26 @@ describeIf(HAS_GOV)("Government Index Search API", () => {
expect(typeof result.position).toBe("number");
}
}, 120000);
+
+ it(
+ "refuses a forced-ZDR team",
+ async () => {
+ const identity = await idmux({
+ name: "gov/forced ZDR refusal",
+ credits: 100,
+ flags: { searchZDR: "forced-zdr" },
+ });
+
+ const res = await researchRaw(
+ PATH,
+ { query: "food labeling requirements", k: 3 },
+ identity,
+ );
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("zero data retention");
+ },
+ scrapeTimeout,
+ );
});
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index ecb18b887..65ae273c6 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -224,4 +224,58 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
}, 60000);
+
+ it(
+ "should serve the research category when searchZDR is forced",
+ async () => {
+ const identity = await idmux({
+ name: "zdr/search-forced-research",
+ credits: 10000,
+ flags: { searchZDR: "forced-zdr" },
+ });
+
+ const response = await searchRaw(
+ { query: "protein folding", limit: 1, categories: ["research"] },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should refuse the developer and gov categories under ZDR",
+ async () => {
+ const forced = await idmux({
+ name: "zdr/search-forced-index-categories",
+ credits: 10000,
+ flags: { searchZDR: "forced-zdr" },
+ });
+ const allowed = await idmux({
+ name: "zdr/search-request-index-categories",
+ credits: 10000,
+ flags: { searchZDR: "allowed" },
+ });
+
+ const attempts = [
+ { identity: forced, enterprise: undefined },
+ { identity: allowed, enterprise: ["zdr" as const] },
+ ];
+ for (const { identity, enterprise } of attempts) {
+ for (const category of ["developer", "gov"] as const) {
+ const response = await searchRaw(
+ { query: "retries", limit: 1, categories: [category], enterprise },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(400);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toContain("zero data retention");
+ }
+ }
+ },
+ scrapeTimeout,
+ );
});
diff --git a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
index 9cf4f69e8..d4994ebc4 100644
--- a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
@@ -137,11 +137,32 @@ describe("/v2/search/gov", () => {
},
);
- it("stays free under zero data retention and marks the logs", async () => {
+ it("refuses a forced-ZDR team ahead of the route handler", async () => {
+ const router: any = createGovRouter();
+ const req = {
+ ...makeReq("GET", { query: "food labeling rules" }, TEAM_ID, {
+ searchZDR: "forced-zdr",
+ }),
+ url: "/",
+ };
+ const res = makeRes();
+ const next = vi.fn();
+ router(req, res, next);
+ await flush();
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(next).not.toHaveBeenCalled();
+ expect(mocks.fetchGovUpstream).not.toHaveBeenCalled();
+ expect(billTeam).not.toHaveBeenCalled();
+ expect(mocks.logRequest).not.toHaveBeenCalled();
+ expect(mocks.logResearchEndpoint).not.toHaveBeenCalled();
+ });
+
+ it("stays free for a forced-anon team and marks the logs", async () => {
const res = makeRes();
await handler("get")(
makeReq("GET", { query: "food labeling rules" }, TEAM_ID, {
- searchZDR: "forced-zdr",
+ searchZDR: "forced-anon",
}),
res,
);
diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
index 6f492a11b..1ec17f4f0 100644
--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts
@@ -34,6 +34,8 @@ vi.mock("../../../lib/logger", () => ({
}));
import { createDeveloperRouter, createResearchRouter } from "../research-proxy";
+import { billTeam } from "../../../services/billing/credit_billing";
+import { logger } from "../../../lib/logger";
const TEAM_ID = "11111111-1111-1111-1111-111111111111";
const flush = () => new Promise(resolve => setImmediate(resolve));
@@ -81,8 +83,39 @@ async function callRoute(
query: Record<string, unknown>,
flags: Record<string, unknown>,
) {
- routeHandler(router, path)(makeReq(query, flags), makeRes(), vi.fn());
+ const res = makeRes();
+ routeHandler(router, path)(makeReq(query, flags), res, vi.fn());
await flush();
+ return res;
+}
+
+async function dispatch(
+ router: any,
+ path: string,
+ query: Record<string, unknown>,
+ flags: Record<string, unknown>,
+) {
+ const req = { ...makeReq(query, flags), url: path };
+ const res = makeRes();
+ const next = vi.fn();
+ router(req, res, next);
+ await flush();
+ return { res, next };
+}
+
+function expectRefused(res: any, next: any) {
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ success: false,
+ error: expect.stringContaining("zero data retention"),
+ }),
+ );
+ expect(next).not.toHaveBeenCalled();
+ expect(mocks.fetchResearchUpstream).not.toHaveBeenCalled();
+ expect(mocks.logRequest).not.toHaveBeenCalled();
+ expect(mocks.logResearchEndpoint).not.toHaveBeenCalled();
+ expect(billTeam).not.toHaveBeenCalled();
}
beforeEach(() => {
@@ -117,10 +150,23 @@ describe.each([
])("standalone ZDR persistence for $name", testCase => {
it.each([
{ searchZDR: "forced-zdr" },
- { searchZDR: "forced-anon" },
+ { searchZDR: "forced" },
{ forceZDR: true },
- ])("marks forced team payloads for redaction and cleanup", async flags => {
- await callRoute(testCase.router(), testCase.path, testCase.query, flags);
+ ])("refuses a forced-ZDR team ahead of every route handler", async flags => {
+ const { res, next } = await dispatch(
+ testCase.router(),
+ testCase.path,
+ testCase.query,
+ flags,
+ );
+
+ expectRefused(res, next);
+ });
+
+ it("marks forced-anon team payloads for redaction and cleanup", async () => {
+ await callRoute(testCase.router(), testCase.path, testCase.query, {
+ searchZDR: "forced-anon",
+ });
expect(mocks.logRequest).toHaveBeenCalledWith(
expect.objectContaining({ zeroDataRetention: true }),
@@ -157,3 +203,29 @@ describe.each([
);
});
});
+
+describe("forced-ZDR refusal ordering", () => {
+ it("refuses on the legacy research mount before its usage log line", async () => {
+ const { res, next } = await dispatch(
+ createResearchRouter({ legacy: true }),
+ "/papers",
+ { query: "private research query" },
+ { searchZDR: "forced-zdr" },
+ );
+
+ expectRefused(res, next);
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+
+ it("refuses GitHub search before its deprecation notice and credit gate", async () => {
+ const { res, next } = await dispatch(
+ createResearchRouter(),
+ "/github",
+ { query: "private code query" },
+ { searchZDR: "forced-zdr" },
+ );
+
+ expectRefused(res, next);
+ expect(res.setHeader).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
index 4c39d630b..00d1ad7bf 100644
--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
@@ -66,6 +66,7 @@ vi.mock("../../../lib/logger", () => ({
}));
import { searchController } from "../search";
+import { billTeam } from "../../../services/billing/credit_billing";
const TEAM_ID = "11111111-1111-1111-1111-111111111111";
@@ -419,3 +420,62 @@ describe("tool discovery availability", () => {
);
});
});
+
+describe("index categories under zero data retention", () => {
+ it.each(["developer", "gov"])(
+ "refuses the %s category for a forced-ZDR team",
+ async category => {
+ const req = makeReq({ query: "http client", categories: [category] });
+ req.acuc = { api_key_id: 7, flags: { searchZDR: "forced-zdr" } };
+ const res = makeRes();
+
+ await searchController(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ success: false,
+ error: expect.stringContaining("zero data retention"),
+ }),
+ );
+ expect(mockExecuteSearch).not.toHaveBeenCalled();
+ expect(mockLogRequest).not.toHaveBeenCalled();
+ expect(mockLogSearch).not.toHaveBeenCalled();
+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();
+ expect(billTeam).not.toHaveBeenCalled();
+ },
+ );
+
+ it("refuses the developer category for a request-scoped ZDR search", async () => {
+ const req = makeReq({
+ query: "http client",
+ categories: ["developer"],
+ enterprise: ["zdr"],
+ });
+ req.acuc = { api_key_id: 7, flags: { searchZDR: "allowed" } };
+ const res = makeRes();
+
+ await searchController(req, res);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(mockExecuteSearch).not.toHaveBeenCalled();
+ expect(mockLogRequest).not.toHaveBeenCalled();
+ expect(mockLogSearch).not.toHaveBeenCalled();
+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();
+ expect(billTeam).not.toHaveBeenCalled();
+ });
+
+ it("keeps serving the research category for a forced-ZDR team", async () => {
+ const req = makeReq({ query: "protein folding", categories: ["research"] });
+ req.acuc = { api_key_id: 7, flags: { searchZDR: "forced-zdr" } };
+ const res = makeRes();
+
+ await searchController(req, res);
+ await flushAsync();
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(mockExecuteSearch).toHaveBeenCalled();
+ expect(mockExecuteSearch.mock.calls[0][0].enterprise).toContain("zdr");
+ expect(mockLogResearchEndpoint).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index 4a226ee23..93beeec10 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -1,4 +1,4 @@
-import express, { Request, Response } from "express";
+import express, { NextFunction, Request, Response } from "express";
import { externalRequestId } from "../../lib/external-request-id";
import { z } from "zod";
import { v7 as uuidv7 } from "uuid";
@@ -37,7 +37,10 @@ const searchCreditGate = () =>
SEARCH_CREDITS_PER_TEN_RESULTS,
SEARCH_CREDITS_FEATURE_ID,
);
-const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;
+// The research, developer and government index services have no zero data
+// retention mode, so forced-ZDR teams are refused before any upstream call.
+const ZDR_UNSUPPORTED_MESSAGE =
+ "Your team has zero data retention enabled. This is not supported on research, developer or government search. Please contact [email protected] to unblock this feature.";
const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;
const GOV_SEARCH_TIMEOUT_MS = 15_000;
@@ -238,11 +241,7 @@ function resultCount(body: any): number {
return Array.isArray(body?.data?.web) ? body.data.web.length : 0;
}
-function creditsFor(
- config: ResearchEndpointConfig,
- body: any,
- req: RequestWithAuth<any, any, any>,
-) {
+function creditsFor(config: ResearchEndpointConfig, body: any) {
// Make certain index accesses free (AI/ML & life-sciences indices and the
// gov index). Research paper and gov endpoints should have no credit cost.
// Explicit exceptions remain billable: GitHub search and developer/code
@@ -257,12 +256,7 @@ function creditsFor(
if (freeResearchKinds.has(config.kind)) return 0;
if (config.billAs === "scrape") return 1;
- const forcedKind = getEffectiveSearchForcedKind(req.acuc?.flags, undefined);
- const perTen =
- forcedKind === "zdr"
- ? ZDR_SEARCH_CREDITS_PER_TEN_RESULTS
- : SEARCH_CREDITS_PER_TEN_RESULTS;
- return Math.ceil(resultCount(body) / 10) * perTen;
+ return Math.ceil(resultCount(body) / 10) * SEARCH_CREDITS_PER_TEN_RESULTS;
}
function researchError(
@@ -278,6 +272,16 @@ function researchError(
});
}
+// First on every router here, so a refused query reaches no log line, credit
+// check or upstream.
+function refuseForcedZdr(req: Request, res: Response, next: NextFunction) {
+ const flags = (req as RequestWithAuth<any, any, any>).acuc?.flags;
+ if (getEffectiveSearchForcedKind(flags, undefined) === "zdr") {
+ return researchError(res, 400, ZDR_UNSUPPORTED_MESSAGE);
+ }
+ next();
+}
+
function isResearchTimeoutError(error: unknown): boolean {
return (
(error instanceof DOMException && error.name === "TimeoutError") ||
@@ -417,7 +421,7 @@ function createResearchController(
}
if (upstream.ok) {
- credits = creditsFor(endpoint, responseBody, authedReq);
+ credits = creditsFor(endpoint, responseBody);
if (credits > 0) {
billTeam(
authedReq.auth.team_id,
@@ -498,6 +502,7 @@ function createResearchController(
export function createResearchRouter(options: { legacy?: boolean } = {}) {
const router = express.Router();
+ router.use(refuseForcedZdr);
if (options.legacy) {
router.use((req, _res, next) => {
@@ -607,6 +612,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {
export function createDeveloperRouter(options: { root?: boolean } = {}) {
const router = express.Router();
+ router.use(refuseForcedZdr);
const controller = wrap(
createResearchController(
@@ -638,6 +644,7 @@ export function createDeveloperRouter(options: { root?: boolean } = {}) {
export function createGovRouter() {
const router = express.Router();
+ router.use(refuseForcedZdr);
const controller = wrap(
createResearchController(govSearchSchema, [], {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 141bc9fb7..aa9a86d8b 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -349,6 +349,15 @@ async function searchControllerInner(
}
}
+ // The developer and gov indexes have no zero data retention mode upstream.
+ if (isZDR && indexCategory) {
+ return res.status(400).json({
+ success: false,
+ error:
+ "The developer and gov search categories are not supported with zero data retention.",
+ });
+ }
+
// Kick off the `requests` row insert without blocking: it queues on the
// Postgres pool and can take seconds under pool pressure. We only need it
// committed before the child-row writes (logSearch et al. below) to keep