Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Forward zero data retention to Exchange from the exchange proxy (#5736)

Developers Digesteaa56e8803b8
diff --git a/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.ts b/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.ts
new file mode 100644
index 000000000..be1d38b9f
--- /dev/null
+++ b/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.ts
@@ -0,0 +1,191 @@
+import express from "express";
+import request from "supertest";
+
+// The real exchangeRouter, authMiddleware and auth.ts; only I/O is mocked.
+const mocks = vi.hoisted(() => ({
+ authChunk: vi.fn(),
+ upstreamFetch: vi.fn(),
+ clickhouse: null as { query: ReturnType<typeof vi.fn> } | null,
+ query: vi.fn(),
+}));
+
+vi.mock("../../lib/clickhouse-client", () => ({
+ get clickhouseClient() {
+ return mocks.clickhouse;
+ },
+ chInsert: vi.fn(),
+}));
+vi.mock("../../lib/billing-country", () => ({
+ billingCountryForTeam: vi.fn(async () => undefined),
+}));
+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));
+vi.mock("../../db/rpc", () => ({
+ authCreditUsageChunk: mocks.authChunk,
+ authCreditUsageChunkFromTeam: vi.fn(),
+ getAgentFreeRequestsLeft: vi.fn(),
+}));
+vi.mock("../../services/redis", () => ({
+ getValue: vi.fn().mockResolvedValue(null),
+ setValue: vi.fn(),
+ deleteKey: vi.fn(),
+}));
+vi.mock("../../services/redlock", () => ({
+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },
+}));
+vi.mock("ioredis", () => ({ default: class {} }));
+vi.mock("../../services/rate-limiter", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),
+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),
+ redisRateLimitClient: {
+ set: vi.fn().mockResolvedValue("OK"),
+ get: vi.fn().mockResolvedValue(null),
+ del: vi.fn().mockResolvedValue(1),
+ },
+}));
+vi.mock("../../services/autumn/autumn.service", async importOriginal => {
+ const actual =
+ await importOriginal<
+ typeof import("../../services/autumn/autumn.service")
+ >();
+ return {
+ ...actual,
+ autumnService: {
+ getTeamLimits: vi.fn().mockResolvedValue({
+ concurrency_limit: 2,
+ rate_limit_multiplier: 1,
+ }),
+ },
+ };
+});
+vi.mock("../../services/agent-sponsor", () => ({
+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),
+}));
+vi.mock("../../services/queue-service", () => ({
+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),
+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),
+}));
+vi.mock("../../services/logging/log_job", () => ({
+ logRequest: vi.fn(),
+ logProviderScrape: vi.fn(),
+}));
+vi.mock("undici", async importOriginal => ({
+ ...(await importOriginal<typeof import("undici")>()),
+ fetch: mocks.upstreamFetch,
+}));
+
+import { config } from "../../config";
+import { exchangeRouter } from "../../routes/exchange";
+
+const KEY = "44444444-4444-4444-8444-444444444444";
+const TEAM_ID = "11111111-1111-4111-8111-111111111111";
+
+const app = express();
+app.use(express.json());
+app.use("/exchange", exchangeRouter);
+
+let flags: Record<string, unknown> = {};
+
+function forwardedHeaders() {
+ const [, init] = mocks.upstreamFetch.mock.calls[0]!;
+ return (init as RequestInit).headers as Record<string, string>;
+}
+
+describe("Exchange proxy under zero data retention", () => {
+ const original = {
+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,
+ };
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ flags = {};
+ config.USE_DB_AUTHENTICATION = true;
+ config.FIRE_EXCHANGE_URL = "https://exchange.test/";
+ mocks.clickhouse = { query: mocks.query };
+ mocks.query.mockResolvedValue({ json: vi.fn().mockResolvedValue([]) });
+ mocks.authChunk.mockImplementation(
+ async (_db: unknown, key: string, purpose = "general") =>
+ key === KEY && purpose === "general"
+ ? [
+ {
+ api_key: KEY,
+ api_key_id: 9,
+ api_key_id_text: "9",
+ team_id: TEAM_ID,
+ org_id: "org-zdr",
+ is_banned: false,
+ flags,
+ },
+ ]
+ : [],
+ );
+ mocks.upstreamFetch.mockImplementation(
+ async () =>
+ new Response(JSON.stringify({ success: true }), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ );
+ });
+
+ afterEach(() => {
+ Object.assign(config, original);
+ });
+
+ const routes = [
+ ["discovery", () => request(app).get("/exchange/discover?q=people")],
+ [
+ "skills",
+ () =>
+ request(app)
+ .post("/exchange/skills/resolve")
+ .send({ urls: ["https://example.com/"] }),
+ ],
+ ["recommendations", () => request(app).get("/exchange/recommendations")],
+ ] as const;
+
+ it.each(routes)(
+ "tells Exchange when the team is held to it (%s)",
+ async (_name, send) => {
+ flags = { scrapeZDR: "forced" };
+ const response = await send().set("authorization", `Bearer ${KEY}`);
+ expect(response.status).toBe(200);
+ expect(forwardedHeaders()["x-exchange-zero-data-retention"]).toBe("true");
+ },
+ );
+
+ it.each([{ forceZDR: true }, { scrapeZDR: "forced" }])(
+ "reads the legacy and current flag alike (%o)",
+ async teamFlags => {
+ flags = teamFlags;
+ await request(app)
+ .get("/exchange/discover?q=people")
+ .set("authorization", `Bearer ${KEY}`);
+ expect(forwardedHeaders()["x-exchange-zero-data-retention"]).toBe("true");
+ },
+ );
+
+ it.each([{}, { scrapeZDR: "allowed" }])(
+ "leaves it off otherwise (%o)",
+ async teamFlags => {
+ flags = teamFlags;
+ await request(app)
+ .get("/exchange/discover?q=people")
+ .set("authorization", `Bearer ${KEY}`);
+ expect(forwardedHeaders()).not.toHaveProperty(
+ "x-exchange-zero-data-retention",
+ );
+ },
+ );
+
+ it("does not take the header from the caller", async () => {
+ await request(app)
+ .get("/exchange/discover?q=people")
+ .set("authorization", `Bearer ${KEY}`)
+ .set("x-exchange-zero-data-retention", "false");
+ expect(forwardedHeaders()).not.toHaveProperty(
+ "x-exchange-zero-data-retention",
+ );
+ });
+});
diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.ts
index 3351db72f..104f8b7df 100644
--- a/apps/api/src/routes/exchange.ts
+++ b/apps/api/src/routes/exchange.ts
@@ -19,6 +19,17 @@ import type { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
import { authMiddleware, wrap } from "./shared";
import { isUsOnly } from "../lib/us-only";
+import { getScrapeZDR } from "../lib/zdr-helpers";
+
+// Tells Exchange the team is held to zero data retention, so it only lists and runs
+// providers that support it.
+function zeroDataRetentionHeader(
+ flags: Parameters<typeof getScrapeZDR>[0],
+): Record<string, string> {
+ return getScrapeZDR(flags) === "forced"
+ ? { "x-exchange-zero-data-retention": "true" }
+ : {};
+}
const DISCOVER_TIMEOUT_MS = 10_000;
const RETRIEVE_TIMEOUT_MS = 50_000;
@@ -192,6 +203,7 @@ function exchangeProxy(
...(isUsOnly(authedReq.acuc?.flags)
? { "x-exchange-us-only": "true" }
: {}),
+ ...zeroDataRetentionHeader(authedReq.acuc?.flags),
},
body: hasBody ? JSON.stringify(req.body ?? {}) : undefined,
signal: AbortSignal.timeout(timeout),
@@ -243,6 +255,7 @@ async function recommendationsController(req: Request, res: Response) {
...(isUsOnly(authedReq.acuc?.flags)
? { "x-exchange-us-only": "true" }
: {}),
+ ...zeroDataRetentionHeader(authedReq.acuc?.flags),
},
body: JSON.stringify(signals),
signal: AbortSignal.timeout(DISCOVER_TIMEOUT_MS),
diff --git a/apps/api/src/search/alexandria.test.ts b/apps/api/src/search/alexandria.test.ts
index a67aa5509..a68e87738 100644
--- a/apps/api/src/search/alexandria.test.ts
+++ b/apps/api/src/search/alexandria.test.ts
@@ -274,6 +274,41 @@ it("reports the tools the Exchange hid from a US-only team", async () => {
expect(result).not.toHaveProperty("warning");
});
+it("asks Exchange for zero-data-retention providers only, on every leg", async () => {
+ request.mockImplementation(async args =>
+ args.path === "/v1/skills/resolve"
+ ? {
+ status: 200,
+ body: {
+ skills: [
+ {
+ id: "sample",
+ matchedDomains: ["example.com"],
+ domainCapabilities: { "example.com": ["records/search"] },
+ },
+ ],
+ },
+ }
+ : {
+ status: 200,
+ body: { success: true, creditsCost: 0, data: { items: [tool] } },
+ },
+ );
+ await discoverTools(
+ {
+ ...input,
+ zeroDataRetention: true,
+ urls: ["https://example.com/records"],
+ },
+ logger,
+ );
+ // The semantic lookup, the domain resolve, and the domain tool lookup.
+ const paths = request.mock.calls.map(([args]) => args.path).sort();
+ expect(paths).toEqual(["/v1/retrieve", "/v1/retrieve", "/v1/skills/resolve"]);
+ for (const [args] of request.mock.calls)
+ expect(args).toMatchObject({ zeroDataRetention: true });
+});
+
it("reports no hidden tools when the Exchange hid nothing", async () => {
request.mockResolvedValue({
status: 200,
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
index 128a1b400..045e052c7 100644
--- a/apps/api/src/search/alexandria.ts
+++ b/apps/api/src/search/alexandria.ts
@@ -87,6 +87,8 @@ export async function discoverTools(
connectors?: boolean;
/** The team may only be served by US-based providers. */
usOnly?: boolean;
+ /** The team is held to zero data retention: only providers that support it are returned. */
+ zeroDataRetention?: boolean;
/** Names each find-tools lookup `<prefix>:<n>` and appends the id to `sent` as it goes out. */
exchangeRequestIds?: { prefix: string; sent: string[] };
},
@@ -116,6 +118,7 @@ export async function discoverTools(
maximumCredits: 0,
connectors: input.connectors === true,
usOnly: input.usOnly === true,
+ zeroDataRetention: input.zeroDataRetention === true,
body: {
provider: "firecrawl",
capability: "find-tools",
@@ -245,6 +248,7 @@ export async function discoverTools(
timeoutMs: remaining(deadline),
connectors: input.connectors === true,
usOnly: input.usOnly === true,
+ zeroDataRetention: input.zeroDataRetention === true,
});
if (result.status !== 200)
throw new Error("Domain discovery unavailable");
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 83756243a..f206f52be 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -292,6 +292,39 @@ describe("executeSearch tool intent", () => {
expect(result.toolsWarning).toBe("2 tools hidden");
});
+ it.each([
+ [{ scrapeZDR: "forced" }, true],
+ [{ forceZDR: true }, true],
+ [{ scrapeZDR: "allowed" }, false],
+ [{}, false],
+ ])(
+ "asks tool and skill discovery for zero-data-retention providers by team flags (%o)",
+ async (flags, zeroDataRetention) => {
+ mocks.search.mockResolvedValue({ web: [webResult] });
+ mocks.discoverTools.mockResolvedValue({ items: [] });
+ await executeSearch(
+ {
+ ...options([]),
+ sources: [
+ { type: "web" },
+ { type: "alexandria" },
+ { type: "skills" },
+ ],
+ },
+ { ...context, flags } as any,
+ logger,
+ );
+ expect(mocks.discoverTools).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention }),
+ logger,
+ );
+ expect(mocks.discoverSkills).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention }),
+ logger,
+ );
+ },
+ );
+
it("leaves a default search alone when the query does not need tools", async () => {
mocks.search.mockResolvedValue({ web: [webResult] });
mocks.inferToolIntent.mockResolvedValue({
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 78cb24afb..e2519aeea 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -65,6 +65,7 @@ import { calculateThreatScanCredits } from "../lib/scrape-billing";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
import { isUsOnly } from "../lib/us-only";
+import { getScrapeZDR } from "../lib/zdr-helpers";
interface SearchOptions {
query: string;
@@ -207,6 +208,9 @@ export async function executeSearch(
const num_results_buffer = Math.floor(limit * 2);
const usOnly = isUsOnly(flags);
+ // A team held to zero data retention for scrapes still gets tools, but only from providers
+ // that support it. Read from team flags only, never from the request.
+ const toolsZeroDataRetention = getScrapeZDR(flags) === "forced";
const usOnlyWarnings: string[] = [];
logger.info("Searching for results");
@@ -225,6 +229,7 @@ export async function executeSearch(
query,
timeoutMs: Math.min(options.timeout, SKILLS_DISCOVERY_TIMEOUT_MS),
usOnly,
+ zeroDataRetention: toolsZeroDataRetention,
},
logger,
).catch(error => {
@@ -254,6 +259,7 @@ export async function executeSearch(
teamId,
connectors: flags?.exchangeRetrieve === true,
usOnly,
+ zeroDataRetention: toolsZeroDataRetention,
limit,
queries: wantsTools
? intent.toolQueries
@@ -304,6 +310,7 @@ export async function executeSearch(
teamId,
connectors: flags?.exchangeRetrieve === true,
usOnly,
+ zeroDataRetention: toolsZeroDataRetention,
limit,
queries: wantsTools ? [query] : undefined,
urls:
diff --git a/apps/api/src/search/skill-cards.test.ts b/apps/api/src/search/skill-cards.test.ts
index 0833b9dfd..24b782816 100644
--- a/apps/api/src/search/skill-cards.test.ts
+++ b/apps/api/src/search/skill-cards.test.ts
@@ -44,6 +44,7 @@ it("maps a successful discover response onto short cards", async () => {
path: "/v1/discover/skill-cards?q=records&limit=3",
timeoutMs: 3000,
usOnly: false,
+ zeroDataRetention: false,
});
expect(request.mock.calls[0][0]).not.toHaveProperty("body");
expect(cards.succeeded).toBe(true);
@@ -131,6 +132,25 @@ it("forwards usOnly so Exchange can hide non-US providers", async () => {
expect(found).toEqual({ skills: [], succeeded: true });
});
+it("forwards zero data retention so Exchange can hide providers without it", async () => {
+ request.mockResolvedValue({
+ status: 200,
+ body: { query: "records", skills: [] },
+ });
+
+ await discoverSkills(
+ {
+ teamId: "team",
+ query: "records",
+ timeoutMs: 1000,
+ zeroDataRetention: true,
+ },
+ logger,
+ );
+
+ expect(request.mock.calls[0][0]).toMatchObject({ zeroDataRetention: true });
+});
+
it("returns only the first three when Exchange sends more", async () => {
const skills = [1, 2, 3, 4].map(n => ({
id: `official/skill-${n}`,
diff --git a/apps/api/src/search/skill-cards.ts b/apps/api/src/search/skill-cards.ts
index d11d33cef..b8c16144a 100644
--- a/apps/api/src/search/skill-cards.ts
+++ b/apps/api/src/search/skill-cards.ts
@@ -69,6 +69,8 @@ export async function discoverSkills(
timeoutMs: number;
/** The team may only be served by US-based providers. */
usOnly?: boolean;
+ /** The team is held to zero data retention: only providers that support it are returned. */
+ zeroDataRetention?: boolean;
},
logger: Logger,
): Promise<SkillDiscovery> {
@@ -88,6 +90,7 @@ export async function discoverSkills(
Math.min(input.timeoutMs, SKILLS_DISCOVERY_TIMEOUT_MS),
),
usOnly: input.usOnly === true,
+ zeroDataRetention: input.zeroDataRetention === true,
});
if (result.status !== 200) {
logger.warn("Skill discovery unavailable", { status: result.status });