Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Forward zero data retention to Exchange from the exchange proxy (#5736)
Developers Digesteaa56e8803b8
diff --git a/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.ts b/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.tsnew file mode 100644index 000000000..be1d38b9f--- /dev/null+++ b/apps/api/src/__tests__/routes/exchange-zero-data-retention.routes.test.ts@@ -0,0 +1,191 @@+import express from "express";+import request from "supertest";++// The real exchangeRouter, authMiddleware and auth.ts; only I/O is mocked.+const mocks = vi.hoisted(() => ({+ authChunk: vi.fn(),+ upstreamFetch: vi.fn(),+ clickhouse: null as { query: ReturnType<typeof vi.fn> } | null,+ query: vi.fn(),+}));++vi.mock("../../lib/clickhouse-client", () => ({+ get clickhouseClient() {+ return mocks.clickhouse;+ },+ chInsert: vi.fn(),+}));+vi.mock("../../lib/billing-country", () => ({+ billingCountryForTeam: vi.fn(async () => undefined),+}));+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));+vi.mock("../../db/rpc", () => ({+ authCreditUsageChunk: mocks.authChunk,+ authCreditUsageChunkFromTeam: vi.fn(),+ getAgentFreeRequestsLeft: vi.fn(),+}));+vi.mock("../../services/redis", () => ({+ getValue: vi.fn().mockResolvedValue(null),+ setValue: vi.fn(),+ deleteKey: vi.fn(),+}));+vi.mock("../../services/redlock", () => ({+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },+}));+vi.mock("ioredis", () => ({ default: class {} }));+vi.mock("../../services/rate-limiter", async importOriginal => ({+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ redisRateLimitClient: {+ set: vi.fn().mockResolvedValue("OK"),+ get: vi.fn().mockResolvedValue(null),+ del: vi.fn().mockResolvedValue(1),+ },+}));+vi.mock("../../services/autumn/autumn.service", async importOriginal => {+ const actual =+ await importOriginal<+ typeof import("../../services/autumn/autumn.service")+ >();+ return {+ ...actual,+ autumnService: {+ getTeamLimits: vi.fn().mockResolvedValue({+ concurrency_limit: 2,+ rate_limit_multiplier: 1,+ }),+ },+ };+});+vi.mock("../../services/agent-sponsor", () => ({+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),+}));+vi.mock("../../services/queue-service", () => ({+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),+}));+vi.mock("../../services/logging/log_job", () => ({+ logRequest: vi.fn(),+ logProviderScrape: vi.fn(),+}));+vi.mock("undici", async importOriginal => ({+ ...(await importOriginal<typeof import("undici")>()),+ fetch: mocks.upstreamFetch,+}));++import { config } from "../../config";+import { exchangeRouter } from "../../routes/exchange";++const KEY = "44444444-4444-4444-8444-444444444444";+const TEAM_ID = "11111111-1111-4111-8111-111111111111";++const app = express();+app.use(express.json());+app.use("/exchange", exchangeRouter);++let flags: Record<string, unknown> = {};++function forwardedHeaders() {+ const [, init] = mocks.upstreamFetch.mock.calls[0]!;+ return (init as RequestInit).headers as Record<string, string>;+}++describe("Exchange proxy under zero data retention", () => {+ const original = {+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,+ };++ beforeEach(() => {+ vi.clearAllMocks();+ flags = {};+ config.USE_DB_AUTHENTICATION = true;+ config.FIRE_EXCHANGE_URL = "https://exchange.test/";+ mocks.clickhouse = { query: mocks.query };+ mocks.query.mockResolvedValue({ json: vi.fn().mockResolvedValue([]) });+ mocks.authChunk.mockImplementation(+ async (_db: unknown, key: string, purpose = "general") =>+ key === KEY && purpose === "general"+ ? [+ {+ api_key: KEY,+ api_key_id: 9,+ api_key_id_text: "9",+ team_id: TEAM_ID,+ org_id: "org-zdr",+ is_banned: false,+ flags,+ },+ ]+ : [],+ );+ mocks.upstreamFetch.mockImplementation(+ async () =>+ new Response(JSON.stringify({ success: true }), {+ status: 200,+ headers: { "content-type": "application/json" },+ }),+ );+ });++ afterEach(() => {+ Object.assign(config, original);+ });++ const routes = [+ ["discovery", () => request(app).get("/exchange/discover?q=people")],+ [+ "skills",+ () =>+ request(app)+ .post("/exchange/skills/resolve")+ .send({ urls: ["https://example.com/"] }),+ ],+ ["recommendations", () => request(app).get("/exchange/recommendations")],+ ] as const;++ it.each(routes)(+ "tells Exchange when the team is held to it (%s)",+ async (_name, send) => {+ flags = { scrapeZDR: "forced" };+ const response = await send().set("authorization", `Bearer ${KEY}`);+ expect(response.status).toBe(200);+ expect(forwardedHeaders()["x-exchange-zero-data-retention"]).toBe("true");+ },+ );++ it.each([{ forceZDR: true }, { scrapeZDR: "forced" }])(+ "reads the legacy and current flag alike (%o)",+ async teamFlags => {+ flags = teamFlags;+ await request(app)+ .get("/exchange/discover?q=people")+ .set("authorization", `Bearer ${KEY}`);+ expect(forwardedHeaders()["x-exchange-zero-data-retention"]).toBe("true");+ },+ );++ it.each([{}, { scrapeZDR: "allowed" }])(+ "leaves it off otherwise (%o)",+ async teamFlags => {+ flags = teamFlags;+ await request(app)+ .get("/exchange/discover?q=people")+ .set("authorization", `Bearer ${KEY}`);+ expect(forwardedHeaders()).not.toHaveProperty(+ "x-exchange-zero-data-retention",+ );+ },+ );++ it("does not take the header from the caller", async () => {+ await request(app)+ .get("/exchange/discover?q=people")+ .set("authorization", `Bearer ${KEY}`)+ .set("x-exchange-zero-data-retention", "false");+ expect(forwardedHeaders()).not.toHaveProperty(+ "x-exchange-zero-data-retention",+ );+ });+});diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.tsindex 3351db72f..104f8b7df 100644--- a/apps/api/src/routes/exchange.ts+++ b/apps/api/src/routes/exchange.ts@@ -19,6 +19,17 @@ import type { RequestWithAuth } from "../controllers/v1/types";import { RateLimiterMode } from "../types";import { authMiddleware, wrap } from "./shared";import { isUsOnly } from "../lib/us-only";+import { getScrapeZDR } from "../lib/zdr-helpers";++// Tells Exchange the team is held to zero data retention, so it only lists and runs+// providers that support it.+function zeroDataRetentionHeader(+ flags: Parameters<typeof getScrapeZDR>[0],+): Record<string, string> {+ return getScrapeZDR(flags) === "forced"+ ? { "x-exchange-zero-data-retention": "true" }+ : {};+}const DISCOVER_TIMEOUT_MS = 10_000;const RETRIEVE_TIMEOUT_MS = 50_000;@@ -192,6 +203,7 @@ function exchangeProxy(...(isUsOnly(authedReq.acuc?.flags)? { "x-exchange-us-only": "true" }: {}),+ ...zeroDataRetentionHeader(authedReq.acuc?.flags),},body: hasBody ? JSON.stringify(req.body ?? {}) : undefined,signal: AbortSignal.timeout(timeout),@@ -243,6 +255,7 @@ async function recommendationsController(req: Request, res: Response) {...(isUsOnly(authedReq.acuc?.flags)? { "x-exchange-us-only": "true" }: {}),+ ...zeroDataRetentionHeader(authedReq.acuc?.flags),},body: JSON.stringify(signals),signal: AbortSignal.timeout(DISCOVER_TIMEOUT_MS),diff --git a/apps/api/src/search/alexandria.test.ts b/apps/api/src/search/alexandria.test.tsindex a67aa5509..a68e87738 100644--- a/apps/api/src/search/alexandria.test.ts+++ b/apps/api/src/search/alexandria.test.ts@@ -274,6 +274,41 @@ it("reports the tools the Exchange hid from a US-only team", async () => {expect(result).not.toHaveProperty("warning");});+it("asks Exchange for zero-data-retention providers only, on every leg", async () => {+ request.mockImplementation(async args =>+ args.path === "/v1/skills/resolve"+ ? {+ status: 200,+ body: {+ skills: [+ {+ id: "sample",+ matchedDomains: ["example.com"],+ domainCapabilities: { "example.com": ["records/search"] },+ },+ ],+ },+ }+ : {+ status: 200,+ body: { success: true, creditsCost: 0, data: { items: [tool] } },+ },+ );+ await discoverTools(+ {+ ...input,+ zeroDataRetention: true,+ urls: ["https://example.com/records"],+ },+ logger,+ );+ // The semantic lookup, the domain resolve, and the domain tool lookup.+ const paths = request.mock.calls.map(([args]) => args.path).sort();+ expect(paths).toEqual(["/v1/retrieve", "/v1/retrieve", "/v1/skills/resolve"]);+ for (const [args] of request.mock.calls)+ expect(args).toMatchObject({ zeroDataRetention: true });+});+it("reports no hidden tools when the Exchange hid nothing", async () => {request.mockResolvedValue({status: 200,diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex 128a1b400..045e052c7 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -87,6 +87,8 @@ export async function discoverTools(connectors?: boolean;/** The team may only be served by US-based providers. */usOnly?: boolean;+ /** The team is held to zero data retention: only providers that support it are returned. */+ zeroDataRetention?: boolean;/** Names each find-tools lookup `<prefix>:<n>` and appends the id to `sent` as it goes out. */exchangeRequestIds?: { prefix: string; sent: string[] };},@@ -116,6 +118,7 @@ export async function discoverTools(maximumCredits: 0,connectors: input.connectors === true,usOnly: input.usOnly === true,+ zeroDataRetention: input.zeroDataRetention === true,body: {provider: "firecrawl",capability: "find-tools",@@ -245,6 +248,7 @@ export async function discoverTools(timeoutMs: remaining(deadline),connectors: input.connectors === true,usOnly: input.usOnly === true,+ zeroDataRetention: input.zeroDataRetention === true,});if (result.status !== 200)throw new Error("Domain discovery unavailable");diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 83756243a..f206f52be 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -292,6 +292,39 @@ describe("executeSearch tool intent", () => {expect(result.toolsWarning).toBe("2 tools hidden");});+ it.each([+ [{ scrapeZDR: "forced" }, true],+ [{ forceZDR: true }, true],+ [{ scrapeZDR: "allowed" }, false],+ [{}, false],+ ])(+ "asks tool and skill discovery for zero-data-retention providers by team flags (%o)",+ async (flags, zeroDataRetention) => {+ mocks.search.mockResolvedValue({ web: [webResult] });+ mocks.discoverTools.mockResolvedValue({ items: [] });+ await executeSearch(+ {+ ...options([]),+ sources: [+ { type: "web" },+ { type: "alexandria" },+ { type: "skills" },+ ],+ },+ { ...context, flags } as any,+ logger,+ );+ expect(mocks.discoverTools).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention }),+ logger,+ );+ expect(mocks.discoverSkills).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention }),+ logger,+ );+ },+ );+it("leaves a default search alone when the query does not need tools", async () => {mocks.search.mockResolvedValue({ web: [webResult] });mocks.inferToolIntent.mockResolvedValue({diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 78cb24afb..e2519aeea 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -65,6 +65,7 @@ import { calculateThreatScanCredits } from "../lib/scrape-billing";import { config } from "../config";import { logger as rootLogger } from "../lib/logger";import { isUsOnly } from "../lib/us-only";+import { getScrapeZDR } from "../lib/zdr-helpers";interface SearchOptions {query: string;@@ -207,6 +208,9 @@ export async function executeSearch(const num_results_buffer = Math.floor(limit * 2);const usOnly = isUsOnly(flags);+ // A team held to zero data retention for scrapes still gets tools, but only from providers+ // that support it. Read from team flags only, never from the request.+ const toolsZeroDataRetention = getScrapeZDR(flags) === "forced";const usOnlyWarnings: string[] = [];logger.info("Searching for results");@@ -225,6 +229,7 @@ export async function executeSearch(query,timeoutMs: Math.min(options.timeout, SKILLS_DISCOVERY_TIMEOUT_MS),usOnly,+ zeroDataRetention: toolsZeroDataRetention,},logger,).catch(error => {@@ -254,6 +259,7 @@ export async function executeSearch(teamId,connectors: flags?.exchangeRetrieve === true,usOnly,+ zeroDataRetention: toolsZeroDataRetention,limit,queries: wantsTools? intent.toolQueries@@ -304,6 +310,7 @@ export async function executeSearch(teamId,connectors: flags?.exchangeRetrieve === true,usOnly,+ zeroDataRetention: toolsZeroDataRetention,limit,queries: wantsTools ? [query] : undefined,urls:diff --git a/apps/api/src/search/skill-cards.test.ts b/apps/api/src/search/skill-cards.test.tsindex 0833b9dfd..24b782816 100644--- a/apps/api/src/search/skill-cards.test.ts+++ b/apps/api/src/search/skill-cards.test.ts@@ -44,6 +44,7 @@ it("maps a successful discover response onto short cards", async () => {path: "/v1/discover/skill-cards?q=records&limit=3",timeoutMs: 3000,usOnly: false,+ zeroDataRetention: false,});expect(request.mock.calls[0][0]).not.toHaveProperty("body");expect(cards.succeeded).toBe(true);@@ -131,6 +132,25 @@ it("forwards usOnly so Exchange can hide non-US providers", async () => {expect(found).toEqual({ skills: [], succeeded: true });});+it("forwards zero data retention so Exchange can hide providers without it", async () => {+ request.mockResolvedValue({+ status: 200,+ body: { query: "records", skills: [] },+ });++ await discoverSkills(+ {+ teamId: "team",+ query: "records",+ timeoutMs: 1000,+ zeroDataRetention: true,+ },+ logger,+ );++ expect(request.mock.calls[0][0]).toMatchObject({ zeroDataRetention: true });+});+it("returns only the first three when Exchange sends more", async () => {const skills = [1, 2, 3, 4].map(n => ({id: `official/skill-${n}`,diff --git a/apps/api/src/search/skill-cards.ts b/apps/api/src/search/skill-cards.tsindex d11d33cef..b8c16144a 100644--- a/apps/api/src/search/skill-cards.ts+++ b/apps/api/src/search/skill-cards.ts@@ -69,6 +69,8 @@ export async function discoverSkills(timeoutMs: number;/** The team may only be served by US-based providers. */usOnly?: boolean;+ /** The team is held to zero data retention: only providers that support it are returned. */+ zeroDataRetention?: boolean;},logger: Logger,): Promise<SkillDiscovery> {@@ -88,6 +90,7 @@ export async function discoverSkills(Math.min(input.timeoutMs, SKILLS_DISCOVERY_TIMEOUT_MS),),usOnly: input.usOnly === true,+ zeroDataRetention: input.zeroDataRetention === true,});if (result.status !== 200) {logger.warn("Skill discovery unavailable", { status: result.status });