Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Alexandria: zero data retention for provider tools (#5178)

Developers Digestf429d6a8c330
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
index f6e6709dc..cbce0da53 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
@@ -38,6 +38,7 @@ const call = {
capability: "categories/category",
options: {},
};
+let flags: Record<string, unknown> = { exchangeRetrieve: true };
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
@@ -46,7 +47,7 @@ app.use((req, _res, next) => {
acuc: {
api_key_id: 12,
org_id: "org",
- flags: { exchangeRetrieve: true },
+ flags,
},
});
next();
@@ -64,6 +65,7 @@ const result = (results: unknown[], executed = true) => ({
beforeEach(() => {
vi.clearAllMocks();
+ flags = { exchangeRetrieve: true };
mocks.log.mockResolvedValue(undefined);
mocks.scrapeLog.mockResolvedValue(undefined);
mocks.retrieve.mockResolvedValue(
@@ -291,3 +293,70 @@ it("executes with supplied arguments but stores only tool identifiers in activit
alexandria: [{ provider: call.provider, capability: call.capability }],
});
});
+
+it("forwards zero data retention for forced-ZDR teams instead of refusing them", async () => {
+ flags = { exchangeRetrieve: true, forceZDR: true };
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call });
+ expect(response.status).toBe(200);
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ );
+ await vi.waitFor(() =>
+ expect(mocks.log).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ ),
+ );
+});
+
+it("accepts zero data retention in the body and forwards it", async () => {
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call, zeroDataRetention: true });
+ expect(response.status).toBe(200);
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ );
+ await vi.waitFor(() =>
+ expect(mocks.log).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ ),
+ );
+});
+
+it("accepts zero data retention on both legacy request shapes", async () => {
+ const single = await request(app)
+ .post("/exchange/retrieve")
+ .send({ ...call, zeroDataRetention: true });
+ expect(single.status).toBe(200);
+ const batch = await request(app)
+ .post("/exchange/retrieve")
+ .send({ requests: [call], zeroDataRetention: true });
+ expect(batch.status).toBe(200);
+ for (const [arg] of mocks.retrieve.mock.calls)
+ expect(arg).toMatchObject({ calls: [call], zeroDataRetention: true });
+});
+
+it("tells the provider activity log about zero data retention", async () => {
+ await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: call, zeroDataRetention: true });
+ await vi.waitFor(() =>
+ expect(mocks.scrapeLog).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: true }),
+ ),
+ );
+});
+
+it("does not ask for zero data retention by default", async () => {
+ await request(app).post("/v2/scrape").send({ alexandria: call });
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: false }),
+ );
+ await vi.waitFor(() =>
+ expect(mocks.log).toHaveBeenCalledWith(
+ expect.objectContaining({ zeroDataRetention: false }),
+ ),
+ );
+});
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index 31c85949e..6299f5b63 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -36,6 +36,7 @@ const providerScrapeSchema = z.strictObject({
.transform(value => Math.min(value, 120_000)),
origin: z.string().default("api"),
integration: z.string().nullable().optional(),
+ zeroDataRetention: z.boolean().optional(),
__agentInterop: z
.strictObject({
auth: z.string(),
@@ -51,17 +52,21 @@ export async function providerScrapeController(
res: Response,
legacy = false,
) {
+ const legacyZeroDataRetention = { zeroDataRetention: z.boolean().optional() };
const legacyBody = legacy
? z
- .union([callSchema, z.strictObject({ requests: callsSchema })])
+ .union([
+ callSchema.extend(legacyZeroDataRetention),
+ z.strictObject({ requests: callsSchema, ...legacyZeroDataRetention }),
+ ])
.parse(req.body)
: null;
const body = providerScrapeSchema.parse(
legacyBody
- ? {
- alexandria:
- "requests" in legacyBody ? legacyBody.requests : [legacyBody],
- }
+ ? (({ zeroDataRetention, ...rest }) => ({
+ alexandria: "requests" in rest ? rest.requests : [rest],
+ zeroDataRetention,
+ }))(legacyBody)
: req.body,
);
@@ -92,11 +97,10 @@ export async function providerScrapeController(
success: false,
error: "This endpoint is not enabled for this team.",
});
- if (getScrapeZDR(req.acuc.flags) === "forced")
- return res.status(403).json({
- success: false,
- error: "Provider tools do not support zero data retention.",
- });
+ // Exchange refuses capabilities that cannot honour ZDR, per call.
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc.flags) === "forced" ||
+ body.zeroDataRetention === true;
if (
(req as any).agentIndexOnly ||
["pending", "blocked"].includes(req.acuc._agentSponsor?.status ?? "")
@@ -137,6 +141,7 @@ export async function providerScrapeController(
scrapeId: uuidv7(),
timeoutMs: body.timeout,
bypassBilling: body.__agentInterop?.shouldBill === false,
+ zeroDataRetention,
clientIp: req.ip,
billingCountry: await billingCountryForTeam(req.auth.team_id),
});
@@ -183,7 +188,7 @@ export async function providerScrapeController(
origin: body.origin,
integration: body.integration ?? null,
target_hint: target,
- zeroDataRetention: false,
+ zeroDataRetention,
jobAccess: false,
});
await logProviderScrape({
@@ -208,6 +213,7 @@ export async function providerScrapeController(
error: successful
? undefined
: (error ?? `Provider request failed (${result.status}).`),
+ zeroDataRetention,
});
})().catch(error =>
logger.warn("Provider activity logging failed", {
diff --git a/apps/api/src/services/alexandria/client.test.ts b/apps/api/src/services/alexandria/client.test.ts
index 5983e01ce..f9b617180 100644
--- a/apps/api/src/services/alexandria/client.test.ts
+++ b/apps/api/src/services/alexandria/client.test.ts
@@ -91,6 +91,27 @@ describe("exchangeRequest", () => {
expect(headersOf()).not.toHaveProperty("x-exchange-connectors");
});
+ it("asks for zero data retention only when set", async () => {
+ await exchangeRequest({
+ teamId: "team",
+ path: "/v1/retrieve",
+ body: {},
+ timeoutMs: 1000,
+ zeroDataRetention: true,
+ });
+ expect(headersOf()).toMatchObject({
+ "x-exchange-zero-data-retention": "true",
+ });
+ mocks.fetch.mockClear();
+ await exchangeRequest({
+ teamId: "team",
+ path: "/v1/retrieve",
+ body: {},
+ timeoutMs: 1000,
+ });
+ expect(headersOf()).not.toHaveProperty("x-exchange-zero-data-retention");
+ });
+
it("sends the deadline as a whole number of milliseconds", async () => {
await exchangeRequest({
teamId: "team",
diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.ts
index 855f49b9f..91b9d6513 100644
--- a/apps/api/src/services/alexandria/client.ts
+++ b/apps/api/src/services/alexandria/client.ts
@@ -26,6 +26,7 @@ export async function exchangeRequest(input: {
* `exchangeRetrieve` flag, which is the connectors whitelist; Exchange reads it as the grant.
*/
connectors?: boolean;
+ zeroDataRetention?: boolean;
}): Promise<ExchangeResponse> {
if (!config.FIRE_EXCHANGE_URL)
throw new Error("Alexandria is not configured");
@@ -53,6 +54,9 @@ export async function exchangeRequest(input: {
? { "x-exchange-billing-country": input.billingCountry }
: {}),
...(input.connectors ? { "x-exchange-connectors": "true" } : {}),
+ ...(input.zeroDataRetention
+ ? { "x-exchange-zero-data-retention": "true" }
+ : {}),
...(input.requestId ? { "x-request-id": input.requestId } : {}),
...(input.maximumCredits === undefined
? {}
diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.ts
index 44c2e645a..ea357ea91 100644
--- a/apps/api/src/services/alexandria/retrieve.test.ts
+++ b/apps/api/src/services/alexandria/retrieve.test.ts
@@ -643,3 +643,53 @@ it("sends no credentials when the caller supplied none", async () => {
});
expect(executions()[0][0].resultAuthorization).toBeUndefined();
});
+
+it("forwards zero data retention to quote and execution and keeps no response", async () => {
+ expect(await run({ zeroDataRetention: true })).toEqual({
+ status: 200,
+ body: answer,
+ executed: true,
+ scrapeId: "scrape-1",
+ });
+ const forwarded = mocks.request.mock.calls.filter(
+ ([r]) => r.path.endsWith("/quote") || r.path === "/v1/retrieve",
+ );
+ expect(forwarded).toHaveLength(2);
+ for (const [request] of forwarded)
+ expect(request.zeroDataRetention).toBe(true);
+ const [stored] = [...mocks.store.values()].map(raw => JSON.parse(raw));
+ expect(stored).toMatchObject({ phase: "done", responseWithheld: true });
+ expect(stored).not.toHaveProperty("response");
+
+ const replay = await run({ zeroDataRetention: true, scrapeId: "scrape-2" });
+ expect(replay).toMatchObject({ status: 409, executed: false });
+ expect(replay.body).toEqual(
+ expect.objectContaining({ code: "response_not_retained" }),
+ );
+ expect(executions()).toHaveLength(1);
+});
+
+it("keeps and replays the response without zero data retention", async () => {
+ await run();
+ expect(
+ mocks.request.mock.calls.every(([r]) => r.zeroDataRetention === undefined),
+ ).toBe(true);
+ const [stored] = [...mocks.store.values()].map(raw => JSON.parse(raw));
+ expect(stored.response).toEqual({ status: 200, body: answer });
+ expect(stored).not.toHaveProperty("responseWithheld");
+ expect(await run()).toMatchObject({
+ status: 200,
+ body: answer,
+ executed: false,
+ });
+});
+
+it("refuses the same x-request-id with a different zero data retention setting", async () => {
+ await run();
+ const other = await run({ zeroDataRetention: true });
+ expect(other.status).toBe(409);
+ expect(other.body).toEqual(
+ expect.objectContaining({ code: "duplicate_request" }),
+ );
+ expect(executions()).toHaveLength(1);
+});
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
index 2261b6750..545a0f997 100644
--- a/apps/api/src/services/alexandria/retrieve.ts
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -40,6 +40,8 @@ type Retrieval = {
lockId?: string;
maximumCredits?: number;
response?: ExchangeResponse;
+ // Set instead of response under zero data retention.
+ responseWithheld?: boolean;
failure?: string;
};
@@ -94,6 +96,7 @@ export async function retrieveProviders(input: {
timeoutMs: number;
bypassBilling?: boolean;
resultAuthorization?: string;
+ zeroDataRetention?: boolean;
clientIp?: string;
billingCountry?: string;
}): Promise<ProviderRetrieval> {
@@ -163,11 +166,14 @@ export async function retrieveProviders(input: {
const billable = !input.bypassBilling;
const id = hash([input.teamId, input.requestId]);
const key = `alexandria:retrieve:${id}`;
- const fingerprint = hash(
- termsOnly
+ const zeroDataRetention = input.zeroDataRetention === true;
+ // Appended only when set so existing non-ZDR fingerprints stay stable.
+ const fingerprint = hash([
+ ...(termsOnly
? [input.calls, billable, termsIdentity ?? null]
- : [input.calls, billable],
- );
+ : [input.calls, billable]),
+ ...(zeroDataRetention ? [{ zeroDataRetention }] : []),
+ ]);
const record: Retrieval = {
fingerprint,
phase: "executing",
@@ -192,6 +198,18 @@ export async function retrieveProviders(input: {
if (claimed !== "OK") {
const raw = await redisRateLimitClient.get(key);
const existing: Retrieval | null = raw ? JSON.parse(raw) : null;
+ if (
+ existing?.phase === "done" &&
+ existing.fingerprint === fingerprint &&
+ existing.responseWithheld
+ )
+ return notExecuted(
+ refusal(
+ 409,
+ "This provider request already completed. Its response is not kept under zero data retention.",
+ { code: "response_not_retained", chargeId: id },
+ ),
+ );
if (existing?.phase === "done" && existing.fingerprint === fingerprint)
return {
...existing.response!,
@@ -273,6 +291,7 @@ export async function retrieveProviders(input: {
clientIp: input.clientIp,
billingCountry: input.billingCountry,
connectors: input.flags?.exchangeRetrieve === true,
+ ...(zeroDataRetention ? { zeroDataRetention } : {}),
}).catch(error => {
logger.warn("Provider quote unavailable", { error, chargeId: id });
return undefined;
@@ -408,6 +427,7 @@ export async function retrieveProviders(input: {
? { resultAuthorization: input.resultAuthorization }
: {}),
...(termsIdentity ? { termsIdentity } : {}),
+ ...(zeroDataRetention ? { zeroDataRetention } : {}),
maximumCredits,
}).catch(error => {
throw new Error(`Alexandria did not answer: ${error?.message ?? error}`);
@@ -463,7 +483,11 @@ export async function retrieveProviders(input: {
});
const done: ExchangeResponse = { status: 200, body: answer };
- await write({ ...record, phase: "done", response: done });
+ await write(
+ zeroDataRetention
+ ? { ...record, phase: "done", responseWithheld: true }
+ : { ...record, phase: "done", response: done },
+ );
return { ...done, executed: true, scrapeId: input.scrapeId };
} catch (error) {
return fail(error instanceof Error ? error.message : String(error));
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 62243c3c0..13f722b7f 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -814,6 +814,7 @@ type LoggedProviderScrape = {
credits_cost: number;
is_successful: boolean;
error?: string;
+ zeroDataRetention: boolean;
};
export async function logProviderScrape(scrape: LoggedProviderScrape) {
@@ -823,7 +824,7 @@ export async function logProviderScrape(scrape: LoggedProviderScrape) {
table: "scrapes",
id: scrape.id,
requestId: scrape.request_id,
- zeroDataRetention: false,
+ zeroDataRetention: scrape.zeroDataRetention,
},
() => logProviderScrapeInternal(scrape),
);
@@ -848,12 +849,18 @@ async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {
{
id: scrape.id,
request_id: scrape.request_id,
- url: scrape.target,
+ url: scrape.zeroDataRetention
+ ? "<redacted due to zero data retention>"
+ : scrape.target,
is_successful: scrape.is_successful,
- error: scrape.error ?? null,
+ error: scrape.zeroDataRetention
+ ? scrape.error === undefined
+ ? null
+ : "<redacted due to zero data retention>"
+ : (scrape.error ?? null),
time_taken: scrape.time_taken,
team_id: storedTeamId,
- options: scrape.options,
+ options: scrape.zeroDataRetention ? null : scrape.options,
cost_tracking: null,
pdf_num_pages: null,
credits_cost: scrape.credits_cost,