Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Alexandria: zero data retention for provider tools (#5178)
Developers Digestf429d6a8c330
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex f6e6709dc..cbce0da53 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -38,6 +38,7 @@ const call = {capability: "categories/category",options: {},};+let flags: Record<string, unknown> = { exchangeRetrieve: true };const app = express();app.use(express.json());app.use((req, _res, next) => {@@ -46,7 +47,7 @@ app.use((req, _res, next) => {acuc: {api_key_id: 12,org_id: "org",- flags: { exchangeRetrieve: true },+ flags,},});next();@@ -64,6 +65,7 @@ const result = (results: unknown[], executed = true) => ({beforeEach(() => {vi.clearAllMocks();+ flags = { exchangeRetrieve: true };mocks.log.mockResolvedValue(undefined);mocks.scrapeLog.mockResolvedValue(undefined);mocks.retrieve.mockResolvedValue(@@ -291,3 +293,70 @@ it("executes with supplied arguments but stores only tool identifiers in activitalexandria: [{ provider: call.provider, capability: call.capability }],});});++it("forwards zero data retention for forced-ZDR teams instead of refusing them", async () => {+ flags = { exchangeRetrieve: true, forceZDR: true };+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call });+ expect(response.status).toBe(200);+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ );+ await vi.waitFor(() =>+ expect(mocks.log).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ ),+ );+});++it("accepts zero data retention in the body and forwards it", async () => {+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call, zeroDataRetention: true });+ expect(response.status).toBe(200);+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ );+ await vi.waitFor(() =>+ expect(mocks.log).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ ),+ );+});++it("accepts zero data retention on both legacy request shapes", async () => {+ const single = await request(app)+ .post("/exchange/retrieve")+ .send({ ...call, zeroDataRetention: true });+ expect(single.status).toBe(200);+ const batch = await request(app)+ .post("/exchange/retrieve")+ .send({ requests: [call], zeroDataRetention: true });+ expect(batch.status).toBe(200);+ for (const [arg] of mocks.retrieve.mock.calls)+ expect(arg).toMatchObject({ calls: [call], zeroDataRetention: true });+});++it("tells the provider activity log about zero data retention", async () => {+ await request(app)+ .post("/v2/scrape")+ .send({ alexandria: call, zeroDataRetention: true });+ await vi.waitFor(() =>+ expect(mocks.scrapeLog).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ ),+ );+});++it("does not ask for zero data retention by default", async () => {+ await request(app).post("/v2/scrape").send({ alexandria: call });+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: false }),+ );+ await vi.waitFor(() =>+ expect(mocks.log).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: false }),+ ),+ );+});diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 31c85949e..6299f5b63 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -36,6 +36,7 @@ const providerScrapeSchema = z.strictObject({.transform(value => Math.min(value, 120_000)),origin: z.string().default("api"),integration: z.string().nullable().optional(),+ zeroDataRetention: z.boolean().optional(),__agentInterop: z.strictObject({auth: z.string(),@@ -51,17 +52,21 @@ export async function providerScrapeController(res: Response,legacy = false,) {+ const legacyZeroDataRetention = { zeroDataRetention: z.boolean().optional() };const legacyBody = legacy? z- .union([callSchema, z.strictObject({ requests: callsSchema })])+ .union([+ callSchema.extend(legacyZeroDataRetention),+ z.strictObject({ requests: callsSchema, ...legacyZeroDataRetention }),+ ]).parse(req.body): null;const body = providerScrapeSchema.parse(legacyBody- ? {- alexandria:- "requests" in legacyBody ? legacyBody.requests : [legacyBody],- }+ ? (({ zeroDataRetention, ...rest }) => ({+ alexandria: "requests" in rest ? rest.requests : [rest],+ zeroDataRetention,+ }))(legacyBody): req.body,);@@ -92,11 +97,10 @@ export async function providerScrapeController(success: false,error: "This endpoint is not enabled for this team.",});- if (getScrapeZDR(req.acuc.flags) === "forced")- return res.status(403).json({- success: false,- error: "Provider tools do not support zero data retention.",- });+ // Exchange refuses capabilities that cannot honour ZDR, per call.+ const zeroDataRetention =+ getScrapeZDR(req.acuc.flags) === "forced" ||+ body.zeroDataRetention === true;if ((req as any).agentIndexOnly ||["pending", "blocked"].includes(req.acuc._agentSponsor?.status ?? "")@@ -137,6 +141,7 @@ export async function providerScrapeController(scrapeId: uuidv7(),timeoutMs: body.timeout,bypassBilling: body.__agentInterop?.shouldBill === false,+ zeroDataRetention,clientIp: req.ip,billingCountry: await billingCountryForTeam(req.auth.team_id),});@@ -183,7 +188,7 @@ export async function providerScrapeController(origin: body.origin,integration: body.integration ?? null,target_hint: target,- zeroDataRetention: false,+ zeroDataRetention,jobAccess: false,});await logProviderScrape({@@ -208,6 +213,7 @@ export async function providerScrapeController(error: successful? undefined: (error ?? `Provider request failed (${result.status}).`),+ zeroDataRetention,});})().catch(error =>logger.warn("Provider activity logging failed", {diff --git a/apps/api/src/services/alexandria/client.test.ts b/apps/api/src/services/alexandria/client.test.tsindex 5983e01ce..f9b617180 100644--- a/apps/api/src/services/alexandria/client.test.ts+++ b/apps/api/src/services/alexandria/client.test.ts@@ -91,6 +91,27 @@ describe("exchangeRequest", () => {expect(headersOf()).not.toHaveProperty("x-exchange-connectors");});+ it("asks for zero data retention only when set", async () => {+ await exchangeRequest({+ teamId: "team",+ path: "/v1/retrieve",+ body: {},+ timeoutMs: 1000,+ zeroDataRetention: true,+ });+ expect(headersOf()).toMatchObject({+ "x-exchange-zero-data-retention": "true",+ });+ mocks.fetch.mockClear();+ await exchangeRequest({+ teamId: "team",+ path: "/v1/retrieve",+ body: {},+ timeoutMs: 1000,+ });+ expect(headersOf()).not.toHaveProperty("x-exchange-zero-data-retention");+ });+it("sends the deadline as a whole number of milliseconds", async () => {await exchangeRequest({teamId: "team",diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.tsindex 855f49b9f..91b9d6513 100644--- a/apps/api/src/services/alexandria/client.ts+++ b/apps/api/src/services/alexandria/client.ts@@ -26,6 +26,7 @@ export async function exchangeRequest(input: {* `exchangeRetrieve` flag, which is the connectors whitelist; Exchange reads it as the grant.*/connectors?: boolean;+ zeroDataRetention?: boolean;}): Promise<ExchangeResponse> {if (!config.FIRE_EXCHANGE_URL)throw new Error("Alexandria is not configured");@@ -53,6 +54,9 @@ export async function exchangeRequest(input: {? { "x-exchange-billing-country": input.billingCountry }: {}),...(input.connectors ? { "x-exchange-connectors": "true" } : {}),+ ...(input.zeroDataRetention+ ? { "x-exchange-zero-data-retention": "true" }+ : {}),...(input.requestId ? { "x-request-id": input.requestId } : {}),...(input.maximumCredits === undefined? {}diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsindex 44c2e645a..ea357ea91 100644--- a/apps/api/src/services/alexandria/retrieve.test.ts+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -643,3 +643,53 @@ it("sends no credentials when the caller supplied none", async () => {});expect(executions()[0][0].resultAuthorization).toBeUndefined();});++it("forwards zero data retention to quote and execution and keeps no response", async () => {+ expect(await run({ zeroDataRetention: true })).toEqual({+ status: 200,+ body: answer,+ executed: true,+ scrapeId: "scrape-1",+ });+ const forwarded = mocks.request.mock.calls.filter(+ ([r]) => r.path.endsWith("/quote") || r.path === "/v1/retrieve",+ );+ expect(forwarded).toHaveLength(2);+ for (const [request] of forwarded)+ expect(request.zeroDataRetention).toBe(true);+ const [stored] = [...mocks.store.values()].map(raw => JSON.parse(raw));+ expect(stored).toMatchObject({ phase: "done", responseWithheld: true });+ expect(stored).not.toHaveProperty("response");++ const replay = await run({ zeroDataRetention: true, scrapeId: "scrape-2" });+ expect(replay).toMatchObject({ status: 409, executed: false });+ expect(replay.body).toEqual(+ expect.objectContaining({ code: "response_not_retained" }),+ );+ expect(executions()).toHaveLength(1);+});++it("keeps and replays the response without zero data retention", async () => {+ await run();+ expect(+ mocks.request.mock.calls.every(([r]) => r.zeroDataRetention === undefined),+ ).toBe(true);+ const [stored] = [...mocks.store.values()].map(raw => JSON.parse(raw));+ expect(stored.response).toEqual({ status: 200, body: answer });+ expect(stored).not.toHaveProperty("responseWithheld");+ expect(await run()).toMatchObject({+ status: 200,+ body: answer,+ executed: false,+ });+});++it("refuses the same x-request-id with a different zero data retention setting", async () => {+ await run();+ const other = await run({ zeroDataRetention: true });+ expect(other.status).toBe(409);+ expect(other.body).toEqual(+ expect.objectContaining({ code: "duplicate_request" }),+ );+ expect(executions()).toHaveLength(1);+});diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 2261b6750..545a0f997 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -40,6 +40,8 @@ type Retrieval = {lockId?: string;maximumCredits?: number;response?: ExchangeResponse;+ // Set instead of response under zero data retention.+ responseWithheld?: boolean;failure?: string;};@@ -94,6 +96,7 @@ export async function retrieveProviders(input: {timeoutMs: number;bypassBilling?: boolean;resultAuthorization?: string;+ zeroDataRetention?: boolean;clientIp?: string;billingCountry?: string;}): Promise<ProviderRetrieval> {@@ -163,11 +166,14 @@ export async function retrieveProviders(input: {const billable = !input.bypassBilling;const id = hash([input.teamId, input.requestId]);const key = `alexandria:retrieve:${id}`;- const fingerprint = hash(- termsOnly+ const zeroDataRetention = input.zeroDataRetention === true;+ // Appended only when set so existing non-ZDR fingerprints stay stable.+ const fingerprint = hash([+ ...(termsOnly? [input.calls, billable, termsIdentity ?? null]- : [input.calls, billable],- );+ : [input.calls, billable]),+ ...(zeroDataRetention ? [{ zeroDataRetention }] : []),+ ]);const record: Retrieval = {fingerprint,phase: "executing",@@ -192,6 +198,18 @@ export async function retrieveProviders(input: {if (claimed !== "OK") {const raw = await redisRateLimitClient.get(key);const existing: Retrieval | null = raw ? JSON.parse(raw) : null;+ if (+ existing?.phase === "done" &&+ existing.fingerprint === fingerprint &&+ existing.responseWithheld+ )+ return notExecuted(+ refusal(+ 409,+ "This provider request already completed. Its response is not kept under zero data retention.",+ { code: "response_not_retained", chargeId: id },+ ),+ );if (existing?.phase === "done" && existing.fingerprint === fingerprint)return {...existing.response!,@@ -273,6 +291,7 @@ export async function retrieveProviders(input: {clientIp: input.clientIp,billingCountry: input.billingCountry,connectors: input.flags?.exchangeRetrieve === true,+ ...(zeroDataRetention ? { zeroDataRetention } : {}),}).catch(error => {logger.warn("Provider quote unavailable", { error, chargeId: id });return undefined;@@ -408,6 +427,7 @@ export async function retrieveProviders(input: {? { resultAuthorization: input.resultAuthorization }: {}),...(termsIdentity ? { termsIdentity } : {}),+ ...(zeroDataRetention ? { zeroDataRetention } : {}),maximumCredits,}).catch(error => {throw new Error(`Alexandria did not answer: ${error?.message ?? error}`);@@ -463,7 +483,11 @@ export async function retrieveProviders(input: {});const done: ExchangeResponse = { status: 200, body: answer };- await write({ ...record, phase: "done", response: done });+ await write(+ zeroDataRetention+ ? { ...record, phase: "done", responseWithheld: true }+ : { ...record, phase: "done", response: done },+ );return { ...done, executed: true, scrapeId: input.scrapeId };} catch (error) {return fail(error instanceof Error ? error.message : String(error));diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 62243c3c0..13f722b7f 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -814,6 +814,7 @@ type LoggedProviderScrape = {credits_cost: number;is_successful: boolean;error?: string;+ zeroDataRetention: boolean;};export async function logProviderScrape(scrape: LoggedProviderScrape) {@@ -823,7 +824,7 @@ export async function logProviderScrape(scrape: LoggedProviderScrape) {table: "scrapes",id: scrape.id,requestId: scrape.request_id,- zeroDataRetention: false,+ zeroDataRetention: scrape.zeroDataRetention,},() => logProviderScrapeInternal(scrape),);@@ -848,12 +849,18 @@ async function logProviderScrapeInternal(scrape: LoggedProviderScrape) {{id: scrape.id,request_id: scrape.request_id,- url: scrape.target,+ url: scrape.zeroDataRetention+ ? "<redacted due to zero data retention>"+ : scrape.target,is_successful: scrape.is_successful,- error: scrape.error ?? null,+ error: scrape.zeroDataRetention+ ? scrape.error === undefined+ ? null+ : "<redacted due to zero data retention>"+ : (scrape.error ?? null),time_taken: scrape.time_taken,team_id: storedTeamId,- options: scrape.options,+ options: scrape.zeroDataRetention ? null : scrape.options,cost_tracking: null,pdf_num_pages: null,credits_cost: scrape.credits_cost,