Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(agent): accept zeroDataRetention on /v2/agent (#5185)
Rakshith Ramprakash3301ea4bab0f
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 964b1b6ec..e68b58293 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -309,6 +309,18 @@ jobs:run: pnpm exec vitest run src/controllers/v2/__tests__/agent-location.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run agent zero data retention regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/controllers/v2/__tests__/agent-zdr.test.ts+ src/controllers/v2/__tests__/agent-zdr-passthrough.test.ts+ src/controllers/v2/__tests__/agent-rollout.test.ts+ src/controllers/v2/__tests__/agent-list.test.ts+ src/controllers/v2/__tests__/agent-status.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run country check route regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=defaultdiff --git a/apps/api/src/controllers/v2/__tests__/agent-list.test.ts b/apps/api/src/controllers/v2/__tests__/agent-list.test.tsindex 44ac36cf5..9e44ca779 100644--- a/apps/api/src/controllers/v2/__tests__/agent-list.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-list.test.ts@@ -141,3 +141,96 @@ describe("agentListController options", () => {expect(options).not.toHaveProperty("threadTurn");});});++describe("agentListController zero data retention rows", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("lists a ZDR row with only model, effort and the flag", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({ ok: true, json: async () => [] }),+ );+ mocks.query.mockImplementation(async ({ query }: { query: string }) => ({+ json: async () =>+ query.includes("FROM requests")+ ? [+ {+ id: AGENT_ID,+ created_at: "2026-09-23 12:00:00.000",+ target_hint: "<redacted due to zero data retention>",+ origin: "api",+ integration: null,+ },+ ]+ : [+ {+ id: AGENT_ID,+ options: JSON.stringify({+ prompt: "secret prompt",+ urls: ["https://example.com"],+ threadId: "thread-1",+ model: "spark-2",+ effort: "high",+ zeroDataRetention: true,+ }),+ is_successful: true,+ error: null,+ },+ ],+ }));++ const res = makeRes();+ await agentListController(makeReq(), res);+ const agent = vi.mocked(res.json).mock.calls[0][0].agents[0];++ expect(agent.targetHint).toBe("<redacted due to zero data retention>");+ expect(agent.status).toBe("completed");+ expect(agent.options).toEqual({+ prompt: "",+ model: "spark-2",+ effort: "high",+ zeroDataRetention: true,+ });+ });++ it("drops content from a recent ZDR run not yet in ClickHouse", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({+ ok: true,+ json: async () => [+ {+ id: AGENT_ID,+ createdAt: "2026-09-23T12:00:00.000Z",+ status: "processing",+ options: {+ prompt: "secret prompt",+ urls: ["https://example.com"],+ modelPreset: "spark-2",+ zeroDataRetention: true,+ },+ },+ ],+ }),+ );+ mocks.query.mockImplementation(async () => ({ json: async () => [] }));++ const res = makeRes();+ await agentListController(makeReq(), res);+ const agent = vi.mocked(res.json).mock.calls[0][0].agents[0];++ expect(agent.targetHint).toBe("<redacted due to zero data retention>");+ expect(agent.options).toEqual({+ prompt: "",+ model: "spark-2",+ effort: "medium",+ zeroDataRetention: true,+ });+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/agent-rollout.test.ts b/apps/api/src/controllers/v2/__tests__/agent-rollout.test.tsindex 7ae496bcd..76718102a 100644--- a/apps/api/src/controllers/v2/__tests__/agent-rollout.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-rollout.test.ts@@ -83,8 +83,8 @@ describe("Agent Alexandria rollout", () => {};const res = { status: vi.fn().mockReturnThis(), json: vi.fn() };await agentController(req as any, res as any);- expect(res.status).toHaveBeenCalledWith(forced ? 400 : 200);- expect(fetch).toHaveBeenCalledTimes(forced ? 0 : 1);+ expect(res.status).toHaveBeenCalledWith(200);+ expect(fetch).toHaveBeenCalledTimes(1);},);});diff --git a/apps/api/src/controllers/v2/__tests__/agent-zdr-passthrough.test.ts b/apps/api/src/controllers/v2/__tests__/agent-zdr-passthrough.test.tsnew file mode 100644index 000000000..895c65a89--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/agent-zdr-passthrough.test.ts@@ -0,0 +1,139 @@+import type { Response } from "express";+import { agentStatusController } from "../agent-status";+import { agentTraceController } from "../agent-trace";+import { agentSnapshotController } from "../agent-snapshot";+import { agentSkillController } from "../agent-skill";+import { getAgentJobAccess } from "../../../lib/operational-job-access";+import {+ ExtractV3AgentStatusError,+ getExtractV3AgentStatus,+} from "../../../lib/extract-v3-status";++vi.mock("../../../config", () => ({+ config: {+ EXTRACT_V3_BETA_URL: "https://agent.example",+ AGENT_INTEROP_SECRET: "test",+ },+}));++vi.mock("../../../lib/operational-job-access", () => ({+ getAgentJobAccess: vi.fn(),+}));++vi.mock("../../../lib/browser-sessions", () => ({+ listActiveBrowserSessionsForRequest: vi.fn().mockResolvedValue([]),+}));++const JOB_ID = "job-123";++function upstream(status: number, body: unknown) {+ vi.mocked(fetch).mockResolvedValue(+ new globalThis.Response(JSON.stringify(body), {+ status,+ headers: { "Content-Type": "application/json" },+ }) as any,+ );+}++function makeReq(params: Record<string, string> = {}) {+ return {+ params: { jobId: JOB_ID, ...params },+ query: {},+ auth: { team_id: "team-123" },+ } as any;+}++function makeRes() {+ return {+ status: vi.fn().mockReturnThis(),+ json: vi.fn(),+ } as unknown as Response;+}++describe("agent passthroughs for zero data retention runs", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.stubGlobal("fetch", vi.fn());+ vi.mocked(getAgentJobAccess).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: Date.now() + 60_000,+ } as any);+ });++ afterEach(() => vi.unstubAllGlobals());++ it("getExtractV3AgentStatus surfaces a 410 zdr_expired as a typed error", async () => {+ upstream(410, { code: "zdr_expired", error: "Result deleted." });++ const error = await getExtractV3AgentStatus(JOB_ID).catch(e => e);++ expect(error).toBeInstanceOf(ExtractV3AgentStatusError);+ expect(error).toMatchObject({+ status: 410,+ code: "zdr_expired",+ message: "Result deleted.",+ });+ });++ it("getExtractV3AgentStatus still throws a plain error for other failures", async () => {+ upstream(500, { error: "boom" });++ const error = await getExtractV3AgentStatus(JOB_ID).catch(e => e);++ expect(error).not.toBeInstanceOf(ExtractV3AgentStatusError);+ expect(error.message).toContain("500");+ });++ it("status forwards 410 zdr_expired", async () => {+ upstream(410, { code: "zdr_expired", error: "Result deleted." });+ const res = makeRes();++ await agentStatusController(makeReq(), res);++ expect(res.status).toHaveBeenCalledWith(410);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ code: "zdr_expired",+ error: "Result deleted.",+ });+ });++ it("trace passes the zeroDataRetention flag through", async () => {+ const body = {+ success: true,+ id: JOB_ID,+ events: [{ type: "run.started" }],+ creditsUsed: 3,+ zeroDataRetention: true,+ };+ upstream(200, body);+ const res = makeRes();++ await agentTraceController(makeReq(), res);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(body);+ });++ it("snapshot forwards 404 zdr_no_snapshots", async () => {+ const body = { code: "zdr_no_snapshots", error: "No snapshots." };+ upstream(404, body);+ const res = makeRes();++ await agentSnapshotController(makeReq({ snapshotId: "snap-1" }), res);++ expect(res.status).toHaveBeenCalledWith(404);+ expect(res.json).toHaveBeenCalledWith(body);+ });++ it("skill forwards 410 zdr_expired", async () => {+ const body = { code: "zdr_expired", error: "Expired." };+ upstream(410, body);+ const res = makeRes();++ await agentSkillController(makeReq(), res);++ expect(res.status).toHaveBeenCalledWith(410);+ expect(res.json).toHaveBeenCalledWith(body);+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.tsnew file mode 100644index 000000000..0dd0e5c9b--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts@@ -0,0 +1,215 @@+vi.mock("../../../lib/crawl-regex", async () => {+ const { z } = await import("zod");+ return {+ addPathRegexIssues: vi.fn(),+ pathPatternsSchema: z.array(z.string()),+ };+});+import { agentController } from "../agent";+import { logRequest } from "../../../services/logging/log_job";+import { logger } from "../../../lib/logger";+import { emitRejectedScrapeActivityEvents } from "../../../lib/siem-logging";+import {+ checkUrlsAgainstThreatPolicy,+ resolveThreatProtection,+} from "../../../lib/threat-protection/request";+import { fetchAgentThread } from "../agent-thread";+vi.mock("../../../lib/logger", () => {+ const logger = { info: vi.fn(), error: vi.fn(), child: vi.fn() };+ logger.child.mockReturnValue(logger);+ return { logger };+});+vi.mock("../../../services/logging/log_job", () => ({ logRequest: vi.fn() }));+vi.mock("../../../lib/external-request-id", () => ({+ externalRequestId: () => undefined,+}));+vi.mock("../../../config", () => ({+ config: {+ EXTRACT_V3_BETA_URL: "https://agent.example",+ USE_DB_AUTHENTICATION: false,+ AGENT_INTEROP_SECRET: "test",+ },+}));+vi.mock("../../../db/rpc", () => ({ agentConsumeFreeRequestIfLeft: vi.fn() }));+vi.mock("../../../services/autumn/autumn.service", () => ({+ autumnService: { checkCredits: vi.fn(), hasAutoTopUp: vi.fn() },+}));+vi.mock("../../../lib/threat-protection/request", () => ({+ resolveThreatProtection: vi.fn().mockResolvedValue({}),+ checkUrlsAgainstThreatPolicy: vi.fn(),+}));+vi.mock("../../../lib/scrape-billing", () => ({+ calculateThreatScanCredits: vi.fn(),+}));+vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: vi.fn(),+}));+vi.mock("../../../lib/siem-logging", () => ({+ emitRejectedScrapeActivityEvents: vi.fn(),+}));+vi.mock("../agent-thread", () => ({+ fetchAgentThread: vi.fn(),+ threadErrorFor: vi.fn(),+}));++function run(body: Record<string, unknown>, flags: Record<string, unknown>) {+ const req = {+ body: { prompt: "Find the price", ...body },+ auth: { team_id: "team-test" },+ acuc: { flags, api_key: "test-key" },+ };+ const res = { status: vi.fn().mockReturnThis(), json: vi.fn() };+ return agentController(req as any, res as any).then(() => res);+}++function forwardedBody() {+ return JSON.parse(vi.mocked(fetch).mock.calls[0][1]!.body as string);+}++describe("agentController zero data retention", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({ status: 200, json: async () => ({}) }),+ );+ });+ afterEach(() => vi.unstubAllGlobals());++ it.each([+ [+ "forced (scrapeZDR)",+ {},+ { scrapeZDR: "forced", searchZDR: "forced-zdr" },+ ],+ ["forced (legacy forceZDR)", {}, { forceZDR: true }],+ [+ "forced with an explicit opt-out",+ { zeroDataRetention: false },+ { scrapeZDR: "forced", searchZDR: "allowed" },+ ],+ [+ "allowed with opt-in",+ { zeroDataRetention: true },+ { scrapeZDR: "allowed", searchZDR: "allowed" },+ ],+ ])("runs with ZDR when %s", async (_name, body, flags) => {+ const res = await run(body, flags);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(forwardedBody().zeroDataRetention).toBe(true);+ expect(logRequest).toHaveBeenCalledWith(+ expect.objectContaining({ kind: "agent", zeroDataRetention: true }),+ );+ expect(logger.child).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: true }),+ );+ });++ it.each([+ ["allowed without opt-in", {}, { scrapeZDR: "allowed" }],+ ["disabled without opt-in", {}, {}],+ ])("runs without ZDR when %s", async (_name, body, flags) => {+ const res = await run(body, flags);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(forwardedBody()).not.toHaveProperty("zeroDataRetention");+ expect(logRequest).toHaveBeenCalledWith(+ expect.objectContaining({ zeroDataRetention: false }),+ );+ });++ it.each([+ ["opt-in, neither enabled", { zeroDataRetention: true }, {}],+ [+ "opt-in, scrape ZDR only",+ { zeroDataRetention: true },+ { scrapeZDR: "allowed" },+ ],+ [+ "opt-in, search ZDR only",+ { zeroDataRetention: true },+ { searchZDR: "allowed" },+ ],+ ["scrape ZDR forced, no search ZDR", {}, { scrapeZDR: "forced" }],+ [+ "scrape ZDR forced, search anonymous only",+ {},+ { scrapeZDR: "forced", searchZDR: "forced-anon" },+ ],+ ])("rejects a ZDR run up front when %s", async (_name, body, flags) => {+ const res = await run(body, flags);++ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith({+ success: false,+ code: "zdr_not_enabled",+ error:+ "Zero Data Retention (ZDR) for agent runs requires both scrape ZDR and search ZDR to be enabled for your team. Contact [email protected] to enable it.",+ });+ expect(fetch).not.toHaveBeenCalled();+ expect(logRequest).not.toHaveBeenCalled();+ });++ it("leaves a run that does not ask for ZDR alone, whatever the search flag", async () => {+ const res = await run({}, { scrapeZDR: "allowed" });++ expect(res.status).toHaveBeenCalledWith(200);+ expect(forwardedBody()).not.toHaveProperty("zeroDataRetention");+ });++ it("rejects a thread follow-up under ZDR", async () => {+ const res = await run(+ { threadId: "0190e3c8-4b2a-7c3d-8e9f-0a1b2c3d4e5f" },+ { scrapeZDR: "forced", searchZDR: "allowed" },+ );++ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ success: false,+ code: "zdr_threads_unsupported",+ }),+ );+ expect(fetchAgentThread).not.toHaveBeenCalled();+ expect(fetch).not.toHaveBeenCalled();+ expect(logRequest).not.toHaveBeenCalled();+ });++ it("rejects exchange options under ZDR", async () => {+ const res = await run(+ { zeroDataRetention: true, exchange: { enabled: true } },+ { scrapeZDR: "allowed", searchZDR: "allowed" },+ );++ expect(res.status).toHaveBeenCalledWith(400);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ success: false,+ code: "zdr_exchange_unsupported",+ }),+ );+ expect(fetch).not.toHaveBeenCalled();+ expect(logRequest).not.toHaveBeenCalled();+ });++ it("marks rejected threat events as ZDR", async () => {+ vi.mocked(resolveThreatProtection).mockResolvedValueOnce({+ policy: {},+ } as any);+ vi.mocked(checkUrlsAgainstThreatPolicy).mockResolvedValueOnce({+ blocked: [{ url: "https://blocked.example", decision: {} }],+ decisionsByUrl: new Map(),+ } as any);++ const res = await run(+ { urls: ["https://blocked.example"] },+ { scrapeZDR: "forced", searchZDR: "allowed" },+ );++ expect(res.status).toHaveBeenCalledWith(403);+ expect(emitRejectedScrapeActivityEvents).toHaveBeenCalledWith([+ expect.objectContaining({ zeroDataRetention: true }),+ ]);+ });+});diff --git a/apps/api/src/controllers/v2/agent-list.ts b/apps/api/src/controllers/v2/agent-list.tsindex 02d8bf423..fa08e332e 100644--- a/apps/api/src/controllers/v2/agent-list.ts+++ b/apps/api/src/controllers/v2/agent-list.ts@@ -46,6 +46,29 @@ function threadTurnOf(value: unknown): number | undefined {return Number.isSafeInteger(turn) ? turn : undefined;}+const ZDR_TARGET_HINT = "<redacted due to zero data retention>";++function isZdrOptions(options: any): boolean {+ return options?.zeroDataRetention === true;+}++// A zero data retention run is listed with its model, effort and the flag+// only, whichever source the row comes from.+function withoutZdrContent(options: any): any {+ return isZdrOptions(options)+ ? {+ model: options.model,+ modelPreset: options.modelPreset,+ effort: options.effort,+ zeroDataRetention: true,+ }+ : options;+}++function zdrOptions(options: any): { zeroDataRetention?: true } {+ return isZdrOptions(options) ? { zeroDataRetention: true } : {};+}+// Each turn of a thread is its own run, so clients need the thread to list// a conversation once.function threadOptions(options: any): {@@ -277,15 +300,19 @@ export async function agentListController(const recent = recentAgents.get(id);const db = dbAgents.get(id);const settings = sessionSettingsMap.get(id);+ const recentOptions = withoutZdrContent(recent?.options);+ const dbOptions = db?.agent ? withoutZdrContent(db.agent.options) : null;agents.push({id: db?.id ?? recent!.id,createdAt: db?.createdAt.toISOString() ?? recent!.createdAt,targetHint:db?.targetHint ??- (recent?.options- ? (recent.options.urls?.[0] ?? recent.options.prompt ?? "")- : ""),+ (isZdrOptions(recentOptions)+ ? ZDR_TARGET_HINT+ : recentOptions+ ? (recentOptions.urls?.[0] ?? recentOptions.prompt ?? "")+ : ""),origin: db?.origin ?? "api", // recent cannot know origin so we lieintegration: db?.integration ?? undefined, // recent cannot know integration so we omitsettings: {@@ -304,29 +331,31 @@ export async function agentListController(: recent!.status === "succeeded"? "completed": "failed",- options: db?.agent+ options: dbOptions? {- urls: db.agent.options.urls ?? undefined,- prompt: db.agent.options.prompt ?? "",- schema: db.agent.options.schema ?? undefined,- model: db.agent.options.model ?? "spark-1-pro",+ urls: dbOptions.urls ?? undefined,+ prompt: dbOptions.prompt ?? "",+ schema: dbOptions.schema ?? undefined,+ model: dbOptions.model ?? "spark-1-pro",effort:- db.agent.options.effort ??- (db.agent.options.model === "spark-2" ? "medium" : undefined),- ...threadOptions(db.agent.options),+ dbOptions.effort ??+ (dbOptions.model === "spark-2" ? "medium" : undefined),+ ...threadOptions(dbOptions),+ ...zdrOptions(dbOptions),}- : recent?.options+ : recentOptions? {- urls: recent.options.urls ?? undefined,- prompt: recent.options.prompt ?? "",- schema: recent.options.schema ?? undefined,- model: recent.options.modelPreset ?? "spark-1-pro",+ urls: recentOptions.urls ?? undefined,+ prompt: recentOptions.prompt ?? "",+ schema: recentOptions.schema ?? undefined,+ model: recentOptions.modelPreset ?? "spark-1-pro",effort:- recent.options.effort ??- (recent.options.modelPreset === "spark-2"+ recentOptions.effort ??+ (recentOptions.modelPreset === "spark-2"? "medium": undefined),- ...threadOptions(recent.options),+ ...threadOptions(recentOptions),+ ...zdrOptions(recentOptions),}: undefined,});diff --git a/apps/api/src/controllers/v2/agent-skill.ts b/apps/api/src/controllers/v2/agent-skill.tsindex e55735c8b..cf0353ed5 100644--- a/apps/api/src/controllers/v2/agent-skill.ts+++ b/apps/api/src/controllers/v2/agent-skill.ts@@ -36,8 +36,9 @@ export async function agentSkillController(const body = (await upstream.json()) as AgentSkillResponse;- // 404 (skills off, or nothing stored) and 409 (run still going, or it- // produced no data) both mean "no skill yet" to the caller, so the upstream- // status carries through rather than collapsing into a 500.+ // 404 (skills off, or nothing stored), 409 (run still going, or it produced+ // no data) and 410 (a zero data retention run past its retention window)+ // all mean "no skill" to the caller, so the upstream status carries through+ // rather than collapsing into a 500.return res.status(upstream.status).json(body);}diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.tsindex f91ff6551..794a9ddc5 100644--- a/apps/api/src/controllers/v2/agent-status.ts+++ b/apps/api/src/controllers/v2/agent-status.ts@@ -8,7 +8,10 @@ import {RequestWithAuth,} from "./types";import { getAgentJobAccess } from "../../lib/operational-job-access";-import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";+import {+ ExtractV3AgentStatusError,+ getExtractV3AgentStatus,+} from "../../lib/extract-v3-status";// python-sdk versions before 4.37.1 validate the status response's `model`// with pydantic as Literal["spark-1-pro", "spark-1-mini"], so any other value@@ -73,7 +76,17 @@ export async function agentStatusController(});}- const agent = await getExtractV3AgentStatus(req.params.jobId);+ let agent: Awaited<ReturnType<typeof getExtractV3AgentStatus>>;+ try {+ agent = await getExtractV3AgentStatus(req.params.jobId);+ } catch (error) {+ if (!(error instanceof ExtractV3AgentStatusError)) throw error;+ return res.status(error.status).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }let model: "spark-1-pro" | "spark-1-mini" | "spark-2" =agent.model ?? "spark-1-pro";diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex a2ee1ba02..7ba757931 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -12,7 +12,7 @@ import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";-import { getScrapeZDR } from "../../lib/zdr-helpers";+import { getScrapeZDR, getSearchZDR } from "../../lib/zdr-helpers";import {checkUrlsAgainstThreatPolicy,resolveThreatProtection,@@ -32,6 +32,33 @@ export async function agentController(res: Response<AgentResponse>,) {const agentId = uuidv7();++ const originalRequest = { ...req.body };+ req.body = agentRequestSchema.parse(req.body);++ // An agent run scrapes and searches under ZDR, so it needs both+ // entitlements up front instead of failing on its first inner call.+ const scrapeZDR = getScrapeZDR(req.acuc?.flags);+ const searchZDR = getSearchZDR(req.acuc?.flags);+ if (+ (req.body.zeroDataRetention === true || scrapeZDR === "forced") &&+ !(+ (scrapeZDR === "allowed" || scrapeZDR === "forced") &&+ (searchZDR === "allowed" || searchZDR === "forced-zdr")+ )+ ) {+ return res.status(400).json({+ success: false,+ code: "zdr_not_enabled",+ error:+ "Zero Data Retention (ZDR) for agent runs requires both scrape ZDR and search ZDR to be enabled for your team. Contact [email protected] to enable it.",+ });+ }++ const zeroDataRetention =+ scrapeZDR === "forced" ||+ (req.body.zeroDataRetention === true && scrapeZDR === "allowed");+const logger = _logger.child({agentId,extractId: agentId,@@ -40,24 +67,30 @@ export async function agentController(team_id: req.auth.team_id,module: "api/v2",method: "agentController",- zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",+ zeroDataRetention,});- const originalRequest = { ...req.body };- req.body = agentRequestSchema.parse(req.body);+ if (zeroDataRetention && req.body.threadId) {+ return res.status(400).json({+ success: false,+ code: "zdr_threads_unsupported",+ error:+ "Agent threads are not supported with zero data retention. Remove threadId to start a new run.",+ });+ }- if (getScrapeZDR(req.acuc?.flags) === "forced") {+ if (zeroDataRetention && req.body.exchange) {return res.status(400).json({success: false,+ code: "zdr_exchange_unsupported",error:- "Your team has zero data retention enabled. This is not supported on extract. Please contact [email protected] to unblock this feature.",+ "Exchange is not supported with zero data retention. Remove the exchange option.",});}- _logger.info("Agent starting...", {+ logger.info("Agent starting...", {request: req.body,originalRequest,- zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced",});// Threat protection: check the agent's starting URLs before handing off to@@ -122,7 +155,7 @@ export async function agentController(threatDecisions: [blockedUrl.decision],origin: req.body.origin ?? "api",integration: req.body.integration,- zeroDataRetention: false,+ zeroDataRetention,})),);return res.status(403).json({@@ -232,7 +265,7 @@ export async function agentController(origin: req.body.origin ?? "api",integration: req.body.integration,target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",- zeroDataRetention: false, // not supported for agent+ zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,creditsShards: AGENT_REQUEST_CREDITS_SHARDS,});@@ -265,6 +298,7 @@ export async function agentController(mode: req.body.mode,exchange: req.body.exchange,location: req.body.location,+ ...(zeroDataRetention ? { zeroDataRetention: true } : {}),}),},);diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 2f50ebce9..d3a8df95f 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -12,7 +12,10 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";-import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";+import {+ ExtractV3AgentStatusError,+ getExtractV3AgentStatus,+} from "../../lib/extract-v3-status";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -50,7 +53,17 @@ export async function extractStatusController(}if (access.kind === "agent") {- const agent = await getExtractV3AgentStatus(req.params.jobId);+ let agent: Awaited<ReturnType<typeof getExtractV3AgentStatus>>;+ try {+ agent = await getExtractV3AgentStatus(req.params.jobId);+ } catch (error) {+ if (!(error instanceof ExtractV3AgentStatusError)) throw error;+ return res.status(error.status).json({+ success: false,+ code: error.code,+ error: error.message,+ });+ }return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 095e2e49c..acaab6812 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1211,6 +1211,7 @@ export const agentRequestSchema = zmode: z.enum(["extract", "chat"]).optional(),exchange: agentExchangeSchema.optional(),location: countryLocationSchema.optional(),+ zeroDataRetention: z.boolean().optional(),})// spark-1 is retired and spark-2 is the default. The spark-1 preset names// remain valid input and silently resolve to spark-2, so every request —@@ -1731,6 +1732,7 @@ export type AgentListResponse =effort?: "low" | "medium" | "high";threadId?: string;threadTurn?: number;+ zeroDataRetention?: boolean;};}[];next?: string;@@ -1950,6 +1952,9 @@ export type AgentTraceResponse =id: string;events: object[];creditsUsed: number;+ // Set by the agent service on a zero data retention run, whose events+ // are limited to content-free lifecycle events.+ zeroDataRetention?: boolean;activeBrowserSessions?: Array<{id: string;liveViewUrl: string;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex aa9ba5b51..b357505ff 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -113,6 +113,13 @@ const errorMap: Record<ErrorCodes, any> = {thread_expired: null,threads_disabled: null,exchange_not_enabled: null,++ // Agent zero data retention rejections are API-level too.+ zdr_threads_unsupported: null,+ zdr_exchange_unsupported: null,+ zdr_expired: null,+ zdr_no_snapshots: null,+ zdr_not_enabled: null,};export function serializeTransportableError(error: TransportableError) {diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex cb815056a..e6b2771f6 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -60,7 +60,13 @@ export type ErrorCodes =| "thread_busy"| "thread_expired"| "threads_disabled"- | "exchange_not_enabled";+ | "exchange_not_enabled"+ // Agent zero data retention. Lowercase to match the agent service's codes.+ | "zdr_threads_unsupported"+ | "zdr_exchange_unsupported"+ | "zdr_expired"+ | "zdr_no_snapshots"+ | "zdr_not_enabled";export class TransportableError extends Error {public readonly code: ErrorCodes;diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.tsindex 5effc65af..6c9384909 100644--- a/apps/api/src/lib/extract-v3-status.ts+++ b/apps/api/src/lib/extract-v3-status.ts@@ -1,4 +1,5 @@import { config } from "../config";+import type { ErrorCodes } from "./error";type ExtractV3AgentStatus = {success: true;@@ -22,6 +23,21 @@ type ExtractV3AgentStatus = {creditsUsed?: number;};+/**+ * The agent service answered with an error the caller should see as is, such+ * as 410 zdr_expired once a zero data retention run's result is deleted.+ */+export class ExtractV3AgentStatusError extends Error {+ constructor(+ public readonly status: number,+ public readonly code: ErrorCodes,+ message: string,+ ) {+ super(message);+ this.name = "ExtractV3AgentStatusError";+ }+}+export async function getExtractV3AgentStatus(id: string,): Promise<ExtractV3AgentStatus> {@@ -40,8 +56,24 @@ export async function getExtractV3AgentStatus(});if (!response.ok) {+ const text = await response.text();+ if (response.status === 410) {+ let body: { code?: unknown; error?: unknown } | null = null;+ try {+ body = JSON.parse(text);+ } catch {}+ if (body?.code === "zdr_expired") {+ throw new ExtractV3AgentStatusError(+ 410,+ "zdr_expired",+ typeof body.error === "string"+ ? body.error+ : "This zero data retention agent run's result has been deleted.",+ );+ }+ }throw new Error(- `Agent status service returned ${response.status}: ${await response.text()}`,+ `Agent status service returned ${response.status}: ${text}`,);}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.tsindex c45e8c3f5..d5ae5fe2f 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts@@ -26,6 +26,7 @@ function prepareAgentPayload(args: {* and search results are localized to it without changing where requests come from. The agent cannot change it.*/location?: { country: string };+ zeroDataRetention?: boolean;}): Record<string, unknown> {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;@@ -48,6 +49,7 @@ function prepareAgentPayload(args: {if (args.mode != null) body.mode = args.mode;if (args.exchange != null) body.exchange = args.exchange;if (args.location != null) body.location = args.location;+ if (args.zeroDataRetention != null) body.zeroDataRetention = args.zeroDataRetention;return body;}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 02b0ec6d4..145ff74c2 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -1965,6 +1965,8 @@ export interface AgentTraceResponse {creditsUsed?: number;/** Present only when the trace was requested with liveView: true. */activeBrowserSessions?: AgentTraceActiveBrowserSession[];+ /** True for a zero data retention run: events are limited to content-free lifecycle events. */+ zeroDataRetention?: boolean;error?: string;}diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 3103e7f29..8091a9df3 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -1553,6 +1553,7 @@ class FirecrawlClient:mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,):"""Start an agent job (non-blocking).@@ -1575,6 +1576,7 @@ class FirecrawlClient:(ISO 3166-1 alpha-2). Scrapes and browser sessions are proxiedfrom it, and search results are localized to it without changingwhere requests come from. The agent cannot change it.+ zero_data_retention: Run the agent with zero data retention (requires the team to have ZDR enabled)Returns:Response payload with job id/status (poll with get_agent_status)"""@@ -1595,6 +1597,7 @@ class FirecrawlClient:mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)def agent(@@ -1617,6 +1620,7 @@ class FirecrawlClient:mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,):"""Run an agent and wait until completion.@@ -1641,6 +1645,7 @@ class FirecrawlClient:(ISO 3166-1 alpha-2). Scrapes and browser sessions are proxiedfrom it, and search results are localized to it without changingwhere requests come from. The agent cannot change it.+ zero_data_retention: Run the agent with zero data retention (requires the team to have ZDR enabled)Returns:Final agent response when completed"""@@ -1663,6 +1668,7 @@ class FirecrawlClient:mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)def get_agent_status(self, job_id: str):diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 7e8adec04..a8de23256 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -877,6 +877,7 @@ class AsyncFirecrawlClient:mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,):return await async_agent.agent(self.async_http_client,@@ -897,6 +898,7 @@ class AsyncFirecrawlClient:mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)async def get_agent_status(self, job_id: str):@@ -920,6 +922,7 @@ class AsyncFirecrawlClient:mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,):return await async_agent.start_agent(self.async_http_client,@@ -938,6 +941,7 @@ class AsyncFirecrawlClient:mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)async def cancel_agent(self, job_id: str) -> bool:diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.pyindex e48e121ad..e0be7eeaf 100644--- a/apps/python-sdk/firecrawl/v2/methods/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py@@ -34,6 +34,7 @@ def _prepare_agent_request(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -81,6 +82,8 @@ def _prepare_agent_request()if location is not None:body["location"] = {"country": location["country"]}+ if zero_data_retention is not None:+ body["zeroDataRetention"] = zero_data_retentionreturn body@@ -111,6 +114,7 @@ def start_agent(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -128,6 +132,7 @@ def start_agent(mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)resp = client.post("/v2/agent", body)if not resp.ok:@@ -208,6 +213,7 @@ def agent(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> AgentResponse:started = start_agent(client,@@ -226,6 +232,7 @@ def agent(mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.pyindex ae03ac07c..d934ccbf7 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py@@ -34,6 +34,7 @@ def _prepare_agent_request(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -81,6 +82,8 @@ def _prepare_agent_request()if location is not None:body["location"] = {"country": location["country"]}+ if zero_data_retention is not None:+ body["zeroDataRetention"] = zero_data_retentionreturn body@@ -111,6 +114,7 @@ async def start_agent(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -128,6 +132,7 @@ async def start_agent(mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)resp = await client.post("/v2/agent", body)if resp.status_code >= 400:@@ -208,6 +213,7 @@ async def agent(mode: Optional[Literal["extract", "chat"]] = None,exchange: Optional[Union[AgentExchangeOptions, Dict[str, Any]]] = None,location: Optional[Dict[str, str]] = None,+ zero_data_retention: Optional[bool] = None,) -> AgentResponse:started = await start_agent(client,@@ -226,6 +232,7 @@ async def agent(mode=mode,exchange=exchange,location=location,+ zero_data_retention=zero_data_retention,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 846c84271..057f780bb 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -2086,6 +2086,9 @@ class AgentTraceResponse(BaseModel):active_browser_sessions: Optional[List[AgentTraceActiveBrowserSession]] = Field(default=None, alias="activeBrowserSessions")+ zero_data_retention: Optional[bool] = Field(+ default=None, alias="zeroDataRetention"+ )error: Optional[str] = None