Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(agent): report a ZDR run's own retention in expiresAt (#5444)

Rakshith Ramprakash33f295dc1f09
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 6f3c3cb97..8461f8f1b 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -323,6 +323,9 @@ jobs:
src/controllers/v2/__tests__/agent-rollout.test.ts
src/controllers/v2/__tests__/agent-list.test.ts
src/controllers/v2/__tests__/agent-status.test.ts
+ src/lib/agent-job-expiry.test.ts
+ src/lib/job-operational-stores.test.ts
+ src/services/logging/log_job.test.ts
--reporter=default
working-directory: apps/api
diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
index 8465f3c49..68214d009 100644
--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts
@@ -520,4 +520,83 @@ describe("agentStatusController terms-required passthrough", () => {
expect(body.exchange).toEqual(exchange);
expect(body.pendingApproval).toEqual(pendingApproval);
});
+
+ describe("expiresAt for zero data retention runs", () => {
+ const MINUTE = 60 * 1000;
+ const completedAt = new Date(Date.now() - 5 * MINUTE).toISOString();
+ const rowExpiresAtMs = Date.now() + 23 * 60 * MINUTE;
+ const baseReq = {
+ params: { jobId: "job-123" },
+ auth: { team_id: "team-123" },
+ } as RequestWithAuth<{ jobId: string }, any, any>;
+ const buildRes = () =>
+ ({
+ status: vi.fn().mockReturnThis(),
+ json: vi.fn(),
+ }) as unknown as Response;
+
+ function status(access: Record<string, unknown>, req = baseReq) {
+ (getAgentJobAccess as Mock).mockResolvedValue({
+ teamId: "team-123",
+ expiresAtMs: rowExpiresAtMs,
+ ...access,
+ });
+ (getExtractV3AgentStatus as Mock).mockResolvedValue({
+ id: "job-123",
+ success: true,
+ status: "success",
+ completedAt,
+ });
+ const res = buildRes();
+ return agentStatusController(req, res).then(() => res);
+ }
+
+ it("reports completion plus the 15 minute tier", async () => {
+ const res = await status({ zeroDataRetentionMinutes: 15 });
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ expiresAt: new Date(
+ Date.parse(completedAt) + 15 * MINUTE,
+ ).toISOString(),
+ }),
+ );
+ });
+
+ it("keeps the retention recorded at start when the flag changed mid-run", async () => {
+ const res = await status({ zeroDataRetentionMinutes: 15 }, {
+ ...baseReq,
+ acuc: { flags: { agentZdrRetentionMinutes: 1440 } },
+ } as any);
+
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ expiresAt: new Date(
+ Date.parse(completedAt) + 15 * MINUTE,
+ ).toISOString(),
+ }),
+ );
+ });
+
+ it("answers 410 zdr_expired past the recorded retention", async () => {
+ const res = await status({ zeroDataRetentionMinutes: 1 });
+
+ expect(res.status).toHaveBeenCalledWith(410);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ success: false, code: "zdr_expired" }),
+ );
+ });
+
+ it("keeps the access row's expiry for a legacy row with no retention", async () => {
+ const res = await status({});
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ expiresAt: new Date(rowExpiresAtMs).toISOString(),
+ }),
+ );
+ });
+ });
});
diff --git a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts
index 7ec53de32..44eed9b44 100644
--- a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts
@@ -254,6 +254,35 @@ describe("agentController zero data retention", () => {
expect(fetch).not.toHaveBeenCalled();
});
+ it.each([
+ ["the team's tier", { agentZdrRetentionMinutes: 15 }, 15],
+ ["24 hours when the flag is unset", {}, 24 * 60],
+ ])(
+ "records %s on the job access row",
+ async (_name, retentionFlags, minutes) => {
+ const res = await run(
+ {},
+ { scrapeZDR: "forced", searchZDR: "forced-zdr", ...retentionFlags },
+ );
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(logRequest).toHaveBeenCalledWith(
+ expect.objectContaining({
+ zeroDataRetention: true,
+ jobAccessZeroDataRetentionMinutes: minutes,
+ }),
+ );
+ },
+ );
+
+ it("records no retention for a run without ZDR", async () => {
+ await run({}, { agentZdrRetentionMinutes: 15 });
+
+ expect(vi.mocked(logRequest).mock.calls[0][0]).not.toHaveProperty(
+ "jobAccessZeroDataRetentionMinutes",
+ );
+ });
+
it("forwards exchange options unchanged without ZDR", async () => {
const exchange = { enabled: true };
const res = await run({ exchange }, {});
diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.ts
index 794a9ddc5..77fab6716 100644
--- a/apps/api/src/controllers/v2/agent-status.ts
+++ b/apps/api/src/controllers/v2/agent-status.ts
@@ -8,6 +8,7 @@ import {
RequestWithAuth,
} from "./types";
import { getAgentJobAccess } from "../../lib/operational-job-access";
+import { agentJobExpiry, ZDR_EXPIRED_ERROR } from "../../lib/agent-job-expiry";
import {
ExtractV3AgentStatusError,
getExtractV3AgentStatus,
@@ -88,6 +89,15 @@ export async function agentStatusController(
});
}
+ const expiry = agentJobExpiry(access, agent);
+ if (expiry.zeroDataRetentionExpired) {
+ return res.status(410).json({
+ success: false,
+ code: "zdr_expired",
+ error: ZDR_EXPIRED_ERROR,
+ });
+ }
+
let model: "spark-1-pro" | "spark-1-mini" | "spark-2" =
agent.model ?? "spark-1-pro";
// The agent service persists the effort of a run that used it. Older rows
@@ -130,7 +140,7 @@ export async function agentStatusController(
suggestions: agent.suggestions as AgentSuggestion[] | undefined,
pendingApproval: agent.pendingApproval as AgentPendingApproval | undefined,
exchange: agent.exchange as AgentExchangeSummary | undefined,
- expiresAt: new Date(access.expiresAtMs).toISOString(),
+ expiresAt: new Date(expiry.expiresAtMs).toISOString(),
creditsUsed: agent.creditsUsed,
});
}
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index d0ef9b9e5..3515d4b58 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -13,6 +13,7 @@ import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";
import {
+ AGENT_ZDR_DEFAULT_RETENTION_MINUTES,
getAgentZdrRetentionMinutes,
getScrapeZDR,
getSearchZDR,
@@ -279,6 +280,12 @@ export async function agentController(
zeroDataRetention,
api_key_id: req.acuc?.api_key_id ?? null,
creditsShards: AGENT_REQUEST_CREDITS_SHARDS,
+ ...(zeroDataRetention
+ ? {
+ jobAccessZeroDataRetentionMinutes:
+ zeroDataRetentionMinutes ?? AGENT_ZDR_DEFAULT_RETENTION_MINUTES,
+ }
+ : {}),
});
const passthrough = await fetch(
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index d3a8df95f..5ebe0233e 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -10,6 +10,7 @@ import {
import { logger as _logger } from "../../lib/logger";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { getExtractJobAccess } from "../../lib/operational-job-access";
+import { agentJobExpiry, ZDR_EXPIRED_ERROR } from "../../lib/agent-job-expiry";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
import {
@@ -65,12 +66,21 @@ export async function extractStatusController(
});
}
+ const expiry = agentJobExpiry(access, agent);
+ if (expiry.zeroDataRetentionExpired) {
+ return res.status(410).json({
+ success: false,
+ code: "zdr_expired",
+ error: ZDR_EXPIRED_ERROR,
+ });
+ }
+
return res.status(200).json({
success: true,
status: agent.status === "success" ? "completed" : agent.status,
error: agent.error,
data: agent.data,
- expiresAt: new Date(access.expiresAtMs).toISOString(),
+ expiresAt: new Date(expiry.expiresAtMs).toISOString(),
creditsUsed: agent.creditsUsed,
});
}
diff --git a/apps/api/src/lib/agent-job-expiry.test.ts b/apps/api/src/lib/agent-job-expiry.test.ts
new file mode 100644
index 000000000..01bd6bc59
--- /dev/null
+++ b/apps/api/src/lib/agent-job-expiry.test.ts
@@ -0,0 +1,88 @@
+import { agentJobExpiry } from "./agent-job-expiry";
+
+const MINUTE = 60 * 1000;
+const STARTED_AT = Date.parse("2026-10-28T20:00:00.000Z");
+const COMPLETED_AT = "2026-10-28T20:20:19.000Z";
+const COMPLETED_AT_MS = Date.parse(COMPLETED_AT);
+const ROW_EXPIRES_AT_MS = STARTED_AT + 24 * 60 * MINUTE;
+
+describe("agentJobExpiry", () => {
+ it("counts a 15 minute retention from completion", () => {
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },
+ { status: "success", completedAt: COMPLETED_AT },
+ COMPLETED_AT_MS + MINUTE,
+ ),
+ ).toEqual({
+ expiresAtMs: COMPLETED_AT_MS + 15 * MINUTE,
+ zeroDataRetentionExpired: false,
+ });
+ });
+
+ it("reports a finished run past its retention as expired", () => {
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },
+ { status: "success", completedAt: COMPLETED_AT },
+ COMPLETED_AT_MS + 15 * MINUTE,
+ ).zeroDataRetentionExpired,
+ ).toBe(true);
+ });
+
+ it("caps the default 24 hour retention at the access row", () => {
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 24 * 60 },
+ { status: "success", completedAt: COMPLETED_AT },
+ COMPLETED_AT_MS + MINUTE,
+ ),
+ ).toEqual({
+ expiresAtMs: ROW_EXPIRES_AT_MS,
+ zeroDataRetentionExpired: false,
+ });
+ });
+
+ it("reports the earliest deadline while a ZDR run is still running", () => {
+ const now = STARTED_AT + 5 * MINUTE;
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },
+ { status: "processing" },
+ now,
+ ),
+ ).toEqual({
+ expiresAtMs: now + 15 * MINUTE,
+ zeroDataRetentionExpired: false,
+ });
+ });
+
+ it("counts from the start for a finished run with no completion time", () => {
+ const later = STARTED_AT + 60 * MINUTE;
+ for (const completedAt of [undefined, "not a date"]) {
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },
+ { status: "failed", completedAt },
+ later,
+ ),
+ ).toEqual({
+ expiresAtMs: STARTED_AT + 15 * MINUTE,
+ zeroDataRetentionExpired: false,
+ });
+ }
+ });
+
+ it("keeps the row's expiry for a legacy row with no retention", () => {
+ expect(
+ agentJobExpiry(
+ { expiresAtMs: ROW_EXPIRES_AT_MS },
+ { status: "success", completedAt: COMPLETED_AT },
+ ROW_EXPIRES_AT_MS - MINUTE,
+ ),
+ ).toEqual({
+ expiresAtMs: ROW_EXPIRES_AT_MS,
+ zeroDataRetentionExpired: false,
+ });
+ });
+});
diff --git a/apps/api/src/lib/agent-job-expiry.ts b/apps/api/src/lib/agent-job-expiry.ts
new file mode 100644
index 000000000..8eec79199
--- /dev/null
+++ b/apps/api/src/lib/agent-job-expiry.ts
@@ -0,0 +1,55 @@
+import { JOB_ACCESS_TTL_MS, type ApiJobAccess } from "./job-access-store";
+
+type AgentJobExpiry = {
+ /** When the run's result stops being served. */
+ expiresAtMs: number;
+ /** A finished ZDR run past its retention. */
+ zeroDataRetentionExpired: boolean;
+};
+
+/**
+ * When an agent run's result stops being served. A ZDR run is kept for the
+ * retention recorded on its job access row when it started, counted from when
+ * it finished, and never past the row itself. Until it finishes, the earliest
+ * possible deadline is reported: its retention from now. A finished run whose
+ * completion time is missing counts from when it started (the row is written
+ * then, with the default TTL), which never overstates the deadline; the agent
+ * service still enforces the real one. Runs without a recorded retention
+ * (non-ZDR runs, and rows written before it was recorded) keep the row's
+ * expiry.
+ */
+export function agentJobExpiry(
+ access: Pick<ApiJobAccess, "expiresAtMs" | "zeroDataRetentionMinutes">,
+ agent: { status: "processing" | "success" | "failed"; completedAt?: string },
+ now = Date.now(),
+): AgentJobExpiry {
+ if (access.zeroDataRetentionMinutes === undefined) {
+ return { expiresAtMs: access.expiresAtMs, zeroDataRetentionExpired: false };
+ }
+ const retentionMs = access.zeroDataRetentionMinutes * 60 * 1000;
+ if (agent.status === "processing") {
+ return {
+ expiresAtMs: Math.min(now + retentionMs, access.expiresAtMs),
+ zeroDataRetentionExpired: false,
+ };
+ }
+ const completedAtMs =
+ agent.completedAt === undefined ? NaN : Date.parse(agent.completedAt);
+ if (!Number.isFinite(completedAtMs)) {
+ return {
+ expiresAtMs: Math.min(
+ access.expiresAtMs - JOB_ACCESS_TTL_MS + retentionMs,
+ access.expiresAtMs,
+ ),
+ zeroDataRetentionExpired: false,
+ };
+ }
+ const deadlineMs = completedAtMs + retentionMs;
+ return {
+ expiresAtMs: Math.min(deadlineMs, access.expiresAtMs),
+ zeroDataRetentionExpired: now >= deadlineMs,
+ };
+}
+
+export const ZDR_EXPIRED_ERROR =
+ "This run used zero data retention; its result has expired";
diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.ts
index 6c9384909..b0cb9e79d 100644
--- a/apps/api/src/lib/extract-v3-status.ts
+++ b/apps/api/src/lib/extract-v3-status.ts
@@ -21,6 +21,8 @@ type ExtractV3AgentStatus = {
pendingApproval?: unknown;
exchange?: unknown;
creditsUsed?: number;
+ /** When a finished run reached its terminal state. */
+ completedAt?: string;
};
/**
diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.ts
index 614ec72c5..8b54239af 100644
--- a/apps/api/src/lib/job-access-store.ts
+++ b/apps/api/src/lib/job-access-store.ts
@@ -35,6 +35,12 @@ export type ApiJobAccess = {
kind: ApiJobKind;
expiresAtMs: number;
clientOrigin?: string;
+ /**
+ * An agent run's zero data retention, in minutes, fixed when the run
+ * started. Set only on ZDR agent runs; rows written before it existed have
+ * none.
+ */
+ zeroDataRetentionMinutes?: number;
};
export function isApiJobKind(kind: string): kind is ApiJobKind {
@@ -53,7 +59,11 @@ function parseApiJobAccess(value: Buffer | string): ApiJobAccess {
!isApiJobKind(row.kind) ||
typeof row.expiresAtMs !== "number" ||
!Number.isFinite(row.expiresAtMs) ||
- (row.clientOrigin !== undefined && typeof row.clientOrigin !== "string")
+ (row.clientOrigin !== undefined && typeof row.clientOrigin !== "string") ||
+ (row.zeroDataRetentionMinutes !== undefined &&
+ (typeof row.zeroDataRetentionMinutes !== "number" ||
+ !Number.isFinite(row.zeroDataRetentionMinutes) ||
+ row.zeroDataRetentionMinutes <= 0))
) {
throw new Error("Invalid Bigtable job access row");
}
@@ -104,6 +114,7 @@ export async function writeApiJobAccess(params: {
expiresAt: Date;
clientOrigin?: string | null;
zeroDataRetention?: boolean;
+ zeroDataRetentionMinutes?: number;
}): Promise<boolean> {
const tableId = config.BIGTABLE_JOB_ACCESS_TABLE;
if (!tableId) return false;
@@ -126,6 +137,11 @@ export async function writeApiJobAccess(params: {
...(params.kind === "agent" && params.clientOrigin
? { clientOrigin: params.clientOrigin }
: {}),
+ ...(params.kind === "agent" &&
+ params.zeroDataRetention &&
+ params.zeroDataRetentionMinutes !== undefined
+ ? { zeroDataRetentionMinutes: params.zeroDataRetentionMinutes }
+ : {}),
}),
);
const table = await getBigtableTable(tableId);
diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.ts
index 96b28d3ad..18e518a58 100644
--- a/apps/api/src/lib/job-operational-stores.test.ts
+++ b/apps/api/src/lib/job-operational-stores.test.ts
@@ -114,6 +114,26 @@ describe("operational Bigtable stores", () => {
});
});
+ it("records a ZDR agent run's retention on its job access row", async () => {
+ const expiresAt = new Date("2026-09-16T12:00:00.000Z");
+ await writeApiJobAccess({
+ id: JOB_ID,
+ teamId: "team-id",
+ kind: "agent",
+ expiresAt,
+ zeroDataRetention: true,
+ zeroDataRetentionMinutes: 15,
+ });
+
+ expect(writtenValue()).toEqual({
+ version: 1,
+ teamId: "team-id",
+ kind: "agent",
+ expiresAtMs: expiresAt.getTime(),
+ zeroDataRetentionMinutes: 15,
+ });
+ });
+
it("reads a job access row", async () => {
getRows.mockResolvedValueOnce([
[
diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.ts
index 15d53c7e5..58194193b 100644
--- a/apps/api/src/lib/operational-job-access.ts
+++ b/apps/api/src/lib/operational-job-access.ts
@@ -11,6 +11,7 @@ type OperationalJobAccess = {
kind: ApiJobKind;
clientOrigin?: string;
expiresAtMs: number;
+ zeroDataRetentionMinutes?: number;
};
/**
diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.ts
index a92864325..5b3a85673 100644
--- a/apps/api/src/lib/zdr-helpers.ts
+++ b/apps/api/src/lib/zdr-helpers.ts
@@ -91,6 +91,9 @@ export function getCustomRobotsAgent(
/** The agent ZDR result retentions an org can be given, in minutes. */
export const AGENT_ZDR_RETENTION_TIERS_MINUTES = [15, 30, 60, 360, 1440];
+/** The agent service's ZDR result retention when the team flag is unset. */
+export const AGENT_ZDR_DEFAULT_RETENTION_MINUTES = 24 * 60;
+
/**
* The agent ZDR result retention from team flags: a tier in minutes, null
* when unset (the agent service's default), or "invalid" for any other value,
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 94771897e..a1bfbe0c0 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -694,6 +694,23 @@ describe("logRequest", () => {
);
});
+ it("records an agent run's ZDR retention on its job access row", async () => {
+ await logRequest({
+ ...makeRequest(null),
+ kind: "agent",
+ zeroDataRetention: true,
+ jobAccessZeroDataRetentionMinutes: 15,
+ });
+
+ expect(writeApiJobAccess).toHaveBeenCalledWith(
+ expect.objectContaining({
+ kind: "agent",
+ zeroDataRetention: true,
+ zeroDataRetentionMinutes: 15,
+ }),
+ );
+ });
+
it("holds the caller until the publish is acknowledged", async () => {
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 13f722b7f..d4a4513a4 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -440,6 +440,8 @@ type LoggedRequest = {
external_request_id?: string | null;
jobAccess?: boolean;
jobAccessExpiresAt?: Date;
+ /** An agent run's ZDR retention, recorded on its job access row. */
+ jobAccessZeroDataRetentionMinutes?: number;
creditsShards?: number;
};
@@ -528,6 +530,7 @@ async function logRequestInternal(request: LoggedRequest) {
new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),
clientOrigin: sanitizedOrigin,
zeroDataRetention: request.zeroDataRetention,
+ zeroDataRetentionMinutes: request.jobAccessZeroDataRetentionMinutes,
});
} catch (error) {
logger.error("Failed to write API job access to Bigtable", {