Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(agent): report a ZDR run's own retention in expiresAt (#5444)
Rakshith Ramprakash33f295dc1f09
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 6f3c3cb97..8461f8f1b 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -323,6 +323,9 @@ jobs:src/controllers/v2/__tests__/agent-rollout.test.tssrc/controllers/v2/__tests__/agent-list.test.tssrc/controllers/v2/__tests__/agent-status.test.ts+ src/lib/agent-job-expiry.test.ts+ src/lib/job-operational-stores.test.ts+ src/services/logging/log_job.test.ts--reporter=defaultworking-directory: apps/apidiff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.tsindex 8465f3c49..68214d009 100644--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts@@ -520,4 +520,83 @@ describe("agentStatusController terms-required passthrough", () => {expect(body.exchange).toEqual(exchange);expect(body.pendingApproval).toEqual(pendingApproval);});++ describe("expiresAt for zero data retention runs", () => {+ const MINUTE = 60 * 1000;+ const completedAt = new Date(Date.now() - 5 * MINUTE).toISOString();+ const rowExpiresAtMs = Date.now() + 23 * 60 * MINUTE;+ const baseReq = {+ params: { jobId: "job-123" },+ auth: { team_id: "team-123" },+ } as RequestWithAuth<{ jobId: string }, any, any>;+ const buildRes = () =>+ ({+ status: vi.fn().mockReturnThis(),+ json: vi.fn(),+ }) as unknown as Response;++ function status(access: Record<string, unknown>, req = baseReq) {+ (getAgentJobAccess as Mock).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: rowExpiresAtMs,+ ...access,+ });+ (getExtractV3AgentStatus as Mock).mockResolvedValue({+ id: "job-123",+ success: true,+ status: "success",+ completedAt,+ });+ const res = buildRes();+ return agentStatusController(req, res).then(() => res);+ }++ it("reports completion plus the 15 minute tier", async () => {+ const res = await status({ zeroDataRetentionMinutes: 15 });++ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ expiresAt: new Date(+ Date.parse(completedAt) + 15 * MINUTE,+ ).toISOString(),+ }),+ );+ });++ it("keeps the retention recorded at start when the flag changed mid-run", async () => {+ const res = await status({ zeroDataRetentionMinutes: 15 }, {+ ...baseReq,+ acuc: { flags: { agentZdrRetentionMinutes: 1440 } },+ } as any);++ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ expiresAt: new Date(+ Date.parse(completedAt) + 15 * MINUTE,+ ).toISOString(),+ }),+ );+ });++ it("answers 410 zdr_expired past the recorded retention", async () => {+ const res = await status({ zeroDataRetentionMinutes: 1 });++ expect(res.status).toHaveBeenCalledWith(410);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ success: false, code: "zdr_expired" }),+ );+ });++ it("keeps the access row's expiry for a legacy row with no retention", async () => {+ const res = await status({});++ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ expiresAt: new Date(rowExpiresAtMs).toISOString(),+ }),+ );+ });+ });});diff --git a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.tsindex 7ec53de32..44eed9b44 100644--- a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts@@ -254,6 +254,35 @@ describe("agentController zero data retention", () => {expect(fetch).not.toHaveBeenCalled();});+ it.each([+ ["the team's tier", { agentZdrRetentionMinutes: 15 }, 15],+ ["24 hours when the flag is unset", {}, 24 * 60],+ ])(+ "records %s on the job access row",+ async (_name, retentionFlags, minutes) => {+ const res = await run(+ {},+ { scrapeZDR: "forced", searchZDR: "forced-zdr", ...retentionFlags },+ );++ expect(res.status).toHaveBeenCalledWith(200);+ expect(logRequest).toHaveBeenCalledWith(+ expect.objectContaining({+ zeroDataRetention: true,+ jobAccessZeroDataRetentionMinutes: minutes,+ }),+ );+ },+ );++ it("records no retention for a run without ZDR", async () => {+ await run({}, { agentZdrRetentionMinutes: 15 });++ expect(vi.mocked(logRequest).mock.calls[0][0]).not.toHaveProperty(+ "jobAccessZeroDataRetentionMinutes",+ );+ });+it("forwards exchange options unchanged without ZDR", async () => {const exchange = { enabled: true };const res = await run({ exchange }, {});diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.tsindex 794a9ddc5..77fab6716 100644--- a/apps/api/src/controllers/v2/agent-status.ts+++ b/apps/api/src/controllers/v2/agent-status.ts@@ -8,6 +8,7 @@ import {RequestWithAuth,} from "./types";import { getAgentJobAccess } from "../../lib/operational-job-access";+import { agentJobExpiry, ZDR_EXPIRED_ERROR } from "../../lib/agent-job-expiry";import {ExtractV3AgentStatusError,getExtractV3AgentStatus,@@ -88,6 +89,15 @@ export async function agentStatusController(});}+ const expiry = agentJobExpiry(access, agent);+ if (expiry.zeroDataRetentionExpired) {+ return res.status(410).json({+ success: false,+ code: "zdr_expired",+ error: ZDR_EXPIRED_ERROR,+ });+ }+let model: "spark-1-pro" | "spark-1-mini" | "spark-2" =agent.model ?? "spark-1-pro";// The agent service persists the effort of a run that used it. Older rows@@ -130,7 +140,7 @@ export async function agentStatusController(suggestions: agent.suggestions as AgentSuggestion[] | undefined,pendingApproval: agent.pendingApproval as AgentPendingApproval | undefined,exchange: agent.exchange as AgentExchangeSummary | undefined,- expiresAt: new Date(access.expiresAtMs).toISOString(),+ expiresAt: new Date(expiry.expiresAtMs).toISOString(),creditsUsed: agent.creditsUsed,});}diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex d0ef9b9e5..3515d4b58 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -13,6 +13,7 @@ import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";import {+ AGENT_ZDR_DEFAULT_RETENTION_MINUTES,getAgentZdrRetentionMinutes,getScrapeZDR,getSearchZDR,@@ -279,6 +280,12 @@ export async function agentController(zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,creditsShards: AGENT_REQUEST_CREDITS_SHARDS,+ ...(zeroDataRetention+ ? {+ jobAccessZeroDataRetentionMinutes:+ zeroDataRetentionMinutes ?? AGENT_ZDR_DEFAULT_RETENTION_MINUTES,+ }+ : {}),});const passthrough = await fetch(diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex d3a8df95f..5ebe0233e 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -10,6 +10,7 @@ import {import { logger as _logger } from "../../lib/logger";import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";+import { agentJobExpiry, ZDR_EXPIRED_ERROR } from "../../lib/agent-job-expiry";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";import {@@ -65,12 +66,21 @@ export async function extractStatusController(});}+ const expiry = agentJobExpiry(access, agent);+ if (expiry.zeroDataRetentionExpired) {+ return res.status(410).json({+ success: false,+ code: "zdr_expired",+ error: ZDR_EXPIRED_ERROR,+ });+ }+return res.status(200).json({success: true,status: agent.status === "success" ? "completed" : agent.status,error: agent.error,data: agent.data,- expiresAt: new Date(access.expiresAtMs).toISOString(),+ expiresAt: new Date(expiry.expiresAtMs).toISOString(),creditsUsed: agent.creditsUsed,});}diff --git a/apps/api/src/lib/agent-job-expiry.test.ts b/apps/api/src/lib/agent-job-expiry.test.tsnew file mode 100644index 000000000..01bd6bc59--- /dev/null+++ b/apps/api/src/lib/agent-job-expiry.test.ts@@ -0,0 +1,88 @@+import { agentJobExpiry } from "./agent-job-expiry";++const MINUTE = 60 * 1000;+const STARTED_AT = Date.parse("2026-10-28T20:00:00.000Z");+const COMPLETED_AT = "2026-10-28T20:20:19.000Z";+const COMPLETED_AT_MS = Date.parse(COMPLETED_AT);+const ROW_EXPIRES_AT_MS = STARTED_AT + 24 * 60 * MINUTE;++describe("agentJobExpiry", () => {+ it("counts a 15 minute retention from completion", () => {+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },+ { status: "success", completedAt: COMPLETED_AT },+ COMPLETED_AT_MS + MINUTE,+ ),+ ).toEqual({+ expiresAtMs: COMPLETED_AT_MS + 15 * MINUTE,+ zeroDataRetentionExpired: false,+ });+ });++ it("reports a finished run past its retention as expired", () => {+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },+ { status: "success", completedAt: COMPLETED_AT },+ COMPLETED_AT_MS + 15 * MINUTE,+ ).zeroDataRetentionExpired,+ ).toBe(true);+ });++ it("caps the default 24 hour retention at the access row", () => {+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 24 * 60 },+ { status: "success", completedAt: COMPLETED_AT },+ COMPLETED_AT_MS + MINUTE,+ ),+ ).toEqual({+ expiresAtMs: ROW_EXPIRES_AT_MS,+ zeroDataRetentionExpired: false,+ });+ });++ it("reports the earliest deadline while a ZDR run is still running", () => {+ const now = STARTED_AT + 5 * MINUTE;+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },+ { status: "processing" },+ now,+ ),+ ).toEqual({+ expiresAtMs: now + 15 * MINUTE,+ zeroDataRetentionExpired: false,+ });+ });++ it("counts from the start for a finished run with no completion time", () => {+ const later = STARTED_AT + 60 * MINUTE;+ for (const completedAt of [undefined, "not a date"]) {+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS, zeroDataRetentionMinutes: 15 },+ { status: "failed", completedAt },+ later,+ ),+ ).toEqual({+ expiresAtMs: STARTED_AT + 15 * MINUTE,+ zeroDataRetentionExpired: false,+ });+ }+ });++ it("keeps the row's expiry for a legacy row with no retention", () => {+ expect(+ agentJobExpiry(+ { expiresAtMs: ROW_EXPIRES_AT_MS },+ { status: "success", completedAt: COMPLETED_AT },+ ROW_EXPIRES_AT_MS - MINUTE,+ ),+ ).toEqual({+ expiresAtMs: ROW_EXPIRES_AT_MS,+ zeroDataRetentionExpired: false,+ });+ });+});diff --git a/apps/api/src/lib/agent-job-expiry.ts b/apps/api/src/lib/agent-job-expiry.tsnew file mode 100644index 000000000..8eec79199--- /dev/null+++ b/apps/api/src/lib/agent-job-expiry.ts@@ -0,0 +1,55 @@+import { JOB_ACCESS_TTL_MS, type ApiJobAccess } from "./job-access-store";++type AgentJobExpiry = {+ /** When the run's result stops being served. */+ expiresAtMs: number;+ /** A finished ZDR run past its retention. */+ zeroDataRetentionExpired: boolean;+};++/**+ * When an agent run's result stops being served. A ZDR run is kept for the+ * retention recorded on its job access row when it started, counted from when+ * it finished, and never past the row itself. Until it finishes, the earliest+ * possible deadline is reported: its retention from now. A finished run whose+ * completion time is missing counts from when it started (the row is written+ * then, with the default TTL), which never overstates the deadline; the agent+ * service still enforces the real one. Runs without a recorded retention+ * (non-ZDR runs, and rows written before it was recorded) keep the row's+ * expiry.+ */+export function agentJobExpiry(+ access: Pick<ApiJobAccess, "expiresAtMs" | "zeroDataRetentionMinutes">,+ agent: { status: "processing" | "success" | "failed"; completedAt?: string },+ now = Date.now(),+): AgentJobExpiry {+ if (access.zeroDataRetentionMinutes === undefined) {+ return { expiresAtMs: access.expiresAtMs, zeroDataRetentionExpired: false };+ }+ const retentionMs = access.zeroDataRetentionMinutes * 60 * 1000;+ if (agent.status === "processing") {+ return {+ expiresAtMs: Math.min(now + retentionMs, access.expiresAtMs),+ zeroDataRetentionExpired: false,+ };+ }+ const completedAtMs =+ agent.completedAt === undefined ? NaN : Date.parse(agent.completedAt);+ if (!Number.isFinite(completedAtMs)) {+ return {+ expiresAtMs: Math.min(+ access.expiresAtMs - JOB_ACCESS_TTL_MS + retentionMs,+ access.expiresAtMs,+ ),+ zeroDataRetentionExpired: false,+ };+ }+ const deadlineMs = completedAtMs + retentionMs;+ return {+ expiresAtMs: Math.min(deadlineMs, access.expiresAtMs),+ zeroDataRetentionExpired: now >= deadlineMs,+ };+}++export const ZDR_EXPIRED_ERROR =+ "This run used zero data retention; its result has expired";diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.tsindex 6c9384909..b0cb9e79d 100644--- a/apps/api/src/lib/extract-v3-status.ts+++ b/apps/api/src/lib/extract-v3-status.ts@@ -21,6 +21,8 @@ type ExtractV3AgentStatus = {pendingApproval?: unknown;exchange?: unknown;creditsUsed?: number;+ /** When a finished run reached its terminal state. */+ completedAt?: string;};/**diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.tsindex 614ec72c5..8b54239af 100644--- a/apps/api/src/lib/job-access-store.ts+++ b/apps/api/src/lib/job-access-store.ts@@ -35,6 +35,12 @@ export type ApiJobAccess = {kind: ApiJobKind;expiresAtMs: number;clientOrigin?: string;+ /**+ * An agent run's zero data retention, in minutes, fixed when the run+ * started. Set only on ZDR agent runs; rows written before it existed have+ * none.+ */+ zeroDataRetentionMinutes?: number;};export function isApiJobKind(kind: string): kind is ApiJobKind {@@ -53,7 +59,11 @@ function parseApiJobAccess(value: Buffer | string): ApiJobAccess {!isApiJobKind(row.kind) ||typeof row.expiresAtMs !== "number" ||!Number.isFinite(row.expiresAtMs) ||- (row.clientOrigin !== undefined && typeof row.clientOrigin !== "string")+ (row.clientOrigin !== undefined && typeof row.clientOrigin !== "string") ||+ (row.zeroDataRetentionMinutes !== undefined &&+ (typeof row.zeroDataRetentionMinutes !== "number" ||+ !Number.isFinite(row.zeroDataRetentionMinutes) ||+ row.zeroDataRetentionMinutes <= 0))) {throw new Error("Invalid Bigtable job access row");}@@ -104,6 +114,7 @@ export async function writeApiJobAccess(params: {expiresAt: Date;clientOrigin?: string | null;zeroDataRetention?: boolean;+ zeroDataRetentionMinutes?: number;}): Promise<boolean> {const tableId = config.BIGTABLE_JOB_ACCESS_TABLE;if (!tableId) return false;@@ -126,6 +137,11 @@ export async function writeApiJobAccess(params: {...(params.kind === "agent" && params.clientOrigin? { clientOrigin: params.clientOrigin }: {}),+ ...(params.kind === "agent" &&+ params.zeroDataRetention &&+ params.zeroDataRetentionMinutes !== undefined+ ? { zeroDataRetentionMinutes: params.zeroDataRetentionMinutes }+ : {}),}),);const table = await getBigtableTable(tableId);diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.tsindex 96b28d3ad..18e518a58 100644--- a/apps/api/src/lib/job-operational-stores.test.ts+++ b/apps/api/src/lib/job-operational-stores.test.ts@@ -114,6 +114,26 @@ describe("operational Bigtable stores", () => {});});+ it("records a ZDR agent run's retention on its job access row", async () => {+ const expiresAt = new Date("2026-09-16T12:00:00.000Z");+ await writeApiJobAccess({+ id: JOB_ID,+ teamId: "team-id",+ kind: "agent",+ expiresAt,+ zeroDataRetention: true,+ zeroDataRetentionMinutes: 15,+ });++ expect(writtenValue()).toEqual({+ version: 1,+ teamId: "team-id",+ kind: "agent",+ expiresAtMs: expiresAt.getTime(),+ zeroDataRetentionMinutes: 15,+ });+ });+it("reads a job access row", async () => {getRows.mockResolvedValueOnce([[diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.tsindex 15d53c7e5..58194193b 100644--- a/apps/api/src/lib/operational-job-access.ts+++ b/apps/api/src/lib/operational-job-access.ts@@ -11,6 +11,7 @@ type OperationalJobAccess = {kind: ApiJobKind;clientOrigin?: string;expiresAtMs: number;+ zeroDataRetentionMinutes?: number;};/**diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex a92864325..5b3a85673 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -91,6 +91,9 @@ export function getCustomRobotsAgent(/** The agent ZDR result retentions an org can be given, in minutes. */export const AGENT_ZDR_RETENTION_TIERS_MINUTES = [15, 30, 60, 360, 1440];+/** The agent service's ZDR result retention when the team flag is unset. */+export const AGENT_ZDR_DEFAULT_RETENTION_MINUTES = 24 * 60;+/*** The agent ZDR result retention from team flags: a tier in minutes, null* when unset (the agent service's default), or "invalid" for any other value,diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 94771897e..a1bfbe0c0 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -694,6 +694,23 @@ describe("logRequest", () => {);});+ it("records an agent run's ZDR retention on its job access row", async () => {+ await logRequest({+ ...makeRequest(null),+ kind: "agent",+ zeroDataRetention: true,+ jobAccessZeroDataRetentionMinutes: 15,+ });++ expect(writeApiJobAccess).toHaveBeenCalledWith(+ expect.objectContaining({+ kind: "agent",+ zeroDataRetention: true,+ zeroDataRetentionMinutes: 15,+ }),+ );+ });+it("holds the caller until the publish is acknowledged", async () => {const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 13f722b7f..d4a4513a4 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -440,6 +440,8 @@ type LoggedRequest = {external_request_id?: string | null;jobAccess?: boolean;jobAccessExpiresAt?: Date;+ /** An agent run's ZDR retention, recorded on its job access row. */+ jobAccessZeroDataRetentionMinutes?: number;creditsShards?: number;};@@ -528,6 +530,7 @@ async function logRequestInternal(request: LoggedRequest) {new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),clientOrigin: sanitizedOrigin,zeroDataRetention: request.zeroDataRetention,+ zeroDataRetentionMinutes: request.jobAccessZeroDataRetentionMinutes,});} catch (error) {logger.error("Failed to write API job access to Bigtable", {