Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(agent): forward the team's agent ZDR retention tier (#5442)
Rakshith Ramprakashb63f4941c529
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 302cfcba2..db0e066a6 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1381,6 +1381,8 @@ export type TeamFlags = {scrapeZDR?: "disabled" | "allowed" | "forced";searchZDR?: "disabled" | "allowed" | "forced" | "forced-zdr" | "forced-anon";zdrCost?: number;+ /** How long an agent run's ZDR result is kept, in minutes. Unset is the default (24 hours). */+ agentZdrRetentionMinutes?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;ipWhitelist?: boolean;diff --git a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.tsindex b55639508..7ec53de32 100644--- a/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-zdr.test.ts@@ -151,6 +151,40 @@ describe("agentController zero data retention", () => {expect(logRequest).not.toHaveBeenCalled();});+ const zdrFlags = { scrapeZDR: "forced", searchZDR: "forced-zdr" };++ it("forwards the team's ZDR retention tier", async () => {+ const res = await run({}, { ...zdrFlags, agentZdrRetentionMinutes: 30 });++ expect(res.status).toHaveBeenCalledWith(200);+ expect(forwardedBody()).toMatchObject({+ zeroDataRetention: true,+ zeroDataRetentionMinutes: 30,+ });+ });++ it("leaves the retention to the agent service's default when the flag is unset", async () => {+ await run({}, zdrFlags);++ expect(forwardedBody()).not.toHaveProperty("zeroDataRetentionMinutes");+ });++ it("does not forward a retention for a run without ZDR", async () => {+ await run({}, { agentZdrRetentionMinutes: 30 });++ expect(forwardedBody()).not.toHaveProperty("zeroDataRetentionMinutes");+ });++ it("refuses a ZDR run whose retention flag is not a tier, rather than keep it longer", async () => {+ const res = await run({}, { ...zdrFlags, agentZdrRetentionMinutes: 45 });++ expect(res.status).toHaveBeenCalledWith(500);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ code: "zdr_retention_misconfigured" }),+ );+ expect(fetch).not.toHaveBeenCalled();+ });+it("leaves a run that does not ask for ZDR alone, whatever the search flag", async () => {const res = await run({}, { scrapeZDR: "allowed" });diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 7f3ebf66a..d0ef9b9e5 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -12,7 +12,11 @@ import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import { agentConsumeFreeRequestIfLeft } from "../../db/rpc";-import { getScrapeZDR, getSearchZDR } from "../../lib/zdr-helpers";+import {+ getAgentZdrRetentionMinutes,+ getScrapeZDR,+ getSearchZDR,+} from "../../lib/zdr-helpers";import {checkUrlsAgainstThreatPolicy,resolveThreatProtection,@@ -79,6 +83,22 @@ export async function agentController(});}+ // Read once, here: the agent service keeps the run's retention from start.+ const zeroDataRetentionMinutes = zeroDataRetention+ ? getAgentZdrRetentionMinutes(req.acuc?.flags)+ : null;+ if (zeroDataRetentionMinutes === "invalid") {+ logger.error("Invalid agentZdrRetentionMinutes team flag", {+ agentZdrRetentionMinutes: req.acuc?.flags?.agentZdrRetentionMinutes,+ });+ return res.status(500).json({+ success: false,+ code: "zdr_retention_misconfigured",+ error:+ "Zero Data Retention is misconfigured for your team. Contact [email protected].",+ });+ }+logger.info("Agent starting...", {request: req.body,originalRequest,@@ -290,6 +310,9 @@ export async function agentController(exchange: req.body.exchange,location: req.body.location,...(zeroDataRetention ? { zeroDataRetention: true } : {}),+ ...(zeroDataRetentionMinutes !== null+ ? { zeroDataRetentionMinutes }+ : {}),}),},);diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 4391f036b..7f969acb9 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2153,6 +2153,8 @@ export type TeamFlags = {scrapeZDR?: "disabled" | "allowed" | "forced";searchZDR?: "disabled" | "allowed" | "forced" | "forced-zdr" | "forced-anon";zdrCost?: number;+ /** How long an agent run's ZDR result is kept, in minutes. Unset is the default (24 hours). */+ agentZdrRetentionMinutes?: number;checkRobotsOnScrape?: boolean;crawlTtlHours?: number;ipWhitelist?: boolean;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 7ecc8d535..0853e630b 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -119,6 +119,7 @@ const errorMap: Record<ErrorCodes, any> = {zdr_expired: null,zdr_no_snapshots: null,zdr_not_enabled: null,+ zdr_retention_misconfigured: null,};export function serializeTransportableError(error: TransportableError) {diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 9a2ba91f9..b647e18e7 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -65,7 +65,8 @@ export type ErrorCodes =| "zdr_threads_unsupported"| "zdr_expired"| "zdr_no_snapshots"- | "zdr_not_enabled";+ | "zdr_not_enabled"+ | "zdr_retention_misconfigured";export class TransportableError extends Error {public readonly code: ErrorCodes;diff --git a/apps/api/src/lib/zdr-helpers.test.ts b/apps/api/src/lib/zdr-helpers.test.tsindex a4509d728..a1e26a382 100644--- a/apps/api/src/lib/zdr-helpers.test.ts+++ b/apps/api/src/lib/zdr-helpers.test.ts@@ -1,4 +1,27 @@-import { isDpaRestricted } from "./zdr-helpers";+import { getAgentZdrRetentionMinutes, isDpaRestricted } from "./zdr-helpers";++describe("getAgentZdrRetentionMinutes", () => {+ it("is null when unset", () => {+ expect(getAgentZdrRetentionMinutes(undefined)).toBeNull();+ expect(getAgentZdrRetentionMinutes({})).toBeNull();+ });++ it("returns each tier", () => {+ for (const minutes of [15, 30, 60, 360, 1440]) {+ expect(+ getAgentZdrRetentionMinutes({ agentZdrRetentionMinutes: minutes }),+ ).toBe(minutes);+ }+ });++ it("marks any other value invalid", () => {+ for (const value of [0, 45, 2880, "30" as unknown as number]) {+ expect(+ getAgentZdrRetentionMinutes({ agentZdrRetentionMinutes: value }),+ ).toBe("invalid");+ }+ });+});describe("isDpaRestricted", () => {it("is false when flags are missing", () => {diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex 9aa9b9375..a92864325 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -88,6 +88,24 @@ export function getCustomRobotsAgent(return "disabled";}+/** The agent ZDR result retentions an org can be given, in minutes. */+export const AGENT_ZDR_RETENTION_TIERS_MINUTES = [15, 30, 60, 360, 1440];++/**+ * The agent ZDR result retention from team flags: a tier in minutes, null+ * when unset (the agent service's default), or "invalid" for any other value,+ * which must never fall back to a longer retention.+ */+export function getAgentZdrRetentionMinutes(+ flags: TeamFlags | undefined,+): number | null | "invalid" {+ const minutes = flags?.agentZdrRetentionMinutes;+ if (minutes === undefined || minutes === null) return null;+ return AGENT_ZDR_RETENTION_TIERS_MINUTES.includes(minutes)+ ? minutes+ : "invalid";+}+/*** Whether the org's DPA (or partner amendment) restricts how its data may be* handled. Absent or any non-`true` value reads as unrestricted.