Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl): keep crawl prompt LLM calls out of traces under zero data retention (#4817)

Abimael Martellefd44d9d90fa
diff --git a/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.ts
new file mode 100644
index 000000000..33922f789
--- /dev/null
+++ b/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.ts
@@ -0,0 +1,296 @@
+import { vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ buildPromptWithWebsiteStructure: vi.fn(),
+ getModel: vi.fn(),
+}));
+
+// Call through to the real AI SDK so its telemetry spans are produced, while
+// still letting the tests read the options each call was made with.
+vi.mock("ai", async importOriginal => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return {
+ ...actual,
+ generateObject: vi.fn(actual.generateObject),
+ generateText: vi.fn(actual.generateText),
+ };
+});
+
+vi.mock("../../../lib/generic-ai", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../../lib/generic-ai")>()),
+ getModel: mocks.getModel,
+}));
+
+vi.mock("../../../lib/map-utils", () => ({
+ buildPromptWithWebsiteStructure: mocks.buildPromptWithWebsiteStructure,
+}));
+
+vi.mock("../../../lib/threat-protection/request", () => ({
+ resolveThreatProtection: vi
+ .fn()
+ .mockResolvedValue({ orgConfig: null, policy: null }),
+}));
+
+vi.mock("../../../lib/key-restriction", async importOriginal => ({
+ ...(await importOriginal<typeof import("../../../lib/key-restriction")>()),
+ checkKeyFormatRestriction: vi.fn().mockResolvedValue({ allowed: true }),
+}));
+
+vi.mock("../../../services/logging/log_job", () => ({
+ logRequest: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../lib/request-credits-store", () => ({
+ initializeRequestCredits: vi.fn().mockResolvedValue(undefined),
+ requestCreditsShards: vi.fn().mockReturnValue(1),
+}));
+
+vi.mock("../../../lib/concurrency-limit", () => ({
+ getEffectiveConcurrencyLimit: vi.fn().mockResolvedValue(10),
+}));
+
+vi.mock("../../../lib/crawl-redis", () => ({
+ crawlToCrawler: vi.fn(() => ({
+ getRobotsTxt: vi.fn().mockResolvedValue(""),
+ })),
+ saveCrawl: vi.fn().mockResolvedValue(undefined),
+ markCrawlActive: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../services/worker/nuq-router", () => ({
+ crawlGroup: { addGroup: vi.fn().mockResolvedValue(undefined) },
+ resolveNewGroupBackend: vi.fn().mockResolvedValue("nuq"),
+}));
+
+vi.mock("../../../services/queue-jobs", () => ({
+ _addScrapeJobToBullMQ: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("../../../lib/logger", () => ({
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+}));
+
+import { context, propagation, trace } from "@opentelemetry/api";
+import {
+ InMemorySpanExporter,
+ type NodeTracerProvider,
+} from "@opentelemetry/sdk-trace-node";
+import { generateObject } from "ai";
+import { MockLanguageModelV3 } from "ai/test";
+import type { Mock } from "vitest";
+import {
+ createTracerProvider,
+ withSpan,
+ withZeroDataRetention,
+} from "../../../lib/otel-tracer";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";
+import { crawlController } from "../crawl";
+import { crawlParamsPreviewController } from "../crawl-params-preview";
+
+const TEAM_ID = "11111111-1111-1111-1111-111111111111";
+const PROMPT = "only crawl the pricing pages of this site";
+
+const noopLogger = {
+ warn: () => {},
+ info: () => {},
+ error: () => {},
+ debug: () => {},
+ child: () => noopLogger,
+} as any;
+
+function mockModel() {
+ return new MockLanguageModelV3({
+ modelId: "gpt-4o-mini",
+ doGenerate: async () => ({
+ content: [{ type: "text", text: '{"includePaths":["pricing/.*"]}' }],
+ finishReason: { unified: "stop", raw: "stop" },
+ usage: {
+ inputTokens: {
+ total: 10,
+ noCache: 10,
+ cacheRead: undefined,
+ cacheWrite: undefined,
+ },
+ outputTokens: { total: 5, text: 5, reasoning: undefined },
+ },
+ warnings: [],
+ }),
+ });
+}
+
+function makeReq(
+ body: Record<string, unknown>,
+ flags: Record<string, unknown>,
+) {
+ return {
+ body,
+ auth: { team_id: TEAM_ID },
+ acuc: { api_key_id: 7, org_id: null, flags },
+ headers: {},
+ protocol: "http",
+ host: "localhost",
+ get: () => undefined,
+ } as any;
+}
+
+function makeRes() {
+ const res: any = { status: vi.fn(), json: vi.fn() };
+ res.status.mockReturnValue(res);
+ res.json.mockReturnValue(res);
+ return res;
+}
+
+function telemetryOfCalls() {
+ return (generateObject as Mock).mock.calls.map(
+ ([options]) => options.experimental_telemetry,
+ );
+}
+
+describe("crawl prompt telemetry under zero data retention", () => {
+ const exporter = new InMemorySpanExporter();
+ let provider: NodeTracerProvider;
+
+ beforeAll(() => {
+ provider = createTracerProvider({ exporter, serviceName: "test-service" });
+ provider.register();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ trace.disable();
+ context.disable();
+ propagation.disable();
+ });
+
+ beforeEach(() => {
+ exporter.reset();
+ (generateObject as Mock).mockClear();
+ mocks.getModel.mockImplementation(() => mockModel());
+ // Stands in for the site map lookup, which starts spans of its own.
+ mocks.buildPromptWithWebsiteStructure.mockImplementation(
+ async ({ basePrompt }: { basePrompt: string }) =>
+ withSpan("test.map_site", async () => ({
+ prompt: `${basePrompt}\n\nhttps://example.com/pricing`,
+ websiteUrls: ["https://example.com/pricing"],
+ })),
+ );
+ });
+
+ async function exportedSpans() {
+ await provider.forceFlush();
+ return exporter.getFinishedSpans();
+ }
+
+ it("exports the prompt step of a regular crawl", async () => {
+ const res = makeRes();
+ await crawlController(
+ makeReq({ url: "https://example.com", prompt: PROMPT }, {}),
+ res,
+ );
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: true }),
+ ]);
+ const spans = await exportedSpans();
+ const names = spans.map(span => span.name);
+ expect(names).toContain("test.map_site");
+ expect(names).toContain("ai.generateObject");
+ expect(JSON.stringify(spans.map(span => span.attributes))).toContain(
+ PROMPT,
+ );
+ });
+
+ it.each([
+ [
+ "team-scoped",
+ { url: "https://example.com", prompt: PROMPT },
+ { scrapeZDR: "forced" },
+ ],
+ [
+ "request-scoped",
+ { url: "https://example.com", prompt: PROMPT, zeroDataRetention: true },
+ { scrapeZDR: "allowed" },
+ ],
+ ])(
+ "records nothing from the prompt step of a %s ZDR crawl",
+ async (_scope, body, flags) => {
+ const res = makeRes();
+ await crawlController(makeReq(body, flags), res);
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({
+ promptGeneratedOptions: { includePaths: ["pricing/.*"] },
+ }),
+ );
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: false }),
+ ]);
+ expect(await exportedSpans()).toEqual([]);
+ },
+ );
+
+ it.each([
+ [
+ "team-scoped",
+ { url: "https://example.com", prompt: PROMPT },
+ { scrapeZDR: "forced" },
+ ],
+ [
+ "request-scoped",
+ { url: "https://example.com", prompt: PROMPT, zeroDataRetention: true },
+ {},
+ ],
+ ])(
+ "records nothing from a %s ZDR params preview",
+ async (_scope, body, flags) => {
+ const res = makeRes();
+ await crawlParamsPreviewController(makeReq(body, flags), res);
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: false }),
+ ]);
+ expect(await exportedSpans()).toEqual([]);
+ },
+ );
+
+ it("exports a params preview for a regular team", async () => {
+ const res = makeRes();
+ await crawlParamsPreviewController(
+ makeReq({ url: "https://example.com", prompt: PROMPT }, {}),
+ res,
+ );
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ const names = (await exportedSpans()).map(span => span.name);
+ expect(names).toContain("ai.generateObject");
+ });
+
+ it("turns off AI SDK telemetry in a ZDR context without an explicit flag", async () => {
+ await withZeroDataRetention(true, () =>
+ generateCompletions({
+ logger: noopLogger,
+ options: { prompt: PROMPT },
+ markdown: "",
+ model: mockModel(),
+ retryModel: mockModel(),
+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
+ metadata: { teamId: TEAM_ID },
+ }),
+ );
+
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: false }),
+ ]);
+ expect(await exportedSpans()).toEqual([]);
+ });
+});
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index fb7539f97..0c6747863 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -5,12 +5,18 @@ import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
import { CostTracking } from "../../lib/cost-tracking";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
+import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { isLockdownZeroDataRetention } from "../../lib/safe-mode";
+import { withZeroDataRetention } from "../../lib/otel-tracer";
// Define the request schema for params preview
// Only url and prompt are required/relevant for preview
const crawlParamsPreviewRequestSchema = z.object({
url: z.url(),
prompt: z.string().max(10000),
+ // Only keeps the preview's own LLM call out of traces, so unlike a crawl it
+ // needs no ZDR permission check.
+ zeroDataRetention: z.boolean().optional(),
});
type CrawlParamsPreviewRequest = z.infer<
@@ -46,10 +52,17 @@ export async function crawlParamsPreviewController(
>,
res: Response<CrawlParamsPreviewResponse>,
) {
+ // Same rule as a crawl: forced ZDR, the request's own flag, or Safe Mode
+ // lockdown. Read before parsing so a rejected body is still covered.
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true ||
+ isLockdownZeroDataRetention(req.acuc?.flags, undefined);
const logger = _logger.child({
module: "api/v2",
method: "crawlParamsPreviewController",
teamId: req.auth.team_id,
+ zeroDataRetention,
});
try {
@@ -61,29 +74,37 @@ export async function crawlParamsPreviewController(
prompt: parsedBody.prompt,
});
- // Build enhanced prompt with website structure
- const { prompt: enhancedPrompt, websiteUrls } =
- await buildPromptWithWebsiteStructure({
- basePrompt: parsedBody.prompt,
- url: parsedBody.url,
- teamId: req.auth.team_id,
- orgId: req.acuc?.org_id ?? null,
- flags: req.acuc?.flags ?? null,
- logger,
- limit: 50,
- includeSubdomains: true,
- allowExternalLinks: false,
- useIndex: true,
- maxFireEngineResults: 500,
- });
+ // The prompt and the discovered URLs end up in LLM telemetry, so keep
+ // this whole step out of traces for zero data retention teams.
+ const { extract, websiteUrls } = await withZeroDataRetention(
+ zeroDataRetention,
+ async () => {
+ // Build enhanced prompt with website structure
+ const { prompt: enhancedPrompt, websiteUrls } =
+ await buildPromptWithWebsiteStructure({
+ basePrompt: parsedBody.prompt,
+ url: parsedBody.url,
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ logger,
+ limit: 50,
+ includeSubdomains: true,
+ allowExternalLinks: false,
+ useIndex: true,
+ maxFireEngineResults: 500,
+ });
- // Generate crawler options from enhanced prompt
- const costTracking = new CostTracking();
- const { extract } = await generateCrawlerOptionsFromPrompt(
- enhancedPrompt,
- logger,
- costTracking,
- { teamId: req.auth.team_id },
+ // Generate crawler options from enhanced prompt
+ const { extract } = await generateCrawlerOptionsFromPrompt(
+ enhancedPrompt,
+ logger,
+ new CostTracking(),
+ { teamId: req.auth.team_id },
+ zeroDataRetention,
+ );
+ return { extract, websiteUrls };
+ },
);
const generatedOptions = extract || {};
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index b3da3c831..4e8b408b0 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -34,6 +34,7 @@ import {
import { logRequest } from "../../services/logging/log_job";
import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { withZeroDataRetention } from "../../lib/otel-tracer";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { checkUrl } from "../../lib/threat-protection";
import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";
@@ -225,27 +226,36 @@ export async function crawlController(
let promptGeneratedOptions = {};
if (req.body.prompt) {
+ const basePrompt = req.body.prompt;
try {
- // Enhance prompt with discovered site URLs (up to 120) to improve option generation
- const { prompt: enhancedPrompt } = await buildPromptWithWebsiteStructure({
- basePrompt: req.body.prompt,
- url: req.body.url,
- teamId: req.auth.team_id,
- orgId: req.acuc?.org_id ?? null,
- flags: req.acuc?.flags ?? null,
- logger,
- limit: 50,
- includeSubdomains: false,
- allowExternalLinks: false,
- useIndex: true,
- maxFireEngineResults: 500,
- });
- const costTracking = new CostTracking();
- const { extract } = await generateCrawlerOptionsFromPrompt(
- enhancedPrompt,
- logger,
- costTracking,
- { teamId: req.auth.team_id, crawlId: id },
+ // The prompt and the discovered URLs end up in LLM telemetry, so keep
+ // this whole step out of traces for zero data retention crawls.
+ const { extract } = await withZeroDataRetention(
+ zeroDataRetention,
+ async () => {
+ // Enhance prompt with discovered site URLs (up to 120) to improve option generation
+ const { prompt: enhancedPrompt } =
+ await buildPromptWithWebsiteStructure({
+ basePrompt,
+ url: req.body.url,
+ teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
+ flags: req.acuc?.flags ?? null,
+ logger,
+ limit: 50,
+ includeSubdomains: false,
+ allowExternalLinks: false,
+ useIndex: true,
+ maxFireEngineResults: 500,
+ });
+ return generateCrawlerOptionsFromPrompt(
+ enhancedPrompt,
+ logger,
+ new CostTracking(),
+ { teamId: req.auth.team_id, crawlId: id },
+ zeroDataRetention,
+ );
+ },
);
promptGeneratedOptions = extract || {};
logger.debug("Generated crawler options from prompt", {
diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.ts
index aaeb53c2f..7efdfb9fb 100644
--- a/apps/api/src/lib/otel-tracer.ts
+++ b/apps/api/src/lib/otel-tracer.ts
@@ -67,6 +67,15 @@ function isZeroDataRetentionContext(ctx: Context): boolean {
return ctx.getValue(ZERO_DATA_RETENTION_CONTEXT_KEY) === true;
}
+/**
+ * Whether the caller runs under `withZeroDataRetention` (or a ZDR `withSpan`),
+ * for code that must also switch off telemetry it controls directly, such as
+ * the AI SDK's `experimental_telemetry`.
+ */
+export function isZeroDataRetentionActive(): boolean {
+ return isZeroDataRetentionContext(context.active());
+}
+
// Traces flagged ZDR after their root span started (a `*.zero_data_retention`
// attribute set from inside a *recording* span). Descendants started afterwards
// are not recorded and spans of these traces still in flight are dropped when
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 9945a8175..87575a806 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -28,6 +28,7 @@ import {
typeIncludes,
} from "../../../lib/openai-strict-schema";
import { CostTracking } from "../../../lib/cost-tracking";
+import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
@@ -345,6 +346,11 @@ export type GenerateCompletionsOptions = {
deepResearchId?: string;
llmsTxtId?: string;
};
+ /**
+ * Turns off AI SDK telemetry for the call. Also on whenever the caller runs
+ * in a zero data retention context, so a caller that forgets to pass it
+ * cannot export prompts.
+ */
zeroDataRetention?: boolean;
};
export async function generateCompletions({
@@ -359,7 +365,7 @@ export async function generateCompletions({
retryModel = getModel("gpt-4.1-mini", "openai"),
costTrackingOptions,
metadata,
- zeroDataRetention = false,
+ zeroDataRetention: zeroDataRetentionOption,
}: GenerateCompletionsOptions): Promise<{
extract: any;
numTokens: number;
@@ -367,6 +373,8 @@ export async function generateCompletions({
totalUsage: TokenUsage;
model: string;
}> {
+ const zeroDataRetention =
+ zeroDataRetentionOption === true || isZeroDataRetentionActive();
let extract: any;
let warning: string | undefined;
let currentModel = model;
@@ -1611,6 +1619,7 @@ export async function generateCrawlerOptionsFromPrompt(
logger: Logger,
costTracking: CostTracking,
metadata: { teamId: string; crawlId?: string },
+ zeroDataRetention = false,
): Promise<{ extract: any }> {
const model = getModel("gpt-4o-mini", "openai");
const retryModel = getModel("gpt-4.1-mini", "openai");
@@ -1657,6 +1666,7 @@ Return a JSON object with only the relevant options for the user's request. Don'
...metadata,
functionId: "generateCrawlerOptionsFromPrompt",
},
+ zeroDataRetention,
});
return { extract };