Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl): keep crawl prompt LLM calls out of traces under zero data retention (#4817)
Abimael Martellefd44d9d90fa
diff --git a/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.tsnew file mode 100644index 000000000..33922f789--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/crawl-prompt-zdr.test.ts@@ -0,0 +1,296 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ buildPromptWithWebsiteStructure: vi.fn(),+ getModel: vi.fn(),+}));++// Call through to the real AI SDK so its telemetry spans are produced, while+// still letting the tests read the options each call was made with.+vi.mock("ai", async importOriginal => {+ const actual = await importOriginal<typeof import("ai")>();+ return {+ ...actual,+ generateObject: vi.fn(actual.generateObject),+ generateText: vi.fn(actual.generateText),+ };+});++vi.mock("../../../lib/generic-ai", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../lib/generic-ai")>()),+ getModel: mocks.getModel,+}));++vi.mock("../../../lib/map-utils", () => ({+ buildPromptWithWebsiteStructure: mocks.buildPromptWithWebsiteStructure,+}));++vi.mock("../../../lib/threat-protection/request", () => ({+ resolveThreatProtection: vi+ .fn()+ .mockResolvedValue({ orgConfig: null, policy: null }),+}));++vi.mock("../../../lib/key-restriction", async importOriginal => ({+ ...(await importOriginal<typeof import("../../../lib/key-restriction")>()),+ checkKeyFormatRestriction: vi.fn().mockResolvedValue({ allowed: true }),+}));++vi.mock("../../../services/logging/log_job", () => ({+ logRequest: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../lib/request-credits-store", () => ({+ initializeRequestCredits: vi.fn().mockResolvedValue(undefined),+ requestCreditsShards: vi.fn().mockReturnValue(1),+}));++vi.mock("../../../lib/concurrency-limit", () => ({+ getEffectiveConcurrencyLimit: vi.fn().mockResolvedValue(10),+}));++vi.mock("../../../lib/crawl-redis", () => ({+ crawlToCrawler: vi.fn(() => ({+ getRobotsTxt: vi.fn().mockResolvedValue(""),+ })),+ saveCrawl: vi.fn().mockResolvedValue(undefined),+ markCrawlActive: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../services/worker/nuq-router", () => ({+ crawlGroup: { addGroup: vi.fn().mockResolvedValue(undefined) },+ resolveNewGroupBackend: vi.fn().mockResolvedValue("nuq"),+}));++vi.mock("../../../services/queue-jobs", () => ({+ _addScrapeJobToBullMQ: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { context, propagation, trace } from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ type NodeTracerProvider,+} from "@opentelemetry/sdk-trace-node";+import { generateObject } from "ai";+import { MockLanguageModelV3 } from "ai/test";+import type { Mock } from "vitest";+import {+ createTracerProvider,+ withSpan,+ withZeroDataRetention,+} from "../../../lib/otel-tracer";+import { CostTracking } from "../../../lib/cost-tracking";+import { generateCompletions } from "../../../scraper/scrapeURL/transformers/llmExtract";+import { crawlController } from "../crawl";+import { crawlParamsPreviewController } from "../crawl-params-preview";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";+const PROMPT = "only crawl the pricing pages of this site";++const noopLogger = {+ warn: () => {},+ info: () => {},+ error: () => {},+ debug: () => {},+ child: () => noopLogger,+} as any;++function mockModel() {+ return new MockLanguageModelV3({+ modelId: "gpt-4o-mini",+ doGenerate: async () => ({+ content: [{ type: "text", text: '{"includePaths":["pricing/.*"]}' }],+ finishReason: { unified: "stop", raw: "stop" },+ usage: {+ inputTokens: {+ total: 10,+ noCache: 10,+ cacheRead: undefined,+ cacheWrite: undefined,+ },+ outputTokens: { total: 5, text: 5, reasoning: undefined },+ },+ warnings: [],+ }),+ });+}++function makeReq(+ body: Record<string, unknown>,+ flags: Record<string, unknown>,+) {+ return {+ body,+ auth: { team_id: TEAM_ID },+ acuc: { api_key_id: 7, org_id: null, flags },+ headers: {},+ protocol: "http",+ host: "localhost",+ get: () => undefined,+ } as any;+}++function makeRes() {+ const res: any = { status: vi.fn(), json: vi.fn() };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ return res;+}++function telemetryOfCalls() {+ return (generateObject as Mock).mock.calls.map(+ ([options]) => options.experimental_telemetry,+ );+}++describe("crawl prompt telemetry under zero data retention", () => {+ const exporter = new InMemorySpanExporter();+ let provider: NodeTracerProvider;++ beforeAll(() => {+ provider = createTracerProvider({ exporter, serviceName: "test-service" });+ provider.register();+ });++ afterAll(async () => {+ await provider.shutdown();+ trace.disable();+ context.disable();+ propagation.disable();+ });++ beforeEach(() => {+ exporter.reset();+ (generateObject as Mock).mockClear();+ mocks.getModel.mockImplementation(() => mockModel());+ // Stands in for the site map lookup, which starts spans of its own.+ mocks.buildPromptWithWebsiteStructure.mockImplementation(+ async ({ basePrompt }: { basePrompt: string }) =>+ withSpan("test.map_site", async () => ({+ prompt: `${basePrompt}\n\nhttps://example.com/pricing`,+ websiteUrls: ["https://example.com/pricing"],+ })),+ );+ });++ async function exportedSpans() {+ await provider.forceFlush();+ return exporter.getFinishedSpans();+ }++ it("exports the prompt step of a regular crawl", async () => {+ const res = makeRes();+ await crawlController(+ makeReq({ url: "https://example.com", prompt: PROMPT }, {}),+ res,+ );++ expect(res.status).toHaveBeenCalledWith(200);+ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: true }),+ ]);+ const spans = await exportedSpans();+ const names = spans.map(span => span.name);+ expect(names).toContain("test.map_site");+ expect(names).toContain("ai.generateObject");+ expect(JSON.stringify(spans.map(span => span.attributes))).toContain(+ PROMPT,+ );+ });++ it.each([+ [+ "team-scoped",+ { url: "https://example.com", prompt: PROMPT },+ { scrapeZDR: "forced" },+ ],+ [+ "request-scoped",+ { url: "https://example.com", prompt: PROMPT, zeroDataRetention: true },+ { scrapeZDR: "allowed" },+ ],+ ])(+ "records nothing from the prompt step of a %s ZDR crawl",+ async (_scope, body, flags) => {+ const res = makeRes();+ await crawlController(makeReq(body, flags), res);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({+ promptGeneratedOptions: { includePaths: ["pricing/.*"] },+ }),+ );+ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: false }),+ ]);+ expect(await exportedSpans()).toEqual([]);+ },+ );++ it.each([+ [+ "team-scoped",+ { url: "https://example.com", prompt: PROMPT },+ { scrapeZDR: "forced" },+ ],+ [+ "request-scoped",+ { url: "https://example.com", prompt: PROMPT, zeroDataRetention: true },+ {},+ ],+ ])(+ "records nothing from a %s ZDR params preview",+ async (_scope, body, flags) => {+ const res = makeRes();+ await crawlParamsPreviewController(makeReq(body, flags), res);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: false }),+ ]);+ expect(await exportedSpans()).toEqual([]);+ },+ );++ it("exports a params preview for a regular team", async () => {+ const res = makeRes();+ await crawlParamsPreviewController(+ makeReq({ url: "https://example.com", prompt: PROMPT }, {}),+ res,+ );++ expect(res.status).toHaveBeenCalledWith(200);+ const names = (await exportedSpans()).map(span => span.name);+ expect(names).toContain("ai.generateObject");+ });++ it("turns off AI SDK telemetry in a ZDR context without an explicit flag", async () => {+ await withZeroDataRetention(true, () =>+ generateCompletions({+ logger: noopLogger,+ options: { prompt: PROMPT },+ markdown: "",+ model: mockModel(),+ retryModel: mockModel(),+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },+ metadata: { teamId: TEAM_ID },+ }),+ );++ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: false }),+ ]);+ expect(await exportedSpans()).toEqual([]);+ });+});diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex fb7539f97..0c6747863 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -5,12 +5,18 @@ import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";import { CostTracking } from "../../lib/cost-tracking";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";+import { getScrapeZDR } from "../../lib/zdr-helpers";+import { isLockdownZeroDataRetention } from "../../lib/safe-mode";+import { withZeroDataRetention } from "../../lib/otel-tracer";// Define the request schema for params preview// Only url and prompt are required/relevant for previewconst crawlParamsPreviewRequestSchema = z.object({url: z.url(),prompt: z.string().max(10000),+ // Only keeps the preview's own LLM call out of traces, so unlike a crawl it+ // needs no ZDR permission check.+ zeroDataRetention: z.boolean().optional(),});type CrawlParamsPreviewRequest = z.infer<@@ -46,10 +52,17 @@ export async function crawlParamsPreviewController(>,res: Response<CrawlParamsPreviewResponse>,) {+ // Same rule as a crawl: forced ZDR, the request's own flag, or Safe Mode+ // lockdown. Read before parsing so a rejected body is still covered.+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true ||+ isLockdownZeroDataRetention(req.acuc?.flags, undefined);const logger = _logger.child({module: "api/v2",method: "crawlParamsPreviewController",teamId: req.auth.team_id,+ zeroDataRetention,});try {@@ -61,29 +74,37 @@ export async function crawlParamsPreviewController(prompt: parsedBody.prompt,});- // Build enhanced prompt with website structure- const { prompt: enhancedPrompt, websiteUrls } =- await buildPromptWithWebsiteStructure({- basePrompt: parsedBody.prompt,- url: parsedBody.url,- teamId: req.auth.team_id,- orgId: req.acuc?.org_id ?? null,- flags: req.acuc?.flags ?? null,- logger,- limit: 50,- includeSubdomains: true,- allowExternalLinks: false,- useIndex: true,- maxFireEngineResults: 500,- });+ // The prompt and the discovered URLs end up in LLM telemetry, so keep+ // this whole step out of traces for zero data retention teams.+ const { extract, websiteUrls } = await withZeroDataRetention(+ zeroDataRetention,+ async () => {+ // Build enhanced prompt with website structure+ const { prompt: enhancedPrompt, websiteUrls } =+ await buildPromptWithWebsiteStructure({+ basePrompt: parsedBody.prompt,+ url: parsedBody.url,+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ logger,+ limit: 50,+ includeSubdomains: true,+ allowExternalLinks: false,+ useIndex: true,+ maxFireEngineResults: 500,+ });- // Generate crawler options from enhanced prompt- const costTracking = new CostTracking();- const { extract } = await generateCrawlerOptionsFromPrompt(- enhancedPrompt,- logger,- costTracking,- { teamId: req.auth.team_id },+ // Generate crawler options from enhanced prompt+ const { extract } = await generateCrawlerOptionsFromPrompt(+ enhancedPrompt,+ logger,+ new CostTracking(),+ { teamId: req.auth.team_id },+ zeroDataRetention,+ );+ return { extract, websiteUrls };+ },);const generatedOptions = extract || {};diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b3da3c831..4e8b408b0 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -34,6 +34,7 @@ import {import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { withZeroDataRetention } from "../../lib/otel-tracer";import { resolveThreatProtection } from "../../lib/threat-protection/request";import { checkUrl } from "../../lib/threat-protection";import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";@@ -225,27 +226,36 @@ export async function crawlController(let promptGeneratedOptions = {};if (req.body.prompt) {+ const basePrompt = req.body.prompt;try {- // Enhance prompt with discovered site URLs (up to 120) to improve option generation- const { prompt: enhancedPrompt } = await buildPromptWithWebsiteStructure({- basePrompt: req.body.prompt,- url: req.body.url,- teamId: req.auth.team_id,- orgId: req.acuc?.org_id ?? null,- flags: req.acuc?.flags ?? null,- logger,- limit: 50,- includeSubdomains: false,- allowExternalLinks: false,- useIndex: true,- maxFireEngineResults: 500,- });- const costTracking = new CostTracking();- const { extract } = await generateCrawlerOptionsFromPrompt(- enhancedPrompt,- logger,- costTracking,- { teamId: req.auth.team_id, crawlId: id },+ // The prompt and the discovered URLs end up in LLM telemetry, so keep+ // this whole step out of traces for zero data retention crawls.+ const { extract } = await withZeroDataRetention(+ zeroDataRetention,+ async () => {+ // Enhance prompt with discovered site URLs (up to 120) to improve option generation+ const { prompt: enhancedPrompt } =+ await buildPromptWithWebsiteStructure({+ basePrompt,+ url: req.body.url,+ teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,+ flags: req.acuc?.flags ?? null,+ logger,+ limit: 50,+ includeSubdomains: false,+ allowExternalLinks: false,+ useIndex: true,+ maxFireEngineResults: 500,+ });+ return generateCrawlerOptionsFromPrompt(+ enhancedPrompt,+ logger,+ new CostTracking(),+ { teamId: req.auth.team_id, crawlId: id },+ zeroDataRetention,+ );+ },);promptGeneratedOptions = extract || {};logger.debug("Generated crawler options from prompt", {diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.tsindex aaeb53c2f..7efdfb9fb 100644--- a/apps/api/src/lib/otel-tracer.ts+++ b/apps/api/src/lib/otel-tracer.ts@@ -67,6 +67,15 @@ function isZeroDataRetentionContext(ctx: Context): boolean {return ctx.getValue(ZERO_DATA_RETENTION_CONTEXT_KEY) === true;}+/**+ * Whether the caller runs under `withZeroDataRetention` (or a ZDR `withSpan`),+ * for code that must also switch off telemetry it controls directly, such as+ * the AI SDK's `experimental_telemetry`.+ */+export function isZeroDataRetentionActive(): boolean {+ return isZeroDataRetentionContext(context.active());+}+// Traces flagged ZDR after their root span started (a `*.zero_data_retention`// attribute set from inside a *recording* span). Descendants started afterwards// are not recorded and spans of these traces still in flight are dropped whendiff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 9945a8175..87575a806 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -28,6 +28,7 @@ import {typeIncludes,} from "../../../lib/openai-strict-schema";import { CostTracking } from "../../../lib/cost-tracking";+import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";@@ -345,6 +346,11 @@ export type GenerateCompletionsOptions = {deepResearchId?: string;llmsTxtId?: string;};+ /**+ * Turns off AI SDK telemetry for the call. Also on whenever the caller runs+ * in a zero data retention context, so a caller that forgets to pass it+ * cannot export prompts.+ */zeroDataRetention?: boolean;};export async function generateCompletions({@@ -359,7 +365,7 @@ export async function generateCompletions({retryModel = getModel("gpt-4.1-mini", "openai"),costTrackingOptions,metadata,- zeroDataRetention = false,+ zeroDataRetention: zeroDataRetentionOption,}: GenerateCompletionsOptions): Promise<{extract: any;numTokens: number;@@ -367,6 +373,8 @@ export async function generateCompletions({totalUsage: TokenUsage;model: string;}> {+ const zeroDataRetention =+ zeroDataRetentionOption === true || isZeroDataRetentionActive();let extract: any;let warning: string | undefined;let currentModel = model;@@ -1611,6 +1619,7 @@ export async function generateCrawlerOptionsFromPrompt(logger: Logger,costTracking: CostTracking,metadata: { teamId: string; crawlId?: string },+ zeroDataRetention = false,): Promise<{ extract: any }> {const model = getModel("gpt-4o-mini", "openai");const retryModel = getModel("gpt-4.1-mini", "openai");@@ -1657,6 +1666,7 @@ Return a JSON object with only the relevant options for the user's request. Don'...metadata,functionId: "generateCrawlerOptionsFromPrompt",},+ zeroDataRetention,});return { extract };