Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): trace monitoring judge LLM calls (#4966)
Abimael Martell96e88e88cf74
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 36d0c3366..d7ca0cb96 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -636,6 +636,11 @@ export type InternalOptions = {v1JSONSystemPrompt?: string;v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility+ /** Attributes the JSON extraction's LLM calls to the feature that asked for+ * the scrape (e.g. a search monitor's judge): replaces the functionId root+ * and adds span metadata. */+ llmTelemetry?: { functionId: string; metadata: Record<string, string> };+isPreCrawl?: boolean; // Whether this scrape is part of a precrawl jobagentIndexOnly?: boolean; // Pre-confirmation agent key: serve from index only, never touch web/Fire EngineisParse?: boolean; // Whether this scrape originated from /v2/parsediff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.tsindex 04a913508..9aed10415 100644--- a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts@@ -211,4 +211,34 @@ describe("generateCompletions telemetry metadata", () => {scrapeId: "test-scrape",});});++ it("adds the requesting feature's span metadata without overriding the job ids", async () => {+ await generateCompletions({+ logger: noopLogger,+ options: {},+ markdown: "page",+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },+ metadata: {+ teamId: "test-team",+ functionId: "monitor/searchJudge",+ scrapeId: "test-scrape",+ telemetry: {+ teamId: "other-team",+ jobId: "test-check",+ jobKind: "monitor",+ feature: "monitor_search_judge",+ },+ },+ });++ const telemetry = lastCall().experimental_telemetry;+ expect(telemetry.functionId).toBe("monitor/searchJudge");+ expect(telemetry.metadata).toMatchObject({+ teamId: "test-team",+ scrapeId: "test-scrape",+ jobId: "test-check",+ jobKind: "monitor",+ feature: "monitor_search_judge",+ });+ });});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex bb824997a..25950002b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -13,6 +13,7 @@ vi.mock("../lib/extractSmartScrape", async importOriginal => {};});import { performLLMExtract, removeDefaultProperty } from "./llmExtract";+import { extractData } from "../lib/extractSmartScrape";import { trimToTokenLimit } from "./llmExtract";import { performSummary } from "./llmExtract";import { performCleanContent } from "./llmExtract";@@ -353,6 +354,93 @@ describe("performCleanContent", () => {});});+function makeJsonMeta(internalOptions: Record<string, unknown> = {}) {+ return {+ options: {+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { rows: { type: "array" } },+ },+ },+ ],+ },+ internalOptions: {+ zeroDataRetention: false,+ teamId: "test-team",+ ...internalOptions,+ },+ logger: {+ child: vi.fn(() => ({+ info: vi.fn(),+ debug: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ })),+ debug: vi.fn(),+ info: vi.fn(),+ },+ costTracking: {},+ id: "test-id",+ url: "https://example.com",+ } as any;+}++describe("performLLMExtract telemetry attribution", () => {+ beforeEach(() => {+ (extractData as Mock).mockClear();+ });++ function extractDataArgs() {+ const calls = (extractData as Mock).mock.calls;+ expect(calls).toHaveLength(1);+ return calls[0][0];+ }++ it("traces plain JSON extraction as performLLMExtract", async () => {+ await performLLMExtract(makeJsonMeta(), {+ markdown: "# page",+ metadata: {},+ } as any);++ const args = extractDataArgs();+ expect(args.extractOptions.metadata).toEqual({+ teamId: "test-team",+ functionId: "performLLMExtract",+ scrapeId: "test-id",+ telemetry: undefined,+ });+ expect(args.metadata.functionId).toBe("performLLMExtract");+ });++ it("traces the extraction as the requesting feature when set", async () => {+ const metadata = {+ teamId: "test-team",+ monitorId: "test-monitor",+ jobId: "test-check",+ jobKind: "monitor",+ feature: "monitor_search_judge",+ };+ await performLLMExtract(+ makeJsonMeta({+ llmTelemetry: { functionId: "monitor/searchJudge", metadata },+ }),+ { markdown: "# page", metadata: {} } as any,+ );++ const args = extractDataArgs();+ expect(args.extractOptions.metadata).toEqual({+ teamId: "test-team",+ functionId: "monitor/searchJudge",+ scrapeId: "test-id",+ telemetry: metadata,+ });+ expect(args.metadata.functionId).toBe("monitor/searchJudge");+ });+});+describe("performLLMExtract empty-result handling", () => {it("keeps the json key (null) and adds a warning when extraction produces nothing", async () => {const mockMeta = {diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 096c63fba..f26ad67f5 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -393,6 +393,8 @@ export type GenerateCompletionsOptions = {deepResearchId?: string;llmsTxtId?: string;crawlId?: string;+ /** Extra span metadata from the feature that requested the call. */+ telemetry?: Record<string, string>;};/*** Turns off AI SDK telemetry for the call. Also on whenever the caller runs@@ -405,6 +407,7 @@ export type GenerateCompletionsOptions = {// Span metadata that ties each call to the job that made it.function telemetryMetadata(metadata: GenerateCompletionsOptions["metadata"]) {return {+ ...metadata.telemetry,teamId: metadata.teamId,...(metadata.extractId? {@@ -1026,6 +1029,8 @@ export async function performLLMExtract(// let generationOptions = { ...originalOptions }; // Start with original optionsconst modelSelection = selectModelForSchema(jsonFormat.schema);+ const llmFunctionId =+ meta.internalOptions.llmTelemetry?.functionId ?? "performLLMExtract";const generationOptions: GenerateCompletionsOptions = {logger: meta.logger.child({@@ -1045,8 +1050,9 @@ export async function performLLMExtract(},metadata: {teamId: meta.internalOptions.teamId,- functionId: "performLLMExtract",+ functionId: llmFunctionId,scrapeId: meta.id,+ telemetry: meta.internalOptions.llmTelemetry?.metadata,},zeroDataRetention: meta.internalOptions.zeroDataRetention,};@@ -1059,7 +1065,7 @@ export async function performLLMExtract(scrapeId: meta.id,metadata: {teamId: meta.internalOptions.teamId,- functionId: "performLLMExtract",+ functionId: llmFunctionId,},});diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.tsindex 064beb5df..fa08f43bb 100644--- a/apps/api/src/services/monitoring/diff-orchestrator.ts+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts@@ -267,6 +267,9 @@ async function runJudge(args: {jsonDiff: args.jsonDiff,markdownDiff: args.markdownDiff,labels: args.labels,+ // Monitors are refused for zero data retention teams and their scrapes+ // run with ZDR off, so the judge call is traced like them.+ zeroDataRetention: false,});} catch (error) {rootLogger.error("Judge call failed", { error });diff --git a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts b/apps/api/src/services/monitoring/judgeChange.telemetry.test.tsnew file mode 100644index 000000000..a718f93bb--- /dev/null+++ b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts@@ -0,0 +1,271 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ doGenerate: vi.fn(),+}));++// Call through to the real AI SDK so its telemetry spans are produced, while+// still letting the tests read the options each call was made with.+vi.mock("ai", async importOriginal => {+ const actual = await importOriginal<typeof import("ai")>();+ return {+ ...actual,+ generateText: vi.fn(actual.generateText),+ };+});++// The judge model is built at import; stand in a mock provider model for it.+vi.mock("../../lib/generic-ai", async importOriginal => {+ const { MockLanguageModelV3 } = await import("ai/test");+ return {+ ...(await importOriginal<typeof import("../../lib/generic-ai")>()),+ getModel: (modelId: string) =>+ new MockLanguageModelV3({+ modelId,+ doGenerate: options => mocks.doGenerate(options),+ }),+ };+});++// Vertex is the production provider and the only one that takes billing labels.+vi.mock("../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../config")>();+ return {+ ...actual,+ config: { ...actual.config, VERTEX_CREDENTIALS: btoa("{}") },+ };+});++import { context, propagation, trace } from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ type NodeTracerProvider,+} from "@opentelemetry/sdk-trace-node";+import { generateText } from "ai";+import type { Mock } from "vitest";+import {+ createTracerProvider,+ withZeroDataRetention,+} from "../../lib/otel-tracer";+import { judgeChange } from "./judgeChange";++const LABELS = {+ teamId: "test-team",+ monitorId: "test-monitor",+ monitorCheckId: "test-check",+};++const MARKDOWN_DIFF = {+ diffText: "@@ -1 +1 @@\n-Pro plan: $19\n+Pro plan: $24",+};++const noopLogger = {+ warn: () => {},+ info: () => {},+ error: () => {},+ debug: () => {},+ child: () => noopLogger,+} as any;++function providerResult(text: string) {+ return {+ content: [{ type: "text", text }],+ finishReason: { unified: "stop", raw: "STOP" },+ usage: {+ inputTokens: { total: 120, noCache: 20, cacheRead: 100, cacheWrite: 0 },+ outputTokens: { total: 30, text: 22, reasoning: 8 },+ },+ warnings: [],+ };+}++const PRICE_CHANGED = JSON.stringify({+ meaningful: true,+ confidence: "high",+ reason: "The Pro price changed from '$19' to '$24'.",+ meaningfulChanges: [+ {+ type: "changed",+ before: "Pro plan: $19",+ after: "Pro plan: $24",+ reason: "The goal tracks the Pro price.",+ },+ // dropped: "added" must have a null before+ { type: "added", before: "x", after: "y", reason: "invalid" },+ ],+});++function judge(overrides: Partial<Parameters<typeof judgeChange>[0]> = {}) {+ return judgeChange({+ logger: noopLogger,+ goal: "Track the Pro plan price",+ markdownDiff: MARKDOWN_DIFF,+ labels: LABELS,+ zeroDataRetention: false,+ ...overrides,+ });+}++function telemetryOfCalls() {+ return (generateText as Mock).mock.calls.map(+ ([options]) => options.experimental_telemetry,+ );+}++describe("judgeChange telemetry", () => {+ const exporter = new InMemorySpanExporter();+ let provider: NodeTracerProvider;++ beforeAll(() => {+ provider = createTracerProvider({ exporter, serviceName: "test-service" });+ provider.register();+ });++ afterAll(async () => {+ await provider.shutdown();+ trace.disable();+ context.disable();+ propagation.disable();+ });++ beforeEach(async () => {+ // Flush first so a span the last test left unexported isn't counted here.+ await provider.forceFlush();+ exporter.reset();+ (generateText as Mock).mockClear();+ mocks.doGenerate.mockReset();+ mocks.doGenerate.mockResolvedValue(providerResult(PRICE_CHANGED));+ // Shortest retry backoff, so the retry tests stay quick.+ vi.spyOn(Math, "random").mockReturnValue(0);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ async function doGenerateSpans() {+ await provider.forceFlush();+ return exporter+ .getFinishedSpans()+ .filter(span => span.name === "ai.generateText.doGenerate");+ }++ it("tags the provider call span with the monitor check and token breakdown", async () => {+ await judge();++ const spans = await doGenerateSpans();+ expect(spans).toHaveLength(1);+ expect(spans[0].attributes).toMatchObject({+ "ai.telemetry.functionId": "monitor/judgeChange",+ "ai.telemetry.metadata.teamId": "test-team",+ "ai.telemetry.metadata.monitorId": "test-monitor",+ "ai.telemetry.metadata.jobId": "test-check",+ "ai.telemetry.metadata.jobKind": "monitor",+ "ai.telemetry.metadata.feature": "monitor_judge",+ "ai.usage.cachedInputTokens": 100,+ "ai.usage.reasoningTokens": 8,+ });+ });++ it("keeps the Vertex billing labels", async () => {+ await judge();++ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);+ expect(mocks.doGenerate.mock.calls[0][0].providerOptions).toEqual({+ vertex: {+ thinkingConfig: { thinkingLevel: "minimal" },+ labels: { functionId: "judgeChange", ...LABELS },+ },+ });+ });++ it("records one span per attempt when a call is retried", async () => {+ mocks.doGenerate+ .mockRejectedValueOnce(new Error("provider unavailable"))+ .mockResolvedValueOnce(providerResult(PRICE_CHANGED));++ const result = await judge();++ expect(result.meaningful).toBe(true);+ const spans = await doGenerateSpans();+ expect(spans).toHaveLength(2);+ for (const span of spans) {+ expect(span.attributes["ai.telemetry.metadata.jobId"]).toBe("test-check");+ }+ });++ it("turns off telemetry for a zero data retention call", async () => {+ await judge({ zeroDataRetention: true });++ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: false }),+ ]);+ expect(await doGenerateSpans()).toEqual([]);+ });++ it("turns off telemetry in a zero data retention context without the flag", async () => {+ await withZeroDataRetention(true, () =>+ judge({ zeroDataRetention: undefined }),+ );++ expect(telemetryOfCalls()).toEqual([+ expect.objectContaining({ isEnabled: false }),+ ]);+ expect(await doGenerateSpans()).toEqual([]);+ });+});++describe("judgeChange behaviour", () => {+ beforeEach(() => {+ mocks.doGenerate.mockReset();+ vi.spyOn(Math, "random").mockReturnValue(0);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("returns the parsed judgment, keeping only valid change events", async () => {+ mocks.doGenerate.mockResolvedValue(providerResult(PRICE_CHANGED));++ expect(await judge()).toEqual({+ meaningful: true,+ confidence: "high",+ reason: "The Pro price changed from '$19' to '$24'.",+ meaningfulChanges: [+ {+ type: "changed",+ before: "Pro plan: $19",+ after: "Pro plan: $24",+ reason: "The goal tracks the Pro price.",+ },+ ],+ });+ });++ it("defaults to meaningful when the response is not JSON", async () => {+ mocks.doGenerate.mockResolvedValue(providerResult("no idea"));++ expect(await judge()).toMatchObject({+ meaningful: true,+ confidence: "low",+ meaningfulChanges: [],+ });+ });++ it("defaults to meaningful after every attempt fails", async () => {+ mocks.doGenerate.mockRejectedValue(new Error("provider unavailable"));++ const result = await judge();++ expect(mocks.doGenerate).toHaveBeenCalledTimes(3);+ expect(result).toMatchObject({ meaningful: true, confidence: "low" });+ expect(result.reason).toContain("provider unavailable");+ });++ it("makes no call without a diff", async () => {+ await judge({ markdownDiff: undefined });++ expect(mocks.doGenerate).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/monitoring/judgeChange.ts b/apps/api/src/services/monitoring/judgeChange.tsindex 0a171945b..bd3adb65a 100644--- a/apps/api/src/services/monitoring/judgeChange.ts+++ b/apps/api/src/services/monitoring/judgeChange.ts@@ -1,8 +1,11 @@import { generateText } from "ai";import type { Logger } from "winston";+import { withUsageTelemetry } from "../../lib/ai-usage-telemetry";+import { isZeroDataRetentionActive } from "../../lib/otel-tracer";import {googleModel,googleProviderOptions,+ monitorTelemetryMetadata,type LlmUsageLabels,} from "./search/tuning";@@ -83,7 +86,11 @@ interface JudgeChangeArgs {};// Vertex billing labels so this LLM call is traceable at the billing level by// function, team, monitor, and monitor check (mirrors the search service).+ // Also tag the call's AI SDK span.labels?: LlmUsageLabels;+ // Turns off AI SDK telemetry for the call, as does running in a zero data+ // retention context.+ zeroDataRetention?: boolean;}function isMeaningfulChangeEvent(@@ -121,11 +128,24 @@ const JUDGE_MODEL_NAME = "gemini-3-flash-preview";const JUDGE_ATTEMPT_TIMEOUT_MS = 30_000;const JUDGE_MAX_ATTEMPTS = 3;const JUDGE_BACKOFF_MS = [300, 800];-const judgeModel = googleModel(JUDGE_MODEL_NAME);+const JUDGE_FUNCTION_ID = "judgeChange";+const judgeModel = withUsageTelemetry(googleModel(JUDGE_MODEL_NAME));++function judgeTelemetry(+ labels: LlmUsageLabels | undefined,+ zeroDataRetention: boolean | undefined,+) {+ return {+ isEnabled: !(zeroDataRetention || isZeroDataRetentionActive()),+ functionId: `monitor/${JUDGE_FUNCTION_ID}`,+ metadata: monitorTelemetryMetadata("monitor_judge", labels),+ };+}async function callGemini(args: {userBlock: string;labels?: LlmUsageLabels;+ zeroDataRetention?: boolean;}): Promise<{ text: string }> {let lastError: unknown;for (let attempt = 1; attempt <= JUDGE_MAX_ATTEMPTS; attempt++) {@@ -141,7 +161,12 @@ async function callGemini(args: {prompt: args.userBlock,temperature: 0,abortSignal: controller.signal,- ...googleProviderOptions("judgeChange", args.labels),+ // Each attempt is its own provider call, so each gets its own span.+ experimental_telemetry: judgeTelemetry(+ args.labels,+ args.zeroDataRetention,+ ),+ ...googleProviderOptions(JUDGE_FUNCTION_ID, args.labels),});return { text: result.text?.trim() ?? "" };} catch (error) {@@ -160,8 +185,15 @@ async function callGemini(args: {export async function judgeChange(args: JudgeChangeArgs,): Promise<JudgmentResult> {- const { logger, goal, extractionPrompt, jsonDiff, markdownDiff, labels } =- args;+ const {+ logger,+ goal,+ extractionPrompt,+ jsonDiff,+ markdownDiff,+ labels,+ zeroDataRetention,+ } = args;const parts: string[] = [`MONITOR GOAL:\n${goal.trim()}`];if (extractionPrompt?.trim()) {@@ -190,7 +222,11 @@ export async function judgeChange(const userBlock = parts.join("\n\n");try {- const { text } = await callGemini({ userBlock, labels });+ const { text } = await callGemini({+ userBlock,+ labels,+ zeroDataRetention,+ });const match = text.match(/\{[\s\S]*\}/);if (!match) {logger.warn("Judge returned unparseable response", {diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 239334436..2d2dd9219 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -72,6 +72,7 @@ import {import { trackMonitorCheckStartedInterest } from "./interest";import { runSearchTarget, type ScrapeSearchResult } from "./search/run";import { verdictJsonSchema } from "./search/judge";+import { monitorTelemetryMetadata } from "./search/tuning";import { computeGoalVersion } from "./search/dedupe";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { getACUCTeam } from "../../controllers/auth";@@ -275,6 +276,7 @@ export function estimateActualCredits(doc: any, options?: any): number {async function scrapeSearchMonitorPage(params: {teamId: string;teamFlags: TeamFlags | null;+ monitorId: string;checkId: string;url: string;judgePrompt: string;@@ -322,6 +324,15 @@ async function scrapeSearchMonitorPage(params: {zeroDataRetention: false,// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.teamFlags: params.teamFlags ?? undefined,+ // The JSON format is the search judge; trace it as part of the check.+ llmTelemetry: {+ functionId: "monitor/searchJudge",+ metadata: monitorTelemetryMetadata("monitor_search_judge", {+ teamId: params.teamId,+ monitorId: params.monitorId,+ monitorCheckId: params.checkId,+ }),+ },},skipNuq: true,origin: "monitor",@@ -861,6 +872,7 @@ async function runMonitorSearchTarget(params: {scrapeSearchMonitorPage({teamId: monitor.team_id,teamFlags,+ monitorId: monitor.id,checkId: check.id,url,judgePrompt,diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.tsindex 63b539272..b306b548c 100644--- a/apps/api/src/services/monitoring/search/tuning.ts+++ b/apps/api/src/services/monitoring/search/tuning.ts@@ -11,6 +11,25 @@ export type LlmUsageLabels = {monitorCheckId: string;};+// AI SDK span metadata for a monitor LLM call: the check is the job, so the LLM+// spend dashboard attributes each provider call to its team, monitor and check.+export function monitorTelemetryMetadata(+ feature: string,+ labels?: LlmUsageLabels,+): Record<string, string> {+ return {+ ...(labels+ ? {+ teamId: labels.teamId,+ monitorId: labels.monitorId,+ jobId: labels.monitorCheckId,+ }+ : {}),+ jobKind: "monitor",+ feature,+ };+}+function geminiApiKey(): string | undefined {return process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY;}