Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): trace monitoring judge LLM calls (#4966)

Abimael Martell96e88e88cf74
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 36d0c3366..d7ca0cb96 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -636,6 +636,11 @@ export type InternalOptions = {
v1JSONSystemPrompt?: string;
v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility
+ /** Attributes the JSON extraction's LLM calls to the feature that asked for
+ * the scrape (e.g. a search monitor's judge): replaces the functionId root
+ * and adds span metadata. */
+ llmTelemetry?: { functionId: string; metadata: Record<string, string> };
+
isPreCrawl?: boolean; // Whether this scrape is part of a precrawl job
agentIndexOnly?: boolean; // Pre-confirmation agent key: serve from index only, never touch web/Fire Engine
isParse?: boolean; // Whether this scrape originated from /v2/parse
diff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
index 04a913508..9aed10415 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
@@ -211,4 +211,34 @@ describe("generateCompletions telemetry metadata", () => {
scrapeId: "test-scrape",
});
});
+
+ it("adds the requesting feature's span metadata without overriding the job ids", async () => {
+ await generateCompletions({
+ logger: noopLogger,
+ options: {},
+ markdown: "page",
+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
+ metadata: {
+ teamId: "test-team",
+ functionId: "monitor/searchJudge",
+ scrapeId: "test-scrape",
+ telemetry: {
+ teamId: "other-team",
+ jobId: "test-check",
+ jobKind: "monitor",
+ feature: "monitor_search_judge",
+ },
+ },
+ });
+
+ const telemetry = lastCall().experimental_telemetry;
+ expect(telemetry.functionId).toBe("monitor/searchJudge");
+ expect(telemetry.metadata).toMatchObject({
+ teamId: "test-team",
+ scrapeId: "test-scrape",
+ jobId: "test-check",
+ jobKind: "monitor",
+ feature: "monitor_search_judge",
+ });
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
index bb824997a..25950002b 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
@@ -13,6 +13,7 @@ vi.mock("../lib/extractSmartScrape", async importOriginal => {
};
});
import { performLLMExtract, removeDefaultProperty } from "./llmExtract";
+import { extractData } from "../lib/extractSmartScrape";
import { trimToTokenLimit } from "./llmExtract";
import { performSummary } from "./llmExtract";
import { performCleanContent } from "./llmExtract";
@@ -353,6 +354,93 @@ describe("performCleanContent", () => {
});
});
+function makeJsonMeta(internalOptions: Record<string, unknown> = {}) {
+ return {
+ options: {
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { rows: { type: "array" } },
+ },
+ },
+ ],
+ },
+ internalOptions: {
+ zeroDataRetention: false,
+ teamId: "test-team",
+ ...internalOptions,
+ },
+ logger: {
+ child: vi.fn(() => ({
+ info: vi.fn(),
+ debug: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ })),
+ debug: vi.fn(),
+ info: vi.fn(),
+ },
+ costTracking: {},
+ id: "test-id",
+ url: "https://example.com",
+ } as any;
+}
+
+describe("performLLMExtract telemetry attribution", () => {
+ beforeEach(() => {
+ (extractData as Mock).mockClear();
+ });
+
+ function extractDataArgs() {
+ const calls = (extractData as Mock).mock.calls;
+ expect(calls).toHaveLength(1);
+ return calls[0][0];
+ }
+
+ it("traces plain JSON extraction as performLLMExtract", async () => {
+ await performLLMExtract(makeJsonMeta(), {
+ markdown: "# page",
+ metadata: {},
+ } as any);
+
+ const args = extractDataArgs();
+ expect(args.extractOptions.metadata).toEqual({
+ teamId: "test-team",
+ functionId: "performLLMExtract",
+ scrapeId: "test-id",
+ telemetry: undefined,
+ });
+ expect(args.metadata.functionId).toBe("performLLMExtract");
+ });
+
+ it("traces the extraction as the requesting feature when set", async () => {
+ const metadata = {
+ teamId: "test-team",
+ monitorId: "test-monitor",
+ jobId: "test-check",
+ jobKind: "monitor",
+ feature: "monitor_search_judge",
+ };
+ await performLLMExtract(
+ makeJsonMeta({
+ llmTelemetry: { functionId: "monitor/searchJudge", metadata },
+ }),
+ { markdown: "# page", metadata: {} } as any,
+ );
+
+ const args = extractDataArgs();
+ expect(args.extractOptions.metadata).toEqual({
+ teamId: "test-team",
+ functionId: "monitor/searchJudge",
+ scrapeId: "test-id",
+ telemetry: metadata,
+ });
+ expect(args.metadata.functionId).toBe("monitor/searchJudge");
+ });
+});
+
describe("performLLMExtract empty-result handling", () => {
it("keeps the json key (null) and adds a warning when extraction produces nothing", async () => {
const mockMeta = {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 096c63fba..f26ad67f5 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -393,6 +393,8 @@ export type GenerateCompletionsOptions = {
deepResearchId?: string;
llmsTxtId?: string;
crawlId?: string;
+ /** Extra span metadata from the feature that requested the call. */
+ telemetry?: Record<string, string>;
};
/**
* Turns off AI SDK telemetry for the call. Also on whenever the caller runs
@@ -405,6 +407,7 @@ export type GenerateCompletionsOptions = {
// Span metadata that ties each call to the job that made it.
function telemetryMetadata(metadata: GenerateCompletionsOptions["metadata"]) {
return {
+ ...metadata.telemetry,
teamId: metadata.teamId,
...(metadata.extractId
? {
@@ -1026,6 +1029,8 @@ export async function performLLMExtract(
// let generationOptions = { ...originalOptions }; // Start with original options
const modelSelection = selectModelForSchema(jsonFormat.schema);
+ const llmFunctionId =
+ meta.internalOptions.llmTelemetry?.functionId ?? "performLLMExtract";
const generationOptions: GenerateCompletionsOptions = {
logger: meta.logger.child({
@@ -1045,8 +1050,9 @@ export async function performLLMExtract(
},
metadata: {
teamId: meta.internalOptions.teamId,
- functionId: "performLLMExtract",
+ functionId: llmFunctionId,
scrapeId: meta.id,
+ telemetry: meta.internalOptions.llmTelemetry?.metadata,
},
zeroDataRetention: meta.internalOptions.zeroDataRetention,
};
@@ -1059,7 +1065,7 @@ export async function performLLMExtract(
scrapeId: meta.id,
metadata: {
teamId: meta.internalOptions.teamId,
- functionId: "performLLMExtract",
+ functionId: llmFunctionId,
},
});
diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.ts
index 064beb5df..fa08f43bb 100644
--- a/apps/api/src/services/monitoring/diff-orchestrator.ts
+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts
@@ -267,6 +267,9 @@ async function runJudge(args: {
jsonDiff: args.jsonDiff,
markdownDiff: args.markdownDiff,
labels: args.labels,
+ // Monitors are refused for zero data retention teams and their scrapes
+ // run with ZDR off, so the judge call is traced like them.
+ zeroDataRetention: false,
});
} catch (error) {
rootLogger.error("Judge call failed", { error });
diff --git a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts
new file mode 100644
index 000000000..a718f93bb
--- /dev/null
+++ b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts
@@ -0,0 +1,271 @@
+import { vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ doGenerate: vi.fn(),
+}));
+
+// Call through to the real AI SDK so its telemetry spans are produced, while
+// still letting the tests read the options each call was made with.
+vi.mock("ai", async importOriginal => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return {
+ ...actual,
+ generateText: vi.fn(actual.generateText),
+ };
+});
+
+// The judge model is built at import; stand in a mock provider model for it.
+vi.mock("../../lib/generic-ai", async importOriginal => {
+ const { MockLanguageModelV3 } = await import("ai/test");
+ return {
+ ...(await importOriginal<typeof import("../../lib/generic-ai")>()),
+ getModel: (modelId: string) =>
+ new MockLanguageModelV3({
+ modelId,
+ doGenerate: options => mocks.doGenerate(options),
+ }),
+ };
+});
+
+// Vertex is the production provider and the only one that takes billing labels.
+vi.mock("../../config", async importOriginal => {
+ const actual = await importOriginal<typeof import("../../config")>();
+ return {
+ ...actual,
+ config: { ...actual.config, VERTEX_CREDENTIALS: btoa("{}") },
+ };
+});
+
+import { context, propagation, trace } from "@opentelemetry/api";
+import {
+ InMemorySpanExporter,
+ type NodeTracerProvider,
+} from "@opentelemetry/sdk-trace-node";
+import { generateText } from "ai";
+import type { Mock } from "vitest";
+import {
+ createTracerProvider,
+ withZeroDataRetention,
+} from "../../lib/otel-tracer";
+import { judgeChange } from "./judgeChange";
+
+const LABELS = {
+ teamId: "test-team",
+ monitorId: "test-monitor",
+ monitorCheckId: "test-check",
+};
+
+const MARKDOWN_DIFF = {
+ diffText: "@@ -1 +1 @@\n-Pro plan: $19\n+Pro plan: $24",
+};
+
+const noopLogger = {
+ warn: () => {},
+ info: () => {},
+ error: () => {},
+ debug: () => {},
+ child: () => noopLogger,
+} as any;
+
+function providerResult(text: string) {
+ return {
+ content: [{ type: "text", text }],
+ finishReason: { unified: "stop", raw: "STOP" },
+ usage: {
+ inputTokens: { total: 120, noCache: 20, cacheRead: 100, cacheWrite: 0 },
+ outputTokens: { total: 30, text: 22, reasoning: 8 },
+ },
+ warnings: [],
+ };
+}
+
+const PRICE_CHANGED = JSON.stringify({
+ meaningful: true,
+ confidence: "high",
+ reason: "The Pro price changed from '$19' to '$24'.",
+ meaningfulChanges: [
+ {
+ type: "changed",
+ before: "Pro plan: $19",
+ after: "Pro plan: $24",
+ reason: "The goal tracks the Pro price.",
+ },
+ // dropped: "added" must have a null before
+ { type: "added", before: "x", after: "y", reason: "invalid" },
+ ],
+});
+
+function judge(overrides: Partial<Parameters<typeof judgeChange>[0]> = {}) {
+ return judgeChange({
+ logger: noopLogger,
+ goal: "Track the Pro plan price",
+ markdownDiff: MARKDOWN_DIFF,
+ labels: LABELS,
+ zeroDataRetention: false,
+ ...overrides,
+ });
+}
+
+function telemetryOfCalls() {
+ return (generateText as Mock).mock.calls.map(
+ ([options]) => options.experimental_telemetry,
+ );
+}
+
+describe("judgeChange telemetry", () => {
+ const exporter = new InMemorySpanExporter();
+ let provider: NodeTracerProvider;
+
+ beforeAll(() => {
+ provider = createTracerProvider({ exporter, serviceName: "test-service" });
+ provider.register();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ trace.disable();
+ context.disable();
+ propagation.disable();
+ });
+
+ beforeEach(async () => {
+ // Flush first so a span the last test left unexported isn't counted here.
+ await provider.forceFlush();
+ exporter.reset();
+ (generateText as Mock).mockClear();
+ mocks.doGenerate.mockReset();
+ mocks.doGenerate.mockResolvedValue(providerResult(PRICE_CHANGED));
+ // Shortest retry backoff, so the retry tests stay quick.
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ async function doGenerateSpans() {
+ await provider.forceFlush();
+ return exporter
+ .getFinishedSpans()
+ .filter(span => span.name === "ai.generateText.doGenerate");
+ }
+
+ it("tags the provider call span with the monitor check and token breakdown", async () => {
+ await judge();
+
+ const spans = await doGenerateSpans();
+ expect(spans).toHaveLength(1);
+ expect(spans[0].attributes).toMatchObject({
+ "ai.telemetry.functionId": "monitor/judgeChange",
+ "ai.telemetry.metadata.teamId": "test-team",
+ "ai.telemetry.metadata.monitorId": "test-monitor",
+ "ai.telemetry.metadata.jobId": "test-check",
+ "ai.telemetry.metadata.jobKind": "monitor",
+ "ai.telemetry.metadata.feature": "monitor_judge",
+ "ai.usage.cachedInputTokens": 100,
+ "ai.usage.reasoningTokens": 8,
+ });
+ });
+
+ it("keeps the Vertex billing labels", async () => {
+ await judge();
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);
+ expect(mocks.doGenerate.mock.calls[0][0].providerOptions).toEqual({
+ vertex: {
+ thinkingConfig: { thinkingLevel: "minimal" },
+ labels: { functionId: "judgeChange", ...LABELS },
+ },
+ });
+ });
+
+ it("records one span per attempt when a call is retried", async () => {
+ mocks.doGenerate
+ .mockRejectedValueOnce(new Error("provider unavailable"))
+ .mockResolvedValueOnce(providerResult(PRICE_CHANGED));
+
+ const result = await judge();
+
+ expect(result.meaningful).toBe(true);
+ const spans = await doGenerateSpans();
+ expect(spans).toHaveLength(2);
+ for (const span of spans) {
+ expect(span.attributes["ai.telemetry.metadata.jobId"]).toBe("test-check");
+ }
+ });
+
+ it("turns off telemetry for a zero data retention call", async () => {
+ await judge({ zeroDataRetention: true });
+
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: false }),
+ ]);
+ expect(await doGenerateSpans()).toEqual([]);
+ });
+
+ it("turns off telemetry in a zero data retention context without the flag", async () => {
+ await withZeroDataRetention(true, () =>
+ judge({ zeroDataRetention: undefined }),
+ );
+
+ expect(telemetryOfCalls()).toEqual([
+ expect.objectContaining({ isEnabled: false }),
+ ]);
+ expect(await doGenerateSpans()).toEqual([]);
+ });
+});
+
+describe("judgeChange behaviour", () => {
+ beforeEach(() => {
+ mocks.doGenerate.mockReset();
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("returns the parsed judgment, keeping only valid change events", async () => {
+ mocks.doGenerate.mockResolvedValue(providerResult(PRICE_CHANGED));
+
+ expect(await judge()).toEqual({
+ meaningful: true,
+ confidence: "high",
+ reason: "The Pro price changed from '$19' to '$24'.",
+ meaningfulChanges: [
+ {
+ type: "changed",
+ before: "Pro plan: $19",
+ after: "Pro plan: $24",
+ reason: "The goal tracks the Pro price.",
+ },
+ ],
+ });
+ });
+
+ it("defaults to meaningful when the response is not JSON", async () => {
+ mocks.doGenerate.mockResolvedValue(providerResult("no idea"));
+
+ expect(await judge()).toMatchObject({
+ meaningful: true,
+ confidence: "low",
+ meaningfulChanges: [],
+ });
+ });
+
+ it("defaults to meaningful after every attempt fails", async () => {
+ mocks.doGenerate.mockRejectedValue(new Error("provider unavailable"));
+
+ const result = await judge();
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(3);
+ expect(result).toMatchObject({ meaningful: true, confidence: "low" });
+ expect(result.reason).toContain("provider unavailable");
+ });
+
+ it("makes no call without a diff", async () => {
+ await judge({ markdownDiff: undefined });
+
+ expect(mocks.doGenerate).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/monitoring/judgeChange.ts b/apps/api/src/services/monitoring/judgeChange.ts
index 0a171945b..bd3adb65a 100644
--- a/apps/api/src/services/monitoring/judgeChange.ts
+++ b/apps/api/src/services/monitoring/judgeChange.ts
@@ -1,8 +1,11 @@
import { generateText } from "ai";
import type { Logger } from "winston";
+import { withUsageTelemetry } from "../../lib/ai-usage-telemetry";
+import { isZeroDataRetentionActive } from "../../lib/otel-tracer";
import {
googleModel,
googleProviderOptions,
+ monitorTelemetryMetadata,
type LlmUsageLabels,
} from "./search/tuning";
@@ -83,7 +86,11 @@ interface JudgeChangeArgs {
};
// Vertex billing labels so this LLM call is traceable at the billing level by
// function, team, monitor, and monitor check (mirrors the search service).
+ // Also tag the call's AI SDK span.
labels?: LlmUsageLabels;
+ // Turns off AI SDK telemetry for the call, as does running in a zero data
+ // retention context.
+ zeroDataRetention?: boolean;
}
function isMeaningfulChangeEvent(
@@ -121,11 +128,24 @@ const JUDGE_MODEL_NAME = "gemini-3-flash-preview";
const JUDGE_ATTEMPT_TIMEOUT_MS = 30_000;
const JUDGE_MAX_ATTEMPTS = 3;
const JUDGE_BACKOFF_MS = [300, 800];
-const judgeModel = googleModel(JUDGE_MODEL_NAME);
+const JUDGE_FUNCTION_ID = "judgeChange";
+const judgeModel = withUsageTelemetry(googleModel(JUDGE_MODEL_NAME));
+
+function judgeTelemetry(
+ labels: LlmUsageLabels | undefined,
+ zeroDataRetention: boolean | undefined,
+) {
+ return {
+ isEnabled: !(zeroDataRetention || isZeroDataRetentionActive()),
+ functionId: `monitor/${JUDGE_FUNCTION_ID}`,
+ metadata: monitorTelemetryMetadata("monitor_judge", labels),
+ };
+}
async function callGemini(args: {
userBlock: string;
labels?: LlmUsageLabels;
+ zeroDataRetention?: boolean;
}): Promise<{ text: string }> {
let lastError: unknown;
for (let attempt = 1; attempt <= JUDGE_MAX_ATTEMPTS; attempt++) {
@@ -141,7 +161,12 @@ async function callGemini(args: {
prompt: args.userBlock,
temperature: 0,
abortSignal: controller.signal,
- ...googleProviderOptions("judgeChange", args.labels),
+ // Each attempt is its own provider call, so each gets its own span.
+ experimental_telemetry: judgeTelemetry(
+ args.labels,
+ args.zeroDataRetention,
+ ),
+ ...googleProviderOptions(JUDGE_FUNCTION_ID, args.labels),
});
return { text: result.text?.trim() ?? "" };
} catch (error) {
@@ -160,8 +185,15 @@ async function callGemini(args: {
export async function judgeChange(
args: JudgeChangeArgs,
): Promise<JudgmentResult> {
- const { logger, goal, extractionPrompt, jsonDiff, markdownDiff, labels } =
- args;
+ const {
+ logger,
+ goal,
+ extractionPrompt,
+ jsonDiff,
+ markdownDiff,
+ labels,
+ zeroDataRetention,
+ } = args;
const parts: string[] = [`MONITOR GOAL:\n${goal.trim()}`];
if (extractionPrompt?.trim()) {
@@ -190,7 +222,11 @@ export async function judgeChange(
const userBlock = parts.join("\n\n");
try {
- const { text } = await callGemini({ userBlock, labels });
+ const { text } = await callGemini({
+ userBlock,
+ labels,
+ zeroDataRetention,
+ });
const match = text.match(/\{[\s\S]*\}/);
if (!match) {
logger.warn("Judge returned unparseable response", {
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 239334436..2d2dd9219 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -72,6 +72,7 @@ import {
import { trackMonitorCheckStartedInterest } from "./interest";
import { runSearchTarget, type ScrapeSearchResult } from "./search/run";
import { verdictJsonSchema } from "./search/judge";
+import { monitorTelemetryMetadata } from "./search/tuning";
import { computeGoalVersion } from "./search/dedupe";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { getACUCTeam } from "../../controllers/auth";
@@ -275,6 +276,7 @@ export function estimateActualCredits(doc: any, options?: any): number {
async function scrapeSearchMonitorPage(params: {
teamId: string;
teamFlags: TeamFlags | null;
+ monitorId: string;
checkId: string;
url: string;
judgePrompt: string;
@@ -322,6 +324,15 @@ async function scrapeSearchMonitorPage(params: {
zeroDataRetention: false,
// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
teamFlags: params.teamFlags ?? undefined,
+ // The JSON format is the search judge; trace it as part of the check.
+ llmTelemetry: {
+ functionId: "monitor/searchJudge",
+ metadata: monitorTelemetryMetadata("monitor_search_judge", {
+ teamId: params.teamId,
+ monitorId: params.monitorId,
+ monitorCheckId: params.checkId,
+ }),
+ },
},
skipNuq: true,
origin: "monitor",
@@ -861,6 +872,7 @@ async function runMonitorSearchTarget(params: {
scrapeSearchMonitorPage({
teamId: monitor.team_id,
teamFlags,
+ monitorId: monitor.id,
checkId: check.id,
url,
judgePrompt,
diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.ts
index 63b539272..b306b548c 100644
--- a/apps/api/src/services/monitoring/search/tuning.ts
+++ b/apps/api/src/services/monitoring/search/tuning.ts
@@ -11,6 +11,25 @@ export type LlmUsageLabels = {
monitorCheckId: string;
};
+// AI SDK span metadata for a monitor LLM call: the check is the job, so the LLM
+// spend dashboard attributes each provider call to its team, monitor and check.
+export function monitorTelemetryMetadata(
+ feature: string,
+ labels?: LlmUsageLabels,
+): Record<string, string> {
+ return {
+ ...(labels
+ ? {
+ teamId: labels.teamId,
+ monitorId: labels.monitorId,
+ jobId: labels.monitorCheckId,
+ }
+ : {}),
+ jobKind: "monitor",
+ feature,
+ };
+}
+
function geminiApiKey(): string | undefined {
return process.env.GOOGLE_GENERATIVE_AI_API_KEY ?? process.env.GEMINI_API_KEY;
}