Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Pass LLM usage labels to judgeChange

Developers Digest18a202968615
diff --git a/apps/api/src/services/monitoring/diff-orchestrator.ts b/apps/api/src/services/monitoring/diff-orchestrator.ts
index b3efbade2..064beb5df 100644
--- a/apps/api/src/services/monitoring/diff-orchestrator.ts
+++ b/apps/api/src/services/monitoring/diff-orchestrator.ts
@@ -13,6 +13,7 @@ import {
formatsRequestJsonExtraction,
} from "./diff";
import { judgeChange } from "./judgeChange";
+import type { LlmUsageLabels } from "./search/tuning";
type MonitorPageDiffStatus = "same" | "new" | "changed" | "error";
@@ -172,6 +173,7 @@ export async function computeAndPersistPageDiff(params: {
diffText: markdownSidecar.text,
}
: undefined,
+ labels: { teamId, monitorId, monitorCheckId: checkId },
})
: undefined;
return {
@@ -235,6 +237,7 @@ export async function computeAndPersistPageDiff(params: {
markdownDiff: {
diffText: diff.text,
},
+ labels: { teamId, monitorId, monitorCheckId: checkId },
})
: undefined;
return {
@@ -254,6 +257,7 @@ async function runJudge(args: {
markdownDiff?: {
diffText?: string;
};
+ labels: LlmUsageLabels;
}): Promise<Judgment | undefined> {
try {
return await judgeChange({
@@ -262,6 +266,7 @@ async function runJudge(args: {
extractionPrompt: args.extractionPrompt ?? undefined,
jsonDiff: args.jsonDiff,
markdownDiff: args.markdownDiff,
+ labels: args.labels,
});
} catch (error) {
rootLogger.error("Judge call failed", { error });
diff --git a/apps/api/src/services/monitoring/judgeChange.ts b/apps/api/src/services/monitoring/judgeChange.ts
index 81d50f353..0a171945b 100644
--- a/apps/api/src/services/monitoring/judgeChange.ts
+++ b/apps/api/src/services/monitoring/judgeChange.ts
@@ -1,6 +1,10 @@
import { generateText } from "ai";
-import { google } from "@ai-sdk/google";
import type { Logger } from "winston";
+import {
+ googleModel,
+ googleProviderOptions,
+ type LlmUsageLabels,
+} from "./search/tuning";
const SYSTEM_PROMPT = `You judge whether a webpage diff matters for the user's monitoring goal.
@@ -77,6 +81,9 @@ interface JudgeChangeArgs {
markdownDiff?: {
diffText?: string;
};
+ // Vertex billing labels so this LLM call is traceable at the billing level by
+ // function, team, monitor, and monitor check (mirrors the search service).
+ labels?: LlmUsageLabels;
}
function isMeaningfulChangeEvent(
@@ -114,10 +121,11 @@ const JUDGE_MODEL_NAME = "gemini-3-flash-preview";
const JUDGE_ATTEMPT_TIMEOUT_MS = 30_000;
const JUDGE_MAX_ATTEMPTS = 3;
const JUDGE_BACKOFF_MS = [300, 800];
-const judgeModel = google(JUDGE_MODEL_NAME);
+const judgeModel = googleModel(JUDGE_MODEL_NAME);
async function callGemini(args: {
userBlock: string;
+ labels?: LlmUsageLabels;
}): Promise<{ text: string }> {
let lastError: unknown;
for (let attempt = 1; attempt <= JUDGE_MAX_ATTEMPTS; attempt++) {
@@ -133,6 +141,7 @@ async function callGemini(args: {
prompt: args.userBlock,
temperature: 0,
abortSignal: controller.signal,
+ ...googleProviderOptions("judgeChange", args.labels),
});
return { text: result.text?.trim() ?? "" };
} catch (error) {
@@ -151,7 +160,8 @@ async function callGemini(args: {
export async function judgeChange(
args: JudgeChangeArgs,
): Promise<JudgmentResult> {
- const { logger, goal, extractionPrompt, jsonDiff, markdownDiff } = args;
+ const { logger, goal, extractionPrompt, jsonDiff, markdownDiff, labels } =
+ args;
const parts: string[] = [`MONITOR GOAL:\n${goal.trim()}`];
if (extractionPrompt?.trim()) {
@@ -180,7 +190,7 @@ export async function judgeChange(
const userBlock = parts.join("\n\n");
try {
- const { text } = await callGemini({ userBlock });
+ const { text } = await callGemini({ userBlock, labels });
const match = text.match(/\{[\s\S]*\}/);
if (!match) {
logger.warn("Judge returned unparseable response", {