Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

monitor search: attach Vertex billing labels to every LLM call

Developers Digeste485f7d93ca5
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 3f4b5dba4..61c443ccf 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -1037,6 +1037,7 @@ async function runMonitorSearchTarget(params: {
maxResults: target.maxResults,
depth: target.depth,
},
+ monitorCheckId: check.id,
goalVersion,
knownPages,
knownEvents,
diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.ts
index ab1c6982d..8eece502a 100644
--- a/apps/api/src/services/monitoring/search/criteria.ts
+++ b/apps/api/src/services/monitoring/search/criteria.ts
@@ -1,6 +1,10 @@
import { generateObject } from "ai";
import { z } from "zod";
-import { googleModel, googleProviderOptions } from "./tuning";
+import {
+ googleModel,
+ googleProviderOptions,
+ type LlmUsageLabels,
+} from "./tuning";
import { recordLlmCall } from "./cost";
import type { CostTracking } from "../../../lib/cost-tracking";
@@ -136,6 +140,7 @@ export async function compileGoalCriteriaWithLlm(params: {
queries: string[];
goalVersion: string;
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<GoalCriteria> {
const deterministic = compileGoalCriteria(params);
const { object, usage } = await generateObject({
@@ -149,7 +154,7 @@ export async function compileGoalCriteriaWithLlm(params: {
queries: params.queries,
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("compileGoalCriteria", params.labels),
});
if (params.costTracking) {
recordLlmCall({
diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.ts
index 3c68f2022..4953ea4bf 100644
--- a/apps/api/src/services/monitoring/search/llm.ts
+++ b/apps/api/src/services/monitoring/search/llm.ts
@@ -1,6 +1,10 @@
import { generateObject } from "ai";
import { z } from "zod";
-import { googleModel, googleProviderOptions } from "./tuning";
+import {
+ googleModel,
+ googleProviderOptions,
+ type LlmUsageLabels,
+} from "./tuning";
import type { GoalCriteria } from "./criteria";
import { recordLlmCall } from "./cost";
import type { CostTracking } from "../../../lib/cost-tracking";
@@ -37,6 +41,7 @@ export async function resolveEvent(params: {
result: { title: string; url: string; evidence: string };
candidates: KnownEvent[];
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<EventResolution> {
const { object, usage } = await generateObject({
model: googleModel(EVENT_MODEL),
@@ -59,7 +64,7 @@ label: when matched, reuse the matched event's label; when new, write a short re
A new article from a different publisher about the same filing/launch/recall is NOT a new event.`,
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("resolveEvent", params.labels),
});
if (params.costTracking) {
@@ -90,6 +95,7 @@ export async function judgeMaterialDevelopment(params: {
eventLabel: string;
result: { title: string; evidence: string };
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<{ material: boolean; reason: string }> {
const { object, usage } = await generateObject({
model: googleModel(EVENT_MODEL),
@@ -107,7 +113,7 @@ export async function judgeMaterialDevelopment(params: {
Set material false when it restates already-known facts, even from a new source or headline.`,
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("judgeMaterialDevelopment", params.labels),
});
if (params.costTracking) {
recordLlmCall({
@@ -149,6 +155,7 @@ export async function reviewAlert(params: {
judgeAnswer: string;
};
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<SkepticVerdict> {
const { object, usage } = await generateObject({
model: googleModel(SKEPTIC_MODEL),
@@ -176,7 +183,7 @@ export async function reviewAlert(params: {
},
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("reviewAlert", params.labels),
});
if (params.costTracking) {
recordLlmCall({
@@ -220,6 +227,7 @@ export async function routeSearchResults(params: {
snippet: string;
}>;
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<RouteDecision[]> {
const candidates = params.candidates.slice(0, 50);
const { object, usage } = await generateObject({
@@ -238,7 +246,7 @@ Never select more than maxResults.`,
candidates,
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("routeSearchResults", params.labels),
});
if (params.costTracking) {
recordLlmCall({
@@ -287,6 +295,7 @@ export async function judgeSnippets(params: {
snippet: string;
}>;
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<SnippetVerdict[]> {
if (params.candidates.length === 0) return [];
const { object, usage } = await generateObject({
@@ -300,7 +309,7 @@ export async function judgeSnippets(params: {
candidates: params.candidates.slice(0, 50),
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("judgeSnippets", params.labels),
});
if (params.costTracking) {
recordLlmCall({
@@ -324,6 +333,7 @@ export async function summarizeRun(params: {
subject: string;
evidence: Array<{ title: string; url: string; rationale: string }>;
costTracking?: CostTracking;
+ labels?: LlmUsageLabels;
}): Promise<{ label: string; summary: string }> {
const { object, usage } = await generateObject({
model: googleModel(SUMMARY_MODEL),
@@ -337,7 +347,7 @@ export async function summarizeRun(params: {
If sources are meaningful but already reported, say they are related evidence, not a new notification.`,
}),
temperature: 0,
- ...googleProviderOptions(),
+ ...googleProviderOptions("summarizeRun", params.labels),
});
if (params.costTracking) {
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index dff89240e..9b56146f1 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -101,6 +101,7 @@ function runParams(
maxResults: 10,
...targetOver,
},
+ monitorCheckId: "check-1",
goalVersion: "v1",
knownPages: new Map(),
knownEvents: [],
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 8a6ca9697..4124a6e72 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -106,6 +106,7 @@ function run(
...over.target,
alertMode: over.alertMode ?? baseTarget.alertMode,
},
+ monitorCheckId: "check-1",
goalVersion: over.goalVersion ?? "gv1",
knownPages: over.knownPages ?? new Map(),
knownEvents: over.knownEvents ?? [],
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index 10153b036..b211c982a 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -174,6 +174,7 @@ export async function runSearchTarget(params: {
judgeEnabled: boolean;
};
target: SearchTargetInput;
+ monitorCheckId: string;
goalVersion: string;
knownPages: Map<string, KnownPage>;
knownEvents: KnownEvent[];
@@ -189,6 +190,12 @@ export async function runSearchTarget(params: {
}
const subject = params.monitor.subject ?? "";
const teamId = params.monitor.teamId;
+ // Vertex billing labels attached to every LLM call for usage tracing.
+ const labels = {
+ teamId,
+ monitorId: params.monitor.id,
+ monitorCheckId: params.monitorCheckId,
+ };
const tbs = windowToTbs(target.searchWindow);
const events: KnownEvent[] = [...params.knownEvents];
@@ -277,6 +284,7 @@ export async function runSearchTarget(params: {
queries: target.queries,
goalVersion,
costTracking,
+ labels,
});
} catch (error) {
logger.warn(
@@ -302,6 +310,7 @@ export async function runSearchTarget(params: {
searchWindow: target.searchWindow,
maxResults: target.maxResults,
costTracking,
+ labels,
candidates: candidates.map((c, i) => ({
id: `result_${i + 1}`,
query: c.query,
@@ -361,6 +370,7 @@ export async function runSearchTarget(params: {
subject,
searchWindow: target.searchWindow,
costTracking,
+ labels,
candidates: snippetCandidates.map((c, i) => ({
id: `result_${i + 1}`,
query: c.query,
@@ -632,6 +642,7 @@ export async function runSearchTarget(params: {
subject,
criteria,
costTracking,
+ labels,
result: {
title: c.title,
url: c.url,
@@ -703,6 +714,7 @@ export async function runSearchTarget(params: {
goal: judgeGoal,
subject,
costTracking,
+ labels,
result: {
title: c.title,
url: c.url,
@@ -740,6 +752,7 @@ export async function runSearchTarget(params: {
subject,
eventLabel,
costTracking,
+ labels,
result: { title: c.title, evidence: verdict.rationale },
});
alreadySatisfied = !dev.material;
@@ -827,6 +840,7 @@ export async function runSearchTarget(params: {
goal: judgeGoal,
subject,
costTracking,
+ labels,
evidence: sources
.filter(s => s.status === "alert")
.map(s => ({
diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.ts
index a3be2a7d2..2e14db3d5 100644
--- a/apps/api/src/services/monitoring/search/tuning.ts
+++ b/apps/api/src/services/monitoring/search/tuning.ts
@@ -1,15 +1,35 @@
import { getModel } from "../../../lib/generic-ai";
import { config } from "../../../config";
-type GoogleThinkingOptions = {
- providerOptions: {
- google: { thinkingConfig: { thinkingLevel: "minimal" } };
- };
+// Vertex billing labels so every LLM call is traceable at the billing level by
+// function, team, monitor, and monitor check (mirrors the Extract code). The
+// GenAI provider ignores labels it doesn't understand.
+export type LlmUsageLabels = {
+ teamId: string;
+ monitorId: string;
+ monitorCheckId: string;
};
-export function googleProviderOptions(): GoogleThinkingOptions {
+export function googleProviderOptions(
+ functionId: string,
+ labels?: LlmUsageLabels,
+) {
return {
- providerOptions: { google: { thinkingConfig: { thinkingLevel: "minimal" } } },
+ providerOptions: {
+ google: {
+ thinkingConfig: { thinkingLevel: "minimal" as const },
+ ...(labels
+ ? {
+ labels: {
+ functionId,
+ teamId: labels.teamId,
+ monitorId: labels.monitorId,
+ monitorCheckId: labels.monitorCheckId,
+ },
+ }
+ : {}),
+ },
+ },
};
}