Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
monitor search: attach Vertex billing labels to every LLM call
Developers Digeste485f7d93ca5
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3f4b5dba4..61c443ccf 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1037,6 +1037,7 @@ async function runMonitorSearchTarget(params: {maxResults: target.maxResults,depth: target.depth,},+ monitorCheckId: check.id,goalVersion,knownPages,knownEvents,diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsindex ab1c6982d..8eece502a 100644--- a/apps/api/src/services/monitoring/search/criteria.ts+++ b/apps/api/src/services/monitoring/search/criteria.ts@@ -1,6 +1,10 @@import { generateObject } from "ai";import { z } from "zod";-import { googleModel, googleProviderOptions } from "./tuning";+import {+ googleModel,+ googleProviderOptions,+ type LlmUsageLabels,+} from "./tuning";import { recordLlmCall } from "./cost";import type { CostTracking } from "../../../lib/cost-tracking";@@ -136,6 +140,7 @@ export async function compileGoalCriteriaWithLlm(params: {queries: string[];goalVersion: string;costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<GoalCriteria> {const deterministic = compileGoalCriteria(params);const { object, usage } = await generateObject({@@ -149,7 +154,7 @@ export async function compileGoalCriteriaWithLlm(params: {queries: params.queries,}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("compileGoalCriteria", params.labels),});if (params.costTracking) {recordLlmCall({diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.tsindex 3c68f2022..4953ea4bf 100644--- a/apps/api/src/services/monitoring/search/llm.ts+++ b/apps/api/src/services/monitoring/search/llm.ts@@ -1,6 +1,10 @@import { generateObject } from "ai";import { z } from "zod";-import { googleModel, googleProviderOptions } from "./tuning";+import {+ googleModel,+ googleProviderOptions,+ type LlmUsageLabels,+} from "./tuning";import type { GoalCriteria } from "./criteria";import { recordLlmCall } from "./cost";import type { CostTracking } from "../../../lib/cost-tracking";@@ -37,6 +41,7 @@ export async function resolveEvent(params: {result: { title: string; url: string; evidence: string };candidates: KnownEvent[];costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<EventResolution> {const { object, usage } = await generateObject({model: googleModel(EVENT_MODEL),@@ -59,7 +64,7 @@ label: when matched, reuse the matched event's label; when new, write a short reA new article from a different publisher about the same filing/launch/recall is NOT a new event.`,}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("resolveEvent", params.labels),});if (params.costTracking) {@@ -90,6 +95,7 @@ export async function judgeMaterialDevelopment(params: {eventLabel: string;result: { title: string; evidence: string };costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<{ material: boolean; reason: string }> {const { object, usage } = await generateObject({model: googleModel(EVENT_MODEL),@@ -107,7 +113,7 @@ export async function judgeMaterialDevelopment(params: {Set material false when it restates already-known facts, even from a new source or headline.`,}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("judgeMaterialDevelopment", params.labels),});if (params.costTracking) {recordLlmCall({@@ -149,6 +155,7 @@ export async function reviewAlert(params: {judgeAnswer: string;};costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<SkepticVerdict> {const { object, usage } = await generateObject({model: googleModel(SKEPTIC_MODEL),@@ -176,7 +183,7 @@ export async function reviewAlert(params: {},}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("reviewAlert", params.labels),});if (params.costTracking) {recordLlmCall({@@ -220,6 +227,7 @@ export async function routeSearchResults(params: {snippet: string;}>;costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<RouteDecision[]> {const candidates = params.candidates.slice(0, 50);const { object, usage } = await generateObject({@@ -238,7 +246,7 @@ Never select more than maxResults.`,candidates,}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("routeSearchResults", params.labels),});if (params.costTracking) {recordLlmCall({@@ -287,6 +295,7 @@ export async function judgeSnippets(params: {snippet: string;}>;costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<SnippetVerdict[]> {if (params.candidates.length === 0) return [];const { object, usage } = await generateObject({@@ -300,7 +309,7 @@ export async function judgeSnippets(params: {candidates: params.candidates.slice(0, 50),}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("judgeSnippets", params.labels),});if (params.costTracking) {recordLlmCall({@@ -324,6 +333,7 @@ export async function summarizeRun(params: {subject: string;evidence: Array<{ title: string; url: string; rationale: string }>;costTracking?: CostTracking;+ labels?: LlmUsageLabels;}): Promise<{ label: string; summary: string }> {const { object, usage } = await generateObject({model: googleModel(SUMMARY_MODEL),@@ -337,7 +347,7 @@ export async function summarizeRun(params: {If sources are meaningful but already reported, say they are related evidence, not a new notification.`,}),temperature: 0,- ...googleProviderOptions(),+ ...googleProviderOptions("summarizeRun", params.labels),});if (params.costTracking) {diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex dff89240e..9b56146f1 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -101,6 +101,7 @@ function runParams(maxResults: 10,...targetOver,},+ monitorCheckId: "check-1",goalVersion: "v1",knownPages: new Map(),knownEvents: [],diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 8a6ca9697..4124a6e72 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -106,6 +106,7 @@ function run(...over.target,alertMode: over.alertMode ?? baseTarget.alertMode,},+ monitorCheckId: "check-1",goalVersion: over.goalVersion ?? "gv1",knownPages: over.knownPages ?? new Map(),knownEvents: over.knownEvents ?? [],diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex 10153b036..b211c982a 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -174,6 +174,7 @@ export async function runSearchTarget(params: {judgeEnabled: boolean;};target: SearchTargetInput;+ monitorCheckId: string;goalVersion: string;knownPages: Map<string, KnownPage>;knownEvents: KnownEvent[];@@ -189,6 +190,12 @@ export async function runSearchTarget(params: {}const subject = params.monitor.subject ?? "";const teamId = params.monitor.teamId;+ // Vertex billing labels attached to every LLM call for usage tracing.+ const labels = {+ teamId,+ monitorId: params.monitor.id,+ monitorCheckId: params.monitorCheckId,+ };const tbs = windowToTbs(target.searchWindow);const events: KnownEvent[] = [...params.knownEvents];@@ -277,6 +284,7 @@ export async function runSearchTarget(params: {queries: target.queries,goalVersion,costTracking,+ labels,});} catch (error) {logger.warn(@@ -302,6 +310,7 @@ export async function runSearchTarget(params: {searchWindow: target.searchWindow,maxResults: target.maxResults,costTracking,+ labels,candidates: candidates.map((c, i) => ({id: `result_${i + 1}`,query: c.query,@@ -361,6 +370,7 @@ export async function runSearchTarget(params: {subject,searchWindow: target.searchWindow,costTracking,+ labels,candidates: snippetCandidates.map((c, i) => ({id: `result_${i + 1}`,query: c.query,@@ -632,6 +642,7 @@ export async function runSearchTarget(params: {subject,criteria,costTracking,+ labels,result: {title: c.title,url: c.url,@@ -703,6 +714,7 @@ export async function runSearchTarget(params: {goal: judgeGoal,subject,costTracking,+ labels,result: {title: c.title,url: c.url,@@ -740,6 +752,7 @@ export async function runSearchTarget(params: {subject,eventLabel,costTracking,+ labels,result: { title: c.title, evidence: verdict.rationale },});alreadySatisfied = !dev.material;@@ -827,6 +840,7 @@ export async function runSearchTarget(params: {goal: judgeGoal,subject,costTracking,+ labels,evidence: sources.filter(s => s.status === "alert").map(s => ({diff --git a/apps/api/src/services/monitoring/search/tuning.ts b/apps/api/src/services/monitoring/search/tuning.tsindex a3be2a7d2..2e14db3d5 100644--- a/apps/api/src/services/monitoring/search/tuning.ts+++ b/apps/api/src/services/monitoring/search/tuning.ts@@ -1,15 +1,35 @@import { getModel } from "../../../lib/generic-ai";import { config } from "../../../config";-type GoogleThinkingOptions = {- providerOptions: {- google: { thinkingConfig: { thinkingLevel: "minimal" } };- };+// Vertex billing labels so every LLM call is traceable at the billing level by+// function, team, monitor, and monitor check (mirrors the Extract code). The+// GenAI provider ignores labels it doesn't understand.+export type LlmUsageLabels = {+ teamId: string;+ monitorId: string;+ monitorCheckId: string;};-export function googleProviderOptions(): GoogleThinkingOptions {+export function googleProviderOptions(+ functionId: string,+ labels?: LlmUsageLabels,+) {return {- providerOptions: { google: { thinkingConfig: { thinkingLevel: "minimal" } } },+ providerOptions: {+ google: {+ thinkingConfig: { thinkingLevel: "minimal" as const },+ ...(labels+ ? {+ labels: {+ functionId,+ teamId: labels.teamId,+ monitorId: labels.monitorId,+ monitorCheckId: labels.monitorCheckId,+ },+ }+ : {}),+ },+ },};}