🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(shared): add toolCallInvocations measure for per-tool invocation counts (#16622)
Trang Nguyen3466904600ff
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.tsindex c4f609622d..0bd408632a 100644--- a/packages/shared/src/features/query/dataModel.ts+++ b/packages/shared/src/features/query/dataModel.ts@@ -489,7 +489,7 @@ export const observationsView: ViewDeclarationType = {explodeArray: true,},calledToolNames: {- sql: "observations.tool_call_names",+ sql: "arrayDistinct(observations.tool_call_names)",alias: "calledToolNames",type: "arrayString",description: "Names of tools that were called by the observation.",@@ -626,6 +626,16 @@ export const observationsView: ViewDeclarationType = {description: "Number of tool calls per observation.",unit: "calls",},+ toolCallInvocations: {+ sql: "countEqual(@@AGG1@@(observations.tool_call_names), calledToolNames)",+ aggs: { agg1: "any" },+ alias: "toolCallInvocations",+ type: "integer",+ requiresDimension: "calledToolNames",+ description:+ "Number of individual tool-call invocations, counting repeated calls to the same tool within one observation. Automatically grouped by the Called Tool Names dimension. Use the Sum aggregation for totals and rankings.",+ unit: "calls",+ },},tableRelations: {traces: {@@ -1315,7 +1325,7 @@ export const eventsObservationsView: ViewDeclarationType = {explodeArray: true,},calledToolNames: {- sql: "events_observations.tool_call_names",+ sql: "arrayDistinct(events_observations.tool_call_names)",alias: "calledToolNames",type: "arrayString",description: "Names of tools that were called by the observation.",@@ -1518,6 +1528,16 @@ export const eventsObservationsView: ViewDeclarationType = {description: "Number of tool calls per observation.",unit: "calls",},+ toolCallInvocations: {+ sql: "countEqual(@@AGG1@@(events_observations.tool_call_names), calledToolNames)",+ aggs: { agg1: "any" },+ alias: "toolCallInvocations",+ type: "integer",+ requiresDimension: "calledToolNames",+ description:+ "Number of individual tool-call invocations, counting repeated calls to the same tool within one observation. Automatically grouped by the Called Tool Names dimension. Use the Sum aggregation for totals and rankings.",+ unit: "calls",+ },costByType: {sql: "cost_value",alias: "costByType",diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.tsindex 115f923028..86f0ead80a 100644--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts@@ -427,3 +427,51 @@ describe("queryBuilder DateTime64 parameter encoding", () => {expect(parameters.fillToDate).toBe("2025-01-02 00:00:00.000");});});++describe("toolCallInvocations measure", () => {+ it("auto-includes distinct calledToolNames in a two-level query", async () => {+ const { query: compiledQuery } = await new QueryBuilder(+ undefined,+ "v1",+ ).build(+ {+ ...baseQuery,+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],+ },+ "test-project",+ );++ expect(compiledQuery).toContain(+ "arrayJoin(arrayDistinct(observations.tool_call_names)) as calledToolNames",+ );+ expect(compiledQuery).toContain(+ "countEqual(any(observations.tool_call_names), calledToolNames)",+ );+ expect(compiledQuery).toContain("sum(toolCallInvocations)");+ });++ it("stays single-level for the events view", async () => {+ const { query: compiledQuery } = await new QueryBuilder(+ undefined,+ "v2",+ ).build(+ {+ ...baseQuery,+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],+ },+ "test-project",+ true,+ );++ expect(compiledQuery).toContain(+ "arrayJoin(arrayDistinct(events_observations.tool_call_names)) as calledToolNames",+ );+ expect(compiledQuery).toContain(+ "sum(countEqual((events_observations.tool_call_names), calledToolNames))",+ );+ expect(compiledQuery).not.toContain(+ "any(events_observations.tool_call_names)",+ );+ expect(compiledQuery.match(/\bSELECT\b/g)).toHaveLength(1);+ });+});diff --git a/packages/shared/src/features/query/server/queryBuilder.ts b/packages/shared/src/features/query/server/queryBuilder.tsindex e02fa96e3e..7ae8d7b394 100644--- a/packages/shared/src/features/query/server/queryBuilder.ts+++ b/packages/shared/src/features/query/server/queryBuilder.ts@@ -1575,9 +1575,12 @@ export class QueryBuilder {const appliedMetrics = this.mapMetrics(query.metrics, view);const appliedBucketingDimension = this.applyBucketingDimension(query, view);- // Auto-include dimensions required by pairExpand-dependent measures.+ // Auto-include dimensions required to evaluate a measure, whether the+ // dimension uses pairExpand or explodeArray.// e.g. costByType.requiresDimension = "costType": without that dimension the// ARRAY JOIN is never emitted and ClickHouse errors with "unknown column cost_value".+ // toolCallInvocations similarly needs calledToolNames so arrayJoin is emitted+ // and its per-tool result alias is in scope.for (const metric of appliedMetrics) {if (metric.requiresDimension &&@@ -1588,6 +1591,7 @@ export class QueryBuilder {appliedDimensions.push({...requiredDimDef,table: requiredDimDef.relationTable || view.name,+ explodeArray: requiredDimDef.explodeArray,pairExpand: requiredDimDef.pairExpand,});}diff --git a/packages/shared/src/features/query/types.ts b/packages/shared/src/features/query/types.tsindex 30a8e454ba..ddc5b08c72 100644--- a/packages/shared/src/features/query/types.ts+++ b/packages/shared/src/features/query/types.ts@@ -46,7 +46,13 @@ export const viewDeclaration = z.object({.optional(),highCardinality: z.boolean().optional(),uiHidden: z.boolean().optional(),+ // Expands dimension.sql independently with arrayJoin(), producing one row+ // per array element. Duplicate elements duplicate the source row; use+ // arrayDistinct(...) in sql when the intended grain is entity + distinct value.explodeArray: z.boolean().optional(),+ // Expands dimension.sql and valuesSql together in one ARRAY JOIN, pairing+ // elements by position and exposing valueAlias to a dependent measure.+ // Only one pairExpand dimension is supported per query, and it cannot be filtered.pairExpand: z.object({valuesSql: z.string(),@@ -78,9 +84,8 @@ export const viewDeclaration = z.object({}),).optional(),- // When set, the query builder will auto-include this dimension if it is absent.- // Used for pairExpand value-alias measures (e.g. costByType requires costType so- // the ARRAY JOIN is emitted and "cost_value" is in scope).+ // Auto-includes a dimension needed to evaluate the measure, including its+ // explodeArray or pairExpand behavior and resulting aliases.requiresDimension: z.string().optional(),}),),diff --git a/web/src/__tests__/server/queryBuilder.servertest.ts b/web/src/__tests__/server/queryBuilder.servertest.tsindex 5ad9d28ac4..c4d65c9640 100644--- a/web/src/__tests__/server/queryBuilder.servertest.ts+++ b/web/src/__tests__/server/queryBuilder.servertest.ts@@ -2807,6 +2807,139 @@ describe("queryBuilder", () => {});describe("observations view", () => {+ it("counts per-tool invocations including same-tool repeats (toolCallInvocations)", async () => {+ const projectId = randomUUID();+ const trace = createTrace({+ project_id: projectId,+ name: "tool-invocations-trace",+ environment: "default",+ timestamp: new Date().getTime(),+ });+ await createTracesCh([trace]);++ const now = new Date().getTime();+ const base = {+ project_id: projectId,+ trace_id: trace.id,+ type: "generation",+ environment: "default",+ start_time: now,+ end_time: now + 100,+ };+ // One observation with three PARALLEL calls to the same tool, one+ // with a single call, one calling a different tool: ground truth is+ // fetch = 4 invocations across 2 observations, search = 1.+ await createObservationsCh([+ createObservation({+ ...base,+ name: "obs-parallel-fetch",+ tool_calls: ['{"id":"c1"}', '{"id":"c2"}', '{"id":"c3"}'],+ tool_call_names: ["fetch", "fetch", "fetch"],+ }),+ createObservation({+ ...base,+ name: "obs-single-fetch",+ tool_calls: ['{"id":"c4"}'],+ tool_call_names: ["fetch"],+ }),+ createObservation({+ ...base,+ name: "obs-search",+ tool_calls: ['{"id":"c5"}'],+ tool_call_names: ["search"],+ }),+ ]);++ const query: QueryType = {+ view: "observations",+ dimensions: [],+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],+ filters: [],+ timeDimension: null,+ fromTimestamp: new Date(now - 86400000).toISOString(),+ toTimestamp: new Date(now + 86400000).toISOString(),+ orderBy: [{ field: "calledToolNames", direction: "asc" }],+ };++ const result = await executeQuery(projectId, query, "v1", false);+ expect(result).toHaveLength(2);+ const fetchRow = result.find((r) => r.calledToolNames === "fetch");+ const searchRow = result.find((r) => r.calledToolNames === "search");+ expect(Number(fetchRow!.sum_toolCallInvocations)).toBe(4);+ expect(Number(searchRow!.sum_toolCallInvocations)).toBe(1);++ // Contrast: counting observation rows dedupes same-tool repeats within+ // one observation, so the same data yields observations-per-tool.+ const countResult = await executeQuery(+ projectId,+ {+ ...query,+ dimensions: [{ field: "calledToolNames" }],+ metrics: [{ measure: "count", aggregation: "count" }],+ },+ "v1",+ false,+ );+ const fetchCountRow = countResult.find(+ (r) => r.calledToolNames === "fetch",+ );+ expect(Number(fetchCountRow!.count_count)).toBe(2);++ // The v1 path normally uses the two-level query, but the declaration+ // also supports the single-level optimization without changing results.+ const resultOptimized = await executeQuery(+ projectId,+ query,+ "v1",+ true,+ );+ expect(resultOptimized).toEqual(result);+ });++ it("counts per-tool invocations on the v2 events view (toolCallInvocations)", async () => {+ const projectId = randomUUID();+ const now = Date.now() * 1000;+ await createEventsCh([+ createEvent({+ project_id: projectId,+ start_time: now,+ end_time: now + 100_000,+ tool_calls: ['{"id":"c1"}', '{"id":"c2"}', '{"id":"c3"}'],+ tool_call_names: ["fetch", "fetch", "fetch"],+ }),+ createEvent({+ project_id: projectId,+ start_time: now,+ end_time: now + 100_000,+ tool_calls: ['{"id":"c4"}'],+ tool_call_names: ["fetch"],+ }),+ ]);++ const result = await executeQuery(+ projectId,+ {+ view: "observations",+ dimensions: [],+ metrics: [+ { measure: "toolCallInvocations", aggregation: "sum" },+ { measure: "count", aggregation: "count" },+ ],+ filters: [],+ timeDimension: null,+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),+ toTimestamp: new Date(Date.now() + 86400000).toISOString(),+ orderBy: null,+ },+ "v2",+ true,+ );+ expect(result).toHaveLength(1);+ expect(result[0].calledToolNames).toBe("fetch");+ expect(Number(result[0].sum_toolCallInvocations)).toBe(4);+ expect(Number(result[0].count_count)).toBe(2);+ });+it("should calculate p95 timeToFirstToken for each trace name using observations view", async () => {// Setupconst projectId = randomUUID();diff --git a/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap b/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snapindex bf2cd5431b..9e26d27c02 100644--- a/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap+++ b/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap@@ -124,6 +124,17 @@ exports[`monitor scalar query — empty project > verifies which parameters retu"observations/tokensPerSecond/p99": null,"observations/tokensPerSecond/sum": null,"observations/tokensPerSecond/uniq": 0,+ "observations/toolCallInvocations/avg": null,+ "observations/toolCallInvocations/count": null,+ "observations/toolCallInvocations/max": null,+ "observations/toolCallInvocations/min": null,+ "observations/toolCallInvocations/p50": null,+ "observations/toolCallInvocations/p75": null,+ "observations/toolCallInvocations/p90": null,+ "observations/toolCallInvocations/p95": null,+ "observations/toolCallInvocations/p99": null,+ "observations/toolCallInvocations/sum": null,+ "observations/toolCallInvocations/uniq": null,"observations/toolCalls/avg": null,"observations/toolCalls/count": 0,"observations/toolCalls/max": null,