Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(shared): add toolCallInvocations measure for per-tool invocation counts (#16622)

Trang Nguyen3466904600ff
diff --git a/packages/shared/src/features/query/dataModel.ts b/packages/shared/src/features/query/dataModel.ts
index c4f609622d..0bd408632a 100644
--- a/packages/shared/src/features/query/dataModel.ts
+++ b/packages/shared/src/features/query/dataModel.ts
@@ -489,7 +489,7 @@ export const observationsView: ViewDeclarationType = {
explodeArray: true,
},
calledToolNames: {
- sql: "observations.tool_call_names",
+ sql: "arrayDistinct(observations.tool_call_names)",
alias: "calledToolNames",
type: "arrayString",
description: "Names of tools that were called by the observation.",
@@ -626,6 +626,16 @@ export const observationsView: ViewDeclarationType = {
description: "Number of tool calls per observation.",
unit: "calls",
},
+ toolCallInvocations: {
+ sql: "countEqual(@@AGG1@@(observations.tool_call_names), calledToolNames)",
+ aggs: { agg1: "any" },
+ alias: "toolCallInvocations",
+ type: "integer",
+ requiresDimension: "calledToolNames",
+ description:
+ "Number of individual tool-call invocations, counting repeated calls to the same tool within one observation. Automatically grouped by the Called Tool Names dimension. Use the Sum aggregation for totals and rankings.",
+ unit: "calls",
+ },
},
tableRelations: {
traces: {
@@ -1315,7 +1325,7 @@ export const eventsObservationsView: ViewDeclarationType = {
explodeArray: true,
},
calledToolNames: {
- sql: "events_observations.tool_call_names",
+ sql: "arrayDistinct(events_observations.tool_call_names)",
alias: "calledToolNames",
type: "arrayString",
description: "Names of tools that were called by the observation.",
@@ -1518,6 +1528,16 @@ export const eventsObservationsView: ViewDeclarationType = {
description: "Number of tool calls per observation.",
unit: "calls",
},
+ toolCallInvocations: {
+ sql: "countEqual(@@AGG1@@(events_observations.tool_call_names), calledToolNames)",
+ aggs: { agg1: "any" },
+ alias: "toolCallInvocations",
+ type: "integer",
+ requiresDimension: "calledToolNames",
+ description:
+ "Number of individual tool-call invocations, counting repeated calls to the same tool within one observation. Automatically grouped by the Called Tool Names dimension. Use the Sum aggregation for totals and rankings.",
+ unit: "calls",
+ },
costByType: {
sql: "cost_value",
alias: "costByType",
diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
index 115f923028..86f0ead80a 100644
--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts
@@ -427,3 +427,51 @@ describe("queryBuilder DateTime64 parameter encoding", () => {
expect(parameters.fillToDate).toBe("2025-01-02 00:00:00.000");
});
});
+
+describe("toolCallInvocations measure", () => {
+ it("auto-includes distinct calledToolNames in a two-level query", async () => {
+ const { query: compiledQuery } = await new QueryBuilder(
+ undefined,
+ "v1",
+ ).build(
+ {
+ ...baseQuery,
+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],
+ },
+ "test-project",
+ );
+
+ expect(compiledQuery).toContain(
+ "arrayJoin(arrayDistinct(observations.tool_call_names)) as calledToolNames",
+ );
+ expect(compiledQuery).toContain(
+ "countEqual(any(observations.tool_call_names), calledToolNames)",
+ );
+ expect(compiledQuery).toContain("sum(toolCallInvocations)");
+ });
+
+ it("stays single-level for the events view", async () => {
+ const { query: compiledQuery } = await new QueryBuilder(
+ undefined,
+ "v2",
+ ).build(
+ {
+ ...baseQuery,
+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],
+ },
+ "test-project",
+ true,
+ );
+
+ expect(compiledQuery).toContain(
+ "arrayJoin(arrayDistinct(events_observations.tool_call_names)) as calledToolNames",
+ );
+ expect(compiledQuery).toContain(
+ "sum(countEqual((events_observations.tool_call_names), calledToolNames))",
+ );
+ expect(compiledQuery).not.toContain(
+ "any(events_observations.tool_call_names)",
+ );
+ expect(compiledQuery.match(/\bSELECT\b/g)).toHaveLength(1);
+ });
+});
diff --git a/packages/shared/src/features/query/server/queryBuilder.ts b/packages/shared/src/features/query/server/queryBuilder.ts
index e02fa96e3e..7ae8d7b394 100644
--- a/packages/shared/src/features/query/server/queryBuilder.ts
+++ b/packages/shared/src/features/query/server/queryBuilder.ts
@@ -1575,9 +1575,12 @@ export class QueryBuilder {
const appliedMetrics = this.mapMetrics(query.metrics, view);
const appliedBucketingDimension = this.applyBucketingDimension(query, view);
- // Auto-include dimensions required by pairExpand-dependent measures.
+ // Auto-include dimensions required to evaluate a measure, whether the
+ // dimension uses pairExpand or explodeArray.
// e.g. costByType.requiresDimension = "costType": without that dimension the
// ARRAY JOIN is never emitted and ClickHouse errors with "unknown column cost_value".
+ // toolCallInvocations similarly needs calledToolNames so arrayJoin is emitted
+ // and its per-tool result alias is in scope.
for (const metric of appliedMetrics) {
if (
metric.requiresDimension &&
@@ -1588,6 +1591,7 @@ export class QueryBuilder {
appliedDimensions.push({
...requiredDimDef,
table: requiredDimDef.relationTable || view.name,
+ explodeArray: requiredDimDef.explodeArray,
pairExpand: requiredDimDef.pairExpand,
});
}
diff --git a/packages/shared/src/features/query/types.ts b/packages/shared/src/features/query/types.ts
index 30a8e454ba..ddc5b08c72 100644
--- a/packages/shared/src/features/query/types.ts
+++ b/packages/shared/src/features/query/types.ts
@@ -46,7 +46,13 @@ export const viewDeclaration = z.object({
.optional(),
highCardinality: z.boolean().optional(),
uiHidden: z.boolean().optional(),
+ // Expands dimension.sql independently with arrayJoin(), producing one row
+ // per array element. Duplicate elements duplicate the source row; use
+ // arrayDistinct(...) in sql when the intended grain is entity + distinct value.
explodeArray: z.boolean().optional(),
+ // Expands dimension.sql and valuesSql together in one ARRAY JOIN, pairing
+ // elements by position and exposing valueAlias to a dependent measure.
+ // Only one pairExpand dimension is supported per query, and it cannot be filtered.
pairExpand: z
.object({
valuesSql: z.string(),
@@ -78,9 +84,8 @@ export const viewDeclaration = z.object({
}),
)
.optional(),
- // When set, the query builder will auto-include this dimension if it is absent.
- // Used for pairExpand value-alias measures (e.g. costByType requires costType so
- // the ARRAY JOIN is emitted and "cost_value" is in scope).
+ // Auto-includes a dimension needed to evaluate the measure, including its
+ // explodeArray or pairExpand behavior and resulting aliases.
requiresDimension: z.string().optional(),
}),
),
diff --git a/web/src/__tests__/server/queryBuilder.servertest.ts b/web/src/__tests__/server/queryBuilder.servertest.ts
index 5ad9d28ac4..c4d65c9640 100644
--- a/web/src/__tests__/server/queryBuilder.servertest.ts
+++ b/web/src/__tests__/server/queryBuilder.servertest.ts
@@ -2807,6 +2807,139 @@ describe("queryBuilder", () => {
});
describe("observations view", () => {
+ it("counts per-tool invocations including same-tool repeats (toolCallInvocations)", async () => {
+ const projectId = randomUUID();
+ const trace = createTrace({
+ project_id: projectId,
+ name: "tool-invocations-trace",
+ environment: "default",
+ timestamp: new Date().getTime(),
+ });
+ await createTracesCh([trace]);
+
+ const now = new Date().getTime();
+ const base = {
+ project_id: projectId,
+ trace_id: trace.id,
+ type: "generation",
+ environment: "default",
+ start_time: now,
+ end_time: now + 100,
+ };
+ // One observation with three PARALLEL calls to the same tool, one
+ // with a single call, one calling a different tool: ground truth is
+ // fetch = 4 invocations across 2 observations, search = 1.
+ await createObservationsCh([
+ createObservation({
+ ...base,
+ name: "obs-parallel-fetch",
+ tool_calls: ['{"id":"c1"}', '{"id":"c2"}', '{"id":"c3"}'],
+ tool_call_names: ["fetch", "fetch", "fetch"],
+ }),
+ createObservation({
+ ...base,
+ name: "obs-single-fetch",
+ tool_calls: ['{"id":"c4"}'],
+ tool_call_names: ["fetch"],
+ }),
+ createObservation({
+ ...base,
+ name: "obs-search",
+ tool_calls: ['{"id":"c5"}'],
+ tool_call_names: ["search"],
+ }),
+ ]);
+
+ const query: QueryType = {
+ view: "observations",
+ dimensions: [],
+ metrics: [{ measure: "toolCallInvocations", aggregation: "sum" }],
+ filters: [],
+ timeDimension: null,
+ fromTimestamp: new Date(now - 86400000).toISOString(),
+ toTimestamp: new Date(now + 86400000).toISOString(),
+ orderBy: [{ field: "calledToolNames", direction: "asc" }],
+ };
+
+ const result = await executeQuery(projectId, query, "v1", false);
+ expect(result).toHaveLength(2);
+ const fetchRow = result.find((r) => r.calledToolNames === "fetch");
+ const searchRow = result.find((r) => r.calledToolNames === "search");
+ expect(Number(fetchRow!.sum_toolCallInvocations)).toBe(4);
+ expect(Number(searchRow!.sum_toolCallInvocations)).toBe(1);
+
+ // Contrast: counting observation rows dedupes same-tool repeats within
+ // one observation, so the same data yields observations-per-tool.
+ const countResult = await executeQuery(
+ projectId,
+ {
+ ...query,
+ dimensions: [{ field: "calledToolNames" }],
+ metrics: [{ measure: "count", aggregation: "count" }],
+ },
+ "v1",
+ false,
+ );
+ const fetchCountRow = countResult.find(
+ (r) => r.calledToolNames === "fetch",
+ );
+ expect(Number(fetchCountRow!.count_count)).toBe(2);
+
+ // The v1 path normally uses the two-level query, but the declaration
+ // also supports the single-level optimization without changing results.
+ const resultOptimized = await executeQuery(
+ projectId,
+ query,
+ "v1",
+ true,
+ );
+ expect(resultOptimized).toEqual(result);
+ });
+
+ it("counts per-tool invocations on the v2 events view (toolCallInvocations)", async () => {
+ const projectId = randomUUID();
+ const now = Date.now() * 1000;
+ await createEventsCh([
+ createEvent({
+ project_id: projectId,
+ start_time: now,
+ end_time: now + 100_000,
+ tool_calls: ['{"id":"c1"}', '{"id":"c2"}', '{"id":"c3"}'],
+ tool_call_names: ["fetch", "fetch", "fetch"],
+ }),
+ createEvent({
+ project_id: projectId,
+ start_time: now,
+ end_time: now + 100_000,
+ tool_calls: ['{"id":"c4"}'],
+ tool_call_names: ["fetch"],
+ }),
+ ]);
+
+ const result = await executeQuery(
+ projectId,
+ {
+ view: "observations",
+ dimensions: [],
+ metrics: [
+ { measure: "toolCallInvocations", aggregation: "sum" },
+ { measure: "count", aggregation: "count" },
+ ],
+ filters: [],
+ timeDimension: null,
+ fromTimestamp: new Date(Date.now() - 86400000).toISOString(),
+ toTimestamp: new Date(Date.now() + 86400000).toISOString(),
+ orderBy: null,
+ },
+ "v2",
+ true,
+ );
+ expect(result).toHaveLength(1);
+ expect(result[0].calledToolNames).toBe("fetch");
+ expect(Number(result[0].sum_toolCallInvocations)).toBe(4);
+ expect(Number(result[0].count_count)).toBe(2);
+ });
+
it("should calculate p95 timeToFirstToken for each trace name using observations view", async () => {
// Setup
const projectId = randomUUID();
diff --git a/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap b/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap
index bf2cd5431b..9e26d27c02 100644
--- a/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap
+++ b/worker/src/__tests__/__snapshots__/monitorProcessorScalarQuery.test.ts.snap
@@ -124,6 +124,17 @@ exports[`monitor scalar query — empty project > verifies which parameters retu
"observations/tokensPerSecond/p99": null,
"observations/tokensPerSecond/sum": null,
"observations/tokensPerSecond/uniq": 0,
+ "observations/toolCallInvocations/avg": null,
+ "observations/toolCallInvocations/count": null,
+ "observations/toolCallInvocations/max": null,
+ "observations/toolCallInvocations/min": null,
+ "observations/toolCallInvocations/p50": null,
+ "observations/toolCallInvocations/p75": null,
+ "observations/toolCallInvocations/p90": null,
+ "observations/toolCallInvocations/p95": null,
+ "observations/toolCallInvocations/p99": null,
+ "observations/toolCallInvocations/sum": null,
+ "observations/toolCallInvocations/uniq": null,
"observations/toolCalls/avg": null,
"observations/toolCalls/count": 0,
"observations/toolCalls/max": null,