🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(filter-options): scan events_core once for exact scores-view facets (#17366)
Niklas Semmler955e29436a3b
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.tsindex 3a75c243d3..8cb5f33e44 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-filter-options.ts@@ -19,8 +19,20 @@ export const EVENTS_FILTER_OPTION_TOP_N = 1000;// Sentinel "column" carrying the approx total observation count in the facet result.export const EVENTS_APPROX_TOTAL_COUNT_MARKER = "__approxTotalCount__";-// Facet tuple format: (option name, option value, value-occurrence count, sort order, display value).-const EVENTS_APPROX_TOTAL_COUNT_TUPLE = `tuple('${EVENTS_APPROX_TOTAL_COUNT_MARKER}', '', toUInt64(approx_total_count), toInt64(0), '')`;+// Named row tuple emitted per facet option. Casting to a named Tuple lets the+// final projection read option.column / option.sortKey etc. instead of opaque+// positional tupleElement(option, N) reads, and forces every facet — plus the+// approx-total sentinel below — to arrayConcat with a structurally identical+// shape (arrayConcat rejects mismatched element types).+const EVENTS_FILTER_OPTION_ROW_TUPLE_TYPE =+ "Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)";++const castEventFilterOptionRowTuple = (tupleExpression: string): string =>+ `CAST(${tupleExpression} AS ${EVENTS_FILTER_OPTION_ROW_TUPLE_TYPE})`;++const EVENTS_APPROX_TOTAL_COUNT_TUPLE = castEventFilterOptionRowTuple(+ `tuple('${EVENTS_APPROX_TOTAL_COUNT_MARKER}', '', toUInt64(approx_total_count), toInt64(0), '')`,+);const EVENTS_FILTER_OPTION_TOP_K_MAX_N = 65_536;@@ -293,27 +305,23 @@ const eventFilterOptionColumnSqlLiteral = (column: EventFilterOptionColumn) =>const stringValueExpression = (expression: string) =>`toString(ifNull(${expression}, ''))`;-const optionValuesArrayExpression = (- column: EventFilterOptionColumn,+// Shared by the sumMap keys and the parallel "ones" counts array so both derive+// their length from one filter and cannot drift apart.+const optionValuesFilteredExpression = (+ definition: Extract<EventFilterOptionDefinition, { kind: "array" }>,): string => {- const definition = EVENTS_FILTER_OPTION_DEFINITIONS[column];-- if (definition.kind === "scalar" || definition.kind === "labeledScalar") {- return `if(${definition.includeWhen}, [${stringValueExpression(definition.expression)}], CAST([], 'Array(String)'))`;- }+ const valuesExpression = definition.distinct+ ? `arrayDistinct(${definition.expression})`+ : definition.expression;- if (definition.kind === "boolean") {- return `[if(${definition.expression}, 'true', 'false')]`;- }-- const valuesExpression =- "distinct" in definition && definition.distinct- ? `arrayDistinct(${definition.expression})`- : definition.expression;-- return `arrayMap(value -> toString(value), arrayFilter(value -> length(toString(value)) > 0, ${valuesExpression}))`;+ return `arrayFilter(value -> length(toString(value)) > 0, ${valuesExpression})`;};+const optionValuesArrayExpression = (+ definition: Extract<EventFilterOptionDefinition, { kind: "array" }>,+): string =>+ `arrayMap(value -> toString(value), ${optionValuesFilteredExpression(definition)})`;+const optionPresenceCondition = (column: EventFilterOptionColumn): string => {const definition = EVENTS_FILTER_OPTION_DEFINITIONS[column];@@ -356,7 +364,7 @@ const optionTopKSelectExpression = (column: EventFilterOptionColumn) => {return `arrayFilter(option -> tupleElement(option, 2) > 0, [tuple('false', countIf(NOT (${definition.expression})), toUInt64(0)), tuple('true', countIf(${definition.expression}), toUInt64(0))]) AS ${optionTopAlias(column)}`;}- return `approx_top_kArray({optionLimit: UInt64})(${optionValuesArrayExpression(column)}) AS ${optionTopAlias(column)}`;+ return `approx_top_kArray({optionLimit: UInt64})(${optionValuesArrayExpression(definition)}) AS ${optionTopAlias(column)}`;};const optionRowsArrayExpression = (column: EventFilterOptionColumn) => {@@ -380,7 +388,9 @@ const optionRowsArrayExpression = (column: EventFilterOptionColumn) => {? "tupleElement(tupleElement(option, 1), 2)": "''";- return `arrayMap(option -> tuple(${eventFilterOptionColumnSqlLiteral(column)}, ${valueExpression}, tupleElement(option, 2), ${sortKeyExpression}, ${displayValueExpression}), ${topAlias})`;+ return `arrayMap(option -> ${castEventFilterOptionRowTuple(+ `tuple(${eventFilterOptionColumnSqlLiteral(column)}, ${valueExpression}, tupleElement(option, 2), ${sortKeyExpression}, ${displayValueExpression})`,+ )}, ${topAlias})`;};const eventFilterOptionScopeCondition = (@@ -444,7 +454,7 @@ export const buildEventsFilterOptionColumnQuery = (params: {? `toString(${definition.expression})`: definition.kind === "boolean"? `if(${definition.expression}, 'true', 'false')`- : `arrayJoin(${optionValuesArrayExpression(column)})`;+ : `arrayJoin(${optionValuesArrayExpression(definition)})`;const queryBuilder = new EventsAggQueryBuilder({projectId: params.projectId,@@ -517,8 +527,8 @@ export const buildEventsFilterOptionsForColumnsQuery = (params: {// approx_top_k counts live inside the tuple, so scale in the outer projection.const sampleFactorRow = sampled ? ",\n sample_factor" : "";const countExpression = sampled- ? "toUInt64(round(tupleElement(option, 3) * sample_factor))"- : "tupleElement(option, 3)";+ ? "toUInt64(round(option.count * sample_factor))"+ : "option.count";const query = `WITH aggregated_options AS (@@ -532,12 +542,185 @@ option_rows AS (FROM aggregated_options)SELECT- tupleElement(option, 1) AS column,- tupleElement(option, 2) AS value,+ option.column AS column,+ option.value AS value,+ ${countExpression} AS count,+ option.displayValue AS displayValue+FROM option_rows+ORDER BY column ASC, option.sortKey ASC, option.value ASC+`.trim();++ return {+ query,+ params: {+ ...aggregatedOptionsParams,+ optionLimit,+ },+ };+};++// Per-column exact aggregate state, computed in the single base scan. Every+// kind resolves to the same alias shape — Array((key, count)) ranked and capped+// to the top-N — so the rows builder below can map any facet uniformly.+// Scalars/arrays aggregate an exact value→count Map (sumMap), then zip it into+// (key, count) pairs; booleans build that pair array directly from two countIf+// branches. The full histogram is held in aggregate-function state, so unlike a+// GROUP BY it does not spill to disk — scan cost is bounded by the base scan's+// sampleRows.+//+// countDesc ranks by (count DESC, value ASC): the value tie-breaker must be+// applied before the optionLimit cap so the top-N stays deterministic at the+// boundary, matching the single-column ORDER BY count() DESC, value ASC. Booleans+// carry only two buckets, always under the cap, so they skip the rank/slice.+//+// Non-boolean facets emit the pipeline as three named columns of the one-row+// aggregate — `<column>_hist` (the value→count Map), `<column>_ranked` (zipped+// and sorted), and `<column>TopOptions` (capped) — so the SQL reads as named+// stages and the histogram is written once instead of inlined per tupleElement+// read. The intermediate columns are only consumed by the next stage's alias;+// the rows builder reads `<column>TopOptions`.+const exactOptionAggSelectExpressions = (+ column: EventFilterOptionColumn,+): string[] => {+ const definition = EVENTS_FILTER_OPTION_DEFINITIONS[column];+ const topAlias = optionTopAlias(column);++ if (definition.kind === "boolean") {+ return [+ `arrayFilter(option -> tupleElement(option, 2) > 0, [tuple('false', countIf(NOT (${definition.expression})), toUInt64(0)), tuple('true', countIf(${definition.expression}), toUInt64(0))]) AS ${topAlias}`,+ ];+ }++ const histAlias = `${column}_hist`;+ const rankedAlias = `${column}_ranked`;++ // The array values expression stays inlined here: it is a per-row (non-+ // aggregate) array, so it cannot be projected as its own CTE column without a+ // GROUP BY. `<column>_hist` is aggregate state and is projectable.+ const histogram =+ definition.kind === "scalar"+ ? `sumMapIf([${stringValueExpression(definition.expression)}], [toUInt64(1)], ${definition.includeWhen})`+ : definition.kind === "labeledScalar"+ ? `sumMapIf([tuple(${stringValueExpression(definition.expression)}, ${stringValueExpression(definition.labelExpression)})], [toUInt64(1)], ${definition.includeWhen})`+ : (() => {+ const filtered = optionValuesFilteredExpression(definition);+ const keys = `arrayMap(value -> toString(value), ${filtered})`;+ const ones = `arrayMap(value -> toUInt64(1), ${filtered})`;+ return `sumMap(${keys}, ${ones})`;+ })();++ const zipped = `arrayZip(tupleElement(${histAlias}, 1), tupleElement(${histAlias}, 2))`;+ const ranked =+ definition.sort === "countDesc"+ ? `arraySort(pair -> tuple(-toInt64(tupleElement(pair, 2)), tupleElement(pair, 1)), ${zipped})`+ : `arraySort(pair -> tupleElement(pair, 1), ${zipped})`;++ return [+ `${histogram} AS ${histAlias}`,+ `${ranked} AS ${rankedAlias}`,+ `arraySlice(${rankedAlias}, 1, {optionLimit: UInt64}) AS ${topAlias}`,+ ];+};++const exactOptionRowsArrayExpression = (+ column: EventFilterOptionColumn,+): string => {+ const definition = EVENTS_FILTER_OPTION_DEFINITIONS[column];+ const alias = optionTopAlias(column);+ const isLabeled = definition.kind === "labeledScalar";++ // alias = array of option tuples, option = (key, count)+ // key = value (scalar / array / boolean)+ // key = (value, label) (labeledScalar)+ // Output row is cast to the named EVENTS_FILTER_OPTION_ROW_TUPLE_TYPE.+ const sortKeyExpression =+ definition.sort === "countDesc"+ ? "-toInt64(tupleElement(option, 2))"+ : definition.sort === "booleanAsc"+ ? "if(tupleElement(option, 1) = 'true', toInt64(1), toInt64(0))"+ : "toInt64(0)";+ const valueExpression = isLabeled+ ? "tupleElement(tupleElement(option, 1), 1)"+ : "tupleElement(option, 1)";+ const displayValueExpression = isLabeled+ ? "tupleElement(tupleElement(option, 1), 2)"+ : "''";++ return `arrayMap(option -> ${castEventFilterOptionRowTuple(+ `tuple(${eventFilterOptionColumnSqlLiteral(column)}, ${valueExpression}, tupleElement(option, 2), ${sortKeyExpression}, ${displayValueExpression})`,+ )}, ${alias})`;+};++/**+ * One events_core scan that materializes the given filter-option facets with+ * exact value→count aggregation (sumMap / countIf), then fans the per-column+ * top-N out with arrayJoin. Used where facet values must stay exact+ * (scores-view facets) instead of the approx_top_k multi-column sketch, without+ * re-scanning or re-evaluating the scope semi-join once per facet.+ */+export const buildEventsExactFilterOptionsForColumnsQuery = (params: {+ projectId: string;+ filter: FilterState;+ columns: readonly EventFilterOptionColumn[];+ limit: number;+ scope?: EventFilterOptionScope;+ sampleRows?: number;+}): { query: string; params: Record<string, unknown> } | null => {+ const columns = uniqueEventFilterOptionColumns(params.columns);+ if (columns.length === 0 || params.limit <= 0) {+ return null;+ }++ const optionLimit = Math.min(params.limit, EVENTS_FILTER_OPTION_TOP_K_MAX_N);+ const { queryBuilder: aggregatedOptionsBuilder } =+ buildEventsObservationRowSelection({+ projectId: params.projectId,+ filter: params.filter,+ });++ const sampleRows = params.sampleRows ?? 0;+ const sampled = sampleRows > 0;++ aggregatedOptionsBuilder.selectRaw(+ ...columns.flatMap(exactOptionAggSelectExpressions),+ ...(sampled ? [EVENTS_SAMPLE_FACTOR_SELECT] : []),+ );+ aggregatedOptionsBuilder.sampleRows(sampleRows);++ if (params.scope) {+ const scopeCondition = eventFilterOptionScopeCondition(params.scope);+ aggregatedOptionsBuilder.whereRaw(+ scopeCondition.condition,+ scopeCondition.params,+ );+ }++ const { query: aggregatedOptionsQuery, params: aggregatedOptionsParams } =+ aggregatedOptionsBuilder.buildWithParams();++ const sampleFactorRow = sampled ? ",\n sample_factor" : "";+ const countExpression = sampled+ ? "toUInt64(round(option.count * sample_factor))"+ : "option.count";++ const query = `+WITH aggregated_options AS (+${aggregatedOptionsQuery}+),+option_rows AS (+ SELECT+ arrayJoin(arrayConcat(+ ${columns.map(exactOptionRowsArrayExpression).join(",\n ")}+ )) AS option${sampleFactorRow}+ FROM aggregated_options+)+SELECT+ option.column AS column,+ option.value AS value,${countExpression} AS count,- tupleElement(option, 5) AS displayValue+ option.displayValue AS displayValueFROM option_rows-ORDER BY column ASC, tupleElement(option, 4) ASC, tupleElement(option, 2) ASC+ORDER BY column ASC, option.sortKey ASC, option.value ASC`.trim();return {diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.tsindex 780cab0d25..fbcbe58e88 100644--- a/packages/shared/src/server/queries/index.ts+++ b/packages/shared/src/server/queries/index.ts@@ -75,6 +75,7 @@ export {export {buildEventsFilterOptionColumnQuery,buildEventsFilterOptionsForColumnsQuery,+ buildEventsExactFilterOptionsForColumnsQuery,buildEventsMetadataValuesQuery,EVENTS_FILTER_OPTION_TOP_N,EVENTS_APPROX_TOTAL_COUNT_MARKER,diff --git a/packages/shared/src/server/query-ast/__snapshots__/event-filter-options.golden.test.ts.snap b/packages/shared/src/server/query-ast/__snapshots__/event-filter-options.golden.test.ts.snapnew file mode 100644index 0000000000..f151b01539--- /dev/null+++ b/packages/shared/src/server/query-ast/__snapshots__/event-filter-options.golden.test.ts.snap@@ -0,0 +1,267 @@+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html++exports[`golden: events.getEventsExactFilterOptionsForColumns > scope=scoredTraces — project + scores.timestamp window semi-join 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": 1000,+ "param2": "golden-project",+ "param3": "2026-01-01 00:00:00.000",+ "param4": "2026-01-01 00:30:00.000",+ },+ "route": "unknown",+ "sql": "WITH+ aggregated_options AS+ (+ SELECT+ sumMapIf(+ [toString(ifNull(e.user_id, ''))],+ [toUInt64(1)],+ (e.user_id IS NOT NULL) AND (length(e.user_id) > 0)+ ) AS userId_hist,+ arraySort(+ pair -> tuple(-toInt64(tupleElement(pair, 2)), tupleElement(pair, 1)),+ arrayZip(tupleElement(userId_hist, 1), tupleElement(userId_hist, 2))+ ) AS userId_ranked,+ arraySlice(userId_ranked, 1, {param1:UInt64}) AS userIdTopOptions,+ sumMapIf(+ [tuple(toString(ifNull(e.experiment_id, '')), toString(ifNull(e.experiment_name, '')))],+ [toUInt64(1)],+ (e.experiment_id IS NOT NULL) AND (length(e.experiment_id) > 0)+ ) AS experimentId_hist,+ arraySort(+ pair -> tuple(-toInt64(tupleElement(pair, 2)), tupleElement(pair, 1)),+ arrayZip(tupleElement(experimentId_hist, 1), tupleElement(experimentId_hist, 2))+ ) AS experimentId_ranked,+ arraySlice(experimentId_ranked, 1, {param1:UInt64}) AS experimentIdTopOptions,+ sumMap(+ arrayMap(+ value -> toString(value),+ arrayFilter(value -> (length(toString(value)) > 0), arrayDistinct(e.tags))+ ),+ arrayMap(+ value -> toUInt64(1),+ arrayFilter(value -> (length(toString(value)) > 0), arrayDistinct(e.tags))+ )+ ) AS traceTags_hist,+ arraySort(+ pair -> tupleElement(pair, 1),+ arrayZip(tupleElement(traceTags_hist, 1), tupleElement(traceTags_hist, 2))+ ) AS traceTags_ranked,+ arraySlice(traceTags_ranked, 1, {param1:UInt64}) AS traceTagsTopOptions,+ arrayFilter(+ option -> (tupleElement(option, 2) > 0),+ [+ tuple('false', countIf(NOT ((e.parent_span_id = '') OR (e.is_app_root = true))), toUInt64(0)),+ tuple('true', countIf((e.parent_span_id = '') OR (e.is_app_root = true)), toUInt64(0))+ ]+ ) AS isRootObservationTopOptions,+ any(e._sample_factor) AS sample_factor+ FROM events_core AS e+ SAMPLE 6000000+ WHERE (e.project_id = {param2:String}) AND (e.trace_id IN (+ SELECT DISTINCT trace_id+ FROM scores+ WHERE (project_id = {param2:String}) AND (timestamp >= {param3:DateTime64(3, 'UTC')}) AND (timestamp <= {param4:DateTime64(3, 'UTC')})+ ))+ ),+ option_rows AS+ (+ SELECT+ arrayJoin(+ arrayConcat(+ arrayMap(+ option -> CAST(+ tuple(+ 'userId',+ tupleElement(option, 1),+ tupleElement(option, 2),+ -toInt64(tupleElement(option, 2)),+ ''+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ userIdTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple(+ 'experimentId',+ tupleElement(tupleElement(option, 1), 1),+ tupleElement(option, 2),+ -toInt64(tupleElement(option, 2)),+ tupleElement(tupleElement(option, 1), 2)+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ experimentIdTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple('traceTags', tupleElement(option, 1), tupleElement(option, 2), toInt64(0), ''),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ traceTagsTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple(+ 'isRootObservation',+ tupleElement(option, 1),+ tupleElement(option, 2),+ if(tupleElement(option, 1) = 'true', toInt64(1), toInt64(0)),+ ''+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ isRootObservationTopOptions+ )+ )+ ) AS option,+ sample_factor+ FROM aggregated_options+ )+SELECT+ option.column AS column,+ option.value AS value,+ toUInt64(round(option.count * sample_factor)) AS count,+ option.displayValue AS displayValue+FROM option_rows+ORDER BY+ column ASC,+ option.sortKey ASC,+ option.value ASC",+ },+]+`;++exports[`golden: events.getEventsExactFilterOptionsForColumns > scope=unset — project-only WHERE, no scores semi-join 1`] = `+[+ {+ "fn": "queryClickhouse",+ "params": {+ "param1": 1000,+ "param2": "golden-project",+ },+ "route": "unknown",+ "sql": "WITH+ aggregated_options AS+ (+ SELECT+ sumMapIf(+ [toString(ifNull(e.user_id, ''))],+ [toUInt64(1)],+ (e.user_id IS NOT NULL) AND (length(e.user_id) > 0)+ ) AS userId_hist,+ arraySort(+ pair -> tuple(-toInt64(tupleElement(pair, 2)), tupleElement(pair, 1)),+ arrayZip(tupleElement(userId_hist, 1), tupleElement(userId_hist, 2))+ ) AS userId_ranked,+ arraySlice(userId_ranked, 1, {param1:UInt64}) AS userIdTopOptions,+ sumMapIf(+ [tuple(toString(ifNull(e.experiment_id, '')), toString(ifNull(e.experiment_name, '')))],+ [toUInt64(1)],+ (e.experiment_id IS NOT NULL) AND (length(e.experiment_id) > 0)+ ) AS experimentId_hist,+ arraySort(+ pair -> tuple(-toInt64(tupleElement(pair, 2)), tupleElement(pair, 1)),+ arrayZip(tupleElement(experimentId_hist, 1), tupleElement(experimentId_hist, 2))+ ) AS experimentId_ranked,+ arraySlice(experimentId_ranked, 1, {param1:UInt64}) AS experimentIdTopOptions,+ sumMap(+ arrayMap(+ value -> toString(value),+ arrayFilter(value -> (length(toString(value)) > 0), arrayDistinct(e.tags))+ ),+ arrayMap(+ value -> toUInt64(1),+ arrayFilter(value -> (length(toString(value)) > 0), arrayDistinct(e.tags))+ )+ ) AS traceTags_hist,+ arraySort(+ pair -> tupleElement(pair, 1),+ arrayZip(tupleElement(traceTags_hist, 1), tupleElement(traceTags_hist, 2))+ ) AS traceTags_ranked,+ arraySlice(traceTags_ranked, 1, {param1:UInt64}) AS traceTagsTopOptions,+ arrayFilter(+ option -> (tupleElement(option, 2) > 0),+ [+ tuple('false', countIf(NOT ((e.parent_span_id = '') OR (e.is_app_root = true))), toUInt64(0)),+ tuple('true', countIf((e.parent_span_id = '') OR (e.is_app_root = true)), toUInt64(0))+ ]+ ) AS isRootObservationTopOptions,+ any(e._sample_factor) AS sample_factor+ FROM events_core AS e+ SAMPLE 6000000+ WHERE e.project_id = {param2:String}+ ),+ option_rows AS+ (+ SELECT+ arrayJoin(+ arrayConcat(+ arrayMap(+ option -> CAST(+ tuple(+ 'userId',+ tupleElement(option, 1),+ tupleElement(option, 2),+ -toInt64(tupleElement(option, 2)),+ ''+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ userIdTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple(+ 'experimentId',+ tupleElement(tupleElement(option, 1), 1),+ tupleElement(option, 2),+ -toInt64(tupleElement(option, 2)),+ tupleElement(tupleElement(option, 1), 2)+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ experimentIdTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple('traceTags', tupleElement(option, 1), tupleElement(option, 2), toInt64(0), ''),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ traceTagsTopOptions+ ),+ arrayMap(+ option -> CAST(+ tuple(+ 'isRootObservation',+ tupleElement(option, 1),+ tupleElement(option, 2),+ if(tupleElement(option, 1) = 'true', toInt64(1), toInt64(0)),+ ''+ ),+ 'Tuple(column String, value String, count UInt64, sortKey Int64, displayValue String)'+ ),+ isRootObservationTopOptions+ )+ )+ ) AS option,+ sample_factor+ FROM aggregated_options+ )+SELECT+ option.column AS column,+ option.value AS value,+ toUInt64(round(option.count * sample_factor)) AS count,+ option.displayValue AS displayValue+FROM option_rows+ORDER BY+ column ASC,+ option.sortKey ASC,+ option.value ASC",+ },+]+`;diff --git a/packages/shared/src/server/query-ast/event-filter-options.golden.test.ts b/packages/shared/src/server/query-ast/event-filter-options.golden.test.tsnew file mode 100644index 0000000000..a8bb61b6a5--- /dev/null+++ b/packages/shared/src/server/query-ast/event-filter-options.golden.test.ts@@ -0,0 +1,121 @@+// Golden SQL for the exact scores-view filter-option facets. The recorded+// snapshot (__snapshots__/event-filter-options.golden.test.ts.snap) is the+// authoritative full-SQL example; this header annotates the array machinery that+// makes it a single-scan, exact, multi-facet query.+//+// Shape: one events_core scan -> one aggregated row of per-facet top-N arrays+// -> arrayJoin fan-out -> tidy (column, value, count, displayValue) rows.+//+// aggregated_options one row, one column per facet, each an+// Array((key, count)) top-N (alias `<column>TopOptions`):+// scalar/array sumMap -> (keys[], counts[]); arrayZip stitches the two+// parallel arrays into (key, count) pairs; arraySort ranks+// (count DESC, value ASC for countDesc; value ASC for alpha)+// BEFORE arraySlice caps to {optionLimit}, so the top-N is+// deterministic at the boundary. The sumMap subexpression+// appears twice (one read per tuple element) but ClickHouse+// dedupes identical aggregate states -> single pass.+// labeledScalar same, but the sumMap key is a (value, label) tuple.+// boolean built directly as [('false', countIf(...)), ('true',+// countIf(...))] — already (key, count) shaped, only two+// buckets, so it skips the zip/rank/slice.+// option_rows arrayConcat glues every facet's array together, arrayJoin+// explodes the single row into one row per option tuple+// (column, value, count, sortKey, displayValue). value/label+// unpack one level deeper for labeledScalar; displayValue is+// '' for the other kinds.+// final SELECT scales count by _sample_factor and orders by+// column, sortKey (the count/alpha/boolean rank), then value.+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";++import { env } from "../../env";+import type { EventFilterOptionColumn } from "../queries/clickhouse-sql/event-filter-options";+import {+ capturedQueries,+ clickhouseFormatAvailable,+ normalizeCapturedQueries,+ resetCaptures,+} from "./goldenHarness";++// Record the exec seam instead of hitting ClickHouse. The factory is hoisted+// above imports, so it pulls the harness in via dynamic import; the captured+// store is a module singleton shared with the assertions below.+vi.mock("../repositories/clickhouse", async (importOriginal) => {+ const actual =+ await importOriginal<typeof import("../repositories/clickhouse")>();+ const { buildClickhouseMock } = await import("./goldenHarness.js");+ return buildClickhouseMock(actual);+});++import { getEventsExactFilterOptionsForColumns } from "../repositories/events";++const FIXED_PROJECT_ID = "golden-project";+// Fixed so the captured scope-window params are deterministic across runs.+const FIXED_FROM_TIMESTAMP = new Date("2026-01-01T00:00:00.000Z");+const FIXED_TO_TIMESTAMP = new Date("2026-01-01T00:30:00.000Z");++// One column of each kind so the snapshot exercises every aggregate branch:+// scalar (userId), labeledScalar (experimentId), array (traceTags), and+// boolean (isRootObservation).+const COLUMNS: readonly EventFilterOptionColumn[] = [+ "userId",+ "experimentId",+ "traceTags",+ "isRootObservation",+];++// clickhouse format is the normalizer; without it a comparison would be+// unnormalized and could pass on cosmetic-only differences. Skip loudly.+const describeWithClickhouse = clickhouseFormatAvailable()+ ? describe+ : describe.skip;++if (!clickhouseFormatAvailable()) {+ console.warn(+ "[golden-harness] `clickhouse format` unavailable — skipping golden SQL tests. Install clickhouse-local to run them.",+ );+}++describeWithClickhouse(+ "golden: events.getEventsExactFilterOptionsForColumns",+ () => {+ // Exact facets are a v4 events_core feature; pin the write mode so the base+ // scan is deterministic regardless of the ambient env.+ const originalWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;++ beforeEach(() => {+ resetCaptures();+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";+ });+ afterAll(() => {+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = originalWriteMode;+ });++ // The scope semi-join is the only input that branches the emitted SQL for a+ // fixed column set, so enumerate present vs. absent.+ it("scope=unset — project-only WHERE, no scores semi-join", async () => {+ await getEventsExactFilterOptionsForColumns({+ projectId: FIXED_PROJECT_ID,+ filter: [],+ columns: COLUMNS,+ });++ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });++ it("scope=scoredTraces — project + scores.timestamp window semi-join", async () => {+ await getEventsExactFilterOptionsForColumns({+ projectId: FIXED_PROJECT_ID,+ filter: [],+ columns: COLUMNS,+ scope: {+ type: "scoredTraces",+ fromTime: { operator: ">=", value: FIXED_FROM_TIMESTAMP },+ toTime: { operator: "<=", value: FIXED_TO_TIMESTAMP },+ },+ });++ expect(normalizeCapturedQueries(capturedQueries)).toMatchSnapshot();+ });+ },+);diff --git a/packages/shared/src/server/query-ast/goldenHarness.ts b/packages/shared/src/server/query-ast/goldenHarness.tsindex 2412481fa8..d761bbb6a3 100644--- a/packages/shared/src/server/query-ast/goldenHarness.ts+++ b/packages/shared/src/server/query-ast/goldenHarness.ts@@ -306,6 +306,112 @@ export function normalizeParams(return { sql: normalizedSql, params: normalizedParams };}+// `clickhouse format` lays queries out clause by clause but never breaks inside a+// single expression, so a deeply-nested facet expression lands as one 500+ char+// line that is unreadable in a snapshot diff. Wrap those — and only those — at+// top-level argument boundaries of the first function/array group on the line.+const MAX_SQL_LINE = 120;+const WRAP_INDENT = " ";++// Lines the formatter opens with a clause keyword are its layout, not an+// expression — breaking them on their first paren mangles `WHERE (a) AND (b)`.+// Leave those alone; only reflow the nested-expression lines.+const CLAUSE_KEYWORD =+ /^(WITH|SELECT|FROM|PREWHERE|WHERE|GROUP|ORDER|HAVING|LIMIT|UNION|SAMPLE|SETTINGS|JOIN|LEFT|RIGHT|INNER|FULL|CROSS|ON|AND|OR)\b/i;++/**+ * Pretty-print any expression line longer than {@link MAX_SQL_LINE} by+ * recursively breaking the first `(...)`/`[...]` group across lines, one+ * top-level argument per line. Shorter lines, and clause-level layout+ * `clickhouse format` already produces, are returned untouched — so this only+ * reflows the giant nested facet expressions. Deterministic and+ * CH-version-independent (pure string work).+ */+function wrapLongSql(sql: string): string {+ return sql+ .split("\n")+ .map((line) => {+ if (line.length <= MAX_SQL_LINE) return line;+ const indent = /^\s*/.exec(line)?.[0] ?? "";+ const body = line.slice(indent.length);+ if (CLAUSE_KEYWORD.test(body)) return line;+ return formatSqlExpr(body, indent);+ })+ .join("\n");+}++/** First top-level `(...)`/`[...]` group on `expr`, split into its args. */+function firstGroup(expr: string): {+ head: string;+ open: string;+ close: string;+ args: string[];+ tail: string;+} | null {+ let inStr = false;+ let open = -1;+ for (let i = 0; i < expr.length; i++) {+ const c = expr[i];+ if (inStr) {+ if (c === "\\") i++;+ else if (c === "'") inStr = false;+ continue;+ }+ if (c === "'") inStr = true;+ else if (c === "(" || c === "[") {+ open = i;+ break;+ }+ }+ if (open === -1) return null;++ const args: string[] = [];+ let depth = 0;+ let argStart = open + 1;+ inStr = false;+ for (let i = open; i < expr.length; i++) {+ const c = expr[i];+ if (inStr) {+ if (c === "\\") i++;+ else if (c === "'") inStr = false;+ continue;+ }+ if (c === "'") inStr = true;+ else if (c === "(" || c === "[") depth++;+ else if (c === ")" || c === "]") {+ depth--;+ if (depth === 0) {+ args.push(expr.slice(argStart, i));+ return {+ head: expr.slice(0, open),+ open: expr[open],+ close: c,+ args,+ tail: expr.slice(i + 1),+ };+ }+ } else if (c === "," && depth === 1) {+ args.push(expr.slice(argStart, i));+ argStart = i + 1;+ }+ }+ return null;+}++function formatSqlExpr(expr: string, indent: string): string {+ if (indent.length + expr.length <= MAX_SQL_LINE) return indent + expr;+ const group = firstGroup(expr);+ if (!group || group.args.every((a) => a.trim() === "")) return indent + expr;+ const childIndent = indent + WRAP_INDENT;+ const lines = [indent + group.head + group.open];+ group.args.forEach((arg, i) => {+ const last = i === group.args.length - 1;+ lines.push(formatSqlExpr(arg.trim(), childIndent) + (last ? "" : ","));+ });+ lines.push(indent + group.close + group.tail);+ return lines.join("\n");+}+function routeOf(tags: unknown): string {if (tags && typeof tags === "object" && "route" in tags) {const route = (tags as { route?: unknown }).route;@@ -325,6 +431,6 @@ export function normalizeCapturedQueries(): GoldenQuery[] {return captured.map((c) => {const { sql, params } = normalizeParams(formatSql(c.query), c.params);- return { fn: c.fn, route: routeOf(c.tags), sql, params };+ return { fn: c.fn, route: routeOf(c.tags), sql: wrapLongSql(sql), params };});}diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 9594c55b61..7a06b21a1a 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -117,6 +117,7 @@ import {import {buildEventsFilterOptionColumnQuery,buildEventsFilterOptionsForColumnsQuery,+ buildEventsExactFilterOptionsForColumnsQuery,buildEventsMetadataValuesQuery,EVENTS_FILTER_OPTION_SAMPLE_ROWS,EVENTS_FILTER_OPTION_TOP_N,@@ -2065,12 +2066,16 @@ type BuiltEventsFilterOptionColumnQuery = NonNullable<type BuiltEventsFilterOptionsForColumnsQuery = NonNullable<ReturnType<typeof buildEventsFilterOptionsForColumnsQuery>>;+type BuiltEventsExactFilterOptionsForColumnsQuery = NonNullable<+ ReturnType<typeof buildEventsExactFilterOptionsForColumnsQuery>+>;const queryEventsFilterOptionRows = async (projectId: string,queryWithParams:| BuiltEventsFilterOptionColumnQuery- | BuiltEventsFilterOptionsForColumnsQuery,+ | BuiltEventsFilterOptionsForColumnsQuery+ | BuiltEventsExactFilterOptionsForColumnsQuery,) => {return queryClickhouse<EventFilterOptionRow>({query: queryWithParams.query,@@ -2144,6 +2149,30 @@ export const getEventsFilterOptionsForColumns = async (params: {limit: params.topN ?? EVENTS_FILTER_OPTION_TOP_N,});+/** Exact per-column facets in one events_core scan: sumMap/countIf aggregate state fanned out with arrayJoin (no GROUP BY / UNION ALL). */+export const getEventsExactFilterOptionsForColumns = async (params: {+ projectId: string;+ filter: FilterState;+ columns: readonly EventFilterOptionColumn[];+ topN?: number;+ scope?: EventFilterOptionScope;+}) => {+ const queryWithParams = buildEventsExactFilterOptionsForColumnsQuery({+ projectId: params.projectId,+ filter: params.filter,+ columns: params.columns,+ limit: params.topN ?? EVENTS_FILTER_OPTION_TOP_N,+ scope: params.scope,+ sampleRows: EVENTS_FILTER_OPTION_SAMPLE_ROWS,+ });++ if (!queryWithParams) {+ return [];+ }++ return queryEventsFilterOptionRows(params.projectId, queryWithParams);+};+// Unsampled: the cursor contract lets MCP agents page the true distinct set.export const getEventsFilterOptionValuesPage = async (params: {projectId: string;@@ -2205,20 +2234,6 @@ const getSingleEventsFilterOptionColumn = async (sampleRows: EVENTS_FILTER_OPTION_SAMPLE_ROWS,});-export const getEventsGroupedByTraceName = async (- projectId: string,- filter: FilterState,- opts?: GroupedEventsFilterOptions,-) => {- const rows = await getSingleEventsFilterOptionColumn(- projectId,- filter,- "traceName",- opts,- );- return rows.map((row) => ({ traceName: row.value, count: row.count }));-};-export const getEventsGroupedByTraceTags = async (projectId: string,filter: FilterState,diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex 442ab05bea..b202bf1ede 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -1,6 +1,7 @@import {buildEventsFilterOptionColumnQuery,buildEventsFilterOptionsForColumnsQuery,+ buildEventsExactFilterOptionsForColumnsQuery,buildEventsMetadataValuesQuery,CTEQueryBuilder,createFilterFromFilterState,@@ -124,7 +125,7 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {expect(built.query).toContain("FROM events_core e SAMPLE 6000000");expect(built.query).toContain("any(e._sample_factor) AS sample_factor");expect(built.query).toContain(- "toUInt64(round(tupleElement(option, 3) * sample_factor)) AS count",+ "toUInt64(round(option.count * sample_factor)) AS count",);});@@ -143,7 +144,7 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {expect(built.query).not.toContain("SAMPLE");expect(built.query).not.toContain("_sample_factor");expect(built.query).not.toContain("sample_factor");- expect(built.query).toContain("tupleElement(option, 3) AS count");+ expect(built.query).toContain("option.count AS count");});it("orders bulk filter option rows by per-column sort key and value", () => {@@ -167,7 +168,7 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {"tuple('isRootObservation', tupleElement(option, 1), tupleElement(option, 2), if(tupleElement(option, 1) = 'true', toInt64(1), toInt64(0)), '')",);expect(built.query).toContain(- "ORDER BY column ASC, tupleElement(option, 4) ASC, tupleElement(option, 2) ASC",+ "ORDER BY column ASC, option.sortKey ASC, option.value ASC",);});@@ -204,7 +205,7 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {expect(built.query).toContain("tuple('experimentId', tupleElement(tupleElement(option, 1), 1), tupleElement(option, 2), -toInt64(tupleElement(option, 2)), tupleElement(tupleElement(option, 1), 2))",);- expect(built.query).toContain("tupleElement(option, 5) AS displayValue");+ expect(built.query).toContain("option.displayValue AS displayValue");});it("applies events filters to the single base scan", () => {@@ -271,6 +272,61 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {expect(Object.values(built.params)).toContain("quality");});+ it("combines scores-view event facets into one exact single-scan query", () => {+ const built = buildEventsExactFilterOptionsForColumnsQuery({+ projectId: "test-project",+ filter: [+ {+ column: "startTime",+ operator: ">=",+ value: new Date("2026-01-01T00:00:00.000Z"),+ type: "datetime",+ },+ ],+ columns: ["traceTags", "traceName", "userId"],+ limit: 1000,+ scope: {+ type: "scoredTraces",+ fromTime: {+ operator: ">=",+ value: new Date("2026-01-01T00:00:00.000Z"),+ },+ toTime: {+ operator: "<=",+ value: new Date("2026-01-01T00:30:00.000Z"),+ },+ },+ });++ expect(built).not.toBeNull();+ if (!built) throw new Error("expected query");++ // One scan of events_core, one scope semi-join, no UNION ALL / GROUP BY /+ // approx sketch: facets are exact aggregates fanned out with arrayJoin.+ expect(built.query.match(/FROM events_core e/g)).toHaveLength(1);+ expect(built.query).not.toContain("UNION ALL");+ expect(built.query).not.toContain("GROUP BY value");+ expect(built.query).not.toContain("approx_top_k");+ expect(built.query.match(/FROM scores WHERE/g)).toHaveLength(1);+ expect(built.query).toContain("sumMapIf(");+ expect(built.query).toContain("sumMap(");+ expect(built.query).toContain("arrayJoin(arrayConcat(");+ expect(built.query).toContain("tuple('traceTags'");+ expect(built.query).toContain("tuple('traceName'");+ expect(built.query).toContain("tuple('userId'");+ expect(built.query).not.toContain("FINAL");+ expect(built.query).not.toMatch(/\bJOIN\b/i);+ expect(built.query).toContain(+ "e.trace_id IN (SELECT DISTINCT trace_id FROM scores WHERE project_id = {projectId: String} AND timestamp >= {scoredTracesFromTime: DateTime64(3, 'UTC')} AND timestamp <= {scoredTracesToTime: DateTime64(3, 'UTC')})",+ );+ expect(built.params).toMatchObject({+ projectId: "test-project",+ optionLimit: 1000,+ scoredTracesFromTime: "2026-01-01 00:00:00.000",+ scoredTracesToTime: "2026-01-01 00:30:00.000",+ });+ });+it("bounds the scored traces scope by the view's both-sided window", () => {const fromTime = new Date("2026-01-01T00:00:00.000Z");const toTime = new Date("2026-01-01T00:30:00.000Z");diff --git a/web/src/__tests__/server/events-filter-options-exact.servertest.ts b/web/src/__tests__/server/events-filter-options-exact.servertest.tsnew file mode 100644index 0000000000..97c05661bd--- /dev/null+++ b/web/src/__tests__/server/events-filter-options-exact.servertest.ts@@ -0,0 +1,126 @@+/**+ * Execution coverage for getEventsExactFilterOptionsForColumns.+ *+ * The golden SQL test (packages/shared/.../event-filter-options.golden.test.ts)+ * pins the generated string; this test runs the query against real ClickHouse to+ * prove the array machinery (sumMap -> arrayZip -> arraySort -> arraySlice, plus+ * the boolean countIf branch and labeledScalar nested key) actually executes and+ * returns exact values/counts for one column of every kind.+ */+import { vi } from "vitest";++// The events tables are created only by the ClickHouse dev-tables setup, which+// runs where the v4 preview opt-in is enabled. -azure / -redis-cluster CI legs+// skip it, so gate on the ORIGINAL opt-in flag captured before any override.+const eventsTableAvailable = vi.hoisted(+ () => process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true",+);++import {+ createEvent,+ createEventsCh,+ getEventsExactFilterOptionsForColumns,+ type EventFilterOptionRow,+} from "@langfuse/shared/src/server";+import { randomUUID } from "crypto";+import waitForExpect from "wait-for-expect";++const maybe = eventsTableAvailable ? describe : describe.skip;++// Keeps the file from hanging (on redis connections opened by the shared server+// imports) when the events-table suite below is skipped.+describe("events exact filter options liveness", () => {+ it("does not hang when the events table is unavailable", () => {});+});++maybe("getEventsExactFilterOptionsForColumns (execution)", () => {+ const valuesFor = (rows: EventFilterOptionRow[], column: string) =>+ rows+ .filter((r) => r.column === column)+ .map((r) => ({+ value: r.value,+ count: r.count,+ displayValue: r.displayValue ?? "",+ }));++ it("returns exact facet values and counts for every column kind", async () => {+ const projectId = randomUUID();++ await createEventsCh([+ // Root observation (is_app_root); contributes user u1, exp1, tags alpha+beta.+ createEvent({+ project_id: projectId,+ user_id: "u1",+ tags: ["alpha", "beta"],+ experiment_id: "exp1",+ experiment_name: "Exp One",+ is_app_root: true,+ }),+ // Non-root; shares user u1, exp1 and tag beta so their counts sum across rows.+ createEvent({+ project_id: projectId,+ user_id: "u1",+ tags: ["beta"],+ experiment_id: "exp1",+ experiment_name: "Exp One",+ parent_span_id: "span-b",+ }),+ // Non-root; distinct user u2, exp2 and tag gamma (each count 1).+ createEvent({+ project_id: projectId,+ user_id: "u2",+ tags: ["gamma"],+ experiment_id: "exp2",+ experiment_name: "Exp Two",+ parent_span_id: "span-c",+ }),+ // Exercises the include filters: null user_id and null experiment_id are+ // excluded; the empty-string tag is dropped by arrayFilter(length > 0)+ // while "delta" survives.+ createEvent({+ project_id: projectId,+ user_id: null,+ tags: ["", "delta"],+ experiment_id: null,+ experiment_name: null,+ parent_span_id: "span-d",+ }),+ ]);++ // events_core is populated from events_full via the events_core_mv MV; poll+ // until it catches up rather than asserting immediately after the insert.+ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId,+ filter: [],+ columns: ["userId", "experimentId", "traceTags", "isRootObservation"],+ });++ // scalar: countDesc, null excluded.+ expect(valuesFor(rows, "userId")).toEqual([+ { value: "u1", count: 2, displayValue: "" },+ { value: "u2", count: 1, displayValue: "" },+ ]);++ // labeledScalar: value + label from the nested key, null excluded.+ expect(valuesFor(rows, "experimentId")).toEqual([+ { value: "exp1", count: 2, displayValue: "Exp One" },+ { value: "exp2", count: 1, displayValue: "Exp Two" },+ ]);++ // array: alpha sort, empty-string tag filtered out, beta summed across rows.+ expect(valuesFor(rows, "traceTags")).toEqual([+ { value: "alpha", count: 1, displayValue: "" },+ { value: "beta", count: 2, displayValue: "" },+ { value: "delta", count: 1, displayValue: "" },+ { value: "gamma", count: 1, displayValue: "" },+ ]);++ // boolean: booleanAsc (false before true).+ expect(valuesFor(rows, "isRootObservation")).toEqual([+ { value: "false", count: 3, displayValue: "" },+ { value: "true", count: 1, displayValue: "" },+ ]);+ });+ });+});diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex 9a9b89bafe..79a1dc0775 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -14,6 +14,7 @@ import {getLatestSdkVersionInfoFromEvents,getTracesIdentifierForSessionFromEvents,getEventsFilterOptionsForColumns,+ getEventsExactFilterOptionsForColumns,getEventsFilterOptionValuesPage,getLatestEvaluatorRunCost,getRecentEvaluatorExecutionTraces,@@ -1940,6 +1941,485 @@ describe("Clickhouse Events Repository Test", () => {});});+ it("returns exact scores-view event facets scoped to scored traces", async () => {+ const uniqueProjectId = randomUUID();+ const scoredTraceA = randomUUID();+ const scoredTraceB = randomUUID();+ const unscoredTrace = randomUUID();+ const now = Date.now();+ const nowMicro = now * 1000;++ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: scoredTraceA,+ type: "SPAN",+ trace_name: "exact-trace-a",+ user_id: "user-a",+ tags: ["alpha", "beta"],+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: scoredTraceA,+ type: "SPAN",+ trace_name: "exact-trace-a",+ user_id: "user-a",+ tags: ["alpha", "beta"],+ start_time: nowMicro + 1_000,+ event_ts: nowMicro + 1_000,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: scoredTraceB,+ type: "SPAN",+ trace_name: "exact-trace-b",+ user_id: "user-b",+ tags: ["beta"],+ start_time: nowMicro + 2_000,+ event_ts: nowMicro + 2_000,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: unscoredTrace,+ type: "SPAN",+ trace_name: "exact-trace-c",+ user_id: "user-c",+ tags: ["gamma"],+ start_time: nowMicro + 3_000,+ event_ts: nowMicro + 3_000,+ }),+ ]);+ await createScoresCh([+ createTraceScore({+ project_id: uniqueProjectId,+ trace_id: scoredTraceA,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now,+ event_ts: now,+ created_at: now,+ updated_at: now,+ }),+ createTraceScore({+ project_id: uniqueProjectId,+ trace_id: scoredTraceB,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now,+ event_ts: now,+ created_at: now,+ updated_at: now,+ }),+ ]);++ const scope = {+ type: "scoredTraces" as const,+ fromTime: {+ operator: ">=" as const,+ value: new Date(now - 60_000),+ },+ toTime: {+ operator: "<=" as const,+ value: new Date(now + 60_000),+ },+ };++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: uniqueProjectId,+ filter: [],+ columns: ["traceTags", "traceName", "userId"],+ scope,+ });++ // Exact per-value counts, scoped to the two scored traces only.+ expect(+ Number(findFilterOption(rows, "traceName", "exact-trace-a")?.count),+ ).toBe(2);+ expect(+ Number(findFilterOption(rows, "traceName", "exact-trace-b")?.count),+ ).toBe(1);+ expect(Number(findFilterOption(rows, "userId", "user-a")?.count)).toBe(+ 2,+ );+ expect(Number(findFilterOption(rows, "userId", "user-b")?.count)).toBe(+ 1,+ );++ // The unscored trace is outside the scope and must not appear.+ expect(+ findFilterOption(rows, "traceName", "exact-trace-c"),+ ).toBeUndefined();+ expect(findFilterOption(rows, "userId", "user-c")).toBeUndefined();++ // Tags: exact distinct set, alphabetical, no gamma.+ expect(+ rows+ .filter((row) => row.column === "traceTags")+ .map((row) => row.value),+ ).toEqual(["alpha", "beta"]);+ });+ });++ it("scopes scored-trace facets by the score timestamp window, not event time", async () => {+ const uniqueProjectId = randomUUID();+ const inWindowTrace = randomUUID();+ const outOfWindowTrace = randomUUID();+ const now = Date.now();+ const nowMicro = now * 1000;++ // Both events sit at "now", so the events themselves are inside any+ // sensible window; the scope must key off the score timestamp instead.+ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: inWindowTrace,+ type: "SPAN",+ trace_name: "in-window",+ user_id: "user-in",+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: outOfWindowTrace,+ type: "SPAN",+ trace_name: "out-window",+ user_id: "user-out",+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ ]);+ await createScoresCh([+ createTraceScore({+ project_id: uniqueProjectId,+ trace_id: inWindowTrace,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now,+ event_ts: now,+ created_at: now,+ updated_at: now,+ }),+ // Scored, but the score predates the window's fromTime.+ createTraceScore({+ project_id: uniqueProjectId,+ trace_id: outOfWindowTrace,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now - 120_000,+ event_ts: now - 120_000,+ created_at: now - 120_000,+ updated_at: now - 120_000,+ }),+ ]);++ const scope = {+ type: "scoredTraces" as const,+ fromTime: { operator: ">=" as const, value: new Date(now - 60_000) },+ toTime: { operator: "<=" as const, value: new Date(now + 60_000) },+ };++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: uniqueProjectId,+ filter: [],+ columns: ["traceName", "userId"],+ scope,+ });++ expect(+ Number(findFilterOption(rows, "traceName", "in-window")?.count),+ ).toBe(1);+ // The event is inside the time window, but its score is older than+ // fromTime, so the scored-trace scope drops it.+ expect(+ findFilterOption(rows, "traceName", "out-window"),+ ).toBeUndefined();+ expect(findFilterOption(rows, "userId", "user-out")).toBeUndefined();+ });+ });++ it("does not leak scored-trace facets across projects sharing a trace_id", async () => {+ const projectA = randomUUID();+ const projectB = randomUUID();+ const sharedTrace = randomUUID();+ const scopedTrace = randomUUID();+ const now = Date.now();+ const nowMicro = now * 1000;++ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: projectA,+ trace_id: sharedTrace,+ type: "SPAN",+ trace_name: "cross-project",+ user_id: "user-cross",+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: projectA,+ trace_id: scopedTrace,+ type: "SPAN",+ trace_name: "same-project",+ user_id: "user-same",+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ ]);+ await createScoresCh([+ // Score for the shared trace lives in project B only.+ createTraceScore({+ project_id: projectB,+ trace_id: sharedTrace,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now,+ event_ts: now,+ created_at: now,+ updated_at: now,+ }),+ createTraceScore({+ project_id: projectA,+ trace_id: scopedTrace,+ observation_id: null,+ name: "quality",+ data_type: "NUMERIC",+ value: 1,+ timestamp: now,+ event_ts: now,+ created_at: now,+ updated_at: now,+ }),+ ]);++ const scope = {+ type: "scoredTraces" as const,+ fromTime: { operator: ">=" as const, value: new Date(now - 60_000) },+ toTime: { operator: "<=" as const, value: new Date(now + 60_000) },+ };++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: projectA,+ filter: [],+ columns: ["traceName", "userId"],+ scope,+ });++ expect(+ Number(findFilterOption(rows, "traceName", "same-project")?.count),+ ).toBe(1);+ // The only score for `sharedTrace` is in project B, so project A's+ // scope subquery (project_id = A) must not match it.+ expect(+ findFilterOption(rows, "traceName", "cross-project"),+ ).toBeUndefined();+ expect(findFilterOption(rows, "userId", "user-cross")).toBeUndefined();+ });+ });++ it("derives traceName from the trace_name/observation-name fallback and never offers empty", async () => {+ const uniqueProjectId = randomUUID();+ const nowMicro = Date.now() * 1000;++ await createEventsCh([+ // Explicit trace name wins.+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ trace_name: "explicit-name",+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ // Root observation with no trace name falls back to the observation name.+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ trace_name: "",+ name: "root-obs-name",+ parent_span_id: "",+ is_app_root: true,+ start_time: nowMicro + 1_000,+ event_ts: nowMicro + 1_000,+ }),+ // Non-root observation with no trace name resolves to NULL, so it+ // contributes no trace name option (and its name is not used).+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ trace_name: "",+ name: "child-obs-name",+ parent_span_id: "parent-span",+ is_app_root: false,+ start_time: nowMicro + 2_000,+ event_ts: nowMicro + 2_000,+ }),+ ]);++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: uniqueProjectId,+ filter: [],+ columns: ["traceName"],+ });++ expect(+ Number(findFilterOption(rows, "traceName", "explicit-name")?.count),+ ).toBe(1);+ expect(+ Number(findFilterOption(rows, "traceName", "root-obs-name")?.count),+ ).toBe(1);+ expect(+ findFilterOption(rows, "traceName", "child-obs-name"),+ ).toBeUndefined();+ // The empty-string trace name is mapped to NULL by the facet+ // expression, so it is never offered as an option.+ expect(findFilterOption(rows, "traceName", "")).toBeUndefined();+ });+ });++ it("counts array facets per event: distinct for traceTags, raw for calledToolNames", async () => {+ const uniqueProjectId = randomUUID();+ const nowMicro = Date.now() * 1000;++ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ tags: ["alpha", "alpha"],+ tool_call_names: ["lookup", "lookup"],+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ tags: ["alpha"],+ tool_call_names: ["lookup"],+ start_time: nowMicro + 1_000,+ event_ts: nowMicro + 1_000,+ }),+ ]);++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: uniqueProjectId,+ filter: [],+ columns: ["traceTags", "calledToolNames"],+ });++ // arrayDistinct collapses the duplicate tag within each event before+ // counting: 1 per event, 2 total.+ expect(+ Number(findFilterOption(rows, "traceTags", "alpha")?.count),+ ).toBe(2);+ // calledToolNames is not distinct, so repeated calls within one event+ // all count: 2 + 1 = 3.+ expect(+ Number(findFilterOption(rows, "calledToolNames", "lookup")?.count),+ ).toBe(3);+ });+ });++ it("suppresses empty boolean buckets in exact facets", async () => {+ const uniqueProjectId = randomUUID();+ const nowMicro = Date.now() * 1000;++ // Every event is a root observation, so the opposite bucket is empty.+ await createEventsCh([+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ parent_span_id: "",+ is_app_root: true,+ start_time: nowMicro,+ event_ts: nowMicro,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ project_id: uniqueProjectId,+ trace_id: randomUUID(),+ type: "SPAN",+ parent_span_id: "",+ is_app_root: true,+ start_time: nowMicro + 1_000,+ event_ts: nowMicro + 1_000,+ }),+ ]);++ await waitForExpect(async () => {+ const rows = await getEventsExactFilterOptionsForColumns({+ projectId: uniqueProjectId,+ filter: [],+ columns: ["isRootObservation", "hasParentObservation"],+ });++ expect(+ Number(findFilterOption(rows, "isRootObservation", "true")?.count),+ ).toBe(2);+ expect(+ findFilterOption(rows, "isRootObservation", "false"),+ ).toBeUndefined();+ expect(+ Number(+ findFilterOption(rows, "hasParentObservation", "false")?.count,+ ),+ ).toBe(2);+ expect(+ findFilterOption(rows, "hasParentObservation", "true"),+ ).toBeUndefined();+ });+ });+it("executes the exact single-column array filter option query with paging", async () => {const uniqueProjectId = randomUUID();const nowMicro = Date.now() * 1000;diff --git a/web/src/__tests__/server/scores-trpc.servertest.ts b/web/src/__tests__/server/scores-trpc.servertest.tsindex d84d87117e..a4d06ba36d 100644--- a/web/src/__tests__/server/scores-trpc.servertest.ts+++ b/web/src/__tests__/server/scores-trpc.servertest.ts@@ -1,15 +1,11 @@const {mockAddScoreDelete,mockAddBatchAction,- mockGetEventsGroupedByTraceTags,- mockGetEventsGroupedByTraceName,- mockGetEventsGroupedByUserId,+ mockGetEventsExactFilterOptionsForColumns,} = vi.hoisted(() => ({mockAddScoreDelete: vi.fn(),mockAddBatchAction: vi.fn(),- mockGetEventsGroupedByTraceTags: vi.fn(async () => []),- mockGetEventsGroupedByTraceName: vi.fn(async () => []),- mockGetEventsGroupedByUserId: vi.fn(async () => []),+ mockGetEventsExactFilterOptionsForColumns: vi.fn(async () => []),}));vi.mock("@langfuse/shared/src/server", async () => {@@ -26,9 +22,8 @@ vi.mock("@langfuse/shared/src/server", async () => {add: mockAddBatchAction,})),},- getEventsGroupedByTraceTags: mockGetEventsGroupedByTraceTags,- getEventsGroupedByTraceName: mockGetEventsGroupedByTraceName,- getEventsGroupedByUserId: mockGetEventsGroupedByUserId,+ getEventsExactFilterOptionsForColumns:+ mockGetEventsExactFilterOptionsForColumns,};});@@ -73,9 +68,7 @@ describe("scores trpc", () => {orgId = setup.orgId;mockAddScoreDelete.mockClear();mockAddBatchAction.mockClear();- mockGetEventsGroupedByTraceTags.mockClear();- mockGetEventsGroupedByTraceName.mockClear();- mockGetEventsGroupedByUserId.mockClear();+ mockGetEventsExactFilterOptionsForColumns.mockClear();const session: Session = {expires: "1",diff --git a/web/src/server/api/routers/scores.ts b/web/src/server/api/routers/scores.tsindex 8c3291f973..4c39039d84 100644--- a/web/src/server/api/routers/scores.ts+++ b/web/src/server/api/routers/scores.ts@@ -45,9 +45,7 @@ import {getTracesGroupedByTags,getTracesGroupedByName,getTracesGroupedByUsers,- getEventsGroupedByTraceName,- getEventsGroupedByTraceTags,- getEventsGroupedByUserId,+ getEventsExactFilterOptionsForColumns,tracesTableUiColumnDefinitions,upsertScore,logger,@@ -426,32 +424,34 @@ export const scoresRouter = createTRPCRouter({toTime: upperBound,};- const [names, tags, traceNames, userIds, stringValues] =- await Promise.all([- getScoreNames(input.projectId, timestampFilter ?? []),- getEventsGroupedByTraceTags(input.projectId, eventsFilter, {- scope,- }),- getEventsGroupedByTraceName(input.projectId, eventsFilter, {- scope,- }),- getEventsGroupedByUserId(input.projectId, eventsFilter, {- scope,- }),- getScoreStringValues(input.projectId, timestampFilter ?? []),- ]);+ const [names, eventFacets, stringValues] = await Promise.all([+ getScoreNames(input.projectId, timestampFilter ?? []),+ getEventsExactFilterOptionsForColumns({+ projectId: input.projectId,+ filter: eventsFilter,+ columns: ["traceTags", "traceName", "userId"],+ scope,+ }),+ getScoreStringValues(input.projectId, timestampFilter ?? []),+ ]);return {name: names.map((i) => ({ value: i.name, count: i.count })),- tags: tags.map((t) => ({ value: t.tag })),- traceName: traceNames.map((tn) => ({- value: tn.traceName,- count: Number(tn.count),- })),- userId: userIds.map((u) => ({- value: u.userId,- count: Number(u.count),- })),+ tags: eventFacets+ .filter((row) => row.column === "traceTags")+ .map((row) => ({ value: row.value })),+ traceName: eventFacets+ .filter((row) => row.column === "traceName")+ .map((row) => ({+ value: row.value,+ count: Number(row.count),+ })),+ userId: eventFacets+ .filter((row) => row.column === "userId")+ .map((row) => ({+ value: row.value,+ count: Number(row.count),+ })),stringValue: stringValues,booleanValue: BOOLEAN_SCORE_VALUE_OPTIONS,};