🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): unify item score filters across levels (#16781)
Nikita Kabardin48369170ccf0
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.tsindex 403ffebe71..1248ee8576 100644--- a/packages/shared/src/server/repositories/experiments.ts+++ b/packages/shared/src/server/repositories/experiments.ts@@ -22,7 +22,6 @@ import {eventsExperimentsForItems,eventsExperiments,eventsExperimentsAggregation,- eventsScoresAggregation,eventsTracesScoresAggregation,scoreBooleansAggregation,} from "../queries/clickhouse-sql/query-fragments";@@ -213,6 +212,85 @@ const experimentScoreCTE = (params: {.buildWithParams();};+/**+ * Per-item score arrays that span BOTH levels: scores recorded on the item's+ * root span and scores recorded on its trace. Keyed by the root span id, which+ * is unique per (experiment, item) - so a join on it scopes to one experiment's+ * run of that item without any experiment predicate of its own.+ *+ * The level is kept in the inner GROUP BY, giving one array entry per+ * (name, level). Score filters compile to array-existence checks, so a positive+ * filter then matches at either level and its negation means "at neither" - the+ * same trick the runs aggregate uses.+ */+const experimentItemScoreCTE = (params: {+ projectId: string;+ startTimeFrom?: string | null;+ itemRootsCTE: CTEWithSchema;+}) => {+ const joinedItemScores = new CTEQueryBuilder()+ .withCTE("item_roots", {+ ...params.itemRootsCTE,+ })+ .withCTE("unit_scores", {+ ...buildScoresCTE({+ projectId: params.projectId,+ startTimeFrom: params.startTimeFrom,+ level: "any",+ }),+ })+ .from("item_roots", "ir")+ .innerJoin(+ "unit_scores",+ "us",+ // Observation-level scores count only when they sit on the item's ROOT+ // span (the pre-existing scope); trace-level ones carry no observation.+ "ON us.project_id = ir.project_id AND us.trace_id = ir.trace_id AND (us.observation_id = ir.root_span_id OR us.observation_id IS NULL)",+ )+ .select(+ "ir.project_id AS project_id",+ "ir.root_span_id AS root_span_id",+ "us.name AS name",+ "us.data_type AS data_type",+ "us.string_value AS string_value",+ "avg(us.avg_value) AS item_avg",+ )+ .groupBy(+ "ir.project_id",+ "ir.root_span_id",+ "us.name",+ "us.data_type",+ "us.string_value",+ // The level discriminator - see the header. Grouped but not selected: the+ // tuples only need to be distinct per level, not to name it.+ "us.observation_id IS NULL",+ )+ .buildWithParams();++ return new CTEQueryBuilder()+ .withCTE("item_scores", {+ ...joinedItemScores,+ schema: [+ "project_id",+ "root_span_id",+ "name",+ "data_type",+ "string_value",+ "item_avg",+ ],+ })+ .from("item_scores", "sc")+ .select(+ "sc.project_id AS project_id",+ "sc.root_span_id AS root_span_id",+ "groupArrayIf(tuple(sc.name, sc.item_avg, sc.data_type, sc.string_value), sc.data_type IN ('NUMERIC', 'BOOLEAN')) AS scores_avg",+ "groupArrayIf(concat(sc.name, ':', sc.string_value), sc.data_type = 'CATEGORICAL' AND notEmpty(sc.string_value)) AS score_categories",+ `${scoreBooleansAggregation("sc.")} AS score_booleans`,+ )+ .groupBy("sc.project_id", "sc.root_span_id")+ .buildWithParams();+};+export const getExperimentsCountFromEvents = async (props: {projectId: string;filter: FilterState;@@ -909,7 +987,7 @@ type QualificationPlan = {where: { query: string; params: Record<string, any> };having: { query: string; params: Record<string, any> } | null;orderBy: string | null;- hasScoreFilters: boolean;+ hasAgnosticScoreFilters: boolean;hasTraceScoreFilters: boolean;};@@ -978,10 +1056,17 @@ const buildQualificationPlan = (});const filters = filterByExperiment.flatMap((f) => f.filters);- const hasScoreFilters = filters.some((f) =>- ["obs_scores_avg", "obs_score_categories", "obs_score_booleans"].includes(- f.column,- ),+ // The canonical ids and their `obs_*` aliases both resolve to the+ // level-agnostic aggregate, so both mount the same CTE.+ const hasAgnosticScoreFilters = filters.some((f) =>+ [+ "scores_avg",+ "score_categories",+ "score_booleans",+ "obs_scores_avg",+ "obs_score_categories",+ "obs_score_booleans",+ ].includes(f.column),);const hasTraceScoreFilters = filters.some((f) =>[@@ -1025,7 +1110,7 @@ const buildQualificationPlan = (}: null,orderBy: `ORDER BY e.experiment_item_id ASC`,- hasScoreFilters,+ hasAgnosticScoreFilters,hasTraceScoreFilters,};};@@ -1056,13 +1141,31 @@ const getExperimentItemsFromEventsGeneric = (params: {offset,} = params;- const { where, having, orderBy, hasScoreFilters, hasTraceScoreFilters } =- buildQualificationPlan({- baseExperimentId,- compExperimentIds,- filterByExperiment,- config,- });+ const {+ where,+ having,+ orderBy,+ hasAgnosticScoreFilters,+ hasTraceScoreFilters,+ } = buildQualificationPlan({+ baseExperimentId,+ compExperimentIds,+ filterByExperiment,+ config,+ });++ // The item roots the agnostic score aggregate is keyed by. Scoped to the+ // experiments in play so the CTE never scans the whole project.+ const itemRoots = eventsExperimentsRootSpans({+ projectId,+ experimentIds: [+ ...(baseExperimentId ? [baseExperimentId] : []),+ ...compExperimentIds,+ ],+ })+ .selectRaw("e.project_id", "e.span_id AS root_span_id", "e.trace_id")+ .limitBy("e.project_id", "e.span_id")+ .buildWithParams();const queryBuilder = new EventsAggQueryBuilder({projectId,@@ -1073,17 +1176,23 @@ const getExperimentItemsFromEventsGeneric = (params: {"e.experiment_item_id as item_id, min(e.start_time) as start_time",}).whereRaw("e.span_id = e.experiment_item_root_span_id")- .when(hasScoreFilters, (b) =>+ .when(hasAgnosticScoreFilters, (b) =>b.withCTE(- "scores_agg",- // Optionally add timestamp >= oldest_selected_experiment_start as a coarse partition prune- eventsScoresAggregation({+ "item_scores_agg",+ experimentItemScoreCTE({projectId,+ itemRootsCTE: {+ ...itemRoots,+ schema: ["project_id", "root_span_id", "trace_id"],+ },}),),)- .when(hasScoreFilters, (b) =>- b.leftJoin("scores_agg AS s", "ON s.observation_id = e.span_id"),+ .when(hasAgnosticScoreFilters, (b) =>+ b.leftJoin(+ "item_scores_agg AS ias",+ "ON ias.project_id = e.project_id AND ias.root_span_id = e.span_id",+ ),).when(hasTraceScoreFilters, (b) =>b.withCTE(diff --git a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.tsindex aee0c02510..2027ccb762 100644--- a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts+++ b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts@@ -2,22 +2,29 @@ import { UiColumnMappings } from "../../tableDefinitions";export const experimentItemsTableNativeUiColumnDefinitions: UiColumnMappings = [{- uiTableName: "Scores (numeric)",- uiTableId: "obs_scores_avg",+ uiTableName: "Numeric Scores",+ uiTableId: "scores_avg",clickhouseTableName: "scores",- clickhouseSelect: "s.scores_avg",+ // The level-agnostic aggregate: one array entry per (name, level), so a+ // filter matches a score recorded on the item's root span OR on its trace.+ // The `obs_*` ids are aliases so existing links and saved views keep+ // resolving - and start matching trace-level scores, which is the fix.+ clickhouseSelect: "ias.scores_avg",+ aliases: ["obs_scores_avg"],},{- uiTableName: "Scores (categorical)",- uiTableId: "obs_score_categories",+ uiTableName: "Categorical Scores",+ uiTableId: "score_categories",clickhouseTableName: "scores",- clickhouseSelect: "s.score_categories",+ clickhouseSelect: "ias.score_categories",+ aliases: ["obs_score_categories"],},{- uiTableName: "Scores (boolean)",- uiTableId: "obs_score_booleans",+ uiTableName: "Boolean Scores",+ uiTableId: "score_booleans",clickhouseTableName: "scores",- clickhouseSelect: "s.score_booleans",+ clickhouseSelect: "ias.score_booleans",+ aliases: ["obs_score_booleans"],},{uiTableName: "Trace Scores (numeric)",diff --git a/web/src/__tests__/server/experiment-score-levels.servertest.ts b/web/src/__tests__/server/experiment-score-levels.servertest.tsindex 667e9446d3..565be5e803 100644--- a/web/src/__tests__/server/experiment-score-levels.servertest.ts+++ b/web/src/__tests__/server/experiment-score-levels.servertest.ts@@ -6,6 +6,8 @@ import {createScoresCh,getExperimentsFromEvents,getExperimentScoreOptions,+ getExperimentItemsCountFromEvents,+ getExperimentItemsFilterOptions,} from "@langfuse/shared/src/server";import { type FilterState } from "@langfuse/shared";import { randomUUID } from "crypto";@@ -75,7 +77,7 @@ const seedExperimentWithScore = async ({}),]);- return { experimentId, experimentName };+ return { experimentId, experimentName, spanId, traceId };};maybeEventTables("level-agnostic experiment score filters", () => {@@ -268,3 +270,213 @@ maybeEventTables("level-agnostic experiment score filters", () => {expect(options.score_name_levels_numeric[traceOnly]).toEqual(["trace"]);});});++maybeEventTables("level-agnostic experiment ITEM score filters", () => {+ const filterFor = (+ experimentId: string,+ filters: FilterState,+ ): { experimentId: string; filters: FilterState }[] => [+ { experimentId, filters },+ ];++ it("qualifies an item whose score sits at either level", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const startTimeMs = Date.now();+ const scoreName = `item-accuracy-${randomUUID().slice(0, 8)}`;++ const observationLevel = await seedExperimentWithScore({+ projectId,+ experimentName: "item-obs-level",+ startTimeMs,+ level: "observation",+ scoreName,+ value: 0.9,+ });+ const traceLevel = await seedExperimentWithScore({+ projectId,+ experimentName: "item-trace-level",+ startTimeMs,+ level: "trace",+ scoreName,+ value: 0.9,+ });+ const belowThreshold = await seedExperimentWithScore({+ projectId,+ experimentName: "item-low-score",+ startTimeMs,+ level: "trace",+ scoreName,+ value: 0.1,+ });++ const above: FilterState = [+ {+ type: "numberObject",+ column: "scores_avg",+ key: scoreName,+ operator: ">",+ value: 0.5,+ },+ ];++ // Each experiment holds exactly one item, so the count IS "did the item+ // qualify" — separating the levels rather than relying on one mixed run.+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [observationLevel.experimentId],+ filterByExperiment: filterFor(observationLevel.experimentId, above),+ }),+ ).resolves.toBe(1);+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [traceLevel.experimentId],+ filterByExperiment: filterFor(traceLevel.experimentId, above),+ }),+ ).resolves.toBe(1);+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [belowThreshold.experimentId],+ filterByExperiment: filterFor(belowThreshold.experimentId, above),+ }),+ ).resolves.toBe(0);+ });++ it("keeps qualifying through the legacy obs_scores_avg column", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const startTimeMs = Date.now();+ const scoreName = `item-legacy-${randomUUID().slice(0, 8)}`;++ const run = await seedExperimentWithScore({+ projectId,+ experimentName: "item-legacy-obs",+ startTimeMs,+ level: "observation",+ scoreName,+ value: 0.9,+ });++ // An unresolved column is dropped silently, which would let the item+ // qualify for the wrong reason — so assert the threshold still bites.+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [run.experimentId],+ filterByExperiment: filterFor(run.experimentId, [+ {+ type: "numberObject",+ column: "obs_scores_avg",+ key: scoreName,+ operator: ">",+ value: 0.5,+ },+ ]),+ }),+ ).resolves.toBe(1);+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [run.experimentId],+ filterByExperiment: filterFor(run.experimentId, [+ {+ type: "numberObject",+ column: "obs_scores_avg",+ key: scoreName,+ operator: ">",+ value: 0.95,+ },+ ]),+ }),+ ).resolves.toBe(0);+ });++ it("excludes a categorical value at whichever level it sits", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const startTimeMs = Date.now();+ const scoreName = `item-verdict-${randomUUID().slice(0, 8)}`;++ const traceLevel = await seedExperimentWithScore({+ projectId,+ experimentName: "item-trace-verdict",+ startTimeMs,+ level: "trace",+ scoreName,+ value: 0,+ stringValue: "bad",+ dataType: "CATEGORICAL",+ });+ const kept = await seedExperimentWithScore({+ projectId,+ experimentName: "item-kept-verdict",+ startTimeMs,+ level: "observation",+ scoreName,+ value: 0,+ stringValue: "good",+ dataType: "CATEGORICAL",+ });++ const excludeBad: FilterState = [+ {+ type: "categoryOptions",+ column: "score_categories",+ key: scoreName,+ operator: "none of",+ value: ["bad"],+ },+ ];++ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [traceLevel.experimentId],+ filterByExperiment: filterFor(traceLevel.experimentId, excludeBad),+ }),+ ).resolves.toBe(0);+ await expect(+ getExperimentItemsCountFromEvents({+ projectId,+ compExperimentIds: [kept.experimentId],+ filterByExperiment: filterFor(kept.experimentId, excludeBad),+ }),+ ).resolves.toBe(1);+ });++ it("offers each item score name once, tagged with its levels", async () => {+ const { projectId } = await createOrgProjectAndApiKey();+ const startTimeMs = Date.now();+ const bothLevels = `item-both-${randomUUID().slice(0, 8)}`;++ const first = await seedExperimentWithScore({+ projectId,+ experimentName: "item-options-obs",+ startTimeMs,+ level: "observation",+ scoreName: bothLevels,+ value: 0.5,+ });+ const second = await seedExperimentWithScore({+ projectId,+ experimentName: "item-options-trace",+ startTimeMs,+ level: "trace",+ scoreName: bothLevels,+ value: 0.5,+ });++ const options = await getExperimentItemsFilterOptions({+ projectId,+ experimentIds: [first.experimentId, second.experimentId],+ });++ expect(options.scores_avg.filter((name) => name === bothLevels)).toEqual([+ bothLevels,+ ]);+ expect(options.score_name_levels_numeric[bothLevels]).toEqual([+ "observation",+ "trace",+ ]);+ });+});diff --git a/web/src/__tests__/table-view-presets.clienttest.ts b/web/src/__tests__/table-view-presets.clienttest.tsindex 3b0d2519e4..f26f9ab2d2 100644--- a/web/src/__tests__/table-view-presets.clienttest.ts+++ b/web/src/__tests__/table-view-presets.clienttest.ts@@ -10,6 +10,7 @@ import {type OrderByState,} from "@langfuse/shared";import { experimentsFilterConfig } from "@/src/features/experiments/components/table/filter-config";+import { experimentItemsFilterConfig } from "@/src/features/experiments/config/experiment-items-filter-config";// Mock data for testingconst mockColumns = [@@ -297,3 +298,56 @@ describe("saved views built before the experiment score facets were unified", ()]);});});++describe("saved views built before the experiment ITEM score facets were unified", () => {+ it.each([+ ["obs_scores_avg", "scores_avg"],+ ["Scores (numeric)", "scores_avg"],+ ["obs_score_categories", "score_categories"],+ ["Scores (categorical)", "score_categories"],+ ["obs_score_booleans", "score_booleans"],+ ["Scores (boolean)", "score_booleans"],+ ] as const)("resolves %s onto %s", (stored, canonical) => {+ const validated = validateFilters(+ [+ {+ type: "numberObject",+ column: stored,+ key: "accuracy",+ operator: ">",+ value: 0.5,+ },+ ],+ experimentItemsFilterConfig.columnDefinitions,+ );++ expect(validated.map((filter) => filter.column)).toEqual([canonical]);+ });++ it("keeps both levels when a saved view carries one of each", () => {+ const validated = validateFilters(+ [+ {+ type: "numberObject",+ column: "Scores (numeric)",+ key: "accuracy",+ operator: ">",+ value: 0.5,+ },+ {+ type: "numberObject",+ column: "trace_scores_avg",+ key: "nps",+ operator: ">",+ value: 5,+ },+ ],+ experimentItemsFilterConfig.columnDefinitions,+ );++ expect(validated.map((filter) => filter.column)).toEqual([+ "scores_avg",+ "trace_scores_avg",+ ]);+ });+});diff --git a/web/src/features/experiments/config/experiment-items-filter-config.ts b/web/src/features/experiments/config/experiment-items-filter-config.tsindex d7319fb544..59391a5d83 100644--- a/web/src/features/experiments/config/experiment-items-filter-config.ts+++ b/web/src/features/experiments/config/experiment-items-filter-config.ts@@ -63,26 +63,35 @@ export const experimentItemsTableCols: ColumnDefinition[] = [internal: "latency_ms",nullable: true,},- {- name: "Scores (numeric)",- id: "obs_scores_avg",+ // Level-agnostic scores: one filter per data type that matches a score+ // whether it was recorded on the item's root span or on its trace. The+ // `obs_*` ids are aliases so existing links and saved views keep resolving -+ // and start matching trace-level scores, which is the fix. The old DISPLAY+ // names are aliases too: a saved view may store a column by its label, and+ // `validateFilters` drops what it cannot resolve.+ {+ name: "Numeric Scores",+ id: "scores_avg",type: "numberObject",- internal: "obs_scores_avg",+ internal: "scores_avg",+ aliases: ["obs_scores_avg", "Scores (numeric)"],},{- name: "Scores (categorical)",- id: "obs_score_categories",+ name: "Categorical Scores",+ id: "score_categories",type: "categoryOptions",- internal: "obs_score_categories",+ internal: "score_categories",options: [],nullable: true,+ aliases: ["obs_score_categories", "Scores (categorical)"],},{- name: "Scores (boolean)",- id: "obs_score_booleans",+ name: "Boolean Scores",+ id: "score_booleans",type: "booleanObject",- internal: "obs_score_booleans",+ internal: "score_booleans",nullable: true,+ aliases: ["obs_score_booleans", "Scores (boolean)"],},{name: "Trace Scores (numeric)",@@ -154,33 +163,18 @@ export const experimentItemsFilterConfig: FilterConfig = {},{type: "keyValue" as const,- column: "obs_score_categories",- label: getExperimentItemsColumnName("obs_score_categories"),- },- {- type: "numericKeyValue" as const,- column: "obs_scores_avg",- label: getExperimentItemsColumnName("obs_scores_avg"),- },- {- type: "booleanKeyValue" as const,- column: "obs_score_booleans",- label: getExperimentItemsColumnName("obs_score_booleans"),- },- {- type: "keyValue" as const,- column: "trace_score_categories",- label: getExperimentItemsColumnName("trace_score_categories"),+ column: "score_categories",+ label: getExperimentItemsColumnName("score_categories"),},{type: "numericKeyValue" as const,- column: "trace_scores_avg",- label: getExperimentItemsColumnName("trace_scores_avg"),+ column: "scores_avg",+ label: getExperimentItemsColumnName("scores_avg"),},{type: "booleanKeyValue" as const,- column: "trace_score_booleans",- label: getExperimentItemsColumnName("trace_score_booleans"),+ column: "score_booleans",+ label: getExperimentItemsColumnName("score_booleans"),},],};diff --git a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.tsindex 9b2d922f73..2307e23a1c 100644--- a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts+++ b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts@@ -20,9 +20,9 @@ const processCategoricalScoreOptions = ();/**- * Hook to fetch experiment item filter options (scores) scoped to specific experiment IDs.- * Returns score filter options for both observation-level and trace-level scores,- * plus full score column definitions for table column visibility.+ * Experiment item filter options (scores) scoped to specific experiment IDs.+ * Returns the level-agnostic score filter options the three facets offer, plus+ * the per-level score column definitions the table's column visibility uses.*/export const useExperimentItemsFilterOptions = ({projectId,@@ -45,26 +45,28 @@ export const useExperimentItemsFilterOptions = ({const transformedOptions = useMemo(() => {if (!filterOptions.data) {return {- obs_scores_avg: undefined,- obs_score_categories: undefined,- obs_score_booleans: undefined,- trace_scores_avg: undefined,- trace_score_categories: undefined,- trace_score_booleans: undefined,+ scores_avg: undefined,+ score_categories: undefined,+ score_booleans: undefined,+ score_name_levels_numeric: undefined,+ score_name_levels_categorical: undefined,+ score_name_levels_boolean: undefined,} satisfies ExperimentItemScoreFilterOptions;}+ // One entry per score name across both levels; the level maps tag each+ // offered name with where it exists. `useSidebarFilterState` looks the maps+ // up by these exact keys.return {- obs_scores_avg: filterOptions.data.obs_scores_avg,- obs_score_categories: processCategoricalScoreOptions(- filterOptions.data.obs_score_categories,+ scores_avg: filterOptions.data.scores_avg,+ score_categories: processCategoricalScoreOptions(+ filterOptions.data.score_categories,),- obs_score_booleans: filterOptions.data.obs_score_booleans,- trace_scores_avg: filterOptions.data.trace_scores_avg,- trace_score_categories: processCategoricalScoreOptions(- filterOptions.data.trace_score_categories,- ),- trace_score_booleans: filterOptions.data.trace_score_booleans,+ score_booleans: filterOptions.data.score_booleans,+ score_name_levels_numeric: filterOptions.data.score_name_levels_numeric,+ score_name_levels_categorical:+ filterOptions.data.score_name_levels_categorical,+ score_name_levels_boolean: filterOptions.data.score_name_levels_boolean,} satisfies ExperimentItemScoreFilterOptions;}, [filterOptions.data]);diff --git a/web/src/features/experiments/types/charts.ts b/web/src/features/experiments/types/charts.tsindex 6d146338a5..49d71579db 100644--- a/web/src/features/experiments/types/charts.ts+++ b/web/src/features/experiments/types/charts.ts@@ -14,13 +14,19 @@ export type ScoreFilterOptions = {experiment_score_booleans?: string[];};+type ScoreNameLevels = Record<string, ("observation" | "trace")[]>;++/**+ * What the three score facets offer, plus the level(s) each offered name exists+ * at so the picker can tag them.+ */export type ExperimentItemScoreFilterOptions = {- obs_scores_avg?: string[];- obs_score_categories?: Record<string, string[]>;- obs_score_booleans?: string[];- trace_scores_avg?: string[];- trace_score_categories?: Record<string, string[]>;- trace_score_booleans?: string[];+ scores_avg?: string[];+ score_categories?: Record<string, string[]>;+ score_booleans?: string[];+ score_name_levels_numeric?: ScoreNameLevels;+ score_name_levels_categorical?: ScoreNameLevels;+ score_name_levels_boolean?: ScoreNameLevels;};export type ScoreLevel = "obs" | "experiment";