🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(experiments): link the error count to failing items (#17243)
Nikita Kabardin3f60d2da04bf
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 94220e145f..3bcde13662 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -1956,7 +1956,9 @@ const EXPERIMENTS_AGGREGATION_FIELDS = {"nullIf(any(e.experiment_dataset_id), '') AS experiment_dataset_id",startTime: "min(e.start_time) AS start_time",itemCount: "uniq(e.experiment_item_id) AS item_count",- errorCount: "countIf(e.level = 'ERROR') AS error_count",+ // Distinct items that carry any ERROR event, so the number matches the+ // items-view Status=ERROR list the badge opens.+ errorCount: "uniqIf(e.experiment_item_id, e.level = 'ERROR') AS error_count",prompts:"groupUniqArrayIf(tuple(e.prompt_name, e.prompt_version), e.prompt_name != '') AS prompts",experimentMetadata:diff --git a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.tsindex 14e3fb8a53..e6684b687b 100644--- a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts@@ -4,6 +4,7 @@import {EventsAggregationQueryBuilder,+ EventsAggQueryBuilder,EventsQueryBuilder,EventsSessionAggregationQueryBuilder,ExperimentsAggregationFieldSetName,@@ -439,6 +440,33 @@ export const eventsExperimentsRootSpans = (params: {"e.experiment_item_root_span_id = e.span_id",);+/**+ * Worst observation level per experiment item, computed across the item's+ * full subtree. Filter tables before joining (query-join-filter-before):+ * scoped to the experiments in play so the CTE never scans the project.+ */+export const experimentItemLevelsAggregation = (params: {+ projectId: string;+ experimentIds: string[];+}): { query: string; params: Record<string, any> } =>+ new EventsAggQueryBuilder({+ projectId: params.projectId,+ groupByColumn: "e.experiment_id, e.experiment_item_id",+ selectExpression: `e.experiment_id AS experiment_id,+ e.experiment_item_id AS experiment_item_id,+ multiIf(+ countIf(e.level = 'ERROR') > 0, 'ERROR',+ countIf(e.level = 'WARNING') > 0, 'WARNING',+ countIf(e.level = 'DEFAULT') > 0, 'DEFAULT',+ 'DEBUG'+ ) AS aggregated_level`,+ })+ .whereRaw("e.experiment_id IN ({itemLevelExperimentIds: Array(String)})", {+ itemLevelExperimentIds: params.experimentIds,+ })+ .whereRaw("e.experiment_id != ''")+ .buildWithParams();+/*** Session-level scores aggregation CTE.* Groups scores by (project_id, session_id), computing numeric/boolean averagesdiff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.tsindex d1fb90fb1b..1c70ec08b8 100644--- a/packages/shared/src/server/repositories/experiments.ts+++ b/packages/shared/src/server/repositories/experiments.ts@@ -24,6 +24,7 @@ import {eventsExperiments,eventsExperimentsAggregation,eventsTracesScoresAggregation,+ experimentItemLevelsAggregation,scoreBooleansAggregation,} from "../queries/clickhouse-sql/query-fragments";import {@@ -984,12 +985,22 @@ type BuildQualificationPlanInput = {};};+const EXPERIMENT_ITEM_LEVEL_FILTER_COLUMNS = [+ "level",+ "Status",+ "Level",+] as const;++const isExperimentItemLevelFilter = (column: string) =>+ (EXPERIMENT_ITEM_LEVEL_FILTER_COLUMNS as readonly string[]).includes(column);+type QualificationPlan = {where: { query: string; params: Record<string, any> };having: { query: string; params: Record<string, any> } | null;orderBy: string | null;hasAgnosticScoreFilters: boolean;hasTraceScoreFilters: boolean;+ hasLevelFilters: boolean;};function combineConditions(@@ -1076,6 +1087,9 @@ const buildQualificationPlan = ("trace_score_booleans",].includes(f.column),);+ const hasLevelFilters = filters.some((f) =>+ isExperimentItemLevelFilter(f.column),+ );const allExperimentIds = [...(baseExperimentId ? [baseExperimentId] : []),@@ -1113,6 +1127,7 @@ const buildQualificationPlan = (orderBy: `ORDER BY e.experiment_item_id ASC`,hasAgnosticScoreFilters,hasTraceScoreFilters,+ hasLevelFilters,};};@@ -1148,6 +1163,7 @@ const getExperimentItemsFromEventsGeneric = (params: {orderBy,hasAgnosticScoreFilters,hasTraceScoreFilters,+ hasLevelFilters,} = buildQualificationPlan({baseExperimentId,compExperimentIds,@@ -1211,6 +1227,24 @@ const getExperimentItemsFromEventsGeneric = (params: {"ON ts.trace_id = e.trace_id AND ts.project_id = e.project_id",),)+ .when(hasLevelFilters, (b) =>+ b.withCTE(+ "item_levels",+ experimentItemLevelsAggregation({+ projectId,+ experimentIds: [+ ...(baseExperimentId ? [baseExperimentId] : []),+ ...compExperimentIds,+ ],+ }),+ ),+ )+ .when(hasLevelFilters, (b) =>+ b.leftJoin(+ "item_levels AS il",+ "ON il.experiment_id = e.experiment_id AND il.experiment_item_id = e.experiment_item_id",+ ),+ ).where(where).when(having !== null, (b) => b.having(having!));diff --git a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.tsindex 2027ccb762..1cadef7a55 100644--- a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts+++ b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts@@ -58,4 +58,13 @@ export const experimentItemsTableNativeUiColumnDefinitions: UiColumnMappings = [clickhouseSelect: "metadata",queryPrefix: "e",},+ {+ uiTableName: "Status",+ uiTableId: "level",+ aliases: ["Level"],+ clickhouseTableName: "events_proto",+ // Worst level across every event on the item, not just the root span.+ // An item whose children carry ERROR still matches Status=ERROR.+ clickhouseSelect: "il.aggregated_level",+ },];diff --git a/packages/shared/src/utils/productUrl.test.ts b/packages/shared/src/utils/productUrl.test.tsindex 56b4b417ee..3183c1fd07 100644--- a/packages/shared/src/utils/productUrl.test.ts+++ b/packages/shared/src/utils/productUrl.test.ts@@ -2,10 +2,41 @@ import { describe, expect, it } from "vitest";import { TableViewPresetTableName } from "../domain/table-view-presets";import {buildCurrentPageSavedViewPermalink,+ buildExperimentPath,parseSavedViewFromURL,tableViewPresetPermalinkUsesCurrentPath,} from "./productUrl";+describe("buildExperimentPath", () => {+ it("links a run to its items view", () => {+ expect(+ buildExperimentPath({+ projectId: "proj_1",+ experimentId: "exp_1",+ }),+ ).toBe("/project/proj_1/experiments/results?baseline=exp_1");+ });++ it("can pre-apply a shareable items filter", () => {+ expect(+ buildExperimentPath({+ projectId: "proj_1",+ experimentId: "exp_1",+ filters: [+ {+ column: "level",+ type: "stringOptions",+ operator: "any of",+ value: ["ERROR"],+ },+ ],+ }),+ ).toBe(+ "/project/proj_1/experiments/results?baseline=exp_1&filter=level%3BstringOptions%3B%3Bany+of%3BERROR",+ );+ });+});+describe("tableViewPresetPermalinkUsesCurrentPath", () => {it("is true only for session detail, which embeds a session id in the path", () => {expect(diff --git a/packages/shared/src/utils/productUrl.ts b/packages/shared/src/utils/productUrl.tsindex def6915796..de87127dd9 100644--- a/packages/shared/src/utils/productUrl.ts+++ b/packages/shared/src/utils/productUrl.ts@@ -192,9 +192,14 @@ export const buildExperimentsPath = (params: { projectId: string }) =>export const buildExperimentPath = (params: {projectId: string;experimentId: string;+ filters?: FilterState;}) =>appendProductPathQuery(`${buildExperimentsPath(params)}/results`, {baseline: params.experimentId,+ filter:+ params.filters && params.filters.length > 0+ ? encodeFiltersGeneric(params.filters)+ : undefined,});export const buildModelsPath = (params: { projectId: string }) =>diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex 1277bab234..442ab05bea 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -993,4 +993,17 @@ describe("ExperimentsAggregationQueryBuilder", () => {startTimeFrom: "2026-01-01 00:00:00.000",});});++ it("counts distinct items that carry an ERROR event", () => {+ const { query } = new ExperimentsAggregationQueryBuilder({+ projectId: "test-project",+ })+ .selectFieldSet("base")+ .whereRaw("e.experiment_id != ''")+ .buildWithParams();++ expect(query).toContain(+ "uniqIf(e.experiment_item_id, e.level = 'ERROR') AS error_count",+ );+ });});diff --git a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.tsindex de810dc94f..f00b303755 100644--- a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts@@ -1473,6 +1473,84 @@ describe("Clickhouse Experiment Items Repository Test", () => {});});+ maybe("getExperimentItemsFromEvents - Level Filtering", () => {+ it("should include items that carry an ERROR event and exclude items that do not", async () => {+ const baselineExpId = randomUUID();+ const datasetId = randomUUID();+ const failingItemId = randomUUID();+ const cleanItemId = randomUUID();+ const failingTraceId = randomUUID();+ const cleanTraceId = randomUUID();+ const failingRootId = randomUUID();+ const failingChildId = randomUUID();+ const cleanRootId = randomUUID();+ const now = Date.now() * 1000;++ await createEventsCh([+ createExperimentEvent({+ project_id: projectId,+ trace_id: failingTraceId,+ span_id: failingRootId,+ experimentId: baselineExpId,+ experimentName: "baseline-exp",+ datasetId,+ itemId: failingItemId,+ experimentItemRootSpanId: failingRootId,+ level: "DEFAULT",+ start_time: now,+ }),+ createExperimentEvent({+ project_id: projectId,+ trace_id: failingTraceId,+ span_id: failingChildId,+ parent_span_id: failingRootId,+ experimentId: baselineExpId,+ experimentName: "baseline-exp",+ datasetId,+ itemId: failingItemId,+ experimentItemRootSpanId: failingRootId,+ level: "ERROR",+ start_time: now + 1,+ }),+ createExperimentEvent({+ project_id: projectId,+ trace_id: cleanTraceId,+ span_id: cleanRootId,+ experimentId: baselineExpId,+ experimentName: "baseline-exp",+ datasetId,+ itemId: cleanItemId,+ experimentItemRootSpanId: cleanRootId,+ level: "DEFAULT",+ start_time: now + 2,+ }),+ ]);++ const result = await getExperimentItemsFromEvents({+ projectId,+ baseExperimentId: baselineExpId,+ compExperimentIds: [],+ filterByExperiment: [+ {+ experimentId: baselineExpId,+ filters: [+ {+ column: "level",+ type: "stringOptions",+ operator: "any of",+ value: ["ERROR"],+ } as FilterCondition,+ ],+ },+ ],+ limit: 10,+ offset: 0,+ });++ expect(result.map((row) => row.itemId)).toEqual([failingItemId]);+ });+ });+maybe("getExperimentItemsBatchIO", () => {it("should fetch IO and truncate to specified length", async () => {// GIVEN: Item with long input/output stringsdiff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.tsindex 7358dbcdf6..c1526bf98f 100644--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts@@ -195,6 +195,91 @@ describe("Clickhouse Experiment Repository Test", () => {expect(experiment?.itemCount).toBe(2);});+ it("should count items that carry an ERROR event, not every ERROR event", async () => {+ const experimentId = randomUUID();+ const experimentName = "experiment-errors-" + randomUUID();+ const datasetId = randomUUID();+ const failingItemId = randomUUID();+ const cleanItemId = randomUUID();+ const failingRootId = randomUUID();+ const failingChildId = randomUUID();+ const cleanRootId = randomUUID();+ const now = Date.now() * 1000;++ await createEventsCh([+ createEvent({+ id: failingRootId,+ span_id: failingRootId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "SPAN",+ experiment_id: experimentId,+ experiment_name: experimentName,+ experiment_dataset_id: datasetId,+ experiment_item_id: failingItemId,+ experiment_item_root_span_id: failingRootId,+ level: "DEFAULT",+ start_time: now,+ }),+ createEvent({+ id: failingChildId,+ span_id: failingChildId,+ parent_span_id: failingRootId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "SPAN",+ experiment_id: experimentId,+ experiment_name: experimentName,+ experiment_dataset_id: datasetId,+ experiment_item_id: failingItemId,+ experiment_item_root_span_id: failingRootId,+ level: "ERROR",+ start_time: now + 1,+ }),+ createEvent({+ id: randomUUID(),+ span_id: randomUUID(),+ parent_span_id: failingRootId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "SPAN",+ experiment_id: experimentId,+ experiment_name: experimentName,+ experiment_dataset_id: datasetId,+ experiment_item_id: failingItemId,+ experiment_item_root_span_id: failingRootId,+ level: "ERROR",+ start_time: now + 2,+ }),+ createEvent({+ id: cleanRootId,+ span_id: cleanRootId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "SPAN",+ experiment_id: experimentId,+ experiment_name: experimentName,+ experiment_dataset_id: datasetId,+ experiment_item_id: cleanItemId,+ experiment_item_root_span_id: cleanRootId,+ level: "DEFAULT",+ start_time: now + 3,+ }),+ ]);++ const result = await getExperimentsFromEvents({+ projectId,+ filter: [],+ limit: 1000,+ page: 0,+ });++ const experiment = result.find((e) => e.id === experimentId);+ expect(experiment).toBeDefined();+ expect(experiment?.itemCount).toBe(2);+ expect(experiment?.errorCount).toBe(1);+ });+it("should order by startTime DESC", async () => {const experimentId1 = randomUUID();const experimentName1 = "experiment-1-" + randomUUID();diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsxindex 21ef7d21ba..250115d804 100644--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx@@ -34,6 +34,7 @@ import {BatchExportTableName,ActionId,BatchActionType,+ buildExperimentPath,} from "@langfuse/shared";import { numberFormatter } from "@/src/utils/numbers";import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";@@ -615,13 +616,42 @@ export default function ExperimentsTable({size: 100,cell: ({ row }) => {const value: number = row.getValue("errorCount");+ const formatted = numberFormatter(value, 0);++ if (value <= 0) {+ return (+ <Badge+ variant="secondary"+ className="max-w-fit rounded-sm px-1 font-normal"+ >+ {formatted}+ </Badge>+ );+ }+return (- <Badge- variant={value > 0 ? "destructive" : "secondary"}- className="max-w-fit rounded-sm px-1 font-normal"+ <Link+ href={buildExperimentPath({+ projectId,+ experimentId: row.id,+ filters: [+ {+ column: "level",+ type: "stringOptions",+ operator: "any of",+ value: ["ERROR"],+ },+ ],+ })}+ aria-label={`View ${formatted} failing items`}>- {numberFormatter(value, 0)}- </Badge>+ <Badge+ variant="destructive"+ className="hover:bg-destructive/80 max-w-fit cursor-pointer rounded-sm px-1 font-normal"+ >+ {formatted}+ </Badge>+ </Link>);},enableHiding: true,diff --git a/web/src/features/experiments/config/experiment-items-filter-config.ts b/web/src/features/experiments/config/experiment-items-filter-config.tsindex 1d84c40813..5778ed5bd1 100644--- a/web/src/features/experiments/config/experiment-items-filter-config.ts+++ b/web/src/features/experiments/config/experiment-items-filter-config.ts@@ -1,4 +1,5 @@import type { FilterConfig } from "@/src/features/filters";+import { renderLevelIcon } from "@/src/components/level-colors";import type { ColumnDefinition, ObservationLevelType } from "@langfuse/shared";/**@@ -151,6 +152,12 @@ export const experimentItemsFilterConfig: FilterConfig = {columnDefinitions: experimentItemsTableCols,facets: [+ {+ type: "categorical" as const,+ column: "level",+ label: getExperimentItemsColumnName("level"),+ renderIcon: renderLevelIcon,+ },{type: "stringKeyValue" as const,column: "itemMetadata",diff --git a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.tsindex 6cabc7cf64..652d3f2156 100644--- a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts+++ b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts@@ -8,6 +8,8 @@ export type ScoreColumnDef = {source: string;};+const EXPERIMENT_ITEM_LEVEL_OPTIONS = ["DEBUG", "DEFAULT", "WARNING", "ERROR"];+const processCategoricalScoreOptions = (categories: Array<{ label: string; values: string[] }>,): Record<string, string[]> =>@@ -53,6 +55,7 @@ export const useExperimentItemsFilterOptions = ({score_name_levels_numeric: undefined,score_name_levels_categorical: undefined,score_name_levels_boolean: undefined,+ level: EXPERIMENT_ITEM_LEVEL_OPTIONS,} satisfies ExperimentItemScoreFilterOptions;}@@ -69,6 +72,7 @@ export const useExperimentItemsFilterOptions = ({score_name_levels_categorical:filterOptions.data.score_name_levels_categorical,score_name_levels_boolean: filterOptions.data.score_name_levels_boolean,+ level: EXPERIMENT_ITEM_LEVEL_OPTIONS,} satisfies ExperimentItemScoreFilterOptions;}, [filterOptions.data]);diff --git a/web/src/features/experiments/types/charts.ts b/web/src/features/experiments/types/charts.tsindex bb8d22b024..3ceeeac398 100644--- a/web/src/features/experiments/types/charts.ts+++ b/web/src/features/experiments/types/charts.ts@@ -48,6 +48,7 @@ export type ExperimentItemScoreFilterOptions = {score_name_levels_numeric?: ScoreNameLevels;score_name_levels_categorical?: ScoreNameLevels;score_name_levels_boolean?: ScoreNameLevels;+ level?: string[];};/**