Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(experiments): link the error count to failing items (#17243)

Nikita Kabardin3f60d2da04bf
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 94220e145f..3bcde13662 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -1956,7 +1956,9 @@ const EXPERIMENTS_AGGREGATION_FIELDS = {
"nullIf(any(e.experiment_dataset_id), '') AS experiment_dataset_id",
startTime: "min(e.start_time) AS start_time",
itemCount: "uniq(e.experiment_item_id) AS item_count",
- errorCount: "countIf(e.level = 'ERROR') AS error_count",
+ // Distinct items that carry any ERROR event, so the number matches the
+ // items-view Status=ERROR list the badge opens.
+ errorCount: "uniqIf(e.experiment_item_id, e.level = 'ERROR') AS error_count",
prompts:
"groupUniqArrayIf(tuple(e.prompt_name, e.prompt_version), e.prompt_name != '') AS prompts",
experimentMetadata:
diff --git a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
index 14e3fb8a53..e6684b687b 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
@@ -4,6 +4,7 @@
import {
EventsAggregationQueryBuilder,
+ EventsAggQueryBuilder,
EventsQueryBuilder,
EventsSessionAggregationQueryBuilder,
ExperimentsAggregationFieldSetName,
@@ -439,6 +440,33 @@ export const eventsExperimentsRootSpans = (params: {
"e.experiment_item_root_span_id = e.span_id",
);
+/**
+ * Worst observation level per experiment item, computed across the item's
+ * full subtree. Filter tables before joining (query-join-filter-before):
+ * scoped to the experiments in play so the CTE never scans the project.
+ */
+export const experimentItemLevelsAggregation = (params: {
+ projectId: string;
+ experimentIds: string[];
+}): { query: string; params: Record<string, any> } =>
+ new EventsAggQueryBuilder({
+ projectId: params.projectId,
+ groupByColumn: "e.experiment_id, e.experiment_item_id",
+ selectExpression: `e.experiment_id AS experiment_id,
+ e.experiment_item_id AS experiment_item_id,
+ multiIf(
+ countIf(e.level = 'ERROR') > 0, 'ERROR',
+ countIf(e.level = 'WARNING') > 0, 'WARNING',
+ countIf(e.level = 'DEFAULT') > 0, 'DEFAULT',
+ 'DEBUG'
+ ) AS aggregated_level`,
+ })
+ .whereRaw("e.experiment_id IN ({itemLevelExperimentIds: Array(String)})", {
+ itemLevelExperimentIds: params.experimentIds,
+ })
+ .whereRaw("e.experiment_id != ''")
+ .buildWithParams();
+
/**
* Session-level scores aggregation CTE.
* Groups scores by (project_id, session_id), computing numeric/boolean averages
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.ts
index d1fb90fb1b..1c70ec08b8 100644
--- a/packages/shared/src/server/repositories/experiments.ts
+++ b/packages/shared/src/server/repositories/experiments.ts
@@ -24,6 +24,7 @@ import {
eventsExperiments,
eventsExperimentsAggregation,
eventsTracesScoresAggregation,
+ experimentItemLevelsAggregation,
scoreBooleansAggregation,
} from "../queries/clickhouse-sql/query-fragments";
import {
@@ -984,12 +985,22 @@ type BuildQualificationPlanInput = {
};
};
+const EXPERIMENT_ITEM_LEVEL_FILTER_COLUMNS = [
+ "level",
+ "Status",
+ "Level",
+] as const;
+
+const isExperimentItemLevelFilter = (column: string) =>
+ (EXPERIMENT_ITEM_LEVEL_FILTER_COLUMNS as readonly string[]).includes(column);
+
type QualificationPlan = {
where: { query: string; params: Record<string, any> };
having: { query: string; params: Record<string, any> } | null;
orderBy: string | null;
hasAgnosticScoreFilters: boolean;
hasTraceScoreFilters: boolean;
+ hasLevelFilters: boolean;
};
function combineConditions(
@@ -1076,6 +1087,9 @@ const buildQualificationPlan = (
"trace_score_booleans",
].includes(f.column),
);
+ const hasLevelFilters = filters.some((f) =>
+ isExperimentItemLevelFilter(f.column),
+ );
const allExperimentIds = [
...(baseExperimentId ? [baseExperimentId] : []),
@@ -1113,6 +1127,7 @@ const buildQualificationPlan = (
orderBy: `ORDER BY e.experiment_item_id ASC`,
hasAgnosticScoreFilters,
hasTraceScoreFilters,
+ hasLevelFilters,
};
};
@@ -1148,6 +1163,7 @@ const getExperimentItemsFromEventsGeneric = (params: {
orderBy,
hasAgnosticScoreFilters,
hasTraceScoreFilters,
+ hasLevelFilters,
} = buildQualificationPlan({
baseExperimentId,
compExperimentIds,
@@ -1211,6 +1227,24 @@ const getExperimentItemsFromEventsGeneric = (params: {
"ON ts.trace_id = e.trace_id AND ts.project_id = e.project_id",
),
)
+ .when(hasLevelFilters, (b) =>
+ b.withCTE(
+ "item_levels",
+ experimentItemLevelsAggregation({
+ projectId,
+ experimentIds: [
+ ...(baseExperimentId ? [baseExperimentId] : []),
+ ...compExperimentIds,
+ ],
+ }),
+ ),
+ )
+ .when(hasLevelFilters, (b) =>
+ b.leftJoin(
+ "item_levels AS il",
+ "ON il.experiment_id = e.experiment_id AND il.experiment_item_id = e.experiment_item_id",
+ ),
+ )
.where(where)
.when(having !== null, (b) => b.having(having!));
diff --git a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts
index 2027ccb762..1cadef7a55 100644
--- a/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts
+++ b/packages/shared/src/server/tableMappings/mapExperimentItemsTable.ts
@@ -58,4 +58,13 @@ export const experimentItemsTableNativeUiColumnDefinitions: UiColumnMappings = [
clickhouseSelect: "metadata",
queryPrefix: "e",
},
+ {
+ uiTableName: "Status",
+ uiTableId: "level",
+ aliases: ["Level"],
+ clickhouseTableName: "events_proto",
+ // Worst level across every event on the item, not just the root span.
+ // An item whose children carry ERROR still matches Status=ERROR.
+ clickhouseSelect: "il.aggregated_level",
+ },
];
diff --git a/packages/shared/src/utils/productUrl.test.ts b/packages/shared/src/utils/productUrl.test.ts
index 56b4b417ee..3183c1fd07 100644
--- a/packages/shared/src/utils/productUrl.test.ts
+++ b/packages/shared/src/utils/productUrl.test.ts
@@ -2,10 +2,41 @@ import { describe, expect, it } from "vitest";
import { TableViewPresetTableName } from "../domain/table-view-presets";
import {
buildCurrentPageSavedViewPermalink,
+ buildExperimentPath,
parseSavedViewFromURL,
tableViewPresetPermalinkUsesCurrentPath,
} from "./productUrl";
+describe("buildExperimentPath", () => {
+ it("links a run to its items view", () => {
+ expect(
+ buildExperimentPath({
+ projectId: "proj_1",
+ experimentId: "exp_1",
+ }),
+ ).toBe("/project/proj_1/experiments/results?baseline=exp_1");
+ });
+
+ it("can pre-apply a shareable items filter", () => {
+ expect(
+ buildExperimentPath({
+ projectId: "proj_1",
+ experimentId: "exp_1",
+ filters: [
+ {
+ column: "level",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["ERROR"],
+ },
+ ],
+ }),
+ ).toBe(
+ "/project/proj_1/experiments/results?baseline=exp_1&filter=level%3BstringOptions%3B%3Bany+of%3BERROR",
+ );
+ });
+});
+
describe("tableViewPresetPermalinkUsesCurrentPath", () => {
it("is true only for session detail, which embeds a session id in the path", () => {
expect(
diff --git a/packages/shared/src/utils/productUrl.ts b/packages/shared/src/utils/productUrl.ts
index def6915796..de87127dd9 100644
--- a/packages/shared/src/utils/productUrl.ts
+++ b/packages/shared/src/utils/productUrl.ts
@@ -192,9 +192,14 @@ export const buildExperimentsPath = (params: { projectId: string }) =>
export const buildExperimentPath = (params: {
projectId: string;
experimentId: string;
+ filters?: FilterState;
}) =>
appendProductPathQuery(`${buildExperimentsPath(params)}/results`, {
baseline: params.experimentId,
+ filter:
+ params.filters && params.filters.length > 0
+ ? encodeFiltersGeneric(params.filters)
+ : undefined,
});
export const buildModelsPath = (params: { projectId: string }) =>
diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.ts
index 1277bab234..442ab05bea 100644
--- a/web/src/__tests__/server/event-query-builder.servertest.ts
+++ b/web/src/__tests__/server/event-query-builder.servertest.ts
@@ -993,4 +993,17 @@ describe("ExperimentsAggregationQueryBuilder", () => {
startTimeFrom: "2026-01-01 00:00:00.000",
});
});
+
+ it("counts distinct items that carry an ERROR event", () => {
+ const { query } = new ExperimentsAggregationQueryBuilder({
+ projectId: "test-project",
+ })
+ .selectFieldSet("base")
+ .whereRaw("e.experiment_id != ''")
+ .buildWithParams();
+
+ expect(query).toContain(
+ "uniqIf(e.experiment_item_id, e.level = 'ERROR') AS error_count",
+ );
+ });
});
diff --git a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
index de810dc94f..f00b303755 100644
--- a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
@@ -1473,6 +1473,84 @@ describe("Clickhouse Experiment Items Repository Test", () => {
});
});
+ maybe("getExperimentItemsFromEvents - Level Filtering", () => {
+ it("should include items that carry an ERROR event and exclude items that do not", async () => {
+ const baselineExpId = randomUUID();
+ const datasetId = randomUUID();
+ const failingItemId = randomUUID();
+ const cleanItemId = randomUUID();
+ const failingTraceId = randomUUID();
+ const cleanTraceId = randomUUID();
+ const failingRootId = randomUUID();
+ const failingChildId = randomUUID();
+ const cleanRootId = randomUUID();
+ const now = Date.now() * 1000;
+
+ await createEventsCh([
+ createExperimentEvent({
+ project_id: projectId,
+ trace_id: failingTraceId,
+ span_id: failingRootId,
+ experimentId: baselineExpId,
+ experimentName: "baseline-exp",
+ datasetId,
+ itemId: failingItemId,
+ experimentItemRootSpanId: failingRootId,
+ level: "DEFAULT",
+ start_time: now,
+ }),
+ createExperimentEvent({
+ project_id: projectId,
+ trace_id: failingTraceId,
+ span_id: failingChildId,
+ parent_span_id: failingRootId,
+ experimentId: baselineExpId,
+ experimentName: "baseline-exp",
+ datasetId,
+ itemId: failingItemId,
+ experimentItemRootSpanId: failingRootId,
+ level: "ERROR",
+ start_time: now + 1,
+ }),
+ createExperimentEvent({
+ project_id: projectId,
+ trace_id: cleanTraceId,
+ span_id: cleanRootId,
+ experimentId: baselineExpId,
+ experimentName: "baseline-exp",
+ datasetId,
+ itemId: cleanItemId,
+ experimentItemRootSpanId: cleanRootId,
+ level: "DEFAULT",
+ start_time: now + 2,
+ }),
+ ]);
+
+ const result = await getExperimentItemsFromEvents({
+ projectId,
+ baseExperimentId: baselineExpId,
+ compExperimentIds: [],
+ filterByExperiment: [
+ {
+ experimentId: baselineExpId,
+ filters: [
+ {
+ column: "level",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["ERROR"],
+ } as FilterCondition,
+ ],
+ },
+ ],
+ limit: 10,
+ offset: 0,
+ });
+
+ expect(result.map((row) => row.itemId)).toEqual([failingItemId]);
+ });
+ });
+
maybe("getExperimentItemsBatchIO", () => {
it("should fetch IO and truncate to specified length", async () => {
// GIVEN: Item with long input/output strings
diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
index 7358dbcdf6..c1526bf98f 100644
--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
@@ -195,6 +195,91 @@ describe("Clickhouse Experiment Repository Test", () => {
expect(experiment?.itemCount).toBe(2);
});
+ it("should count items that carry an ERROR event, not every ERROR event", async () => {
+ const experimentId = randomUUID();
+ const experimentName = "experiment-errors-" + randomUUID();
+ const datasetId = randomUUID();
+ const failingItemId = randomUUID();
+ const cleanItemId = randomUUID();
+ const failingRootId = randomUUID();
+ const failingChildId = randomUUID();
+ const cleanRootId = randomUUID();
+ const now = Date.now() * 1000;
+
+ await createEventsCh([
+ createEvent({
+ id: failingRootId,
+ span_id: failingRootId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "SPAN",
+ experiment_id: experimentId,
+ experiment_name: experimentName,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: failingItemId,
+ experiment_item_root_span_id: failingRootId,
+ level: "DEFAULT",
+ start_time: now,
+ }),
+ createEvent({
+ id: failingChildId,
+ span_id: failingChildId,
+ parent_span_id: failingRootId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "SPAN",
+ experiment_id: experimentId,
+ experiment_name: experimentName,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: failingItemId,
+ experiment_item_root_span_id: failingRootId,
+ level: "ERROR",
+ start_time: now + 1,
+ }),
+ createEvent({
+ id: randomUUID(),
+ span_id: randomUUID(),
+ parent_span_id: failingRootId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "SPAN",
+ experiment_id: experimentId,
+ experiment_name: experimentName,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: failingItemId,
+ experiment_item_root_span_id: failingRootId,
+ level: "ERROR",
+ start_time: now + 2,
+ }),
+ createEvent({
+ id: cleanRootId,
+ span_id: cleanRootId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "SPAN",
+ experiment_id: experimentId,
+ experiment_name: experimentName,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: cleanItemId,
+ experiment_item_root_span_id: cleanRootId,
+ level: "DEFAULT",
+ start_time: now + 3,
+ }),
+ ]);
+
+ const result = await getExperimentsFromEvents({
+ projectId,
+ filter: [],
+ limit: 1000,
+ page: 0,
+ });
+
+ const experiment = result.find((e) => e.id === experimentId);
+ expect(experiment).toBeDefined();
+ expect(experiment?.itemCount).toBe(2);
+ expect(experiment?.errorCount).toBe(1);
+ });
+
it("should order by startTime DESC", async () => {
const experimentId1 = randomUUID();
const experimentName1 = "experiment-1-" + randomUUID();
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsx
index 21ef7d21ba..250115d804 100644
--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx
@@ -34,6 +34,7 @@ import {
BatchExportTableName,
ActionId,
BatchActionType,
+ buildExperimentPath,
} from "@langfuse/shared";
import { numberFormatter } from "@/src/utils/numbers";
import { useOrderByState } from "@/src/features/orderBy/hooks/useOrderByState";
@@ -615,13 +616,42 @@ export default function ExperimentsTable({
size: 100,
cell: ({ row }) => {
const value: number = row.getValue("errorCount");
+ const formatted = numberFormatter(value, 0);
+
+ if (value <= 0) {
+ return (
+ <Badge
+ variant="secondary"
+ className="max-w-fit rounded-sm px-1 font-normal"
+ >
+ {formatted}
+ </Badge>
+ );
+ }
+
return (
- <Badge
- variant={value > 0 ? "destructive" : "secondary"}
- className="max-w-fit rounded-sm px-1 font-normal"
+ <Link
+ href={buildExperimentPath({
+ projectId,
+ experimentId: row.id,
+ filters: [
+ {
+ column: "level",
+ type: "stringOptions",
+ operator: "any of",
+ value: ["ERROR"],
+ },
+ ],
+ })}
+ aria-label={`View ${formatted} failing items`}
>
- {numberFormatter(value, 0)}
- </Badge>
+ <Badge
+ variant="destructive"
+ className="hover:bg-destructive/80 max-w-fit cursor-pointer rounded-sm px-1 font-normal"
+ >
+ {formatted}
+ </Badge>
+ </Link>
);
},
enableHiding: true,
diff --git a/web/src/features/experiments/config/experiment-items-filter-config.ts b/web/src/features/experiments/config/experiment-items-filter-config.ts
index 1d84c40813..5778ed5bd1 100644
--- a/web/src/features/experiments/config/experiment-items-filter-config.ts
+++ b/web/src/features/experiments/config/experiment-items-filter-config.ts
@@ -1,4 +1,5 @@
import type { FilterConfig } from "@/src/features/filters";
+import { renderLevelIcon } from "@/src/components/level-colors";
import type { ColumnDefinition, ObservationLevelType } from "@langfuse/shared";
/**
@@ -151,6 +152,12 @@ export const experimentItemsFilterConfig: FilterConfig = {
columnDefinitions: experimentItemsTableCols,
facets: [
+ {
+ type: "categorical" as const,
+ column: "level",
+ label: getExperimentItemsColumnName("level"),
+ renderIcon: renderLevelIcon,
+ },
{
type: "stringKeyValue" as const,
column: "itemMetadata",
diff --git a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts
index 6cabc7cf64..652d3f2156 100644
--- a/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts
+++ b/web/src/features/experiments/hooks/useExperimentItemsFilterOptions.ts
@@ -8,6 +8,8 @@ export type ScoreColumnDef = {
source: string;
};
+const EXPERIMENT_ITEM_LEVEL_OPTIONS = ["DEBUG", "DEFAULT", "WARNING", "ERROR"];
+
const processCategoricalScoreOptions = (
categories: Array<{ label: string; values: string[] }>,
): Record<string, string[]> =>
@@ -53,6 +55,7 @@ export const useExperimentItemsFilterOptions = ({
score_name_levels_numeric: undefined,
score_name_levels_categorical: undefined,
score_name_levels_boolean: undefined,
+ level: EXPERIMENT_ITEM_LEVEL_OPTIONS,
} satisfies ExperimentItemScoreFilterOptions;
}
@@ -69,6 +72,7 @@ export const useExperimentItemsFilterOptions = ({
score_name_levels_categorical:
filterOptions.data.score_name_levels_categorical,
score_name_levels_boolean: filterOptions.data.score_name_levels_boolean,
+ level: EXPERIMENT_ITEM_LEVEL_OPTIONS,
} satisfies ExperimentItemScoreFilterOptions;
}, [filterOptions.data]);
diff --git a/web/src/features/experiments/types/charts.ts b/web/src/features/experiments/types/charts.ts
index bb8d22b024..3ceeeac398 100644
--- a/web/src/features/experiments/types/charts.ts
+++ b/web/src/features/experiments/types/charts.ts
@@ -48,6 +48,7 @@ export type ExperimentItemScoreFilterOptions = {
score_name_levels_numeric?: ScoreNameLevels;
score_name_levels_categorical?: ScoreNameLevels;
score_name_levels_boolean?: ScoreNameLevels;
+ level?: string[];
};
/**