Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): unify experiment score filters across levels (#16764)

Nikita Kabardin86fa3a31c812
diff --git a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
index 32233ea07e..14e3fb8a53 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/query-fragments.ts
@@ -557,7 +557,21 @@ export const buildScoreRowsCTE = (params: BaseScoresParams): CTEWithSchema => {
};
};
-export const buildScoresCTE = (params: BaseScoresParams): CTEWithSchema => {
+/**
+ * `level: "any"` keeps BOTH levels in one CTE, for the level-agnostic score
+ * filters (a score matches whether it was recorded on an observation or on the
+ * trace). A caller that aggregates must keep the level in its GROUP BY, or a
+ * name present at both levels collapses into a single averaged value and a
+ * numeric comparison stops meaning "at either level".
+ *
+ * Only this CTE understands "any" — the other score fragments branch on
+ * `level === "trace"` and would silently treat it as observation-level.
+ */
+type ScoresCTEParams = Omit<BaseScoresParams, "level"> & {
+ level: BaseScoresParams["level"] | "any";
+};
+
+export const buildScoresCTE = (params: ScoresCTEParams): CTEWithSchema => {
const queryParams: Record<string, any> = {
projectId: params.projectId,
};
@@ -566,10 +580,12 @@ export const buildScoresCTE = (params: BaseScoresParams): CTEWithSchema => {
queryParams.startTimeFrom = params.startTimeFrom;
}
- const isTraceLevel = params.level === "trace";
- const observationFilter = isTraceLevel
- ? "AND observation_id IS NULL"
- : "AND observation_id IS NOT NULL";
+ const observationFilter =
+ params.level === "any"
+ ? ""
+ : params.level === "trace"
+ ? "AND observation_id IS NULL"
+ : "AND observation_id IS NOT NULL";
const query = `
SELECT
diff --git a/packages/shared/src/server/repositories/experimentScoreOptions.ts b/packages/shared/src/server/repositories/experimentScoreOptions.ts
new file mode 100644
index 0000000000..5fee21a858
--- /dev/null
+++ b/packages/shared/src/server/repositories/experimentScoreOptions.ts
@@ -0,0 +1,102 @@
+/**
+ * The level-agnostic score-option shape shared by the experiment runs and items
+ * surfaces. Kept free of query imports so it stays a pure, directly testable
+ * projection.
+ */
+
+export type ScoreColumnDefinition = {
+ name: string;
+ dataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+ source: string;
+};
+
+type ProcessedScoreFilterOptions = {
+ numeric: string[];
+ boolean: string[];
+ categorical: Array<{ label: string; values: string[] }>;
+ scoreColumns: ScoreColumnDefinition[];
+};
+
+export type ScoreNameLevels = Record<string, ("observation" | "trace")[]>;
+
+export type AgnosticScoreFilterOptions = {
+ scores_avg: string[];
+ score_categories: Array<{ label: string; values: string[] }>;
+ score_booleans: string[];
+ score_columns: ScoreColumnDefinition[];
+ score_name_levels_numeric: ScoreNameLevels;
+ score_name_levels_categorical: ScoreNameLevels;
+ score_name_levels_boolean: ScoreNameLevels;
+};
+
+export type { ProcessedScoreFilterOptions };
+
+/**
+ * Projects the per-level score options into the one level-agnostic set the
+ * three score facets offer, plus the level map that tags each offered name.
+ *
+ * The per-level shape stays the source — the run charts need it, because a
+ * merged metric identity cannot say which level's series to plot. This is the
+ * projection for filtering, where the level is not what the user is asking
+ * about.
+ *
+ * Offered-set == matchable-set: every name here is one the level-agnostic
+ * filter can match, and nothing else is offered. Experiment-level (dataset-run)
+ * scores are deliberately absent — the observation-or-trace union can never
+ * match them, so offering them would advertise a filter that cannot fire.
+ *
+ * The level map is SPLIT PER DATA TYPE because a name can be reused across
+ * types at different levels (a numeric observation-level "accuracy" beside an
+ * unrelated categorical trace-level one); a name-only map would mislabel both.
+ */
+export const toAgnosticScoreFilterOptions = (
+ observation: ProcessedScoreFilterOptions,
+ trace: ProcessedScoreFilterOptions,
+): AgnosticScoreFilterOptions => {
+ const levels = (
+ pick: (options: ProcessedScoreFilterOptions) => string[],
+ ): ScoreNameLevels => {
+ const out: ScoreNameLevels = {};
+ for (const name of pick(observation))
+ (out[name] ??= []).push("observation");
+ for (const name of pick(trace)) (out[name] ??= []).push("trace");
+ return out;
+ };
+
+ const mergedCategorical = new Map<string, Set<string>>();
+ for (const { label, values } of [
+ ...observation.categorical,
+ ...trace.categorical,
+ ]) {
+ const set = mergedCategorical.get(label) ?? new Set<string>();
+ for (const value of values) set.add(value);
+ mergedCategorical.set(label, set);
+ }
+
+ return {
+ scores_avg: Array.from(
+ new Set([...observation.numeric, ...trace.numeric]),
+ ).sort(),
+ score_booleans: Array.from(
+ new Set([...observation.boolean, ...trace.boolean]),
+ ).sort(),
+ score_categories: Array.from(mergedCategorical, ([label, values]) => ({
+ label,
+ values: Array.from(values).sort(),
+ })).sort((a, b) => a.label.localeCompare(b.label)),
+ // Deduped: a name that exists at both levels is ONE column, not two.
+ score_columns: Array.from(
+ new Map(
+ [...observation.scoreColumns, ...trace.scoreColumns].map((column) => [
+ `${column.name}-${column.source}-${column.dataType}`,
+ column,
+ ]),
+ ).values(),
+ ),
+ score_name_levels_numeric: levels((options) => options.numeric),
+ score_name_levels_categorical: levels((options) =>
+ options.categorical.map((entry) => entry.label),
+ ),
+ score_name_levels_boolean: levels((options) => options.boolean),
+ };
+};
diff --git a/packages/shared/src/server/repositories/experiments.test.ts b/packages/shared/src/server/repositories/experiments.test.ts
new file mode 100644
index 0000000000..2ae2a68a20
--- /dev/null
+++ b/packages/shared/src/server/repositories/experiments.test.ts
@@ -0,0 +1,99 @@
+import { describe, expect, it } from "vitest";
+import { toAgnosticScoreFilterOptions } from "./experimentScoreOptions";
+import { experimentCols } from "../tableMappings/mapExperimentTable";
+import { matchesUiColumnMapping } from "../../tableDefinitions/types";
+
+const options = (
+ partial: Partial<Parameters<typeof toAgnosticScoreFilterOptions>[0]> = {},
+): Parameters<typeof toAgnosticScoreFilterOptions>[0] => ({
+ numeric: [],
+ categorical: [],
+ boolean: [],
+ scoreColumns: [],
+ ...partial,
+});
+
+describe("toAgnosticScoreFilterOptions", () => {
+ it("unions the names across levels and tags each with the levels it exists at", () => {
+ const result = toAgnosticScoreFilterOptions(
+ options({ numeric: ["accuracy", "obs-only"] }),
+ options({ numeric: ["accuracy", "trace-only"] }),
+ );
+
+ expect(result.scores_avg).toEqual(["accuracy", "obs-only", "trace-only"]);
+ expect(result.score_name_levels_numeric).toEqual({
+ accuracy: ["observation", "trace"],
+ "obs-only": ["observation"],
+ "trace-only": ["trace"],
+ });
+ });
+
+ it("keeps the level maps per data type, so a name reused across types is not mislabeled", () => {
+ const result = toAgnosticScoreFilterOptions(
+ options({ numeric: ["quality"] }),
+ options({ categorical: [{ label: "quality", values: ["good"] }] }),
+ );
+
+ // Same name, different data type, different level: the numeric facet must
+ // not claim it also exists at trace level.
+ expect(result.score_name_levels_numeric).toEqual({
+ quality: ["observation"],
+ });
+ expect(result.score_name_levels_categorical).toEqual({
+ quality: ["trace"],
+ });
+ });
+
+ it("unions categorical values of a name present at both levels", () => {
+ const result = toAgnosticScoreFilterOptions(
+ options({ categorical: [{ label: "tone", values: ["warm", "flat"] }] }),
+ options({ categorical: [{ label: "tone", values: ["flat", "sharp"] }] }),
+ );
+
+ expect(result.score_categories).toEqual([
+ { label: "tone", values: ["flat", "sharp", "warm"] },
+ ]);
+ });
+
+ it("dedupes a score column that exists at both levels", () => {
+ const column = {
+ name: "accuracy",
+ dataType: "NUMERIC" as const,
+ source: "API",
+ };
+
+ const result = toAgnosticScoreFilterOptions(
+ options({ scoreColumns: [column] }),
+ options({ scoreColumns: [column] }),
+ );
+
+ expect(result.score_columns).toEqual([column]);
+ });
+});
+
+describe("experiments table score column mapping", () => {
+ // Filters whose column the repository cannot resolve are dropped silently, so
+ // a lost alias would turn an existing saved view into "no filter at all".
+ it.each([
+ ["obs_scores_avg", "scores_avg"],
+ ["obs_score_categories", "score_categories"],
+ ["obs_score_booleans", "score_booleans"],
+ ])("resolves the legacy %s alias onto %s", (legacy, canonical) => {
+ const match = experimentCols.find((column) =>
+ matchesUiColumnMapping(column, legacy),
+ );
+
+ expect(match?.uiTableId).toBe(canonical);
+ });
+
+ it.each(["scores_avg", "score_categories", "score_booleans"])(
+ "resolves the canonical %s column",
+ (canonical) => {
+ const match = experimentCols.find((column) =>
+ matchesUiColumnMapping(column, canonical),
+ );
+
+ expect(match?.uiTableId).toBe(canonical);
+ },
+ );
+});
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.ts
index 2670b1b084..403ffebe71 100644
--- a/packages/shared/src/server/repositories/experiments.ts
+++ b/packages/shared/src/server/repositories/experiments.ts
@@ -1,3 +1,4 @@
+import { matchesUiColumnMapping } from "../../tableDefinitions";
import { env } from "../../env";
import { type ScoreSourceType } from "../../domain";
import { type OrderByState } from "../../interfaces/orderBy";
@@ -36,6 +37,20 @@ import {
experimentOrderByCols,
} from "../tableMappings/mapExperimentTable";
+import {
+ toAgnosticScoreFilterOptions,
+ type AgnosticScoreFilterOptions,
+ type ProcessedScoreFilterOptions,
+ type ScoreColumnDefinition,
+} from "./experimentScoreOptions";
+
+export {
+ toAgnosticScoreFilterOptions,
+ type AgnosticScoreFilterOptions,
+ type ScoreColumnDefinition,
+ type ScoreNameLevels,
+} from "./experimentScoreOptions";
+
export type ExperimentEventsDataReturnType = {
experiment_id: string;
experiment_name: string;
@@ -102,14 +117,39 @@ export const getDatasetExperimentMetricsFromEvents = async (props: {
}));
};
+/**
+ * One experiment-score CTE, aggregated into the arrays the score filters read
+ * as a HAVING.
+ *
+ * `level: "any"` is the level-agnostic mode: a score matches whether it was
+ * recorded on an observation or on the trace, which is what users mean by
+ * "groundedness is low" — they neither know nor care which level carried it.
+ *
+ * The level stays in the inner GROUP BY so the arrays hold ONE ENTRY PER
+ * (name, level). That is load-bearing twice over, because the filters compile
+ * to array-existence checks:
+ * - a comparison matches if EITHER level satisfies it (the OR semantics),
+ * where a merged average would instead test the mean of two different
+ * measurements;
+ * - an exclusion matches only when NO entry does, which is "at neither
+ * level" — De Morgan, without composing two predicates.
+ */
const experimentScoreCTE = (params: {
projectId: string;
startTimeFrom?: string | null;
- level: "observation" | "trace";
+ level: "observation" | "trace" | "any";
eventKeysCTE: CTEWithSchema;
filters: FilterList;
}) => {
- const prefix = params.level === "observation" ? "obs" : "trace";
+ // The agnostic arrays carry the canonical column names the level-agnostic
+ // filters target; the trace-only mode keeps its prefix so legacy
+ // `trace_*` filters still resolve against a trace-only aggregate.
+ const prefix =
+ params.level === "any"
+ ? ""
+ : params.level === "observation"
+ ? "obs_"
+ : "trace_";
const joinedEventScores = new CTEQueryBuilder()
.withCTE("event_keys", {
@@ -142,6 +182,9 @@ const experimentScoreCTE = (params: {
"us.name",
"us.data_type",
"us.string_value",
+ // The level discriminator — see the header. Grouped but not selected: the
+ // tuples only need to be distinct per level, not to name it.
+ "us.observation_id IS NULL",
)
.buildWithParams();
@@ -161,9 +204,9 @@ const experimentScoreCTE = (params: {
.select(
"s.project_id AS project_id",
"s.experiment_id AS experiment_id",
- `groupArrayIf(tuple(s.name, s.exp_avg, s.data_type, s.string_value), s.data_type IN ('NUMERIC', 'BOOLEAN')) AS ${prefix}_scores_avg`,
- `groupArrayIf(concat(s.name, ':', s.string_value), s.data_type = 'CATEGORICAL' AND notEmpty(s.string_value)) AS ${prefix}_score_categories`,
- `${scoreBooleansAggregation("s.")} AS ${prefix}_score_booleans`,
+ `groupArrayIf(tuple(s.name, s.exp_avg, s.data_type, s.string_value), s.data_type IN ('NUMERIC', 'BOOLEAN')) AS ${prefix}scores_avg`,
+ `groupArrayIf(concat(s.name, ':', s.string_value), s.data_type = 'CATEGORICAL' AND notEmpty(s.string_value)) AS ${prefix}score_categories`,
+ `${scoreBooleansAggregation("s.")} AS ${prefix}score_booleans`,
)
.groupBy("s.project_id", "s.experiment_id")
.having(params.filters.apply())
@@ -269,8 +312,11 @@ const getExperimentsFromEventsGeneric = async <T>(
const preAggFilterState = filter.filter((f) =>
experimentPreAggCols.some((col) => col.uiTableId === f.column),
);
+ // Alias-aware: the legacy `obs_*` ids reach the agnostic columns as aliases,
+ // and matching only on uiTableId would drop them on the floor (this partition
+ // silently discards anything it does not recognise).
const scoreAggFilterState = filter.filter((f) =>
- experimentScoreAggCols.some((col) => col.uiTableId === f.column),
+ experimentScoreAggCols.some((col) => matchesUiColumnMapping(col, f.column)),
);
const preAggFilters = new FilterList(
@@ -301,10 +347,8 @@ const getExperimentsFromEventsGeneric = async <T>(
"trace_score_booleans",
].includes(f.field),
);
- const hasObsScoreFilter = scoreAggFilters.some((f) =>
- ["obs_scores_avg", "obs_score_categories", "obs_score_booleans"].includes(
- f.field,
- ),
+ const hasAgnosticScoreFilter = scoreAggFilters.some((f) =>
+ ["scores_avg", "score_categories", "score_booleans"].includes(f.field),
);
const experimentIds = experimentIdFilter?.values;
@@ -322,10 +366,10 @@ const getExperimentsFromEventsGeneric = async <T>(
experimentIds,
})
.applyFilters(preAggFilters)
- .when(hasObsScoreFilter, (b) => {
+ .when(hasAgnosticScoreFilter, (b) => {
return b
.withCTE(
- "matching_obs_experiments",
+ "matching_agnostic_experiments",
experimentScoreCTE({
projectId,
startTimeFrom,
@@ -334,18 +378,16 @@ const getExperimentsFromEventsGeneric = async <T>(
schema: ["project_id", "experiment_id", "trace_id"],
},
filters: scoreAggFilters.filter((f) =>
- [
- "obs_scores_avg",
- "obs_score_categories",
- "obs_score_booleans",
- ].includes(f.field),
+ ["scores_avg", "score_categories", "score_booleans"].includes(
+ f.field,
+ ),
),
- level: "observation",
+ level: "any",
}),
)
.innerJoin(
- "matching_obs_experiments AS moe",
- "ON moe.project_id = e.project_id AND moe.experiment_id = e.experiment_id",
+ "matching_agnostic_experiments AS mae",
+ "ON mae.project_id = e.project_id AND mae.experiment_id = e.experiment_id",
);
})
.when(hasTraceScoreFilter, (b) => {
@@ -618,19 +660,6 @@ const buildScoreFilterOptionsQuery = (params: {
return queryBuilder.buildWithParams();
};
-export type ScoreColumnDefinition = {
- name: string;
- dataType: "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
- source: string;
-};
-
-type ProcessedScoreFilterOptions = {
- numeric: string[];
- boolean: string[];
- categorical: Array<{ label: string; values: string[] }>;
- scoreColumns: ScoreColumnDefinition[];
-};
-
const processScoreFilterOptionsResults = (
rows: ScoreFilterOptionsRow[],
): ProcessedScoreFilterOptions => {
@@ -684,6 +713,7 @@ type ExperimentItemScoreOptionsByLevel = {
type ExperimentScoreOptionsByLevel = {
observation: ProcessedScoreFilterOptions;
+ trace: ProcessedScoreFilterOptions;
experiment: ProcessedScoreFilterOptions;
};
@@ -735,16 +765,18 @@ const getExperimentItemScoreOptionsByLevel = async ({
export const getExperimentItemsFilterOptions = async (
props: ExperimentItemsFilterOptionsInput,
-): Promise<{
- obs_scores_avg: string[];
- obs_score_categories: Array<{ label: string; values: string[] }>;
- obs_score_booleans: string[];
- obs_score_columns: ScoreColumnDefinition[];
- trace_scores_avg: string[];
- trace_score_categories: Array<{ label: string; values: string[] }>;
- trace_score_booleans: string[];
- trace_score_columns: ScoreColumnDefinition[];
-}> => {
+): Promise<
+ AgnosticScoreFilterOptions & {
+ obs_scores_avg: string[];
+ obs_score_categories: Array<{ label: string; values: string[] }>;
+ obs_score_booleans: string[];
+ obs_score_columns: ScoreColumnDefinition[];
+ trace_scores_avg: string[];
+ trace_score_categories: Array<{ label: string; values: string[] }>;
+ trace_score_booleans: string[];
+ trace_score_columns: ScoreColumnDefinition[];
+ }
+> => {
const { observation, trace } =
await getExperimentItemScoreOptionsByLevel(props);
@@ -757,6 +789,7 @@ export const getExperimentItemsFilterOptions = async (
trace_score_categories: trace.categorical,
trace_score_booleans: trace.boolean,
trace_score_columns: trace.scoreColumns,
+ ...toAgnosticScoreFilterOptions(observation, trace),
};
};
@@ -769,6 +802,7 @@ const getExperimentScoreOptionsByLevel = async ({
if (uniqueExperimentIds.length === 0) {
return {
observation: emptyScoreFilterOptions(),
+ trace: emptyScoreFilterOptions(),
experiment: emptyScoreFilterOptions(),
};
}
@@ -779,18 +813,32 @@ const getExperimentScoreOptionsByLevel = async ({
level: "observation",
});
+ // Trace-level names are half of what the level-agnostic facets offer; without
+ // this a score recorded on the trace was never even suggested.
+ const traceQuery = buildScoreFilterOptionsQuery({
+ projectId,
+ experimentIds: uniqueExperimentIds,
+ level: "trace",
+ });
+
const runQuery = buildExperimentRunScoreFilterOptionsQuery({
projectId,
experimentIds: uniqueExperimentIds,
});
- const [obsResults, runResults] = await Promise.all([
+ const [obsResults, traceResults, runResults] = await Promise.all([
queryClickhouse<ScoreFilterOptionsRow>({
query: obsQuery.query,
params: obsQuery.params,
tags: { projectId },
preferredClickhouseService: "ReadOnly",
}),
+ queryClickhouse<ScoreFilterOptionsRow>({
+ query: traceQuery.query,
+ params: traceQuery.params,
+ tags: { projectId },
+ preferredClickhouseService: "ReadOnly",
+ }),
queryClickhouse<ScoreFilterOptionsRow>({
query: runQuery.query,
params: runQuery.params,
@@ -801,27 +849,42 @@ const getExperimentScoreOptionsByLevel = async ({
return {
observation: processScoreFilterOptionsResults(obsResults),
+ trace: processScoreFilterOptionsResults(traceResults),
experiment: processScoreFilterOptionsResults(runResults),
};
};
export const getExperimentScoreOptions = async (
props: ExperimentScoreOptionsInput,
-): Promise<{
- obs_scores_avg: string[];
- obs_score_categories: Array<{ label: string; values: string[] }>;
- obs_score_columns: ScoreColumnDefinition[];
- experiment_scores_avg: string[];
- experiment_score_categories: Array<{ label: string; values: string[] }>;
- experiment_score_columns: ScoreColumnDefinition[];
-}> => {
- const { observation, experiment } =
+): Promise<
+ AgnosticScoreFilterOptions & {
+ obs_scores_avg: string[];
+ obs_score_categories: Array<{ label: string; values: string[] }>;
+ obs_score_columns: ScoreColumnDefinition[];
+ trace_scores_avg: string[];
+ trace_score_categories: Array<{ label: string; values: string[] }>;
+ trace_score_columns: ScoreColumnDefinition[];
+ experiment_scores_avg: string[];
+ experiment_score_categories: Array<{ label: string; values: string[] }>;
+ experiment_score_columns: ScoreColumnDefinition[];
+ }
+> => {
+ const { observation, trace, experiment } =
await getExperimentScoreOptionsByLevel(props);
return {
+ // Per-level stays the SOURCE: the run charts select a metric per level,
+ // because a merged identity cannot say which level's series to plot.
obs_scores_avg: observation.numeric,
obs_score_categories: observation.categorical,
obs_score_columns: observation.scoreColumns,
+ trace_scores_avg: trace.numeric,
+ trace_score_categories: trace.categorical,
+ trace_score_columns: trace.scoreColumns,
+ // …and the agnostic projection is what the score FACETS offer.
+ ...toAgnosticScoreFilterOptions(observation, trace),
+ // Experiment-level scores grade the RUN, not an item, so they stay their own
+ // thing rather than being folded into the item-score filters.
experiment_scores_avg: experiment.numeric,
experiment_score_categories: experiment.categorical,
experiment_score_columns: experiment.scoreColumns,
diff --git a/packages/shared/src/server/tableMappings/mapExperimentTable.ts b/packages/shared/src/server/tableMappings/mapExperimentTable.ts
index 1ea89ce21d..c321225760 100644
--- a/packages/shared/src/server/tableMappings/mapExperimentTable.ts
+++ b/packages/shared/src/server/tableMappings/mapExperimentTable.ts
@@ -56,26 +56,35 @@ export const experimentPreAggCols: UiColumnMappings = [
* Score aggregation column mappings for experiments.
*/
export const experimentScoreAggCols: UiColumnMappings = [
- // Observation-level scores
+ // Level-agnostic scores: these match a score whether it was recorded on an
+ // observation or on the trace. The legacy `obs_*` ids map here too, so links
+ // and saved views written against them keep working — and start matching
+ // trace-level scores, which is the point (a score recorded on the trace used
+ // to return nothing).
{
uiTableName: "Scores (numeric)",
- uiTableId: "obs_scores_avg",
+ uiTableId: "scores_avg",
clickhouseTableName: "scores",
- clickhouseSelect: "obs_scores_avg",
+ clickhouseSelect: "scores_avg",
+ aliases: ["obs_scores_avg"],
},
{
uiTableName: "Scores (categorical)",
- uiTableId: "obs_score_categories",
+ uiTableId: "score_categories",
clickhouseTableName: "scores",
- clickhouseSelect: "obs_score_categories",
+ clickhouseSelect: "score_categories",
+ aliases: ["obs_score_categories"],
},
{
uiTableName: "Scores (boolean)",
- uiTableId: "obs_score_booleans",
+ uiTableId: "score_booleans",
clickhouseTableName: "scores",
- clickhouseSelect: "obs_score_booleans",
+ clickhouseSelect: "score_booleans",
+ aliases: ["obs_score_booleans"],
},
- // Trace-level scores
+ // Trace-only, kept resolving for saved views that deliberately scoped to the
+ // trace level. Not offered as a facet: aliasing these onto the agnostic
+ // arrays would silently WIDEN such a filter.
{
uiTableName: "Trace Scores (numeric)",
uiTableId: "trace_scores_avg",
diff --git a/web/src/__tests__/server/experiment-score-levels.servertest.ts b/web/src/__tests__/server/experiment-score-levels.servertest.ts
new file mode 100644
index 0000000000..667e9446d3
--- /dev/null
+++ b/web/src/__tests__/server/experiment-score-levels.servertest.ts
@@ -0,0 +1,270 @@
+import {
+ createEvent,
+ createEventsCh,
+ createOrgProjectAndApiKey,
+ createTraceScore,
+ createScoresCh,
+ getExperimentsFromEvents,
+ getExperimentScoreOptions,
+} from "@langfuse/shared/src/server";
+import { type FilterState } from "@langfuse/shared";
+import { randomUUID } from "crypto";
+import { env } from "@/src/env.mjs";
+
+const maybeEventTables =
+ env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true"
+ ? describe
+ : describe.skip;
+
+/**
+ * One experiment, one item, and one score placed at the level the caller asks
+ * for. Observation level means the score points at the item's root span;
+ * trace level means it points only at the trace.
+ */
+const seedExperimentWithScore = async ({
+ projectId,
+ experimentName,
+ startTimeMs,
+ level,
+ scoreName,
+ value,
+ stringValue,
+ dataType = "NUMERIC",
+}: {
+ projectId: string;
+ experimentName: string;
+ startTimeMs: number;
+ level: "observation" | "trace";
+ scoreName: string;
+ value: number;
+ stringValue?: string;
+ dataType?: "NUMERIC" | "CATEGORICAL" | "BOOLEAN";
+}) => {
+ const spanId = randomUUID();
+ const traceId = randomUUID();
+ const experimentId = `exp-${randomUUID()}`;
+
+ await createEventsCh([
+ createEvent({
+ id: spanId,
+ span_id: spanId,
+ trace_id: traceId,
+ project_id: projectId,
+ name: `${experimentName}-root`,
+ type: "SPAN",
+ start_time: startTimeMs * 1000,
+ end_time: (startTimeMs + 100) * 1000,
+ experiment_id: experimentId,
+ experiment_name: experimentName,
+ experiment_dataset_id: "dataset-score-levels",
+ experiment_item_id: randomUUID(),
+ experiment_item_root_span_id: spanId,
+ }),
+ ]);
+
+ await createScoresCh([
+ createTraceScore({
+ project_id: projectId,
+ trace_id: traceId,
+ observation_id: level === "observation" ? spanId : null,
+ name: scoreName,
+ value,
+ string_value: stringValue ?? null,
+ data_type: dataType,
+ timestamp: startTimeMs,
+ }),
+ ]);
+
+ return { experimentId, experimentName };
+};
+
+maybeEventTables("level-agnostic experiment score filters", () => {
+ it("matches an experiment whose score sits at either level", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const startTimeMs = Date.now();
+ const scoreName = `accuracy-${randomUUID().slice(0, 8)}`;
+
+ const observationLevel = await seedExperimentWithScore({
+ projectId,
+ experimentName: "obs-level-run",
+ startTimeMs,
+ level: "observation",
+ scoreName,
+ value: 0.9,
+ });
+ const traceLevel = await seedExperimentWithScore({
+ projectId,
+ experimentName: "trace-level-run",
+ startTimeMs,
+ level: "trace",
+ scoreName,
+ value: 0.9,
+ });
+ // Same score name, below the threshold: proves the filter compares values
+ // rather than merely detecting the name's presence.
+ const belowThreshold = await seedExperimentWithScore({
+ projectId,
+ experimentName: "low-score-run",
+ startTimeMs,
+ level: "trace",
+ scoreName,
+ value: 0.1,
+ });
+
+ const filter: FilterState = [
+ {
+ type: "numberObject",
+ column: "scores_avg",
+ key: scoreName,
+ operator: ">",
+ value: 0.5,
+ },
+ ];
+
+ const rows = await getExperimentsFromEvents({ projectId, filter });
+ const matchedIds = rows.map((row) => row.id);
+
+ expect(matchedIds).toContain(observationLevel.experimentId);
+ expect(matchedIds).toContain(traceLevel.experimentId);
+ expect(matchedIds).not.toContain(belowThreshold.experimentId);
+ });
+
+ it("keeps matching through the legacy obs_scores_avg column", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const startTimeMs = Date.now();
+ const scoreName = `legacy-${randomUUID().slice(0, 8)}`;
+
+ const observationLevel = await seedExperimentWithScore({
+ projectId,
+ experimentName: "legacy-obs-run",
+ startTimeMs,
+ level: "observation",
+ scoreName,
+ value: 0.9,
+ });
+
+ // A second run without that score, so a silently dropped column would show
+ // up as an extra row rather than passing by having nothing to exclude.
+ await seedExperimentWithScore({
+ projectId,
+ experimentName: "legacy-unscored-run",
+ startTimeMs,
+ level: "observation",
+ scoreName: `other-${randomUUID().slice(0, 8)}`,
+ value: 0.9,
+ });
+
+ // Saved views and bookmarked URLs still carry the old column id. An
+ // unresolved column is dropped silently, which would widen the result set
+ // instead of failing loudly - so assert the filter still bites.
+ const rows = await getExperimentsFromEvents({
+ projectId,
+ filter: [
+ {
+ type: "numberObject",
+ column: "obs_scores_avg",
+ key: scoreName,
+ operator: ">",
+ value: 0.5,
+ },
+ ],
+ });
+
+ expect(rows.map((row) => row.id)).toEqual([observationLevel.experimentId]);
+ });
+
+ it("excludes a categorical value at whichever level it sits", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const startTimeMs = Date.now();
+ const scoreName = `verdict-${randomUUID().slice(0, 8)}`;
+
+ const traceLevel = await seedExperimentWithScore({
+ projectId,
+ experimentName: "trace-verdict-run",
+ startTimeMs,
+ level: "trace",
+ scoreName,
+ value: 0,
+ stringValue: "bad",
+ dataType: "CATEGORICAL",
+ });
+ const keptRun = await seedExperimentWithScore({
+ projectId,
+ experimentName: "kept-verdict-run",
+ startTimeMs,
+ level: "observation",
+ scoreName,
+ value: 0,
+ stringValue: "good",
+ dataType: "CATEGORICAL",
+ });
+
+ const rows = await getExperimentsFromEvents({
+ projectId,
+ filter: [
+ {
+ type: "categoryOptions",
+ column: "score_categories",
+ key: scoreName,
+ operator: "none of",
+ value: ["bad"],
+ },
+ ],
+ });
+ const matchedIds = rows.map((row) => row.id);
+
+ expect(matchedIds).toContain(keptRun.experimentId);
+ expect(matchedIds).not.toContain(traceLevel.experimentId);
+ });
+
+ it("offers each score name once, tagged with the levels it exists at", async () => {
+ const { projectId } = await createOrgProjectAndApiKey();
+ const startTimeMs = Date.now();
+ const bothLevels = `both-${randomUUID().slice(0, 8)}`;
+ const traceOnly = `trace-only-${randomUUID().slice(0, 8)}`;
+
+ const first = await seedExperimentWithScore({
+ projectId,
+ experimentName: "options-obs-run",
+ startTimeMs,
+ level: "observation",
+ scoreName: bothLevels,
+ value: 0.5,
+ });
+ const second = await seedExperimentWithScore({
+ projectId,
+ experimentName: "options-trace-run",
+ startTimeMs,
+ level: "trace",
+ scoreName: bothLevels,
+ value: 0.5,
+ });
+ const third = await seedExperimentWithScore({
+ projectId,
+ experimentName: "options-trace-only-run",
+ startTimeMs,
+ level: "trace",
+ scoreName: traceOnly,
+ value: 0.5,
+ });
+
+ const options = await getExperimentScoreOptions({
+ projectId,
+ experimentIds: [
+ first.experimentId,
+ second.experimentId,
+ third.experimentId,
+ ],
+ });
+
+ expect(options.scores_avg.filter((name) => name === bothLevels)).toEqual([
+ bothLevels,
+ ]);
+ expect(options.scores_avg).toContain(traceOnly);
+ expect(options.score_name_levels_numeric[bothLevels]).toEqual([
+ "observation",
+ "trace",
+ ]);
+ expect(options.score_name_levels_numeric[traceOnly]).toEqual(["trace"]);
+ });
+});
diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
index b6f408f726..bc64ed0797 100644
--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
@@ -1380,6 +1380,9 @@ describe("Clickhouse Experiment Repository Test", () => {
experimentIds: [],
});
+ // The per-level lists stay the source (the charts read them); the
+ // level-agnostic set and its level maps are the projection the three
+ // score facets offer.
expect(result).toEqual({
obs_scores_avg: [],
obs_score_categories: [],
@@ -1389,6 +1392,13 @@ describe("Clickhouse Experiment Repository Test", () => {
trace_score_categories: [],
trace_score_booleans: [],
trace_score_columns: [],
+ scores_avg: [],
+ score_categories: [],
+ score_booleans: [],
+ score_columns: [],
+ score_name_levels_numeric: {},
+ score_name_levels_categorical: {},
+ score_name_levels_boolean: {},
});
});
@@ -1398,6 +1408,9 @@ describe("Clickhouse Experiment Repository Test", () => {
experimentIds: [randomUUID()],
});
+ // The per-level lists stay the source (the charts read them); the
+ // level-agnostic set and its level maps are the projection the three
+ // score facets offer.
expect(result).toEqual({
obs_scores_avg: [],
obs_score_categories: [],
@@ -1407,6 +1420,13 @@ describe("Clickhouse Experiment Repository Test", () => {
trace_score_categories: [],
trace_score_booleans: [],
trace_score_columns: [],
+ scores_avg: [],
+ score_categories: [],
+ score_booleans: [],
+ score_columns: [],
+ score_name_levels_numeric: {},
+ score_name_levels_categorical: {},
+ score_name_levels_boolean: {},
});
});
@@ -1739,13 +1759,20 @@ describe("Clickhouse Experiment Repository Test", () => {
obs_scores_avg: [],
obs_score_categories: [],
obs_score_columns: [],
+ trace_scores_avg: [],
+ trace_score_categories: [],
+ trace_score_columns: [],
experiment_scores_avg: [],
experiment_score_categories: [],
experiment_score_columns: [],
+ scores_avg: [],
+ score_categories: [],
+ score_booleans: [],
+ score_columns: [],
+ score_name_levels_numeric: {},
+ score_name_levels_categorical: {},
+ score_name_levels_boolean: {},
});
- expect(result).not.toHaveProperty("trace_scores_avg");
- expect(result).not.toHaveProperty("trace_score_categories");
- expect(result).not.toHaveProperty("trace_score_columns");
});
it("should return experiment-run score filter options", async () => {
diff --git a/web/src/__tests__/table-view-presets.clienttest.ts b/web/src/__tests__/table-view-presets.clienttest.ts
index dbaa516b4f..3b0d2519e4 100644
--- a/web/src/__tests__/table-view-presets.clienttest.ts
+++ b/web/src/__tests__/table-view-presets.clienttest.ts
@@ -9,6 +9,7 @@ import {
type FilterState,
type OrderByState,
} from "@langfuse/shared";
+import { experimentsFilterConfig } from "@/src/features/experiments/components/table/filter-config";
// Mock data for testing
const mockColumns = [
@@ -217,3 +218,82 @@ describe("table view presets validation functions", () => {
});
});
});
+
+describe("saved views built before the experiment score facets were unified", () => {
+ // `validateFilters` drops any filter it cannot resolve to a column id, so a
+ // renamed facet silently loses a saved view's filter unless the OLD id and
+ // the OLD label are both aliases.
+ const legacyColumns = [
+ ["obs_scores_avg", "scores_avg"],
+ ["Scores (numeric)", "scores_avg"],
+ ["obs_score_categories", "score_categories"],
+ ["Scores (categorical)", "score_categories"],
+ ["obs_score_booleans", "score_booleans"],
+ ["Scores (boolean)", "score_booleans"],
+ ] as const;
+
+ it.each(legacyColumns)("resolves %s onto %s", (stored, canonical) => {
+ const validated = validateFilters(
+ [
+ {
+ type: "numberObject",
+ column: stored,
+ key: "accuracy",
+ operator: ">",
+ value: 0.5,
+ },
+ ],
+ experimentsFilterConfig.columnDefinitions,
+ );
+
+ expect(validated.map((filter) => filter.column)).toEqual([canonical]);
+ });
+
+ it.each([
+ "trace_scores_avg",
+ "trace_score_categories",
+ "trace_score_booleans",
+ ])("keeps a saved %s filter, which is no longer offered", (column) => {
+ const validated = validateFilters(
+ [
+ {
+ type: "numberObject",
+ column,
+ key: "accuracy",
+ operator: ">",
+ value: 0.5,
+ },
+ ],
+ experimentsFilterConfig.columnDefinitions,
+ );
+
+ expect(validated.map((filter) => filter.column)).toEqual([column]);
+ });
+
+ it("keeps both levels when a saved view carries one of each", () => {
+ const validated = validateFilters(
+ [
+ {
+ type: "numberObject",
+ column: "Scores (numeric)",
+ key: "accuracy",
+ operator: ">",
+ value: 0.5,
+ },
+ {
+ type: "numberObject",
+ column: "Trace Scores (numeric)",
+ key: "nps",
+ operator: ">",
+ value: 5,
+ },
+ ],
+ experimentsFilterConfig.columnDefinitions,
+ );
+
+ expect(validated.map((filter) => filter.column)).toEqual([
+ "scores_avg",
+ "trace_scores_avg",
+ ]);
+ });
+});
diff --git a/web/src/features/experiments/components/table/filter-config.ts b/web/src/features/experiments/components/table/filter-config.ts
index 4521c4c720..20885f44b4 100644
--- a/web/src/features/experiments/components/table/filter-config.ts
+++ b/web/src/features/experiments/components/table/filter-config.ts
@@ -96,27 +96,35 @@ export const experimentsTableCols: ColumnDefinition[] = [
type: "number",
internal: "error_count",
},
- // Observation-level scores (eos.* alias in backend)
+ // Level-agnostic scores: one filter per data type that matches a score
+ // whether it was recorded on an observation or on the trace. The `obs_*` ids
+ // are aliases so existing links and saved views keep resolving — and start
+ // matching trace-level scores, which is the fix. The old DISPLAY names are
+ // aliases too: a saved view may store a column by its label, and
+ // `validateFilters` drops what it cannot resolve.
{
- name: "Scores (numeric)",
- id: "obs_scores_avg",
+ name: "Numeric Scores",
+ id: "scores_avg",
type: "numberObject",
- internal: "obs_scores_avg",
+ internal: "scores_avg",
+ aliases: ["obs_scores_avg", "Scores (numeric)"],
},
{
- name: "Scores (categorical)",
- id: "obs_score_categories",
+ name: "Categorical Scores",
+ id: "score_categories",
type: "categoryOptions",
- internal: "obs_score_categories",
+ internal: "score_categories",
options: [],
nullable: true,
+ aliases: ["obs_score_categories", "Scores (categorical)"],
},
{
- name: "Scores (boolean)",
- id: "obs_score_booleans",
+ name: "Boolean Scores",
+ id: "score_booleans",
type: "booleanObject",
- internal: "obs_score_booleans",
+ internal: "score_booleans",
nullable: true,
+ aliases: ["obs_score_booleans", "Scores (boolean)"],
},
// Trace-level scores (ets.* alias in backend)
{
@@ -234,37 +242,23 @@ export const experimentsFilterConfig: FilterConfig = {
column: "metadata",
label: getExperimentsColumnName("metadata"),
},
- // Observation-level scores
+ // One facet per score data type. Each offered name is tagged with the
+ // level(s) it exists at (ScoreTag), so the distinction stays visible where
+ // it is actionable instead of being duplicated across six facets.
{
type: "keyValue" as const,
- column: "obs_score_categories",
- label: getExperimentsColumnName("obs_score_categories"),
+ column: "score_categories",
+ label: getExperimentsColumnName("score_categories"),
},
{
type: "numericKeyValue" as const,
- column: "obs_scores_avg",
- label: getExperimentsColumnName("obs_scores_avg"),
+ column: "scores_avg",
+ label: getExperimentsColumnName("scores_avg"),
},
{
type: "booleanKeyValue" as const,
- column: "obs_score_booleans",
- label: getExperimentsColumnName("obs_score_booleans"),
- },
- // Trace-level scores
- {
- type: "keyValue" as const,
- column: "trace_score_categories",
- label: getExperimentsColumnName("trace_score_categories"),
- },
- {
- type: "numericKeyValue" as const,
- column: "trace_scores_avg",
- label: getExperimentsColumnName("trace_scores_avg"),
- },
- {
- type: "booleanKeyValue" as const,
- column: "trace_score_booleans",
- label: getExperimentsColumnName("trace_score_booleans"),
+ column: "score_booleans",
+ label: getExperimentsColumnName("score_booleans"),
},
],
};
diff --git a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts b/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
index b0b029a999..ea9decbe28 100644
--- a/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
+++ b/web/src/features/experiments/hooks/useExperimentFilterOptions.ts
@@ -77,19 +77,20 @@ export function useExperimentFilterOptions({
const experimentFilterOptions = useMemo(() => {
return {
experimentDatasetName: usedDatasets?.map((d) => ({ value: d.name })),
- // Observation-level score options
- obs_scores_avg: filterOptions.data?.obs_scores_avg ?? undefined,
- obs_score_categories: processScoreCategories(
- filterOptions.data?.obs_score_categories,
+ // The three level-agnostic score facets, plus the per-data-type level maps
+ // that tag each offered name with the level(s) it exists at (ScoreTag).
+ // `useSidebarFilterState` looks the maps up by these exact keys.
+ scores_avg: filterOptions.data?.scores_avg ?? undefined,
+ score_categories: processScoreCategories(
+ filterOptions.data?.score_categories,
),
- obs_score_booleans: filterOptions.data?.obs_score_booleans ?? undefined,
- // Trace-level score options
- trace_scores_avg: filterOptions.data?.trace_scores_avg ?? undefined,
- trace_score_categories: processScoreCategories(
- filterOptions.data?.trace_score_categories,
- ),
- trace_score_booleans:
- filterOptions.data?.trace_score_booleans ?? undefined,
+ score_booleans: filterOptions.data?.score_booleans ?? undefined,
+ score_name_levels_numeric:
+ filterOptions.data?.score_name_levels_numeric ?? undefined,
+ score_name_levels_categorical:
+ filterOptions.data?.score_name_levels_categorical ?? undefined,
+ score_name_levels_boolean:
+ filterOptions.data?.score_name_levels_boolean ?? undefined,
};
}, [usedDatasets, filterOptions.data]);
diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.ts
index 9f81b58982..952f30869c 100644
--- a/web/src/features/experiments/server/router.ts
+++ b/web/src/features/experiments/server/router.ts
@@ -473,30 +473,78 @@ export const experimentsRouter = createTRPCRouter({
}))
: [];
+ // Offered-set == matchable-set: the level-agnostic score filters match a
+ // score only if it rolls up into a trace, so discovery is scoped the same
+ // way. Session- and dataset-run scores can never match, and offering them
+ // would advertise a filter that cannot fire.
+ const nullCondition = (
+ column: "traceId" | "observationId",
+ operator: "is null" | "is not null",
+ ) => ({ type: "null" as const, column, operator, value: "" as const });
+ const traceScopedFilter = [
+ nullCondition("traceId", "is not null"),
+ ...(traceTimestampFilters ?? []),
+ ];
+ // Discovered per level as well, to tag each offered name with the level(s)
+ // it exists at rather than duplicating the facets per level.
+ const observationScopedFilter = [
+ ...traceScopedFilter,
+ nullCondition("observationId", "is not null"),
+ ];
+ const traceLevelScopedFilter = [
+ ...traceScopedFilter,
+ nullCondition("observationId", "is null"),
+ ];
+
const [
numericScoreNames,
categoricalScoreNames,
booleanScoreNames,
+ observationLevelNumeric,
+ observationLevelCategorical,
+ observationLevelBoolean,
+ traceLevelNumeric,
+ traceLevelCategorical,
+ traceLevelBoolean,
experimentDatasetIds,
] = await Promise.all([
- getNumericScoresGroupedByName(
- input.projectId,
- traceTimestampFilters ?? [],
- ),
+ getNumericScoresGroupedByName(input.projectId, traceScopedFilter),
+ getCategoricalScoresGroupedByName(input.projectId, traceScopedFilter),
+ getBooleanScoresGroupedByName(input.projectId, traceScopedFilter),
+ getNumericScoresGroupedByName(input.projectId, observationScopedFilter),
getCategoricalScoresGroupedByName(
input.projectId,
- traceTimestampFilters ?? [],
+ observationScopedFilter,
),
- getBooleanScoresGroupedByName(
+ getBooleanScoresGroupedByName(input.projectId, observationScopedFilter),
+ getNumericScoresGroupedByName(input.projectId, traceLevelScopedFilter),
+ getCategoricalScoresGroupedByName(
input.projectId,
- traceTimestampFilters ?? [],
+ traceLevelScopedFilter,
),
+ getBooleanScoresGroupedByName(input.projectId, traceLevelScopedFilter),
getEventsGroupedByExperimentDatasetId(
input.projectId,
input.startTimeFilter ?? [],
),
]);
+ /**
+ * Split per data type: a name can be reused across types at different
+ * levels (a numeric observation-level "accuracy" beside an unrelated
+ * categorical trace-level one), and a name-only map would mislabel both.
+ */
+ const levelsOf = (
+ observationNames: string[],
+ traceNames: string[],
+ ): Record<string, ("observation" | "trace")[]> => {
+ const out: Record<string, ("observation" | "trace")[]> = {};
+ for (const name of observationNames)
+ (out[name] ??= []).push("observation");
+ for (const name of traceNames) (out[name] ??= []).push("trace");
+ return out;
+ };
+
const experimentDatasetIdSet = new Set<string>();
for (const { experimentDatasetId } of experimentDatasetIds) {
if (experimentDatasetId !== null) {
@@ -510,14 +558,23 @@ export const experimentsRouter = createTRPCRouter({
const booleanScoreOptions = booleanScoreNames.map((score) => score.name);
return {
- // Observation-level score options (eos.*)
- obs_scores_avg: numericScoreOptions,
- obs_score_categories: categoricalScoreNames,
- obs_score_booleans: booleanScoreOptions,
- // Trace-level score options (ets.*)
- trace_scores_avg: numericScoreOptions,
- trace_score_categories: categoricalScoreNames,
- trace_score_booleans: booleanScoreOptions,
+ // What the three score facets offer, with the level(s) each name exists
+ // at so the picker can tag them.
+ scores_avg: numericScoreOptions,
+ score_categories: categoricalScoreNames,
+ score_booleans: booleanScoreOptions,
+ score_name_levels_numeric: levelsOf(
+ observationLevelNumeric.map((score) => score.name),
+ traceLevelNumeric.map((score) => score.name),
+ ),
+ score_name_levels_categorical: levelsOf(
+ observationLevelCategorical.map((score) => score.label),
+ traceLevelCategorical.map((score) => score.label),
+ ),
+ score_name_levels_boolean: levelsOf(
+ observationLevelBoolean.map((score) => score.name),
+ traceLevelBoolean.map((score) => score.name),
+ ),
experimentDatasetIds: Array.from(experimentDatasetIdSet),
};
}),
diff --git a/web/src/features/search-bar/lib/fields.ts b/web/src/features/search-bar/lib/fields.ts
index ec8ba49ea6..940889fb4d 100644
--- a/web/src/features/search-bar/lib/fields.ts
+++ b/web/src/features/search-bar/lib/fields.ts
@@ -149,12 +149,13 @@ export function fieldRegistryFromColumns(
// `*Object` columns and the categorical score column are keyed dot-paths
// (`metadata.<key>`, `scores.<name>`), never plain fields — a derived
// `score_categories` field would lower to a keyless categoryOptions filter
- // the backend cannot answer. They stay in `columns` so the reverse adapter
- // still resolves them.
+ // the backend cannot answer. That holds whether or not this view exposes
+ // the `scores.` namespace: a view whose sidebar owns score filters still
+ // has the column in its facets, and it is sidebar-only, not a bare field.
+ // They stay in `columns` so the reverse adapter still resolves them.
.filter(
(column) =>
- !column.type.endsWith("Object") &&
- !(scores && isKeyedScoreColumn(column.id)),
+ !column.type.endsWith("Object") && !isKeyedScoreColumn(column.id),
)
.map((column): FieldDef => {
const fieldOverlay = overlay.fields?.[column.id];
diff --git a/web/src/features/search-bar/lib/filter-state-to-query.ts b/web/src/features/search-bar/lib/filter-state-to-query.ts
index f3fd91a22e..deee391030 100644
--- a/web/src/features/search-bar/lib/filter-state-to-query.ts
+++ b/web/src/features/search-bar/lib/filter-state-to-query.ts
@@ -39,7 +39,16 @@ function negate(node: FilterNode): ASTNode {
return { kind: "not", child: node };
}
-function scorePathOf(column: string, key: string): string | null {
+function scorePathOf(
+ column: string,
+ key: string,
+ registry: FieldRegistry,
+): string | null {
+ // A view can own score filters in its sidebar without exposing them in the
+ // bar. Emitting the path anyway would render text the parser rejects as an
+ // unknown field, so a score space the registry does not expose is left to the
+ // sidebar — reported as skipped, and preserved across commits.
+ //
// Quote the score name iff it has grammar chars so it re-lexes as one token
// (`scores."Rouge Score"`); resolveField unquotes it on the way back.
if (
@@ -47,14 +56,14 @@ function scorePathOf(column: string, key: string): string | null {
column === SCORE_COLUMNS.observation.categorical ||
column === SCORE_COLUMNS.observation.boolean
) {
- return `scores.${quoteIfNeeded(key)}`;
+ return registry.scores ? `scores.${quoteIfNeeded(key)}` : null;
}
if (
column === SCORE_COLUMNS.trace.numeric ||
column === SCORE_COLUMNS.trace.categorical ||
column === SCORE_COLUMNS.trace.boolean
) {
- return `traceScores.${quoteIfNeeded(key)}`;
+ return registry.traceScores ? `traceScores.${quoteIfNeeded(key)}` : null;
}
return null;
}
@@ -209,19 +218,19 @@ function lowerSingle(
return filterNode(key, op, [filter.value]);
}
case "numberObject": {
- const path = scorePathOf(filter.column, filter.key);
+ const path = scorePathOf(filter.column, filter.key, registry);
if (path === null) return null;
const op = filter.operator === "=" ? "=" : filter.operator;
return filterNode(path, op, [String(filter.value)]);
}
case "booleanObject": {
- const path = scorePathOf(filter.column, filter.key);
+ const path = scorePathOf(filter.column, filter.key, registry);
if (path === null) return null;
const node = filterNode(path, "=", [String(filter.value)]);
return filter.operator === "<>" ? negate(node) : node;
}
case "categoryOptions": {
- const path = scorePathOf(filter.column, filter.key);
+ const path = scorePathOf(filter.column, filter.key, registry);
if (path === null || filter.value.length === 0) return null;
const node = filterNode(path, "=", filter.value);
return filter.operator === "none of" ? negate(node) : node;
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
index bf444a76ed..5b1df669c1 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.clienttest.ts
@@ -97,6 +97,26 @@ const eventsView: RegistryUnderTest = {
"(grouped)",
'has "quote"',
],
+ sidebarFilters: [
+ [
+ {
+ type: "categoryOptions",
+ column: "score_categories",
+ key: "verdict",
+ operator: "any of",
+ value: ["good"],
+ },
+ ],
+ [
+ {
+ type: "numberObject",
+ column: "trace_scores_avg",
+ key: "nps",
+ operator: ">",
+ value: 5,
+ },
+ ],
+ ],
};
const evaluationRulesView: RegistryUnderTest = {
@@ -436,6 +456,27 @@ const sessionsView: RegistryUnderTest = {
// Free text is rewritten onto `id`, not dropped, so the quoting/reserved-word
// mirror still has to hold on this registry.
freeTextValues: ["hello", "refund policy", "or", "and", "!important", "-foo"],
+ sidebarFilters: [
+ [
+ {
+ type: "numberObject",
+ column: "scores_avg",
+ key: "accuracy",
+ operator: ">",
+ value: 0.5,
+ },
+ ],
+ // Trace scores are not offered here, so this one must stay sidebar-only.
+ [
+ {
+ type: "numberObject",
+ column: "trace_scores_avg",
+ key: "nps",
+ operator: ">",
+ value: 5,
+ },
+ ],
+ ],
};
describe("search bar invariants — sessions registry", () => {
@@ -668,6 +709,55 @@ const experimentsView: RegistryUnderTest = {
scoreContexts: [],
fieldValues: ["x", "sonnet", "5", "0.8", "a b"],
freeTextValues: ["hello", "run 12", "or", "!important"],
+ // Scores stay in the sidebar on this view, and after the score unification
+ // they sit on the SAME canonical columns the bar's `scores.` path recognizes —
+ // so the serializer has to be told they are not the bar's to render.
+ sidebarFilters: [
+ [
+ {
+ type: "categoryOptions",
+ column: "score_categories",
+ key: "verified_article_status",
+ operator: "any of",
+ value: ["none-verified"],
+ },
+ ],
+ [
+ {
+ type: "numberObject",
+ column: "scores_avg",
+ key: "groundedness",
+ operator: ">",
+ value: 0.5,
+ },
+ ],
+ [
+ {
+ type: "booleanObject",
+ column: "score_booleans",
+ key: "in_force",
+ operator: "=",
+ value: true,
+ },
+ ],
+ // A filter the bar DOES own, alongside one it does not: the owned half must
+ // still render.
+ [
+ {
+ type: "stringOptions",
+ column: "experimentDatasetName",
+ operator: "any of",
+ value: ["legal-answer-quality"],
+ },
+ {
+ type: "categoryOptions",
+ column: "score_categories",
+ key: "verified_article_status",
+ operator: "any of",
+ value: ["none-verified"],
+ },
+ ],
+ ],
};
describe("search bar invariants — experiments registry", () => {
diff --git a/web/src/features/search-bar/lib/searchBarInvariants.ts b/web/src/features/search-bar/lib/searchBarInvariants.ts
index 941f3fe0c8..68337a1ac8 100644
--- a/web/src/features/search-bar/lib/searchBarInvariants.ts
+++ b/web/src/features/search-bar/lib/searchBarInvariants.ts
@@ -13,6 +13,11 @@
// INV-2 no silent drop/rewrite — FilterState → text → FilterState is stable:
// every filter round-trips unchanged or is reported in skippedFilters,
// and nothing is rewritten into a different filter.
+// INV-4 derived text is parseable — text derived FROM a FilterState is
+// always valid for that registry. INV-2 enters from text, so it only
+// ever sees filters the bar can already express; a sidebar can put a
+// column in the URL that the bar does not expose, and emitting a token
+// for it renders an "unknown field" the user cannot fix.
// INV-3 serialize ↔ parse symmetry — a free-text value always re-parses to
// itself and stays valid. (Regressed in #4: serializeValue emitted a
// bare reserved token — `or`, `!important` — that the parser rejects.)
@@ -56,6 +61,12 @@ export type RegistryUnderTest = {
* leading operators, quotes) — these are what break serialize↔parse.
*/
freeTextValues: string[];
+ /**
+ * Filter states the view's SIDEBAR can put in the URL, including columns the
+ * bar deliberately does not expose (INV-4). Without these the harness only
+ * ever sees the bar's own field surface and cannot catch a one-sided gate.
+ */
+ sidebarFilters?: FilterState[];
};
export type InvariantFailure = {
@@ -192,7 +203,30 @@ function checkSerializeSymmetry(
}
/**
- * Run all three invariants over a view's registry and return every failure.
+ * INV-4: text derived from a FilterState always parses for that registry.
+ *
+ * Deliberately context-free: this is about whether the registry EXPOSES the
+ * field at all, not about how an observed score type routes. Pinning a score
+ * context here would only re-test INV-2's ground and could fail on a
+ * value/type mismatch that says nothing about field exposure.
+ */
+function checkDerivedTextValidity(
+ filters: FilterState,
+ registry: FieldRegistry,
+): InvariantFailure | null {
+ const derived = filterStateToQueryText(filters, {}, registry);
+ if (validateQuery(derived.text, undefined, registry).valid) return null;
+ return {
+ invariant: "INV-4 derived text is parseable",
+ case: stable(filters),
+ detail: `derived "${derived.text}" which the same registry rejects; a column the bar does not expose belongs in skippedFilters (skipped: ${stable(
+ derived.skippedFilters,
+ )})`,
+ };
+}
+
+/**
+ * Run all invariants over a view's registry and return every failure.
* The caller (a *.clienttest.ts) asserts the list is empty.
*/
export function runSearchBarInvariants(
@@ -210,6 +244,10 @@ export function runSearchBarInvariants(
if (roundTrip) failures.push(roundTrip);
}
}
+ for (const filters of view.sidebarFilters ?? []) {
+ const derived = checkDerivedTextValidity(filters, view.registry);
+ if (derived) failures.push(derived);
+ }
for (const value of view.freeTextValues) {
const symmetry = checkSerializeSymmetry(value, view.registry);
if (symmetry) failures.push(symmetry);