Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): a worse-than-comparison filter and a score-by-run matrix (#16918)

Nikita Kabardin79b2c6644338
diff --git a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx b/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
index fe6898ac0d..1ce02420c3 100644
--- a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
+++ b/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
@@ -86,6 +86,12 @@ export function ExperimentDisplaySettings({
>
Side by side β€” a column per experiment
</OptionItem>
+ <OptionItem
+ selected={layout === "matrix"}
+ onSelect={() => onLayoutChange("matrix")}
+ >
+ Score matrix β€” scores as rows, runs as columns
+ </OptionItem>
<DropdownMenuSeparator />
diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
index 872c447aa9..8bcb4e778b 100644
--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
@@ -93,6 +93,23 @@ import {
type ScoreColumnSummary,
} from "@/src/features/experiments/fns/summariseScoreColumn";
import { ScoreColumnHeaderSummary } from "./ScoreColumnHeaderSummary";
+import {
+ ScoreColumnFilterMenu,
+ ScoreColumnFilterMenuTrigger,
+} from "./ScoreColumnFilterMenu";
+import { ScoreComparisonFilterPills } from "./ScoreComparisonFilterPills";
+import {
+ ExperimentScoreMatrix,
+ type ScoreMatrixRow,
+} from "./ExperimentScoreMatrix";
+import { useScoreComparisonFilters } from "@/src/features/experiments/hooks/useScoreComparisonFilters";
+import {
+ describeEmptyScoreComparison,
+ rowPassesScoreComparisonFilters,
+ scoreFieldForLevel,
+ type ScoreComparisonFilter,
+ type ScoreLevel,
+} from "@/src/features/experiments/fns/scoreComparisonFilter";
import { resetStaleDefaultColumnOrder } from "@/src/features/experiments/fns/experimentItemsColumnOrder";
const renderExperimentSpecificHeader = (label: string) => (
<span className="text-muted-foreground">{label}</span>
@@ -694,6 +711,16 @@ export default function ExperimentItemsTable({
() => allExperimentIds.join(","),
[allExperimentIds],
);
+ // The selection arrives as a fresh array on every render, so anything that
+ // reaches a hook dependency has to come off the key, which is stable by
+ // value. `rows` below is one of those: it feeds the effect that publishes the
+ // peek navigation list, and publishing re-renders every consumer of that
+ // provider β€” this table included. Depending on the array identity there
+ // leaves the two re-triggering each other for as long as the page is open.
+ const comparableExperimentIds = useMemo(
+ () => (experimentSelectionKey ? experimentSelectionKey.split(",") : []),
+ [experimentSelectionKey],
+ );
const [expectedOutputSeenFor, setExpectedOutputSeenFor] = useState<
string | null
>(null);
@@ -810,6 +837,47 @@ export default function ExperimentItemsTable({
scoreColumnDefs.traceScoreColumns,
]);
+ const scoreNamesByKey = useMemo(() => {
+ const build = (columns: ScoreColumnDef[]) =>
+ new Map(toScoreColumnInput(columns).map(({ key, name }) => [key, name]));
+ return {
+ observationScores: build(scoreColumnDefs.observationScoreColumns),
+ traceScores: build(scoreColumnDefs.traceScoreColumns),
+ };
+ }, [
+ scoreColumnDefs.observationScoreColumns,
+ scoreColumnDefs.traceScoreColumns,
+ ]);
+
+ const {
+ filters: scoreComparisonFilters,
+ setFilter: setScoreComparisonFilter,
+ removeFilter: removeScoreComparisonFilter,
+ } = useScoreComparisonFilters();
+
+ // The runs a score can be read against, the auto-selected comparison first so
+ // the menu's default is the one the column header already reports.
+ const comparisonTargets = useMemo(() => {
+ const others = selectedExperimentNames.filter(
+ (exp) => exp.experimentId !== primaryExperimentId,
+ );
+ return [
+ ...others.filter((exp) => exp.experimentId === primaryComparisonId),
+ ...others.filter((exp) => exp.experimentId !== primaryComparisonId),
+ ].map(({ experimentId, experimentName }) => ({
+ experimentId,
+ experimentName,
+ }));
+ }, [selectedExperimentNames, primaryExperimentId, primaryComparisonId]);
+
+ const scoreDataTypeFor = useCallback(
+ (filter: ScoreComparisonFilter) =>
+ scoreDataTypesByKey[scoreFieldForLevel(filter.level)].get(
+ filter.scoreKey,
+ ),
+ [scoreDataTypesByKey],
+ );
+
const scoreColumnSummaries = useMemo(() => {
const rowsInView = items.rows ?? [];
return {
@@ -852,6 +920,13 @@ export default function ExperimentItemsTable({
typeof scoreCol.header === "string"
? scoreCol.header
: (scoreCol.accessorKey ?? "");
+ const level: ScoreLevel =
+ scoreField === "traceScores" ? "trace" : "observation";
+ const activeComparisonFilter = key
+ ? scoreComparisonFilters.find(
+ (filter) => filter.level === level && filter.scoreKey === key,
+ )
+ : undefined;
return {
...scoreCol,
@@ -868,6 +943,42 @@ export default function ExperimentItemsTable({
dataType={dataType}
summary={summary}
comparisonName={primaryComparisonName}
+ filterMenu={
+ comparisonTargets.length === 0 ? undefined : (
+ <ScoreColumnFilterMenu
+ targets={comparisonTargets}
+ hasOrder={dataType !== "CATEGORICAL"}
+ active={activeComparisonFilter}
+ onSelect={(operator, comparisonExperimentId) => {
+ if (!key) return;
+ const nextFilter = {
+ level,
+ scoreKey: key,
+ operator,
+ comparisonExperimentId,
+ };
+ setScoreComparisonFilter(nextFilter);
+ }}
+ onClear={() =>
+ activeComparisonFilter &&
+ removeScoreComparisonFilter(activeComparisonFilter)
+ }
+ >
+ {({ Trigger }) => (
+ // The header cell sorts on click, so opening the menu
+ // must not also reorder the table.
+ <Trigger
+ asChild
+ onClick={(event) => event.stopPropagation()}
+ >
+ <ScoreColumnFilterMenuTrigger
+ isActive={Boolean(activeComparisonFilter)}
+ />
+ </Trigger>
+ )}
+ </ScoreColumnFilterMenu>
+ )
+ }
/>
),
}
@@ -971,6 +1082,10 @@ export default function ExperimentItemsTable({
scoreColumnSummaries,
scoreDataTypesByKey,
showComparisonDiff,
+ comparisonTargets,
+ scoreComparisonFilters,
+ setScoreComparisonFilter,
+ removeScoreComparisonFilter,
runNameOf,
],
);
@@ -1366,6 +1481,59 @@ export default function ExperimentItemsTable({
columnOrderMigrations,
);
+ // The transposed layout's axes: score columns as rows β€”
+ // respecting what the column picker hid β€” and the selected runs as columns,
+ // baseline first. Both are a re-read of what the grid already has, so the
+ // layout needs no query of its own.
+ const matrixScoreRows = useMemo<ScoreMatrixRow[]>(() => {
+ const build = (
+ scoreCols: LangfuseColumnDef<ExperimentItemData>[],
+ level: ScoreLevel,
+ ): ScoreMatrixRow[] =>
+ scoreCols.flatMap((scoreCol) => {
+ const accessorKey = scoreCol.accessorKey;
+ if (!accessorKey || columnVisibility[accessorKey] === false) return [];
+ const scoreKey = accessorKey.replace(/^Trace-/, "");
+ const dataType =
+ scoreDataTypesByKey[scoreFieldForLevel(level)].get(scoreKey);
+ if (!dataType) return [];
+ return [
+ {
+ scoreKey,
+ level,
+ dataType,
+ label:
+ typeof scoreCol.header === "string" ? scoreCol.header : scoreKey,
+ },
+ ];
+ });
+ return [
+ ...build(observationScoreColumns, "observation"),
+ ...build(traceScoreColumns, "trace"),
+ ];
+ }, [
+ observationScoreColumns,
+ traceScoreColumns,
+ scoreDataTypesByKey,
+ columnVisibility,
+ ]);
+
+ const matrixExperiments = useMemo(
+ () =>
+ [
+ ...(primaryExperimentId ? [primaryExperimentId] : []),
+ ...allExperimentIds.filter((id) => id !== primaryExperimentId),
+ ].map((experimentId) => ({
+ experimentId,
+ experimentName:
+ selectedExperimentNames.find(
+ (exp) => exp.experimentId === experimentId,
+ )?.experimentName ?? experimentId.slice(0, 8),
+ isBaseline: experimentId === primaryExperimentId,
+ })),
+ [allExperimentIds, primaryExperimentId, selectedExperimentNames],
+ );
+
const peekNavigationProps = usePeekNavigation({
queryParams: [
"observation",
@@ -1425,12 +1593,38 @@ export default function ExperimentItemsTable({
};
}, [peekNavigationProps, canUsePeek]);
- const rows: ExperimentItemsTableRow[] = useMemo(() => {
+ // The page as fetched. The score column header aggregates β€” and the score
+ // matrix, which reads the same ones β€” deliberately describe this whole page,
+ // so the movement a comparison filter was built from stays readable while
+ // that filter is applied.
+ const unfilteredRows: ExperimentItemsTableRow[] = useMemo(() => {
if (items.status !== "success" || !items.rows) return [];
// Add 'id' field for DataTable row identification (peek view requires it)
return items.rows.map((row) => ({ ...row, id: row.itemId }));
}, [items]);
+ // The score comparison filters narrow the page here rather than in the
+ // query β€” see `useScoreComparisonFilters` for why.
+ const rows: ExperimentItemsTableRow[] = useMemo(
+ () =>
+ unfilteredRows.filter((row) =>
+ rowPassesScoreComparisonFilters({
+ filters: scoreComparisonFilters,
+ experiments: row.experiments,
+ baselineExperimentId: primaryExperimentId,
+ comparableExperimentIds,
+ dataTypeFor: scoreDataTypeFor,
+ }),
+ ),
+ [
+ unfilteredRows,
+ scoreComparisonFilters,
+ primaryExperimentId,
+ comparableExperimentIds,
+ scoreDataTypeFor,
+ ],
+ );
+
useEffect(() => {
if (items.status === "success") {
// Store all experiment targets for peek navigation
@@ -1471,6 +1665,38 @@ export default function ExperimentItemsTable({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [items.status, rows, baselineId]);
+ // Plain English for the active comparisons, and for the table when they leave
+ // nothing: "no regressions on this score" is an answer, not a broken table.
+ const scoreComparisonPills = useMemo(
+ () =>
+ scoreComparisonFilters.map((filter) => ({
+ filter,
+ scoreName:
+ scoreNamesByKey[scoreFieldForLevel(filter.level)].get(
+ filter.scoreKey,
+ ) ?? filter.scoreKey,
+ comparisonName:
+ selectedExperimentNames.find(
+ (exp) => exp.experimentId === filter.comparisonExperimentId,
+ )?.experimentName ?? filter.comparisonExperimentId,
+ })),
+ [scoreComparisonFilters, scoreNamesByKey, selectedExperimentNames],
+ );
+
+ const scoreComparisonEmptyMessage = useMemo(() => {
+ if (rows.length > 0 || (items.rows ?? []).length === 0) return undefined;
+ if (scoreComparisonPills.length !== 1)
+ return scoreComparisonPills.length > 1
+ ? "No item on this page matches every score comparison."
+ : undefined;
+ const [pill] = scoreComparisonPills;
+ return describeEmptyScoreComparison({
+ operator: pill.filter.operator,
+ scoreName: pill.scoreName,
+ comparisonName: pill.comparisonName,
+ });
+ }, [rows.length, items.rows, scoreComparisonPills]);
+
const pagination = useMemo(
() => ({
totalCount: totalCount ?? null,
@@ -1640,6 +1866,14 @@ export default function ExperimentItemsTable({
/>
)}
+ {/* Score comparison filters β€” evaluated over the loaded page */}
+ {scoreComparisonPills.length > 0 && (
+ <ScoreComparisonFilterPills
+ pills={scoreComparisonPills}
+ onRemove={removeScoreComparisonFilter}
+ />
+ )}
+
{/* Filter Pills with Experiment Targeting */}
{filtersByExperiment.length > 0 && (
<ExperimentFilterPills
@@ -1662,7 +1896,23 @@ export default function ExperimentItemsTable({
)}
<div className="flex flex-1 flex-col overflow-hidden">
- {layout === "grid" ? (
+ {layout === "matrix" ? (
+ hasSelectedRuns ? (
+ <ExperimentScoreMatrix
+ rows={unfilteredRows}
+ scoreRows={matrixScoreRows}
+ experiments={matrixExperiments}
+ isLoading={items.status === "loading" || isViewLoading}
+ pagination={pagination}
+ />
+ ) : (
+ <div className="flex flex-1 items-center justify-center">
+ <span className="text-muted-foreground text-sm">
+ Please select a baseline experiment.
+ </span>
+ </div>
+ )
+ ) : layout === "grid" ? (
hasSelectedRuns ? (
<ExperimentGridView
projectId={projectId}
@@ -1689,6 +1939,13 @@ export default function ExperimentItemsTable({
rowSelection={selectedRows}
setRowSelection={setSelectedRows}
highlightAllRows={selectAll}
+ noResultsMessage={
+ scoreComparisonEmptyMessage ? (
+ <span className="text-muted-foreground text-sm">
+ {scoreComparisonEmptyMessage}
+ </span>
+ ) : undefined
+ }
/>
) : (
<div className="flex flex-1 items-center justify-center">
@@ -1720,6 +1977,10 @@ export default function ExperimentItemsTable({
<span className="text-muted-foreground text-sm">
Please select a baseline experiment.
</span>
+ ) : scoreComparisonEmptyMessage ? (
+ <span className="text-muted-foreground text-sm">
+ {scoreComparisonEmptyMessage}
+ </span>
) : undefined
}
highlightAllRows={selectAll}
diff --git a/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsx b/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsx
new file mode 100644
index 0000000000..ebe7c4e273
--- /dev/null
+++ b/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsx
@@ -0,0 +1,102 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+import type { PaginationState } from "@tanstack/react-table";
+import { TooltipProvider } from "@/src/components/ui/tooltip";
+import {
+ ExperimentScoreMatrix,
+ type ScoreMatrixColumn,
+ type ScoreMatrixRow,
+} from "./ExperimentScoreMatrix";
+import type { ExperimentItemsTableRow } from "./types";
+
+vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({
+ usePostHogClientCapture: () => vi.fn(),
+}));
+
+const scoreKey = "groundedness-EVAL-NUMERIC";
+
+const scoreRows: ScoreMatrixRow[] = [
+ { scoreKey, level: "trace", dataType: "NUMERIC", label: "# groundedness" },
+];
+
+const experiments: ScoreMatrixColumn[] = [
+ { experimentId: "base", experimentName: "baseline-run", isBaseline: true },
+ { experimentId: "cand", experimentName: "candidate-run", isBaseline: false },
+];
+
+const itemRow = (id: string, value: number): ExperimentItemsTableRow =>
+ ({
+ id,
+ experiments: experiments.map((experiment) => ({
+ experimentId: experiment.experimentId,
+ traceScores: { [scoreKey]: { type: "NUMERIC", average: value } },
+ })),
+ }) as unknown as ExperimentItemsTableRow;
+
+const renderMatrix = ({
+ rows,
+ state,
+ onChange,
+ totalCount,
+}: {
+ rows: ExperimentItemsTableRow[];
+ state: PaginationState;
+ onChange: (next: unknown) => void;
+ totalCount: number | null;
+}) =>
+ render(
+ <TooltipProvider>
+ <ExperimentScoreMatrix
+ rows={rows}
+ scoreRows={scoreRows}
+ experiments={experiments}
+ isLoading={false}
+ pagination={{ totalCount, onChange, state }}
+ />
+ </TooltipProvider>,
+ );
+
+describe("ExperimentScoreMatrix pagination", () => {
+ it("offers page controls and reports the window it aggregated", () => {
+ const onChange = vi.fn();
+ renderMatrix({
+ rows: [itemRow("a", 0.5), itemRow("b", 0.7)],
+ state: { pageIndex: 0, pageSize: 2 },
+ onChange,
+ totalCount: 6,
+ });
+
+ // The footnote has to name the window, since the cells above it are an
+ // aggregate over exactly these rows and not over the whole run.
+ expect(
+ screen.getByText(/aggregated over the 2 items on this page/),
+ ).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole("button", { name: /go to next page/i }));
+ expect(onChange).toHaveBeenCalled();
+ });
+
+ it("keeps the page controls when no score has a value on this page", () => {
+ render(
+ <TooltipProvider>
+ <ExperimentScoreMatrix
+ rows={[itemRow("a", 0.5)]}
+ scoreRows={[]}
+ experiments={experiments}
+ isLoading={false}
+ pagination={{
+ totalCount: 6,
+ onChange: vi.fn(),
+ state: { pageIndex: 1, pageSize: 1 },
+ }}
+ />
+ </TooltipProvider>,
+ );
+
+ expect(
+ screen.getByText(/No score columns are visible for the items in view/),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByRole("button", { name: /go to next page/i }),
+ ).toBeInTheDocument();
+ });
+});
diff --git a/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsx b/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsx
new file mode 100644
index 0000000000..61ef62ad7f
--- /dev/null
+++ b/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsx
@@ -0,0 +1,379 @@
+import { useMemo } from "react";
+import {
+ getCoreRowModel,
+ useReactTable,
+ type ColumnDef,
+ type OnChangeFn,
+ type PaginationState,
+ type Table as TanStackTable,
+} from "@tanstack/react-table";
+import { Badge } from "@/src/components/ui/badge";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { DataTablePagination } from "@/src/components/table/data-table-pagination";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { DiffLabel } from "@/src/features/datasets/components/DiffLabel";
+import {
+ getScoreDataTypeExplanation,
+ splitScoreDataTypeIcon,
+} from "@/src/features/scores/lib/scoreColumns";
+import {
+ formatScoreColumnAggregate,
+ formatScoreValue,
+} from "@/src/features/experiments/fns/formatScoreColumnAggregate";
+import {
+ summariseScoreColumn,
+ type ScoreColumnDataType,
+ type ScoreColumnSummary,
+} from "@/src/features/experiments/fns/summariseScoreColumn";
+import {
+ scoreFieldForLevel,
+ type ScoreLevel,
+} from "@/src/features/experiments/fns/scoreComparisonFilter";
+import {
+ getExperimentColorStyles,
+ type ExperimentItemsTableRow,
+} from "./types";
+import { cn } from "@/src/utils/tailwind";
+
+/**
+ * The pagination instance never renders cells, only counts rows, so it needs
+ * no column defs. Module-level so the reference stays stable across renders.
+ */
+const NO_COLUMNS: ColumnDef<ExperimentItemsTableRow>[] = [];
+
+/** One row of the matrix: a score column, read at its own level. */
+export type ScoreMatrixRow = {
+ scoreKey: string;
+ level: ScoreLevel;
+ dataType: ScoreColumnDataType;
+ /** The column header, e.g. `Trace: # groundedness (eval)`. */
+ label: string;
+};
+
+/** One column of the matrix: a selected experiment, the baseline first. */
+export type ScoreMatrixColumn = {
+ experimentId: string;
+ experimentName: string;
+ isBaseline: boolean;
+};
+
+const MatrixCell = ({
+ summary,
+ hasBaselineColumn,
+}: {
+ summary: ScoreColumnSummary;
+ /** False for the baseline's own column, which has nothing to move against. */
+ hasBaselineColumn: boolean;
+}) => {
+ const { baseline, delta, movement } = summary;
+ const notComparable = movement?.notComparable ?? 0;
+
+ if (!baseline)
+ return <span className="text-muted-foreground">not scored</span>;
+
+ return (
+ <span className="flex flex-wrap items-center gap-x-1 gap-y-0.5 tabular-nums">
+ <span className="font-bold">{formatScoreColumnAggregate(baseline)}</span>
+ {hasBaselineColumn && delta !== null && delta !== 0 && (
+ <DiffLabel
+ diff={{
+ type: "NUMERIC",
+ absoluteDifference: Math.abs(delta),
+ direction: delta > 0 ? "+" : "-",
+ }}
+ formatValue={formatScoreValue}
+ />
+ )}
+ {hasBaselineColumn && movement && movement.changed > 0 && (
+ <span className="text-muted-foreground">↻{movement.changed}</span>
+ )}
+ {/* Items only one of the two runs scored: never folded into the delta. */}
+ {hasBaselineColumn && notComparable > 0 && (
+ <span className="text-muted-foreground opacity-70">
+ {notComparable} n/a
+ </span>
+ )}
+ </span>
+ );
+};
+
+/**
+ * The transpose of the grid: **scores as rows, experiments as columns**, every
+ * cell the run's aggregate plus its move against the baseline column. Answers
+ * "how did all my runs move on all my metrics" in one screen β€” what the deleted
+ * Analytics tab was going to be.
+ *
+ * It needs no query of its own: the aggregates are the same ones the score
+ * column headers compute, over the items already loaded for this page. Which
+ * page that is stays the user's choice β€” see `paginationTable`.
+ */
+export const ExperimentScoreMatrix = ({
+ rows,
+ scoreRows,
+ experiments,
+ isLoading,
+ pagination,
+}: {
+ rows: ExperimentItemsTableRow[];
+ scoreRows: ScoreMatrixRow[];
+ experiments: ScoreMatrixColumn[];
+ isLoading: boolean;
+ pagination: {
+ totalCount: number | null;
+ onChange: OnChangeFn<PaginationState>;
+ state: PaginationState;
+ options?: number[];
+ };
+}) => {
+ const baselineExperimentId = experiments.find(
+ (exp) => exp.isBaseline,
+ )?.experimentId;
+
+ const summaries = useMemo(() => {
+ const scoresOf = (
+ row: ExperimentItemsTableRow,
+ experimentId: string,
+ level: ScoreLevel,
+ ) =>
+ row.experiments.find((exp) => exp.experimentId === experimentId)?.[
+ scoreFieldForLevel(level)
+ ];
+
+ // Cell (score, experiment) reads the experiment as the thing the delta
+ // describes and the baseline column as what it moved against β€” the same
+ // frame as a score column header.
+ const byScore = new Map<string, Map<string, ScoreColumnSummary>>();
+ for (const scoreRow of scoreRows) {
+ const byExperiment = new Map<string, ScoreColumnSummary>();
+ for (const experiment of experiments) {
+ byExperiment.set(
+ experiment.experimentId,
+ summariseScoreColumn({
+ pairs: rows.map((row) => ({
+ baseline:
+ scoresOf(row, experiment.experimentId, scoreRow.level)?.[
+ scoreRow.scoreKey
+ ] ?? null,
+ comparison:
+ baselineExperimentId === undefined
+ ? null
+ : (scoresOf(row, baselineExperimentId, scoreRow.level)?.[
+ scoreRow.scoreKey
+ ] ?? null),
+ })),
+ dataType: scoreRow.dataType,
+ hasComparison:
+ baselineExperimentId !== undefined &&
+ experiment.experimentId !== baselineExperimentId,
+ }),
+ );
+ }
+ byScore.set(`${scoreRow.level}-${scoreRow.scoreKey}`, byExperiment);
+ }
+ return byScore;
+ }, [rows, scoreRows, experiments, baselineExperimentId]);
+
+ // The matrix aggregates the items on the page instead of listing them, so it
+ // has no table of its own to drive the pagination bar. A headless instance
+ // over the same rows and the same URL-held state does, which keeps the
+ // aggregate's window under the user's control rather than pinning it to
+ // whichever page was open first.
+ const paginationTable = useReactTable({
+ data: rows,
+ columns: NO_COLUMNS,
+ getCoreRowModel: getCoreRowModel(),
+ manualPagination: true,
+ pageCount:
+ pagination.totalCount === null
+ ? -1
+ : Math.ceil(Number(pagination.totalCount) / pagination.state.pageSize),
+ onPaginationChange: pagination.onChange,
+ // The page lives in the URL and the rows arrive per page, so TanStack's
+ // default reset-on-data-change would write page 1 back on every fetch and
+ // re-fetch from it. The data tables opt out for the same reason.
+ autoResetPageIndex: false,
+ state: { pagination: pagination.state },
+ });
+
+ if (isLoading)
+ return (
+ <div className="flex flex-col gap-2 p-4">
+ {Array.from({ length: 6 }).map((_, index) => (
+ <Skeleton key={index} className="h-6 w-full" />
+ ))}
+ </div>
+ );
+
+ const allExperimentIds = experiments.map((exp) => exp.experimentId);
+
+ // The empty state shares the footer, so the page controls stay reachable
+ // when the items on this page happen to carry none of the visible scores.
+ if (scoreRows.length === 0)
+ return (
+ <div className="flex min-h-0 flex-1 flex-col">
+ <div className="flex flex-1 items-center justify-center p-4">
+ <span className="text-muted-foreground text-sm">
+ No score columns are visible for the items in view.
+ </span>
+ </div>
+ <MatrixFooter
+ scoreCount={0}
+ experimentCount={experiments.length}
+ itemCount={rows.length}
+ paginationTable={paginationTable}
+ paginationOptions={pagination.options}
+ isLoading={isLoading}
+ />
+ </div>
+ );
+
+ return (
+ <div className="flex min-h-0 flex-1 flex-col">
+ <div className="min-h-0 flex-1 overflow-auto">
+ <table className="w-full min-w-max border-separate border-spacing-0 text-xs">
+ <thead>
+ <tr>
+ <th className="bg-background text-muted-foreground sticky top-0 left-0 z-20 min-w-[220px] border-b p-2 text-left font-normal">
+ Score
+ </th>
+ {experiments.map((experiment) => {
+ const colorStyles = getExperimentColorStyles(
+ experiment.experimentId,
+ allExperimentIds,
+ );
+ return (
+ <th
+ key={experiment.experimentId}
+ className="bg-background sticky top-0 z-10 min-w-[150px] border-b p-2 text-left font-normal"
+ >
+ <span className="flex min-w-0 items-center gap-1">
+ <span
+ className={cn(
+ "block h-3 w-0.5 shrink-0 rounded-full",
+ colorStyles.markerClass,
+ )}
+ />
+ <span
+ className="truncate font-bold"
+ title={experiment.experimentName}
+ >
+ {experiment.experimentName}
+ </span>
+ {experiment.isBaseline && (
+ <Badge
+ size="sm"
+ variant="secondary"
+ className="shrink-0"
+ >
+ baseline
+ </Badge>
+ )}
+ </span>
+ </th>
+ );
+ })}
+ </tr>
+ </thead>
+ <tbody>
+ {scoreRows.map((scoreRow) => {
+ const { icon, label } = splitScoreDataTypeIcon(scoreRow.label);
+ const byExperiment = summaries.get(
+ `${scoreRow.level}-${scoreRow.scoreKey}`,
+ );
+ return (
+ <tr key={`${scoreRow.level}-${scoreRow.scoreKey}`}>
+ <th
+ scope="row"
+ className="bg-background sticky left-0 z-10 min-w-[220px] border-b p-2 text-left font-normal"
+ >
+ <span className="flex min-w-0 items-baseline gap-1">
+ {icon && (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-muted-foreground shrink-0 cursor-default">
+ {icon}
+ </span>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[280px]">
+ {getScoreDataTypeExplanation(scoreRow.dataType)}
+ </TooltipContent>
+ </Tooltip>
+ )}
+ <span className="truncate" title={scoreRow.label}>
+ {label}
+ </span>
+ </span>
+ </th>
+ {experiments.map((experiment) => {
+ const summary = byExperiment?.get(experiment.experimentId);
+ return (
+ <td
+ key={experiment.experimentId}
+ className="min-w-[150px] border-b p-2 align-top"
+ >
+ {summary ? (
+ <MatrixCell
+ summary={summary}
+ hasBaselineColumn={!experiment.isBaseline}
+ />
+ ) : null}
+ </td>
+ );
+ })}
+ </tr>
+ );
+ })}
+ </tbody>
+ </table>
+ </div>
+ <MatrixFooter
+ scoreCount={scoreRows.length}
+ experimentCount={experiments.length}
+ itemCount={rows.length}
+ paginationTable={paginationTable}
+ paginationOptions={pagination.options}
+ isLoading={isLoading}
+ />
+ </div>
+ );
+};
+
+/**
+ * What the numbers above cover, and the control that changes it. The scope
+ * sentence and the page controls sit on one line deliberately: the aggregate
+ * is a window over the run, so reading it without seeing which window is
+ * selected invites the wrong conclusion.
+ */
+const MatrixFooter = ({
+ scoreCount,
+ experimentCount,
+ itemCount,
+ paginationTable,
+ paginationOptions,
+ isLoading,
+}: {
+ scoreCount: number;
+ experimentCount: number;
+ itemCount: number;
+ paginationTable: TanStackTable<ExperimentItemsTableRow>;
+ paginationOptions?: number[];
+ isLoading: boolean;
+}) => (
+ <div className="flex flex-wrap items-center justify-between gap-x-4 gap-y-1 border-t px-2 py-1.5">
+ <span className="text-muted-foreground text-[10px]">
+ {scoreCount} score{scoreCount === 1 ? "" : "s"} Γ— {experimentCount} run
+ {experimentCount === 1 ? "" : "s"}, aggregated over the {itemCount} item
+ {itemCount === 1 ? "" : "s"} on this page. Deltas are measured against the
+ baseline column.
+ </span>
+ <DataTablePagination
+ table={paginationTable}
+ isLoading={isLoading}
+ paginationOptions={paginationOptions}
+ />
+ </div>
+);
diff --git a/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsx b/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsx
new file mode 100644
index 0000000000..e4e5d64cc1
--- /dev/null
+++ b/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsx
@@ -0,0 +1,179 @@
+import {
+ forwardRef,
+ type ComponentProps,
+ type ComponentPropsWithoutRef,
+} from "react";
+import {
+ DropdownMenuController,
+ DropdownMenuItem,
+ DropdownMenuLabel,
+ DropdownMenuPortal,
+ DropdownMenuSeparator,
+ DropdownMenuSub,
+ DropdownMenuSubContent,
+ DropdownMenuSubTrigger,
+} from "@/src/components/ui/dropdown-menu";
+import { Check, ListFilter } from "lucide-react";
+import { type ScoreComparisonOperator } from "@/src/features/experiments/fns/scoreComparisonFilter";
+import { cn } from "@/src/utils/tailwind";
+
+export type ScoreComparisonTarget = {
+ experimentId: string;
+ experimentName: string;
+};
+
+const OPERATOR_LABELS: Record<ScoreComparisonOperator, string> = {
+ lower: "Worse than",
+ higher: "Better than",
+ differs: "Different from",
+};
+
+const ORDERED_OPERATORS: ScoreComparisonOperator[] = [
+ "lower",
+ "higher",
+ "differs",
+];
+
+/**
+ * The score header's filter affordance. Spreads and forwards so a
+ * `<Trigger asChild>` at the call site can hand it Radix's open state and
+ * handlers, and takes its look from an explicit variant rather than a style
+ * prop.
+ */
+export const ScoreColumnFilterMenuTrigger = forwardRef<
+ HTMLButtonElement,
+ Omit<
+ ComponentPropsWithoutRef<"button">,
+ "className" | "style" | "children" | "aria-label"
+ > & {
+ /** A filter is set on this score, so the control stops hiding itself. */
+ isActive: boolean;
+ }
+>(({ isActive, ...triggerProps }, ref) => (
+ <button
+ ref={ref}
+ aria-label="Filter items by this score"
+ className={cn(
+ "hover:bg-muted mt-0.5 shrink-0 rounded-sm p-0.5",
+ isActive
+ ? "text-primary-accent"
+ : "text-muted-foreground opacity-0 group-hover:opacity-100",
+ )}
+ {...triggerProps}
+ >
+ <ListFilter className="h-3 w-3" />
+ </button>
+));
+ScoreColumnFilterMenuTrigger.displayName = "ScoreColumnFilterMenuTrigger";
+
+/**
+ * The score column's own way into the comparison filter: "show
+ * only the items worse than <comparison> on this score", from the header of the
+ * score in question, with the comparison to read against picked here rather than
+ * assumed.
+ */
+export const ScoreColumnFilterMenu = ({
+ targets,
+ active,
+ hasOrder,
+ onSelect,
+ onClear,
+ children,
+}: {
+ /** The experiments this score can be read against, default first. */
+ targets: ScoreComparisonTarget[];
+ active?: {
+ operator: ScoreComparisonOperator;
+ comparisonExperimentId: string;
+ };
+ /** Categorical scores have no order, so only "different from" applies. */
+ hasOrder: boolean;
+ onSelect: (
+ operator: ScoreComparisonOperator,
+ comparisonExperimentId: string,
+ ) => void;
+ onClear: () => void;
+ /** Renders the trigger, so its presentation stays with the caller. */
+ children: ComponentProps<typeof DropdownMenuController>["children"];
+}) => {
+ const operators = hasOrder ? ORDERED_OPERATORS : (["differs"] as const);
+ const isActive = (operator: ScoreComparisonOperator, experimentId: string) =>
+ active?.operator === operator &&
+ active?.comparisonExperimentId === experimentId;
+
+ return (
+ <DropdownMenuController
+ align="end"
+ maxWidth="16rem"
+ renderMenu={() => (
+ <>
+ <DropdownMenuLabel>Show only items</DropdownMenuLabel>
+ {operators.map((operator) =>
+ targets.length === 1 ? (
+ <DropdownMenuItem
+ key={operator}
+ onClick={() => onSelect(operator, targets[0].experimentId)}
+ >
+ {isActive(operator, targets[0].experimentId) ? (
+ <Check className="mr-2 h-4 w-4 shrink-0" />
+ ) : (
+ <span className="mr-2 h-4 w-4 shrink-0" />
+ )}
+ <span
+ className="truncate"
+ title={`${OPERATOR_LABELS[operator]} ${targets[0].experimentName}`}
+ >
+ {OPERATOR_LABELS[operator]} {targets[0].experimentName}
+ </span>
+ </DropdownMenuItem>
+ ) : (
+ <DropdownMenuSub key={operator}>
+ <DropdownMenuSubTrigger>
+ {OPERATOR_LABELS[operator]}…
+ </DropdownMenuSubTrigger>
+ <DropdownMenuPortal>
+ <DropdownMenuSubContent className="w-56">
+ {targets.map((target) => (
+ <DropdownMenuItem
+ key={target.experimentId}
+ onClick={() => onSelect(operator, target.experimentId)}
+ >
+ {isActive(operator, target.experimentId) ? (
+ <Check className="mr-2 h-4 w-4 shrink-0" />
+ ) : (
+ <span className="mr-2 h-4 w-4 shrink-0" />
+ )}
+ <span
+ className="truncate"
+ title={target.experimentName}
+ >
+ {target.experimentName}
+ </span>
+ </DropdownMenuItem>
+ ))}
+ </DropdownMenuSubContent>
+ </DropdownMenuPortal>
+ </DropdownMenuSub>
+ ),
+ )}
+ {active && (
+ <>
+ <DropdownMenuSeparator />
+ <DropdownMenuItem onClick={onClear}>
+ <span className="mr-2 h-4 w-4 shrink-0" />
+ Remove this score&apos;s filter
+ </DropdownMenuItem>
+ </>
+ )}
+ <DropdownMenuSeparator />
+ <span className="text-muted-foreground block px-2 py-1.5 text-[10px]">
+ Compares the items loaded on this page β€” page through to check the
+ rest of the run.
+ </span>
+ </>
+ )}
+ >
+ {children}
+ </DropdownMenuController>
+ );
+};
diff --git a/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsx b/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsx
new file mode 100644
index 0000000000..c5cf71b7f5
--- /dev/null
+++ b/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsx
@@ -0,0 +1,79 @@
+import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { GitCompareArrows, X } from "lucide-react";
+import {
+ describeScoreComparisonFilter,
+ type ScoreComparisonFilter,
+} from "@/src/features/experiments/fns/scoreComparisonFilter";
+
+export type ScoreComparisonPill = {
+ filter: ScoreComparisonFilter;
+ scoreName: string;
+ comparisonName: string;
+};
+
+/**
+ * The active score comparisons, in plain English ("Worse groundedness than
+ * judge-haiku-baseline") and removable β€” the same affordance the score filters
+ * get, for the one predicate that is evaluated in the client.
+ */
+export const ScoreComparisonFilterPills = ({
+ pills,
+ onRemove,
+}: {
+ pills: ScoreComparisonPill[];
+ onRemove: (filter: ScoreComparisonFilter) => void;
+}) => {
+ return (
+ <div className="flex flex-wrap items-center gap-1.5 border-b px-2 py-2">
+ {pills.map(({ filter, scoreName, comparisonName }) => {
+ const label = describeScoreComparisonFilter({
+ operator: filter.operator,
+ scoreName,
+ comparisonName,
+ });
+ return (
+ <Badge
+ key={`${filter.level}-${filter.scoreKey}`}
+ variant="secondary"
+ className="flex max-w-full items-center gap-1 px-2 py-1 text-xs"
+ >
+ <GitCompareArrows className="h-3 w-3 shrink-0" />
+ <span className="truncate" title={label}>
+ {label}
+ </span>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-muted-foreground shrink-0 cursor-default underline decoration-dotted">
+ this page
+ </span>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[280px]">
+ Comparing one experiment&apos;s score against another&apos;s is
+ not something the items query can express, so this narrows the
+ items loaded on the current page. Page through to check the rest
+ of the run.
+ </TooltipContent>
+ </Tooltip>
+ <Button
+ variant="ghost"
+ size="sm"
+ className="h-4 w-4 shrink-0 p-0 hover:bg-transparent"
+ onClick={(event) => {
+ event.stopPropagation();
+ onRemove(filter);
+ }}
+ >
+ <X className="h-3 w-3" />
+ </Button>
+ </Badge>
+ );
+ })}
+ </div>
+ );
+};
diff --git a/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.ts b/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.ts
new file mode 100644
index 0000000000..4177a901be
--- /dev/null
+++ b/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.ts
@@ -0,0 +1,131 @@
+import { type AggregatedScoreData } from "@langfuse/shared";
+import {
+ decodeScoreComparisonFilter,
+ encodeScoreComparisonFilter,
+ matchesScoreComparisonFilter,
+} from "./scoreComparisonFilter";
+
+const numeric = (value: number): AggregatedScoreData => ({
+ type: "NUMERIC",
+ values: [value],
+ average: value,
+});
+
+const categorical = (value: string): AggregatedScoreData => ({
+ type: "CATEGORICAL",
+ values: [value],
+ valueCounts: [{ value, count: 1 }],
+});
+
+describe("matchesScoreComparisonFilter", () => {
+ it("keeps only the items the experiment you opened scored lower on", () => {
+ const lower = (baseline: number, comparison: number) =>
+ matchesScoreComparisonFilter({
+ operator: "lower",
+ dataType: "NUMERIC",
+ baseline: numeric(baseline),
+ comparison: numeric(comparison),
+ });
+
+ expect(lower(0.2, 0.9)).toBe(true);
+ expect(lower(0.9, 0.2)).toBe(false);
+ expect(lower(0.5, 0.5)).toBe(false);
+ });
+
+ it("never keeps an item only one of the two experiments scored", () => {
+ for (const operator of ["lower", "higher", "differs"] as const) {
+ expect(
+ matchesScoreComparisonFilter({
+ operator,
+ dataType: "NUMERIC",
+ baseline: numeric(0.2),
+ comparison: null,
+ }),
+ ).toBe(false);
+ expect(
+ matchesScoreComparisonFilter({
+ operator,
+ dataType: "NUMERIC",
+ baseline: undefined,
+ comparison: numeric(0.2),
+ }),
+ ).toBe(false);
+ }
+ });
+
+ it("reads a boolean as false < true", () => {
+ expect(
+ matchesScoreComparisonFilter({
+ operator: "lower",
+ dataType: "BOOLEAN",
+ baseline: categorical("false"),
+ comparison: categorical("true"),
+ }),
+ ).toBe(true);
+ });
+
+ it("only answers 'differs' for a categorical score, which has no order", () => {
+ const args = {
+ dataType: "CATEGORICAL" as const,
+ baseline: categorical("weak"),
+ comparison: categorical("grounded"),
+ };
+ expect(matchesScoreComparisonFilter({ ...args, operator: "lower" })).toBe(
+ false,
+ );
+ expect(matchesScoreComparisonFilter({ ...args, operator: "differs" })).toBe(
+ true,
+ );
+ expect(
+ matchesScoreComparisonFilter({
+ ...args,
+ operator: "differs",
+ comparison: categorical("weak"),
+ }),
+ ).toBe(false);
+ });
+});
+
+describe("score comparison filter URL state", () => {
+ it("round-trips through the URL and rejects anything malformed", () => {
+ const filter = {
+ level: "trace" as const,
+ scoreKey: "groundedness-EVAL-NUMERIC",
+ operator: "lower" as const,
+ comparisonExperimentId: "1b0a2c3d-4e5f",
+ };
+
+ expect(
+ decodeScoreComparisonFilter(encodeScoreComparisonFilter(filter)),
+ ).toEqual(filter);
+ expect(decodeScoreComparisonFilter("trace:key:sideways:id")).toBeNull();
+ expect(decodeScoreComparisonFilter("session:key:lower:id")).toBeNull();
+ expect(decodeScoreComparisonFilter("")).toBeNull();
+ });
+
+ // The score key carries the score's name, which is user-supplied: a name
+ // holding the field separator or the one between several filters used to
+ // land in the wrong slot and drop the filter without a word.
+ it("round-trips a score name holding the separators", () => {
+ for (const scoreKey of [
+ "ratio:accuracy-EVAL-NUMERIC",
+ "either|or-EVAL-NUMERIC",
+ "a:b|c-EVAL-NUMERIC",
+ ]) {
+ const filter = {
+ level: "observation" as const,
+ scoreKey,
+ operator: "differs" as const,
+ comparisonExperimentId: "1b0a2c3d-4e5f",
+ };
+ const encoded = encodeScoreComparisonFilter(filter);
+ expect(encoded).not.toContain("|");
+ expect(encoded.split(":")).toHaveLength(4);
+ expect(decodeScoreComparisonFilter(encoded)).toEqual(filter);
+ }
+ });
+
+ it("rejects a malformed escape instead of throwing", () => {
+ expect(decodeScoreComparisonFilter("trace:%zz:lower:id")).toBeNull();
+ });
+});
diff --git a/web/src/features/experiments/fns/scoreComparisonFilter.ts b/web/src/features/experiments/fns/scoreComparisonFilter.ts
new file mode 100644
index 0000000000..ad76e19166
--- /dev/null
+++ b/web/src/features/experiments/fns/scoreComparisonFilter.ts
@@ -0,0 +1,215 @@
+import { type AggregatedScoreData } from "@langfuse/shared";
+import {
+ readOrderedScoreValue,
+ type ScoreColumnDataType,
+} from "./summariseScoreColumn";
+
+/** Which score family a filtered column belongs to. */
+export type ScoreLevel = "observation" | "trace";
+
+export type ScoreComparisonOperator = "lower" | "higher" | "differs";
+
+/**
+ * "Keep the items whose score is lower than the comparison experiment's" β€” the
+ * one predicate the item filters could not express, because it is about a pair
+ * of experiments rather than a column against a literal.
+ */
+export type ScoreComparisonFilter = {
+ level: ScoreLevel;
+ /** Aggregate score key, e.g. `groundedness-EVAL-NUMERIC`. */
+ scoreKey: string;
+ operator: ScoreComparisonOperator;
+ /** The experiment the score is read against. */
+ comparisonExperimentId: string;
+};
+
+const OPERATORS: ScoreComparisonOperator[] = ["lower", "higher", "differs"];
+
+/**
+ * `level:scoreKey:operator:experimentId`, each field percent-encoded. A score
+ * key carries the score's name, which is user-supplied and may hold a colon or
+ * the pipe that separates several of these in the URL β€” an unescaped field
+ * would land in the wrong slot and the filter would silently vanish.
+ */
+export const encodeScoreComparisonFilter = (
+ filter: ScoreComparisonFilter,
+): string =>
+ [
+ filter.level,
+ filter.scoreKey,
+ filter.operator,
+ filter.comparisonExperimentId,
+ ]
+ .map(encodeURIComponent)
+ .join(":");
+
+export const decodeScoreComparisonFilter = (
+ encoded: string | null | undefined,
+): ScoreComparisonFilter | null => {
+ if (!encoded) return null;
+ const parts = encoded.split(":");
+ if (parts.length !== 4) return null;
+ let level: string;
+ let scoreKey: string;
+ let operator: string;
+ let comparisonExperimentId: string;
+ try {
+ [level, scoreKey, operator, comparisonExperimentId] =
+ parts.map(decodeURIComponent);
+ } catch {
+ // A hand-edited or truncated URL can hold a malformed escape.
+ return null;
+ }
+ if (level !== "observation" && level !== "trace") return null;
+ if (!scoreKey || !comparisonExperimentId) return null;
+ if (!OPERATORS.includes(operator as ScoreComparisonOperator)) return null;
+ return {
+ level,
+ scoreKey,
+ operator: operator as ScoreComparisonOperator,
+ comparisonExperimentId,
+ };
+};
+
+export const isSameScoreComparisonTarget = (
+ a: ScoreComparisonFilter,
+ b: ScoreComparisonFilter,
+) => a.level === b.level && a.scoreKey === b.scoreKey;
+
+/**
+ * Whether one item passes the filter, read the same way the column header reads
+ * it: an item only one of the two experiments scored is not comparable, so it is
+ * never a regression and never kept. A categorical score has no order, so only
+ * `differs` can match it.
+ */
+export const matchesScoreComparisonFilter = ({
+ operator,
+ dataType,
+ baseline,
+ comparison,
+}: {
+ operator: ScoreComparisonOperator;
+ dataType: ScoreColumnDataType;
+ baseline: AggregatedScoreData | null | undefined;
+ comparison: AggregatedScoreData | null | undefined;
+}): boolean => {
+ if (!baseline || !comparison) return false;
+
+ if (dataType === "CATEGORICAL") {
+ // No order, so only "differs" means anything on a categorical score.
+ if (operator !== "differs") return false;
+ if (baseline.type !== "CATEGORICAL" || comparison.type !== "CATEGORICAL")
+ return false;
+ return (
+ [...baseline.values].sort().join("|") !==
+ [...comparison.values].sort().join("|")
+ );
+ }
+
+ const baselineValue = readOrderedScoreValue(baseline, dataType);
+ const comparisonValue = readOrderedScoreValue(comparison, dataType);
+ if (baselineValue === null || comparisonValue === null) return false;
+
+ if (operator === "differs") return baselineValue !== comparisonValue;
+ return operator === "lower"
+ ? baselineValue < comparisonValue
+ : baselineValue > comparisonValue;
+};
+
+const OPERATOR_WORDS: Record<ScoreComparisonOperator, string> = {
+ lower: "Worse",
+ higher: "Better",
+ differs: "Different",
+};
+
+/** The chip's plain English: "Worse groundedness than judge-haiku-baseline". */
+export const describeScoreComparisonFilter = ({
+ operator,
+ scoreName,
+ comparisonName,
+}: {
+ operator: ScoreComparisonOperator;
+ scoreName: string;
+ comparisonName: string;
+}): string =>
+ `${OPERATOR_WORDS[operator]} ${scoreName} ${
+ operator === "differs" ? "vs" : "than"
+ } ${comparisonName}`;
+
+/** Which of a row's two score maps a level's scores live in. */
+export const scoreFieldForLevel = (level: ScoreLevel) =>
+ level === "trace" ? ("traceScores" as const) : ("observationScores" as const);
+
+/**
+ * Whether one item survives every active score comparison. A filter that cannot
+ * be evaluated does not hide rows: its score type is not known yet (the column
+ * definitions are still loading), or its target is no longer one of the runs on
+ * screen. Reading either as "nothing matches" would empty the table, which
+ * looks like a broken page rather than a filter with nothing to point at.
+ */
+export const rowPassesScoreComparisonFilters = ({
+ filters,
+ experiments,
+ baselineExperimentId,
+ comparableExperimentIds,
+ dataTypeFor,
+}: {
+ filters: ScoreComparisonFilter[];
+ experiments: Array<{
+ experimentId: string;
+ observationScores: Record<string, AggregatedScoreData>;
+ traceScores: Record<string, AggregatedScoreData>;
+ }>;
+ baselineExperimentId?: string;
+ /** The runs currently being compared, if the caller knows them. */
+ comparableExperimentIds?: readonly string[];
+ dataTypeFor: (
+ filter: ScoreComparisonFilter,
+ ) => ScoreColumnDataType | undefined;
+}): boolean => {
+ if (filters.length === 0) return true;
+ if (!baselineExperimentId) return true;
+
+ const scoresOf = (experimentId: string, level: ScoreLevel) =>
+ experiments.find(
+ (experiment) => experiment.experimentId === experimentId,
+ )?.[scoreFieldForLevel(level)];
+
+ return filters.every((filter) => {
+ const dataType = dataTypeFor(filter);
+ if (!dataType) return true;
+ // The target was deselected, or is now the baseline it would be read
+ // against β€” either way there is no second run left to compare with.
+ if (filter.comparisonExperimentId === baselineExperimentId) return true;
+ if (
+ comparableExperimentIds &&
+ !comparableExperimentIds.includes(filter.comparisonExperimentId)
+ )
+ return true;
+ return matchesScoreComparisonFilter({
+ operator: filter.operator,
+ dataType,
+ baseline: scoresOf(baselineExperimentId, filter.level)?.[filter.scoreKey],
+ comparison: scoresOf(filter.comparisonExperimentId, filter.level)?.[
+ filter.scoreKey
+ ],
+ });
+ });
+};
+
+/** What an emptied table says, so it reads as an answer and not as a failure. */
+export const describeEmptyScoreComparison = ({
+ operator,
+ scoreName,
+ comparisonName,
+}: {
+ operator: ScoreComparisonOperator;
+ scoreName: string;
+ comparisonName: string;
+}): string => {
+ if (operator === "differs")
+ return `No item on this page scored ${scoreName} differently from ${comparisonName}.`;
+ if (operator === "lower")
+ return `No regressions on this score β€” no item on this page scored lower on ${scoreName} than ${comparisonName}.`;
+ return `No item on this page scored higher on ${scoreName} than ${comparisonName}.`;
+};
diff --git a/web/src/features/experiments/fns/summariseScoreColumn.ts b/web/src/features/experiments/fns/summariseScoreColumn.ts
index aa20c466ae..97d5e285fa 100644
--- a/web/src/features/experiments/fns/summariseScoreColumn.ts
+++ b/web/src/features/experiments/fns/summariseScoreColumn.ts
@@ -54,7 +54,7 @@ const countValues = (aggregate: AggregatedScoreData) =>
* The score read as a number, so two items can be ordered. A boolean's ordered
* reading is its true-rate (false < true); a categorical has none.
*/
-const readOrderedScoreValue = (
+export const readOrderedScoreValue = (
aggregate: AggregatedScoreData | null,
dataType: ScoreColumnDataType,
): number | null => {
diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.ts b/web/src/features/experiments/hooks/useExperimentResultsState.ts
index c488aa4894..d4ce999cd7 100644
--- a/web/src/features/experiments/hooks/useExperimentResultsState.ts
+++ b/web/src/features/experiments/hooks/useExperimentResultsState.ts
@@ -10,15 +10,18 @@ import useLocalStorage from "@/src/components/useLocalStorage";
/**
* How the selected experiments are laid out against each other: one row per item
- * ("list") or one column per experiment ("grid").
+ * ("list"), one column per experiment ("grid"), or the transpose β€” scores as
+ * rows and experiments as columns ("matrix").
*/
-export type ExperimentResultsLayout = "grid" | "list";
+export type ExperimentResultsLayout = "grid" | "list" | "matrix";
/** What a diff cell's extra line is measured against. */
export type ExperimentDiffMode = "comparison" | "expected" | "off";
const asLayout = (value: unknown): ExperimentResultsLayout | undefined =>
- value === "grid" || value === "list" ? value : undefined;
+ value === "grid" || value === "list" || value === "matrix"
+ ? value
+ : undefined;
const asDiffMode = (value: unknown): ExperimentDiffMode | undefined =>
value === "comparison" || value === "expected" || value === "off"
diff --git a/web/src/features/experiments/hooks/useScoreComparisonFilters.ts b/web/src/features/experiments/hooks/useScoreComparisonFilters.ts
new file mode 100644
index 0000000000..4be7d71777
--- /dev/null
+++ b/web/src/features/experiments/hooks/useScoreComparisonFilters.ts
@@ -0,0 +1,86 @@
+import { useCallback, useMemo } from "react";
+import { ArrayParam, useQueryParams, withDefault } from "use-query-params";
+import {
+ decodeScoreComparisonFilter,
+ encodeScoreComparisonFilter,
+ isSameScoreComparisonTarget,
+ type ScoreComparisonFilter,
+} from "@/src/features/experiments/fns/scoreComparisonFilter";
+
+/**
+ * "Show only the items this run scored worse on than <comparison>" β€” Annabell's
+ * top ask, as a filter rather than a report.
+ *
+ * The predicate is about a *pair* of experiments, which the items query cannot
+ * express: `compileExperimentFilter` translates each condition into a
+ * `column op literal` WHERE clause on a single experiment's rows, so comparing
+ * one experiment's score against another's for the same item would need a new
+ * HAVING path over the `experiment_item_id` group β€” new query machinery we are
+ * explicitly not building. So the comparison is evaluated in the client over the
+ * items already fetched for the current page, and the UI says so. Everything
+ * else about it behaves like a filter: it lives in the URL, it is shareable, it
+ * survives a reload, and it shows as a removable chip.
+ *
+ * One filter per score column: picking an operator on a column replaces that
+ * column's filter instead of stacking a second one.
+ */
+const NO_FILTERS: string[] = [];
+
+export const useScoreComparisonFilters = () => {
+ const [state, setState] = useQueryParams({
+ scoreCompare: withDefault(ArrayParam, NO_FILTERS),
+ });
+
+ // Keyed on the encoded filters rather than the array `use-query-params` hands
+ // back, whose identity changes on every render and would make every consumer
+ // of `filters` β€” the filtered rows above all β€” churn with it.
+ const encoded = (
+ (state.scoreCompare as (string | null)[] | undefined) ?? NO_FILTERS
+ )
+ .filter(Boolean)
+ .join("|");
+
+ const filters = useMemo(
+ () =>
+ encoded
+ .split("|")
+ .map(decodeScoreComparisonFilter)
+ .filter(Boolean) as ScoreComparisonFilter[],
+ [encoded],
+ );
+
+ const write = useCallback(
+ (next: ScoreComparisonFilter[]) =>
+ setState({
+ scoreCompare: next.length
+ ? next.map(encodeScoreComparisonFilter)
+ : undefined,
+ }),
+ [setState],
+ );
+
+ const setFilter = useCallback(
+ (filter: ScoreComparisonFilter) =>
+ write([
+ ...filters.filter(
+ (existing) => !isSameScoreComparisonTarget(existing, filter),
+ ),
+ filter,
+ ]),
+ [filters, write],
+ );
+
+ const removeFilter = useCallback(
+ (filter: ScoreComparisonFilter) =>
+ write(
+ filters.filter(
+ (existing) => !isSameScoreComparisonTarget(existing, filter),
+ ),
+ ),
+ [filters, write],
+ );
+
+ const clearFilters = useCallback(() => write([]), [write]);
+
+ return { filters, setFilter, removeFilter, clearFilters };
+};