πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): a worse-than-comparison filter and a score-by-run matrix (#16918)
Nikita Kabardin79b2c6644338
diff --git a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx b/web/src/features/experiments/components/ExperimentDisplaySettings.tsxindex fe6898ac0d..1ce02420c3 100644--- a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx+++ b/web/src/features/experiments/components/ExperimentDisplaySettings.tsx@@ -86,6 +86,12 @@ export function ExperimentDisplaySettings({>Side by side β a column per experiment</OptionItem>+ <OptionItem+ selected={layout === "matrix"}+ onSelect={() => onLayoutChange("matrix")}+ >+ Score matrix β scores as rows, runs as columns+ </OptionItem><DropdownMenuSeparator />diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsxindex 872c447aa9..8bcb4e778b 100644--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx@@ -93,6 +93,23 @@ import {type ScoreColumnSummary,} from "@/src/features/experiments/fns/summariseScoreColumn";import { ScoreColumnHeaderSummary } from "./ScoreColumnHeaderSummary";+import {+ ScoreColumnFilterMenu,+ ScoreColumnFilterMenuTrigger,+} from "./ScoreColumnFilterMenu";+import { ScoreComparisonFilterPills } from "./ScoreComparisonFilterPills";+import {+ ExperimentScoreMatrix,+ type ScoreMatrixRow,+} from "./ExperimentScoreMatrix";+import { useScoreComparisonFilters } from "@/src/features/experiments/hooks/useScoreComparisonFilters";+import {+ describeEmptyScoreComparison,+ rowPassesScoreComparisonFilters,+ scoreFieldForLevel,+ type ScoreComparisonFilter,+ type ScoreLevel,+} from "@/src/features/experiments/fns/scoreComparisonFilter";import { resetStaleDefaultColumnOrder } from "@/src/features/experiments/fns/experimentItemsColumnOrder";const renderExperimentSpecificHeader = (label: string) => (<span className="text-muted-foreground">{label}</span>@@ -694,6 +711,16 @@ export default function ExperimentItemsTable({() => allExperimentIds.join(","),[allExperimentIds],);+ // The selection arrives as a fresh array on every render, so anything that+ // reaches a hook dependency has to come off the key, which is stable by+ // value. `rows` below is one of those: it feeds the effect that publishes the+ // peek navigation list, and publishing re-renders every consumer of that+ // provider β this table included. Depending on the array identity there+ // leaves the two re-triggering each other for as long as the page is open.+ const comparableExperimentIds = useMemo(+ () => (experimentSelectionKey ? experimentSelectionKey.split(",") : []),+ [experimentSelectionKey],+ );const [expectedOutputSeenFor, setExpectedOutputSeenFor] = useState<string | null>(null);@@ -810,6 +837,47 @@ export default function ExperimentItemsTable({scoreColumnDefs.traceScoreColumns,]);+ const scoreNamesByKey = useMemo(() => {+ const build = (columns: ScoreColumnDef[]) =>+ new Map(toScoreColumnInput(columns).map(({ key, name }) => [key, name]));+ return {+ observationScores: build(scoreColumnDefs.observationScoreColumns),+ traceScores: build(scoreColumnDefs.traceScoreColumns),+ };+ }, [+ scoreColumnDefs.observationScoreColumns,+ scoreColumnDefs.traceScoreColumns,+ ]);++ const {+ filters: scoreComparisonFilters,+ setFilter: setScoreComparisonFilter,+ removeFilter: removeScoreComparisonFilter,+ } = useScoreComparisonFilters();++ // The runs a score can be read against, the auto-selected comparison first so+ // the menu's default is the one the column header already reports.+ const comparisonTargets = useMemo(() => {+ const others = selectedExperimentNames.filter(+ (exp) => exp.experimentId !== primaryExperimentId,+ );+ return [+ ...others.filter((exp) => exp.experimentId === primaryComparisonId),+ ...others.filter((exp) => exp.experimentId !== primaryComparisonId),+ ].map(({ experimentId, experimentName }) => ({+ experimentId,+ experimentName,+ }));+ }, [selectedExperimentNames, primaryExperimentId, primaryComparisonId]);++ const scoreDataTypeFor = useCallback(+ (filter: ScoreComparisonFilter) =>+ scoreDataTypesByKey[scoreFieldForLevel(filter.level)].get(+ filter.scoreKey,+ ),+ [scoreDataTypesByKey],+ );+const scoreColumnSummaries = useMemo(() => {const rowsInView = items.rows ?? [];return {@@ -852,6 +920,13 @@ export default function ExperimentItemsTable({typeof scoreCol.header === "string"? scoreCol.header: (scoreCol.accessorKey ?? "");+ const level: ScoreLevel =+ scoreField === "traceScores" ? "trace" : "observation";+ const activeComparisonFilter = key+ ? scoreComparisonFilters.find(+ (filter) => filter.level === level && filter.scoreKey === key,+ )+ : undefined;return {...scoreCol,@@ -868,6 +943,42 @@ export default function ExperimentItemsTable({dataType={dataType}summary={summary}comparisonName={primaryComparisonName}+ filterMenu={+ comparisonTargets.length === 0 ? undefined : (+ <ScoreColumnFilterMenu+ targets={comparisonTargets}+ hasOrder={dataType !== "CATEGORICAL"}+ active={activeComparisonFilter}+ onSelect={(operator, comparisonExperimentId) => {+ if (!key) return;+ const nextFilter = {+ level,+ scoreKey: key,+ operator,+ comparisonExperimentId,+ };+ setScoreComparisonFilter(nextFilter);+ }}+ onClear={() =>+ activeComparisonFilter &&+ removeScoreComparisonFilter(activeComparisonFilter)+ }+ >+ {({ Trigger }) => (+ // The header cell sorts on click, so opening the menu+ // must not also reorder the table.+ <Trigger+ asChild+ onClick={(event) => event.stopPropagation()}+ >+ <ScoreColumnFilterMenuTrigger+ isActive={Boolean(activeComparisonFilter)}+ />+ </Trigger>+ )}+ </ScoreColumnFilterMenu>+ )+ }/>),}@@ -971,6 +1082,10 @@ export default function ExperimentItemsTable({scoreColumnSummaries,scoreDataTypesByKey,showComparisonDiff,+ comparisonTargets,+ scoreComparisonFilters,+ setScoreComparisonFilter,+ removeScoreComparisonFilter,runNameOf,],);@@ -1366,6 +1481,59 @@ export default function ExperimentItemsTable({columnOrderMigrations,);+ // The transposed layout's axes: score columns as rows β+ // respecting what the column picker hid β and the selected runs as columns,+ // baseline first. Both are a re-read of what the grid already has, so the+ // layout needs no query of its own.+ const matrixScoreRows = useMemo<ScoreMatrixRow[]>(() => {+ const build = (+ scoreCols: LangfuseColumnDef<ExperimentItemData>[],+ level: ScoreLevel,+ ): ScoreMatrixRow[] =>+ scoreCols.flatMap((scoreCol) => {+ const accessorKey = scoreCol.accessorKey;+ if (!accessorKey || columnVisibility[accessorKey] === false) return [];+ const scoreKey = accessorKey.replace(/^Trace-/, "");+ const dataType =+ scoreDataTypesByKey[scoreFieldForLevel(level)].get(scoreKey);+ if (!dataType) return [];+ return [+ {+ scoreKey,+ level,+ dataType,+ label:+ typeof scoreCol.header === "string" ? scoreCol.header : scoreKey,+ },+ ];+ });+ return [+ ...build(observationScoreColumns, "observation"),+ ...build(traceScoreColumns, "trace"),+ ];+ }, [+ observationScoreColumns,+ traceScoreColumns,+ scoreDataTypesByKey,+ columnVisibility,+ ]);++ const matrixExperiments = useMemo(+ () =>+ [+ ...(primaryExperimentId ? [primaryExperimentId] : []),+ ...allExperimentIds.filter((id) => id !== primaryExperimentId),+ ].map((experimentId) => ({+ experimentId,+ experimentName:+ selectedExperimentNames.find(+ (exp) => exp.experimentId === experimentId,+ )?.experimentName ?? experimentId.slice(0, 8),+ isBaseline: experimentId === primaryExperimentId,+ })),+ [allExperimentIds, primaryExperimentId, selectedExperimentNames],+ );+const peekNavigationProps = usePeekNavigation({queryParams: ["observation",@@ -1425,12 +1593,38 @@ export default function ExperimentItemsTable({};}, [peekNavigationProps, canUsePeek]);- const rows: ExperimentItemsTableRow[] = useMemo(() => {+ // The page as fetched. The score column header aggregates β and the score+ // matrix, which reads the same ones β deliberately describe this whole page,+ // so the movement a comparison filter was built from stays readable while+ // that filter is applied.+ const unfilteredRows: ExperimentItemsTableRow[] = useMemo(() => {if (items.status !== "success" || !items.rows) return [];// Add 'id' field for DataTable row identification (peek view requires it)return items.rows.map((row) => ({ ...row, id: row.itemId }));}, [items]);+ // The score comparison filters narrow the page here rather than in the+ // query β see `useScoreComparisonFilters` for why.+ const rows: ExperimentItemsTableRow[] = useMemo(+ () =>+ unfilteredRows.filter((row) =>+ rowPassesScoreComparisonFilters({+ filters: scoreComparisonFilters,+ experiments: row.experiments,+ baselineExperimentId: primaryExperimentId,+ comparableExperimentIds,+ dataTypeFor: scoreDataTypeFor,+ }),+ ),+ [+ unfilteredRows,+ scoreComparisonFilters,+ primaryExperimentId,+ comparableExperimentIds,+ scoreDataTypeFor,+ ],+ );+useEffect(() => {if (items.status === "success") {// Store all experiment targets for peek navigation@@ -1471,6 +1665,38 @@ export default function ExperimentItemsTable({// eslint-disable-next-line react-hooks/exhaustive-deps}, [items.status, rows, baselineId]);+ // Plain English for the active comparisons, and for the table when they leave+ // nothing: "no regressions on this score" is an answer, not a broken table.+ const scoreComparisonPills = useMemo(+ () =>+ scoreComparisonFilters.map((filter) => ({+ filter,+ scoreName:+ scoreNamesByKey[scoreFieldForLevel(filter.level)].get(+ filter.scoreKey,+ ) ?? filter.scoreKey,+ comparisonName:+ selectedExperimentNames.find(+ (exp) => exp.experimentId === filter.comparisonExperimentId,+ )?.experimentName ?? filter.comparisonExperimentId,+ })),+ [scoreComparisonFilters, scoreNamesByKey, selectedExperimentNames],+ );++ const scoreComparisonEmptyMessage = useMemo(() => {+ if (rows.length > 0 || (items.rows ?? []).length === 0) return undefined;+ if (scoreComparisonPills.length !== 1)+ return scoreComparisonPills.length > 1+ ? "No item on this page matches every score comparison."+ : undefined;+ const [pill] = scoreComparisonPills;+ return describeEmptyScoreComparison({+ operator: pill.filter.operator,+ scoreName: pill.scoreName,+ comparisonName: pill.comparisonName,+ });+ }, [rows.length, items.rows, scoreComparisonPills]);+const pagination = useMemo(() => ({totalCount: totalCount ?? null,@@ -1640,6 +1866,14 @@ export default function ExperimentItemsTable({/>)}+ {/* Score comparison filters β evaluated over the loaded page */}+ {scoreComparisonPills.length > 0 && (+ <ScoreComparisonFilterPills+ pills={scoreComparisonPills}+ onRemove={removeScoreComparisonFilter}+ />+ )}+{/* Filter Pills with Experiment Targeting */}{filtersByExperiment.length > 0 && (<ExperimentFilterPills@@ -1662,7 +1896,23 @@ export default function ExperimentItemsTable({)}<div className="flex flex-1 flex-col overflow-hidden">- {layout === "grid" ? (+ {layout === "matrix" ? (+ hasSelectedRuns ? (+ <ExperimentScoreMatrix+ rows={unfilteredRows}+ scoreRows={matrixScoreRows}+ experiments={matrixExperiments}+ isLoading={items.status === "loading" || isViewLoading}+ pagination={pagination}+ />+ ) : (+ <div className="flex flex-1 items-center justify-center">+ <span className="text-muted-foreground text-sm">+ Please select a baseline experiment.+ </span>+ </div>+ )+ ) : layout === "grid" ? (hasSelectedRuns ? (<ExperimentGridViewprojectId={projectId}@@ -1689,6 +1939,13 @@ export default function ExperimentItemsTable({rowSelection={selectedRows}setRowSelection={setSelectedRows}highlightAllRows={selectAll}+ noResultsMessage={+ scoreComparisonEmptyMessage ? (+ <span className="text-muted-foreground text-sm">+ {scoreComparisonEmptyMessage}+ </span>+ ) : undefined+ }/>) : (<div className="flex flex-1 items-center justify-center">@@ -1720,6 +1977,10 @@ export default function ExperimentItemsTable({<span className="text-muted-foreground text-sm">Please select a baseline experiment.</span>+ ) : scoreComparisonEmptyMessage ? (+ <span className="text-muted-foreground text-sm">+ {scoreComparisonEmptyMessage}+ </span>) : undefined}highlightAllRows={selectAll}diff --git a/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsx b/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsxnew file mode 100644index 0000000000..ebe7c4e273--- /dev/null+++ b/web/src/features/experiments/components/table/ExperimentScoreMatrix.clienttest.tsx@@ -0,0 +1,102 @@+import { fireEvent, render, screen } from "@testing-library/react";+import type { PaginationState } from "@tanstack/react-table";+import { TooltipProvider } from "@/src/components/ui/tooltip";+import {+ ExperimentScoreMatrix,+ type ScoreMatrixColumn,+ type ScoreMatrixRow,+} from "./ExperimentScoreMatrix";+import type { ExperimentItemsTableRow } from "./types";++vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({+ usePostHogClientCapture: () => vi.fn(),+}));++const scoreKey = "groundedness-EVAL-NUMERIC";++const scoreRows: ScoreMatrixRow[] = [+ { scoreKey, level: "trace", dataType: "NUMERIC", label: "# groundedness" },+];++const experiments: ScoreMatrixColumn[] = [+ { experimentId: "base", experimentName: "baseline-run", isBaseline: true },+ { experimentId: "cand", experimentName: "candidate-run", isBaseline: false },+];++const itemRow = (id: string, value: number): ExperimentItemsTableRow =>+ ({+ id,+ experiments: experiments.map((experiment) => ({+ experimentId: experiment.experimentId,+ traceScores: { [scoreKey]: { type: "NUMERIC", average: value } },+ })),+ }) as unknown as ExperimentItemsTableRow;++const renderMatrix = ({+ rows,+ state,+ onChange,+ totalCount,+}: {+ rows: ExperimentItemsTableRow[];+ state: PaginationState;+ onChange: (next: unknown) => void;+ totalCount: number | null;+}) =>+ render(+ <TooltipProvider>+ <ExperimentScoreMatrix+ rows={rows}+ scoreRows={scoreRows}+ experiments={experiments}+ isLoading={false}+ pagination={{ totalCount, onChange, state }}+ />+ </TooltipProvider>,+ );++describe("ExperimentScoreMatrix pagination", () => {+ it("offers page controls and reports the window it aggregated", () => {+ const onChange = vi.fn();+ renderMatrix({+ rows: [itemRow("a", 0.5), itemRow("b", 0.7)],+ state: { pageIndex: 0, pageSize: 2 },+ onChange,+ totalCount: 6,+ });++ // The footnote has to name the window, since the cells above it are an+ // aggregate over exactly these rows and not over the whole run.+ expect(+ screen.getByText(/aggregated over the 2 items on this page/),+ ).toBeInTheDocument();++ fireEvent.click(screen.getByRole("button", { name: /go to next page/i }));+ expect(onChange).toHaveBeenCalled();+ });++ it("keeps the page controls when no score has a value on this page", () => {+ render(+ <TooltipProvider>+ <ExperimentScoreMatrix+ rows={[itemRow("a", 0.5)]}+ scoreRows={[]}+ experiments={experiments}+ isLoading={false}+ pagination={{+ totalCount: 6,+ onChange: vi.fn(),+ state: { pageIndex: 1, pageSize: 1 },+ }}+ />+ </TooltipProvider>,+ );++ expect(+ screen.getByText(/No score columns are visible for the items in view/),+ ).toBeInTheDocument();+ expect(+ screen.getByRole("button", { name: /go to next page/i }),+ ).toBeInTheDocument();+ });+});diff --git a/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsx b/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsxnew file mode 100644index 0000000000..61ef62ad7f--- /dev/null+++ b/web/src/features/experiments/components/table/ExperimentScoreMatrix.tsx@@ -0,0 +1,379 @@+import { useMemo } from "react";+import {+ getCoreRowModel,+ useReactTable,+ type ColumnDef,+ type OnChangeFn,+ type PaginationState,+ type Table as TanStackTable,+} from "@tanstack/react-table";+import { Badge } from "@/src/components/ui/badge";+import { Skeleton } from "@/src/components/ui/skeleton";+import { DataTablePagination } from "@/src/components/table/data-table-pagination";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/src/components/ui/tooltip";+import { DiffLabel } from "@/src/features/datasets/components/DiffLabel";+import {+ getScoreDataTypeExplanation,+ splitScoreDataTypeIcon,+} from "@/src/features/scores/lib/scoreColumns";+import {+ formatScoreColumnAggregate,+ formatScoreValue,+} from "@/src/features/experiments/fns/formatScoreColumnAggregate";+import {+ summariseScoreColumn,+ type ScoreColumnDataType,+ type ScoreColumnSummary,+} from "@/src/features/experiments/fns/summariseScoreColumn";+import {+ scoreFieldForLevel,+ type ScoreLevel,+} from "@/src/features/experiments/fns/scoreComparisonFilter";+import {+ getExperimentColorStyles,+ type ExperimentItemsTableRow,+} from "./types";+import { cn } from "@/src/utils/tailwind";++/**+ * The pagination instance never renders cells, only counts rows, so it needs+ * no column defs. Module-level so the reference stays stable across renders.+ */+const NO_COLUMNS: ColumnDef<ExperimentItemsTableRow>[] = [];++/** One row of the matrix: a score column, read at its own level. */+export type ScoreMatrixRow = {+ scoreKey: string;+ level: ScoreLevel;+ dataType: ScoreColumnDataType;+ /** The column header, e.g. `Trace: # groundedness (eval)`. */+ label: string;+};++/** One column of the matrix: a selected experiment, the baseline first. */+export type ScoreMatrixColumn = {+ experimentId: string;+ experimentName: string;+ isBaseline: boolean;+};++const MatrixCell = ({+ summary,+ hasBaselineColumn,+}: {+ summary: ScoreColumnSummary;+ /** False for the baseline's own column, which has nothing to move against. */+ hasBaselineColumn: boolean;+}) => {+ const { baseline, delta, movement } = summary;+ const notComparable = movement?.notComparable ?? 0;++ if (!baseline)+ return <span className="text-muted-foreground">not scored</span>;++ return (+ <span className="flex flex-wrap items-center gap-x-1 gap-y-0.5 tabular-nums">+ <span className="font-bold">{formatScoreColumnAggregate(baseline)}</span>+ {hasBaselineColumn && delta !== null && delta !== 0 && (+ <DiffLabel+ diff={{+ type: "NUMERIC",+ absoluteDifference: Math.abs(delta),+ direction: delta > 0 ? "+" : "-",+ }}+ formatValue={formatScoreValue}+ />+ )}+ {hasBaselineColumn && movement && movement.changed > 0 && (+ <span className="text-muted-foreground">β»{movement.changed}</span>+ )}+ {/* Items only one of the two runs scored: never folded into the delta. */}+ {hasBaselineColumn && notComparable > 0 && (+ <span className="text-muted-foreground opacity-70">+ {notComparable} n/a+ </span>+ )}+ </span>+ );+};++/**+ * The transpose of the grid: **scores as rows, experiments as columns**, every+ * cell the run's aggregate plus its move against the baseline column. Answers+ * "how did all my runs move on all my metrics" in one screen β what the deleted+ * Analytics tab was going to be.+ *+ * It needs no query of its own: the aggregates are the same ones the score+ * column headers compute, over the items already loaded for this page. Which+ * page that is stays the user's choice β see `paginationTable`.+ */+export const ExperimentScoreMatrix = ({+ rows,+ scoreRows,+ experiments,+ isLoading,+ pagination,+}: {+ rows: ExperimentItemsTableRow[];+ scoreRows: ScoreMatrixRow[];+ experiments: ScoreMatrixColumn[];+ isLoading: boolean;+ pagination: {+ totalCount: number | null;+ onChange: OnChangeFn<PaginationState>;+ state: PaginationState;+ options?: number[];+ };+}) => {+ const baselineExperimentId = experiments.find(+ (exp) => exp.isBaseline,+ )?.experimentId;++ const summaries = useMemo(() => {+ const scoresOf = (+ row: ExperimentItemsTableRow,+ experimentId: string,+ level: ScoreLevel,+ ) =>+ row.experiments.find((exp) => exp.experimentId === experimentId)?.[+ scoreFieldForLevel(level)+ ];++ // Cell (score, experiment) reads the experiment as the thing the delta+ // describes and the baseline column as what it moved against β the same+ // frame as a score column header.+ const byScore = new Map<string, Map<string, ScoreColumnSummary>>();+ for (const scoreRow of scoreRows) {+ const byExperiment = new Map<string, ScoreColumnSummary>();+ for (const experiment of experiments) {+ byExperiment.set(+ experiment.experimentId,+ summariseScoreColumn({+ pairs: rows.map((row) => ({+ baseline:+ scoresOf(row, experiment.experimentId, scoreRow.level)?.[+ scoreRow.scoreKey+ ] ?? null,+ comparison:+ baselineExperimentId === undefined+ ? null+ : (scoresOf(row, baselineExperimentId, scoreRow.level)?.[+ scoreRow.scoreKey+ ] ?? null),+ })),+ dataType: scoreRow.dataType,+ hasComparison:+ baselineExperimentId !== undefined &&+ experiment.experimentId !== baselineExperimentId,+ }),+ );+ }+ byScore.set(`${scoreRow.level}-${scoreRow.scoreKey}`, byExperiment);+ }+ return byScore;+ }, [rows, scoreRows, experiments, baselineExperimentId]);++ // The matrix aggregates the items on the page instead of listing them, so it+ // has no table of its own to drive the pagination bar. A headless instance+ // over the same rows and the same URL-held state does, which keeps the+ // aggregate's window under the user's control rather than pinning it to+ // whichever page was open first.+ const paginationTable = useReactTable({+ data: rows,+ columns: NO_COLUMNS,+ getCoreRowModel: getCoreRowModel(),+ manualPagination: true,+ pageCount:+ pagination.totalCount === null+ ? -1+ : Math.ceil(Number(pagination.totalCount) / pagination.state.pageSize),+ onPaginationChange: pagination.onChange,+ // The page lives in the URL and the rows arrive per page, so TanStack's+ // default reset-on-data-change would write page 1 back on every fetch and+ // re-fetch from it. The data tables opt out for the same reason.+ autoResetPageIndex: false,+ state: { pagination: pagination.state },+ });++ if (isLoading)+ return (+ <div className="flex flex-col gap-2 p-4">+ {Array.from({ length: 6 }).map((_, index) => (+ <Skeleton key={index} className="h-6 w-full" />+ ))}+ </div>+ );++ const allExperimentIds = experiments.map((exp) => exp.experimentId);++ // The empty state shares the footer, so the page controls stay reachable+ // when the items on this page happen to carry none of the visible scores.+ if (scoreRows.length === 0)+ return (+ <div className="flex min-h-0 flex-1 flex-col">+ <div className="flex flex-1 items-center justify-center p-4">+ <span className="text-muted-foreground text-sm">+ No score columns are visible for the items in view.+ </span>+ </div>+ <MatrixFooter+ scoreCount={0}+ experimentCount={experiments.length}+ itemCount={rows.length}+ paginationTable={paginationTable}+ paginationOptions={pagination.options}+ isLoading={isLoading}+ />+ </div>+ );++ return (+ <div className="flex min-h-0 flex-1 flex-col">+ <div className="min-h-0 flex-1 overflow-auto">+ <table className="w-full min-w-max border-separate border-spacing-0 text-xs">+ <thead>+ <tr>+ <th className="bg-background text-muted-foreground sticky top-0 left-0 z-20 min-w-[220px] border-b p-2 text-left font-normal">+ Score+ </th>+ {experiments.map((experiment) => {+ const colorStyles = getExperimentColorStyles(+ experiment.experimentId,+ allExperimentIds,+ );+ return (+ <th+ key={experiment.experimentId}+ className="bg-background sticky top-0 z-10 min-w-[150px] border-b p-2 text-left font-normal"+ >+ <span className="flex min-w-0 items-center gap-1">+ <span+ className={cn(+ "block h-3 w-0.5 shrink-0 rounded-full",+ colorStyles.markerClass,+ )}+ />+ <span+ className="truncate font-bold"+ title={experiment.experimentName}+ >+ {experiment.experimentName}+ </span>+ {experiment.isBaseline && (+ <Badge+ size="sm"+ variant="secondary"+ className="shrink-0"+ >+ baseline+ </Badge>+ )}+ </span>+ </th>+ );+ })}+ </tr>+ </thead>+ <tbody>+ {scoreRows.map((scoreRow) => {+ const { icon, label } = splitScoreDataTypeIcon(scoreRow.label);+ const byExperiment = summaries.get(+ `${scoreRow.level}-${scoreRow.scoreKey}`,+ );+ return (+ <tr key={`${scoreRow.level}-${scoreRow.scoreKey}`}>+ <th+ scope="row"+ className="bg-background sticky left-0 z-10 min-w-[220px] border-b p-2 text-left font-normal"+ >+ <span className="flex min-w-0 items-baseline gap-1">+ {icon && (+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="text-muted-foreground shrink-0 cursor-default">+ {icon}+ </span>+ </TooltipTrigger>+ <TooltipContent className="max-w-[280px]">+ {getScoreDataTypeExplanation(scoreRow.dataType)}+ </TooltipContent>+ </Tooltip>+ )}+ <span className="truncate" title={scoreRow.label}>+ {label}+ </span>+ </span>+ </th>+ {experiments.map((experiment) => {+ const summary = byExperiment?.get(experiment.experimentId);+ return (+ <td+ key={experiment.experimentId}+ className="min-w-[150px] border-b p-2 align-top"+ >+ {summary ? (+ <MatrixCell+ summary={summary}+ hasBaselineColumn={!experiment.isBaseline}+ />+ ) : null}+ </td>+ );+ })}+ </tr>+ );+ })}+ </tbody>+ </table>+ </div>+ <MatrixFooter+ scoreCount={scoreRows.length}+ experimentCount={experiments.length}+ itemCount={rows.length}+ paginationTable={paginationTable}+ paginationOptions={pagination.options}+ isLoading={isLoading}+ />+ </div>+ );+};++/**+ * What the numbers above cover, and the control that changes it. The scope+ * sentence and the page controls sit on one line deliberately: the aggregate+ * is a window over the run, so reading it without seeing which window is+ * selected invites the wrong conclusion.+ */+const MatrixFooter = ({+ scoreCount,+ experimentCount,+ itemCount,+ paginationTable,+ paginationOptions,+ isLoading,+}: {+ scoreCount: number;+ experimentCount: number;+ itemCount: number;+ paginationTable: TanStackTable<ExperimentItemsTableRow>;+ paginationOptions?: number[];+ isLoading: boolean;+}) => (+ <div className="flex flex-wrap items-center justify-between gap-x-4 gap-y-1 border-t px-2 py-1.5">+ <span className="text-muted-foreground text-[10px]">+ {scoreCount} score{scoreCount === 1 ? "" : "s"} Γ {experimentCount} run+ {experimentCount === 1 ? "" : "s"}, aggregated over the {itemCount} item+ {itemCount === 1 ? "" : "s"} on this page. Deltas are measured against the+ baseline column.+ </span>+ <DataTablePagination+ table={paginationTable}+ isLoading={isLoading}+ paginationOptions={paginationOptions}+ />+ </div>+);diff --git a/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsx b/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsxnew file mode 100644index 0000000000..e4e5d64cc1--- /dev/null+++ b/web/src/features/experiments/components/table/ScoreColumnFilterMenu.tsx@@ -0,0 +1,179 @@+import {+ forwardRef,+ type ComponentProps,+ type ComponentPropsWithoutRef,+} from "react";+import {+ DropdownMenuController,+ DropdownMenuItem,+ DropdownMenuLabel,+ DropdownMenuPortal,+ DropdownMenuSeparator,+ DropdownMenuSub,+ DropdownMenuSubContent,+ DropdownMenuSubTrigger,+} from "@/src/components/ui/dropdown-menu";+import { Check, ListFilter } from "lucide-react";+import { type ScoreComparisonOperator } from "@/src/features/experiments/fns/scoreComparisonFilter";+import { cn } from "@/src/utils/tailwind";++export type ScoreComparisonTarget = {+ experimentId: string;+ experimentName: string;+};++const OPERATOR_LABELS: Record<ScoreComparisonOperator, string> = {+ lower: "Worse than",+ higher: "Better than",+ differs: "Different from",+};++const ORDERED_OPERATORS: ScoreComparisonOperator[] = [+ "lower",+ "higher",+ "differs",+];++/**+ * The score header's filter affordance. Spreads and forwards so a+ * `<Trigger asChild>` at the call site can hand it Radix's open state and+ * handlers, and takes its look from an explicit variant rather than a style+ * prop.+ */+export const ScoreColumnFilterMenuTrigger = forwardRef<+ HTMLButtonElement,+ Omit<+ ComponentPropsWithoutRef<"button">,+ "className" | "style" | "children" | "aria-label"+ > & {+ /** A filter is set on this score, so the control stops hiding itself. */+ isActive: boolean;+ }+>(({ isActive, ...triggerProps }, ref) => (+ <button+ ref={ref}+ aria-label="Filter items by this score"+ className={cn(+ "hover:bg-muted mt-0.5 shrink-0 rounded-sm p-0.5",+ isActive+ ? "text-primary-accent"+ : "text-muted-foreground opacity-0 group-hover:opacity-100",+ )}+ {...triggerProps}+ >+ <ListFilter className="h-3 w-3" />+ </button>+));+ScoreColumnFilterMenuTrigger.displayName = "ScoreColumnFilterMenuTrigger";++/**+ * The score column's own way into the comparison filter: "show+ * only the items worse than <comparison> on this score", from the header of the+ * score in question, with the comparison to read against picked here rather than+ * assumed.+ */+export const ScoreColumnFilterMenu = ({+ targets,+ active,+ hasOrder,+ onSelect,+ onClear,+ children,+}: {+ /** The experiments this score can be read against, default first. */+ targets: ScoreComparisonTarget[];+ active?: {+ operator: ScoreComparisonOperator;+ comparisonExperimentId: string;+ };+ /** Categorical scores have no order, so only "different from" applies. */+ hasOrder: boolean;+ onSelect: (+ operator: ScoreComparisonOperator,+ comparisonExperimentId: string,+ ) => void;+ onClear: () => void;+ /** Renders the trigger, so its presentation stays with the caller. */+ children: ComponentProps<typeof DropdownMenuController>["children"];+}) => {+ const operators = hasOrder ? ORDERED_OPERATORS : (["differs"] as const);+ const isActive = (operator: ScoreComparisonOperator, experimentId: string) =>+ active?.operator === operator &&+ active?.comparisonExperimentId === experimentId;++ return (+ <DropdownMenuController+ align="end"+ maxWidth="16rem"+ renderMenu={() => (+ <>+ <DropdownMenuLabel>Show only items</DropdownMenuLabel>+ {operators.map((operator) =>+ targets.length === 1 ? (+ <DropdownMenuItem+ key={operator}+ onClick={() => onSelect(operator, targets[0].experimentId)}+ >+ {isActive(operator, targets[0].experimentId) ? (+ <Check className="mr-2 h-4 w-4 shrink-0" />+ ) : (+ <span className="mr-2 h-4 w-4 shrink-0" />+ )}+ <span+ className="truncate"+ title={`${OPERATOR_LABELS[operator]} ${targets[0].experimentName}`}+ >+ {OPERATOR_LABELS[operator]} {targets[0].experimentName}+ </span>+ </DropdownMenuItem>+ ) : (+ <DropdownMenuSub key={operator}>+ <DropdownMenuSubTrigger>+ {OPERATOR_LABELS[operator]}β¦+ </DropdownMenuSubTrigger>+ <DropdownMenuPortal>+ <DropdownMenuSubContent className="w-56">+ {targets.map((target) => (+ <DropdownMenuItem+ key={target.experimentId}+ onClick={() => onSelect(operator, target.experimentId)}+ >+ {isActive(operator, target.experimentId) ? (+ <Check className="mr-2 h-4 w-4 shrink-0" />+ ) : (+ <span className="mr-2 h-4 w-4 shrink-0" />+ )}+ <span+ className="truncate"+ title={target.experimentName}+ >+ {target.experimentName}+ </span>+ </DropdownMenuItem>+ ))}+ </DropdownMenuSubContent>+ </DropdownMenuPortal>+ </DropdownMenuSub>+ ),+ )}+ {active && (+ <>+ <DropdownMenuSeparator />+ <DropdownMenuItem onClick={onClear}>+ <span className="mr-2 h-4 w-4 shrink-0" />+ Remove this score's filter+ </DropdownMenuItem>+ </>+ )}+ <DropdownMenuSeparator />+ <span className="text-muted-foreground block px-2 py-1.5 text-[10px]">+ Compares the items loaded on this page β page through to check the+ rest of the run.+ </span>+ </>+ )}+ >+ {children}+ </DropdownMenuController>+ );+};diff --git a/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsx b/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsxnew file mode 100644index 0000000000..c5cf71b7f5--- /dev/null+++ b/web/src/features/experiments/components/table/ScoreComparisonFilterPills.tsx@@ -0,0 +1,79 @@+import { Badge } from "@/src/components/ui/badge";+import { Button } from "@/src/components/ui/button";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/src/components/ui/tooltip";+import { GitCompareArrows, X } from "lucide-react";+import {+ describeScoreComparisonFilter,+ type ScoreComparisonFilter,+} from "@/src/features/experiments/fns/scoreComparisonFilter";++export type ScoreComparisonPill = {+ filter: ScoreComparisonFilter;+ scoreName: string;+ comparisonName: string;+};++/**+ * The active score comparisons, in plain English ("Worse groundedness than+ * judge-haiku-baseline") and removable β the same affordance the score filters+ * get, for the one predicate that is evaluated in the client.+ */+export const ScoreComparisonFilterPills = ({+ pills,+ onRemove,+}: {+ pills: ScoreComparisonPill[];+ onRemove: (filter: ScoreComparisonFilter) => void;+}) => {+ return (+ <div className="flex flex-wrap items-center gap-1.5 border-b px-2 py-2">+ {pills.map(({ filter, scoreName, comparisonName }) => {+ const label = describeScoreComparisonFilter({+ operator: filter.operator,+ scoreName,+ comparisonName,+ });+ return (+ <Badge+ key={`${filter.level}-${filter.scoreKey}`}+ variant="secondary"+ className="flex max-w-full items-center gap-1 px-2 py-1 text-xs"+ >+ <GitCompareArrows className="h-3 w-3 shrink-0" />+ <span className="truncate" title={label}>+ {label}+ </span>+ <Tooltip>+ <TooltipTrigger asChild>+ <span className="text-muted-foreground shrink-0 cursor-default underline decoration-dotted">+ this page+ </span>+ </TooltipTrigger>+ <TooltipContent className="max-w-[280px]">+ Comparing one experiment's score against another's is+ not something the items query can express, so this narrows the+ items loaded on the current page. Page through to check the rest+ of the run.+ </TooltipContent>+ </Tooltip>+ <Button+ variant="ghost"+ size="sm"+ className="h-4 w-4 shrink-0 p-0 hover:bg-transparent"+ onClick={(event) => {+ event.stopPropagation();+ onRemove(filter);+ }}+ >+ <X className="h-3 w-3" />+ </Button>+ </Badge>+ );+ })}+ </div>+ );+};diff --git a/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.ts b/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.tsnew file mode 100644index 0000000000..4177a901be--- /dev/null+++ b/web/src/features/experiments/fns/scoreComparisonFilter.clienttest.ts@@ -0,0 +1,131 @@+import { type AggregatedScoreData } from "@langfuse/shared";+import {+ decodeScoreComparisonFilter,+ encodeScoreComparisonFilter,+ matchesScoreComparisonFilter,+} from "./scoreComparisonFilter";++const numeric = (value: number): AggregatedScoreData => ({+ type: "NUMERIC",+ values: [value],+ average: value,+});++const categorical = (value: string): AggregatedScoreData => ({+ type: "CATEGORICAL",+ values: [value],+ valueCounts: [{ value, count: 1 }],+});++describe("matchesScoreComparisonFilter", () => {+ it("keeps only the items the experiment you opened scored lower on", () => {+ const lower = (baseline: number, comparison: number) =>+ matchesScoreComparisonFilter({+ operator: "lower",+ dataType: "NUMERIC",+ baseline: numeric(baseline),+ comparison: numeric(comparison),+ });++ expect(lower(0.2, 0.9)).toBe(true);+ expect(lower(0.9, 0.2)).toBe(false);+ expect(lower(0.5, 0.5)).toBe(false);+ });++ it("never keeps an item only one of the two experiments scored", () => {+ for (const operator of ["lower", "higher", "differs"] as const) {+ expect(+ matchesScoreComparisonFilter({+ operator,+ dataType: "NUMERIC",+ baseline: numeric(0.2),+ comparison: null,+ }),+ ).toBe(false);+ expect(+ matchesScoreComparisonFilter({+ operator,+ dataType: "NUMERIC",+ baseline: undefined,+ comparison: numeric(0.2),+ }),+ ).toBe(false);+ }+ });++ it("reads a boolean as false < true", () => {+ expect(+ matchesScoreComparisonFilter({+ operator: "lower",+ dataType: "BOOLEAN",+ baseline: categorical("false"),+ comparison: categorical("true"),+ }),+ ).toBe(true);+ });++ it("only answers 'differs' for a categorical score, which has no order", () => {+ const args = {+ dataType: "CATEGORICAL" as const,+ baseline: categorical("weak"),+ comparison: categorical("grounded"),+ };+ expect(matchesScoreComparisonFilter({ ...args, operator: "lower" })).toBe(+ false,+ );+ expect(matchesScoreComparisonFilter({ ...args, operator: "differs" })).toBe(+ true,+ );+ expect(+ matchesScoreComparisonFilter({+ ...args,+ operator: "differs",+ comparison: categorical("weak"),+ }),+ ).toBe(false);+ });+});++describe("score comparison filter URL state", () => {+ it("round-trips through the URL and rejects anything malformed", () => {+ const filter = {+ level: "trace" as const,+ scoreKey: "groundedness-EVAL-NUMERIC",+ operator: "lower" as const,+ comparisonExperimentId: "1b0a2c3d-4e5f",+ };++ expect(+ decodeScoreComparisonFilter(encodeScoreComparisonFilter(filter)),+ ).toEqual(filter);+ expect(decodeScoreComparisonFilter("trace:key:sideways:id")).toBeNull();+ expect(decodeScoreComparisonFilter("session:key:lower:id")).toBeNull();+ expect(decodeScoreComparisonFilter("")).toBeNull();+ });++ // The score key carries the score's name, which is user-supplied: a name+ // holding the field separator or the one between several filters used to+ // land in the wrong slot and drop the filter without a word.+ it("round-trips a score name holding the separators", () => {+ for (const scoreKey of [+ "ratio:accuracy-EVAL-NUMERIC",+ "either|or-EVAL-NUMERIC",+ "a:b|c-EVAL-NUMERIC",+ ]) {+ const filter = {+ level: "observation" as const,+ scoreKey,+ operator: "differs" as const,+ comparisonExperimentId: "1b0a2c3d-4e5f",+ };+ const encoded = encodeScoreComparisonFilter(filter);+ expect(encoded).not.toContain("|");+ expect(encoded.split(":")).toHaveLength(4);+ expect(decodeScoreComparisonFilter(encoded)).toEqual(filter);+ }+ });++ it("rejects a malformed escape instead of throwing", () => {+ expect(decodeScoreComparisonFilter("trace:%zz:lower:id")).toBeNull();+ });+});diff --git a/web/src/features/experiments/fns/scoreComparisonFilter.ts b/web/src/features/experiments/fns/scoreComparisonFilter.tsnew file mode 100644index 0000000000..ad76e19166--- /dev/null+++ b/web/src/features/experiments/fns/scoreComparisonFilter.ts@@ -0,0 +1,215 @@+import { type AggregatedScoreData } from "@langfuse/shared";+import {+ readOrderedScoreValue,+ type ScoreColumnDataType,+} from "./summariseScoreColumn";++/** Which score family a filtered column belongs to. */+export type ScoreLevel = "observation" | "trace";++export type ScoreComparisonOperator = "lower" | "higher" | "differs";++/**+ * "Keep the items whose score is lower than the comparison experiment's" β the+ * one predicate the item filters could not express, because it is about a pair+ * of experiments rather than a column against a literal.+ */+export type ScoreComparisonFilter = {+ level: ScoreLevel;+ /** Aggregate score key, e.g. `groundedness-EVAL-NUMERIC`. */+ scoreKey: string;+ operator: ScoreComparisonOperator;+ /** The experiment the score is read against. */+ comparisonExperimentId: string;+};++const OPERATORS: ScoreComparisonOperator[] = ["lower", "higher", "differs"];++/**+ * `level:scoreKey:operator:experimentId`, each field percent-encoded. A score+ * key carries the score's name, which is user-supplied and may hold a colon or+ * the pipe that separates several of these in the URL β an unescaped field+ * would land in the wrong slot and the filter would silently vanish.+ */+export const encodeScoreComparisonFilter = (+ filter: ScoreComparisonFilter,+): string =>+ [+ filter.level,+ filter.scoreKey,+ filter.operator,+ filter.comparisonExperimentId,+ ]+ .map(encodeURIComponent)+ .join(":");++export const decodeScoreComparisonFilter = (+ encoded: string | null | undefined,+): ScoreComparisonFilter | null => {+ if (!encoded) return null;+ const parts = encoded.split(":");+ if (parts.length !== 4) return null;+ let level: string;+ let scoreKey: string;+ let operator: string;+ let comparisonExperimentId: string;+ try {+ [level, scoreKey, operator, comparisonExperimentId] =+ parts.map(decodeURIComponent);+ } catch {+ // A hand-edited or truncated URL can hold a malformed escape.+ return null;+ }+ if (level !== "observation" && level !== "trace") return null;+ if (!scoreKey || !comparisonExperimentId) return null;+ if (!OPERATORS.includes(operator as ScoreComparisonOperator)) return null;+ return {+ level,+ scoreKey,+ operator: operator as ScoreComparisonOperator,+ comparisonExperimentId,+ };+};++export const isSameScoreComparisonTarget = (+ a: ScoreComparisonFilter,+ b: ScoreComparisonFilter,+) => a.level === b.level && a.scoreKey === b.scoreKey;++/**+ * Whether one item passes the filter, read the same way the column header reads+ * it: an item only one of the two experiments scored is not comparable, so it is+ * never a regression and never kept. A categorical score has no order, so only+ * `differs` can match it.+ */+export const matchesScoreComparisonFilter = ({+ operator,+ dataType,+ baseline,+ comparison,+}: {+ operator: ScoreComparisonOperator;+ dataType: ScoreColumnDataType;+ baseline: AggregatedScoreData | null | undefined;+ comparison: AggregatedScoreData | null | undefined;+}): boolean => {+ if (!baseline || !comparison) return false;++ if (dataType === "CATEGORICAL") {+ // No order, so only "differs" means anything on a categorical score.+ if (operator !== "differs") return false;+ if (baseline.type !== "CATEGORICAL" || comparison.type !== "CATEGORICAL")+ return false;+ return (+ [...baseline.values].sort().join("|") !==+ [...comparison.values].sort().join("|")+ );+ }++ const baselineValue = readOrderedScoreValue(baseline, dataType);+ const comparisonValue = readOrderedScoreValue(comparison, dataType);+ if (baselineValue === null || comparisonValue === null) return false;++ if (operator === "differs") return baselineValue !== comparisonValue;+ return operator === "lower"+ ? baselineValue < comparisonValue+ : baselineValue > comparisonValue;+};++const OPERATOR_WORDS: Record<ScoreComparisonOperator, string> = {+ lower: "Worse",+ higher: "Better",+ differs: "Different",+};++/** The chip's plain English: "Worse groundedness than judge-haiku-baseline". */+export const describeScoreComparisonFilter = ({+ operator,+ scoreName,+ comparisonName,+}: {+ operator: ScoreComparisonOperator;+ scoreName: string;+ comparisonName: string;+}): string =>+ `${OPERATOR_WORDS[operator]} ${scoreName} ${+ operator === "differs" ? "vs" : "than"+ } ${comparisonName}`;++/** Which of a row's two score maps a level's scores live in. */+export const scoreFieldForLevel = (level: ScoreLevel) =>+ level === "trace" ? ("traceScores" as const) : ("observationScores" as const);++/**+ * Whether one item survives every active score comparison. A filter that cannot+ * be evaluated does not hide rows: its score type is not known yet (the column+ * definitions are still loading), or its target is no longer one of the runs on+ * screen. Reading either as "nothing matches" would empty the table, which+ * looks like a broken page rather than a filter with nothing to point at.+ */+export const rowPassesScoreComparisonFilters = ({+ filters,+ experiments,+ baselineExperimentId,+ comparableExperimentIds,+ dataTypeFor,+}: {+ filters: ScoreComparisonFilter[];+ experiments: Array<{+ experimentId: string;+ observationScores: Record<string, AggregatedScoreData>;+ traceScores: Record<string, AggregatedScoreData>;+ }>;+ baselineExperimentId?: string;+ /** The runs currently being compared, if the caller knows them. */+ comparableExperimentIds?: readonly string[];+ dataTypeFor: (+ filter: ScoreComparisonFilter,+ ) => ScoreColumnDataType | undefined;+}): boolean => {+ if (filters.length === 0) return true;+ if (!baselineExperimentId) return true;++ const scoresOf = (experimentId: string, level: ScoreLevel) =>+ experiments.find(+ (experiment) => experiment.experimentId === experimentId,+ )?.[scoreFieldForLevel(level)];++ return filters.every((filter) => {+ const dataType = dataTypeFor(filter);+ if (!dataType) return true;+ // The target was deselected, or is now the baseline it would be read+ // against β either way there is no second run left to compare with.+ if (filter.comparisonExperimentId === baselineExperimentId) return true;+ if (+ comparableExperimentIds &&+ !comparableExperimentIds.includes(filter.comparisonExperimentId)+ )+ return true;+ return matchesScoreComparisonFilter({+ operator: filter.operator,+ dataType,+ baseline: scoresOf(baselineExperimentId, filter.level)?.[filter.scoreKey],+ comparison: scoresOf(filter.comparisonExperimentId, filter.level)?.[+ filter.scoreKey+ ],+ });+ });+};++/** What an emptied table says, so it reads as an answer and not as a failure. */+export const describeEmptyScoreComparison = ({+ operator,+ scoreName,+ comparisonName,+}: {+ operator: ScoreComparisonOperator;+ scoreName: string;+ comparisonName: string;+}): string => {+ if (operator === "differs")+ return `No item on this page scored ${scoreName} differently from ${comparisonName}.`;+ if (operator === "lower")+ return `No regressions on this score β no item on this page scored lower on ${scoreName} than ${comparisonName}.`;+ return `No item on this page scored higher on ${scoreName} than ${comparisonName}.`;+};diff --git a/web/src/features/experiments/fns/summariseScoreColumn.ts b/web/src/features/experiments/fns/summariseScoreColumn.tsindex aa20c466ae..97d5e285fa 100644--- a/web/src/features/experiments/fns/summariseScoreColumn.ts+++ b/web/src/features/experiments/fns/summariseScoreColumn.ts@@ -54,7 +54,7 @@ const countValues = (aggregate: AggregatedScoreData) =>* The score read as a number, so two items can be ordered. A boolean's ordered* reading is its true-rate (false < true); a categorical has none.*/-const readOrderedScoreValue = (+export const readOrderedScoreValue = (aggregate: AggregatedScoreData | null,dataType: ScoreColumnDataType,): number | null => {diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.ts b/web/src/features/experiments/hooks/useExperimentResultsState.tsindex c488aa4894..d4ce999cd7 100644--- a/web/src/features/experiments/hooks/useExperimentResultsState.ts+++ b/web/src/features/experiments/hooks/useExperimentResultsState.ts@@ -10,15 +10,18 @@ import useLocalStorage from "@/src/components/useLocalStorage";/*** How the selected experiments are laid out against each other: one row per item- * ("list") or one column per experiment ("grid").+ * ("list"), one column per experiment ("grid"), or the transpose β scores as+ * rows and experiments as columns ("matrix").*/-export type ExperimentResultsLayout = "grid" | "list";+export type ExperimentResultsLayout = "grid" | "list" | "matrix";/** What a diff cell's extra line is measured against. */export type ExperimentDiffMode = "comparison" | "expected" | "off";const asLayout = (value: unknown): ExperimentResultsLayout | undefined =>- value === "grid" || value === "list" ? value : undefined;+ value === "grid" || value === "list" || value === "matrix"+ ? value+ : undefined;const asDiffMode = (value: unknown): ExperimentDiffMode | undefined =>value === "comparison" || value === "expected" || value === "off"diff --git a/web/src/features/experiments/hooks/useScoreComparisonFilters.ts b/web/src/features/experiments/hooks/useScoreComparisonFilters.tsnew file mode 100644index 0000000000..4be7d71777--- /dev/null+++ b/web/src/features/experiments/hooks/useScoreComparisonFilters.ts@@ -0,0 +1,86 @@+import { useCallback, useMemo } from "react";+import { ArrayParam, useQueryParams, withDefault } from "use-query-params";+import {+ decodeScoreComparisonFilter,+ encodeScoreComparisonFilter,+ isSameScoreComparisonTarget,+ type ScoreComparisonFilter,+} from "@/src/features/experiments/fns/scoreComparisonFilter";++/**+ * "Show only the items this run scored worse on than <comparison>" β Annabell's+ * top ask, as a filter rather than a report.+ *+ * The predicate is about a *pair* of experiments, which the items query cannot+ * express: `compileExperimentFilter` translates each condition into a+ * `column op literal` WHERE clause on a single experiment's rows, so comparing+ * one experiment's score against another's for the same item would need a new+ * HAVING path over the `experiment_item_id` group β new query machinery we are+ * explicitly not building. So the comparison is evaluated in the client over the+ * items already fetched for the current page, and the UI says so. Everything+ * else about it behaves like a filter: it lives in the URL, it is shareable, it+ * survives a reload, and it shows as a removable chip.+ *+ * One filter per score column: picking an operator on a column replaces that+ * column's filter instead of stacking a second one.+ */+const NO_FILTERS: string[] = [];++export const useScoreComparisonFilters = () => {+ const [state, setState] = useQueryParams({+ scoreCompare: withDefault(ArrayParam, NO_FILTERS),+ });++ // Keyed on the encoded filters rather than the array `use-query-params` hands+ // back, whose identity changes on every render and would make every consumer+ // of `filters` β the filtered rows above all β churn with it.+ const encoded = (+ (state.scoreCompare as (string | null)[] | undefined) ?? NO_FILTERS+ )+ .filter(Boolean)+ .join("|");++ const filters = useMemo(+ () =>+ encoded+ .split("|")+ .map(decodeScoreComparisonFilter)+ .filter(Boolean) as ScoreComparisonFilter[],+ [encoded],+ );++ const write = useCallback(+ (next: ScoreComparisonFilter[]) =>+ setState({+ scoreCompare: next.length+ ? next.map(encodeScoreComparisonFilter)+ : undefined,+ }),+ [setState],+ );++ const setFilter = useCallback(+ (filter: ScoreComparisonFilter) =>+ write([+ ...filters.filter(+ (existing) => !isSameScoreComparisonTarget(existing, filter),+ ),+ filter,+ ]),+ [filters, write],+ );++ const removeFilter = useCallback(+ (filter: ScoreComparisonFilter) =>+ write(+ filters.filter(+ (existing) => !isSameScoreComparisonTarget(existing, filter),+ ),+ ),+ [filters, write],+ );++ const clearFilters = useCallback(() => write([]), [write]);++ return { filters, setFilter, removeFilter, clearFilters };+};