🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(experiments): make the comparison selectable, grouped and pre-picked (#16916)
Nikita Kabardin07af5596579f
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.tsindex 1248ee8576..84abab63fb 100644--- a/packages/shared/src/server/repositories/experiments.ts+++ b/packages/shared/src/server/repositories/experiments.ts@@ -1497,16 +1497,27 @@ export const getExperimentItemsBatchIO = async (props: {});};+/**+ * Experiment options for the baseline / comparison pickers: one row per run,+ * with the dataset it ran on and when it started so the UI can group by dataset+ * and order by recency.+ */export const getExperimentNamesFromEvents = async (props: {projectId: string;}) => {const queryBuilder = new EventsAggQueryBuilder({projectId: props.projectId,+ // Grouped by experiment id: two runs sharing a name are two runs, not one+ // option pointing at an arbitrary id.groupByColumn: "e.experiment_id",- selectExpression:- "any(e.experiment_name) as experimentName, e.experiment_id as experimentId, nullIf(any(e.experiment_dataset_id), '') as datasetId",+ selectExpression: `any(e.experiment_name) as experimentName,+ e.experiment_id as experimentId,+ nullIf(any(e.experiment_dataset_id), '') as datasetId,+ min(e.start_time) as startTime`,}).whereRaw("e.experiment_name IS NOT NULL AND length(e.experiment_name) > 0")+ // Keeps the bound meaningful: the newest 1000 runs, not an arbitrary 1000.+ .orderBy("ORDER BY min(e.start_time) DESC").limit(1000, 0);const { query, params } = queryBuilder.buildWithParams();@@ -1515,6 +1526,7 @@ export const getExperimentNamesFromEvents = async (props: {experimentName: string;experimentId: string;datasetId: string | null;+ startTime: string;}>({query,params,@@ -1522,5 +1534,10 @@ export const getExperimentNamesFromEvents = async (props: {preferredClickhouseService: "EventsReadOnly",});- return res;+ return res.map((row) => ({+ experimentId: row.experimentId,+ experimentName: row.experimentName,+ datasetId: row.datasetId,+ startTime: parseClickhouseUTCDateTimeFormat(row.startTime),+ }));};diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.tsindex bc64ed0797..7358dbcdf6 100644--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts@@ -1355,17 +1355,21 @@ describe("Clickhouse Experiment Repository Test", () => {projectId: isolatedProjectId,});+ // `startTime` (the run's first event) came with the recency ordering the+ // comparison picker needs — the rows are otherwise unchanged.expect(result).toEqual(expect.arrayContaining([{experimentId: experimentIdA,experimentName: sharedName,datasetId: datasetIdA,+ startTime: expect.any(Date),},{experimentId: experimentIdB,experimentName: sharedName,datasetId: datasetIdB,+ startTime: expect.any(Date),},]),);@@ -1827,4 +1831,56 @@ describe("Clickhouse Experiment Repository Test", () => {);});});+ maybe("getExperimentNamesFromEvents", () => {+ it("should return one option per run when two runs share a name", async () => {+ // Own project so the assertion sees only the runs created here.+ const ownProjectId = randomUUID();+ const sharedName = "shared-name-" + randomUUID();+ const datasetId = randomUUID();+ const olderExperimentId = randomUUID();+ const newerExperimentId = randomUUID();+ const now = new Date();+ const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000);++ const eventFor = (experimentId: string, startTime: Date) => {+ const rootSpanId = randomUUID();+ return createEvent({+ id: randomUUID(),+ span_id: rootSpanId,+ project_id: ownProjectId,+ trace_id: randomUUID(),+ type: "GENERATION",+ name: "test-generation",+ experiment_id: experimentId,+ experiment_name: sharedName,+ experiment_metadata_names: [],+ experiment_metadata_values: [],+ experiment_dataset_id: datasetId,+ experiment_item_id: randomUUID(),+ experiment_item_version: null,+ experiment_item_root_span_id: rootSpanId,+ start_time: startTime.getTime() * 1000,+ });+ };++ await createEventsCh([+ eventFor(olderExperimentId, yesterday),+ eventFor(newerExperimentId, now),+ ]);++ const options = await getExperimentNamesFromEvents({+ projectId: ownProjectId,+ });++ expect(options).toHaveLength(2);+ expect(options.map((option) => option.experimentId)).toEqual([+ newerExperimentId,+ olderExperimentId,+ ]);+ options.forEach((option) => {+ expect(option.experimentName).toBe(sharedName);+ expect(option.datasetId).toBe(datasetId);+ });+ });+ });});diff --git a/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx b/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsxindex 6927e4c927..fcfd487521 100644--- a/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx+++ b/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx@@ -13,6 +13,8 @@ vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({usePostHogClientCapture: () => h.capture,}));+// The rebuilt control groups its options by dataset and dates each run, so the+// options carry a dataset name and a start time.vi.mock("@/src/features/experiments/hooks/useExperimentNames", () => ({useExperimentNames: () => ({experimentNames: [@@ -20,11 +22,15 @@ vi.mock("@/src/features/experiments/hooks/useExperimentNames", () => ({experimentId: "exp-a",experimentName: "My Experiment A",datasetId: "ds-1",+ datasetName: "My Dataset One",+ startTime: new Date("2026-08-26T10:00:00Z"),},{experimentId: "exp-b",experimentName: "My Experiment B",datasetId: "ds-2",+ datasetName: "My Dataset Two",+ startTime: new Date("2026-08-25T10:00:00Z"),},],isLoading: false,@@ -104,7 +110,9 @@ describe("ExperimentBaselineControls analytics", () => {);fireEvent.click(screen.getByRole("combobox"));- fireEvent.click(screen.getByRole("option", { name: "My Experiment A" }));+ // Each option now carries a relative-date badge, so its accessible name is+ // the run name plus that badge rather than the name alone.+ fireEvent.click(screen.getByRole("option", { name: /My Experiment A/ }));expect(h.capture).not.toHaveBeenCalled();expect(h.onBaselineChange).not.toHaveBeenCalled();diff --git a/web/src/features/experiments/components/ExperimentBaselineControls.tsx b/web/src/features/experiments/components/ExperimentBaselineControls.tsxindex b68f3dbcf5..b06b387b2d 100644--- a/web/src/features/experiments/components/ExperimentBaselineControls.tsx+++ b/web/src/features/experiments/components/ExperimentBaselineControls.tsx@@ -1,7 +1,17 @@+import { useMemo } from "react";import { Button } from "@/src/components/ui/button";-import { Combobox } from "@/src/components/ui/combobox";+import {+ Combobox,+ type ComboboxOptionGroup,+} from "@/src/components/ui/combobox";import { X } from "lucide-react";import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";+import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";+import {+ NO_DATASET_KEY,+ NO_DATASET_LABEL,+ UNNAMED_DATASET_LABEL,+} from "@/src/features/experiments/constants/comparison";import { cn } from "@/src/utils/tailwind";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { baselineChangedProps } from "@/src/features/experiments/lib/analytics";@@ -21,20 +31,41 @@ export function ExperimentBaselineControls({onBaselineChange,onBaselineClear,}: ExperimentBaselineControlsProps) {- const capture = usePostHogClientCapture();const { experimentNames, isLoading } = useExperimentNames({projectId,});- const baselineOptions = experimentNames.map((exp) => ({- value: exp.experimentId,- label: exp.experimentName,- }));+ const capture = usePostHogClientCapture();++ // Grouped by dataset and dated, so two runs sharing a name are two readable+ // options rather than the same label twice.+ const baselineOptionGroups = useMemo<ComboboxOptionGroup<string>[]>(() => {+ const byDataset = new Map<string, ComboboxOptionGroup<string>>();++ for (const experiment of experimentNames) {+ const datasetKey = experiment.datasetId ?? NO_DATASET_KEY;+ const group = byDataset.get(datasetKey) ?? {+ heading:+ experiment.datasetId === null+ ? NO_DATASET_LABEL+ : (experiment.datasetName ?? UNNAMED_DATASET_LABEL),+ options: [],+ };+ group.options.push({+ value: experiment.experimentId,+ label: experiment.experimentName,+ badge: formatRunRecency(experiment.startTime),+ });+ byDataset.set(datasetKey, group);+ }++ return [...byDataset.values()];+ }, [experimentNames]);return (<div className="flex min-w-0 items-center"><div className="min-w-0 flex-1"><Combobox- options={baselineOptions}+ options={baselineOptionGroups}value={baselineId}onValueChange={(id) => {if (id === baselineId) return;diff --git a/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx b/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsxindex 72249c4a1a..813418eb75 100644--- a/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx+++ b/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx@@ -6,21 +6,30 @@ const h = vi.hoisted(() => ({capture: vi.fn(),onSelectedIdsChange: vi.fn(),searchQuery: "",+ onAutoSelectEnabledChange: vi.fn(),+ // Newest first, the order the picker's own query returns. `startTime` and+ // `datasetName` came with the dataset grouping and recency ordering.searchResults: [{experimentId: "exp-a",experimentName: "My Experiment A",datasetId: "ds-1",+ datasetName: "My Dataset One",+ startTime: new Date("2026-08-26T10:00:00Z"),},{experimentId: "exp-b",experimentName: "My Experiment B",datasetId: "ds-1",+ datasetName: "My Dataset One",+ startTime: new Date("2026-08-25T10:00:00Z"),},{experimentId: "exp-c",experimentName: "My Experiment C",datasetId: "ds-2",+ datasetName: "My Dataset Two",+ startTime: new Date("2026-08-24T10:00:00Z"),},],}));@@ -34,6 +43,7 @@ vi.mock("@/src/features/experiments/hooks/useExperimentSearch", () => ({searchResults: h.searchResults,searchQuery: h.searchQuery,setSearchQuery: vi.fn(),+ isSearchActive: false,isLoading: false,availableExperimentNames: h.searchResults,}),@@ -45,6 +55,7 @@ describe("ExperimentComparisonSelector analytics", () => {beforeEach(() => {h.capture.mockClear();h.onSelectedIdsChange.mockClear();+ h.onAutoSelectEnabledChange.mockClear();h.searchQuery = "";});@@ -56,6 +67,8 @@ describe("ExperimentComparisonSelector analytics", () => {selectedIds={[]}selectedExperimentCount={1}onSelectedIdsChange={h.onSelectedIdsChange}+ isAutoSelectEnabled={true}+ onAutoSelectEnabledChange={h.onAutoSelectEnabledChange}/>,);@@ -86,6 +99,8 @@ describe("ExperimentComparisonSelector analytics", () => {selectedIds={[]}selectedExperimentCount={1}onSelectedIdsChange={h.onSelectedIdsChange}+ isAutoSelectEnabled={true}+ onAutoSelectEnabledChange={h.onAutoSelectEnabledChange}/>,);diff --git a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx b/web/src/features/experiments/components/ExperimentComparisonSelector.tsxindex 2b5e545b74..10912a2620 100644--- a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx+++ b/web/src/features/experiments/components/ExperimentComparisonSelector.tsx@@ -1,19 +1,71 @@-import { useMemo } from "react";-import { Check, X } from "lucide-react";+import { useCallback, useMemo, useRef, useState } from "react";+import { Check, ChevronDown, ChevronRight, X } from "lucide-react";import { MultiSelectCombobox } from "@/src/components/ui/multi-select-combobox";import { Badge } from "@/src/components/ui/badge";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/src/components/ui/tooltip";import { useExperimentSearch } from "@/src/features/experiments/hooks/useExperimentSearch";-import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";+import { type ExperimentNameOption } from "@/src/features/experiments/hooks/useExperimentNames";+import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import {comparisonChangedProps,comparisonPickerOpenedProps,} from "@/src/features/experiments/lib/analytics";+import {+ MAX_SELECTED_EXPERIMENTS,+ MAX_VISIBLE_COMPARISON_CHIPS,+ NO_DATASET_KEY,+ NO_DATASET_LABEL,+ UNNAMED_DATASET_LABEL,+} from "@/src/features/experiments/constants/comparison";++export type ExperimentOption = Omit<ExperimentNameOption, "startTime"> & {+ /** null when an id in the URL no longer resolves to a run. */+ startTime: Date | null;+};++/**+ * The dropdown is a flat list of rows so the shared combobox can render it, but+ * the rows carry the dataset grouping: a header per dataset, its runs beneath.+ */+type ComparisonRow =+ | { kind: "preference" }+ | {+ kind: "group";+ datasetKey: string;+ label: string;+ runCount: number;+ isBaselineDataset: boolean;+ isExpanded: boolean;+ }+ | {+ kind: "option";+ option: ExperimentOption;+ isBaseline: boolean;+ isLatest: boolean;+ };++const datasetKeyOf = (option: { datasetId: string | null }) =>+ option.datasetId ?? NO_DATASET_KEY;-export type ExperimentOption = {- experimentId: string;- experimentName: string;- datasetId: string | null;+const datasetLabelOf = (option: ExperimentOption) =>+ option.datasetId === null+ ? NO_DATASET_LABEL+ : (option.datasetName ?? UNNAMED_DATASET_LABEL);++const rowKeyOf = (row: ComparisonRow) => {+ switch (row.kind) {+ case "preference":+ return "preference";+ case "group":+ return `group:${row.datasetKey}`;+ default:+ return `experiment:${row.option.experimentId}`;+ }};type ExperimentComparisonSelectorProps = {@@ -22,6 +74,8 @@ type ExperimentComparisonSelectorProps = {selectedIds: string[];selectedExperimentCount: number;onSelectedIdsChange: (ids: string[]) => void;+ isAutoSelectEnabled: boolean;+ onAutoSelectEnabledChange: (isEnabled: boolean) => void;};export function ExperimentComparisonSelector({@@ -30,40 +84,195 @@ export function ExperimentComparisonSelector({selectedIds,selectedExperimentCount,onSelectedIdsChange,+ isAutoSelectEnabled,+ onAutoSelectEnabledChange,}: ExperimentComparisonSelectorProps) {- const capture = usePostHogClientCapture();const {searchResults,searchQuery,setSearchQuery,+ isSearchActive,isLoading,availableExperimentNames,} = useExperimentSearch({projectId,});- // Filter out baseline experiment from options- const searchResultsExclBaseline = useMemo(() => {- return baselineExperimentId- ? searchResults.filter((exp) => exp.experimentId !== baselineExperimentId)- : searchResults;- }, [baselineExperimentId, searchResults]);-- // Map selected IDs to full experiment objects- const selectedExperiments = useMemo(() => {- return selectedIds.map((id) => {- const found = availableExperimentNames.find(- (exp) => exp.experimentId === id,+ const [expandedOverrides, setExpandedOverrides] = useState<+ Record<string, boolean>+ >({});+ const capture = usePostHogClientCapture();++ // How long is the list people choose from, and how many datasets are mixed+ // into it? Counted over the whole available list, not the filtered rows, so+ // it describes the picker rather than the current search.+ const isPickerOpenRef = useRef(false);+ const handleOpenChange = useCallback(+ (isOpen: boolean) => {+ // The combobox re-announces "open" on every input focus, including the+ // refocus after picking a row; only the closed-to-open edge is an open.+ if (!isOpen) {+ isPickerOpenRef.current = false;+ return;+ }+ if (isPickerOpenRef.current) return;+ isPickerOpenRef.current = true;++ // Counted over the whole available list rather than the filtered rows, so+ // it describes the pool the user chooses from; the baseline is not one of+ // its options. The query itself is never sent — only whether there is one+ // and how long it is.+ const options = availableExperimentNames.filter(+ (experiment) => experiment.experimentId !== baselineExperimentId,+ );+ capture(+ "experiment:comparison_picker_opened",+ comparisonPickerOpenedProps({+ tableName: "experiment-items",+ optionCount: options.length,+ datasetIds: options.map((experiment) => experiment.datasetId),+ queryLength: searchQuery.length,+ }),+ );+ },+ [capture, availableExperimentNames, baselineExperimentId, searchQuery],+ );++ const baselineDatasetKey = useMemo(() => {+ const baseline = availableExperimentNames.find(+ (experiment) => experiment.experimentId === baselineExperimentId,+ );+ return baseline ? datasetKeyOf(baseline) : null;+ }, [availableExperimentNames, baselineExperimentId]);++ // The newest run on the baseline's dataset, taken from the full list rather+ // than the search results so the badge does not move when filtering. Absent+ // when the baseline is itself the newest run.+ const latestExperimentId = useMemo(+ () =>+ baselineDatasetKey+ ? availableExperimentNames.find(+ (experiment) => datasetKeyOf(experiment) === baselineDatasetKey,+ )?.experimentId+ : undefined,+ [availableExperimentNames, baselineDatasetKey],+ );++ const groups = useMemo(() => {+ const byDataset = new Map<+ string,+ { datasetKey: string; label: string; options: ExperimentNameOption[] }+ >();++ // searchResults arrive newest-first, so the runs inside a group already are.+ for (const option of searchResults) {+ const datasetKey = datasetKeyOf(option);+ const group = byDataset.get(datasetKey) ?? {+ datasetKey,+ label: datasetLabelOf(option),+ options: [],+ };+ group.options.push(option);+ byDataset.set(datasetKey, group);+ }++ // The baseline's dataset first: comparing across datasets compares+ // different items, so it must never be what the list offers first.+ return [...byDataset.values()].sort((a, b) => {+ if (a.datasetKey === baselineDatasetKey) return -1;+ if (b.datasetKey === baselineDatasetKey) return 1;+ return (+ (b.options[0]?.startTime.getTime() ?? 0) -+ (a.options[0]?.startTime.getTime() ?? 0));- if (found) return found;- // If not in current results, create placeholder with ID as name- return { experimentId: id, experimentName: id, datasetId: null };});- }, [selectedIds, availableExperimentNames]);+ }, [searchResults, baselineDatasetKey]);++ const rows = useMemo(() => {+ // With no baseline yet there is no "current" dataset, so open the one+ // holding the most recent run instead of showing nothing but headers.+ const defaultExpandedKey = baselineDatasetKey ?? groups[0]?.datasetKey;+ // The preference row is chrome, not a result: seeding it unconditionally+ // would make the list never look empty, and the combobox's "no results"+ // state — which keys off the row count — could never be reached.+ const result: ComparisonRow[] =+ groups.length > 0 ? [{ kind: "preference" }] : [];++ for (const group of groups) {+ // While searching, every group with a match is open: `??` falls through+ // only on nullish, so a group the user collapsed earlier would keep its+ // explicit `false` and swallow the rows the search just found.+ const isExpanded = isSearchActive+ ? true+ : (expandedOverrides[group.datasetKey] ??+ group.datasetKey === defaultExpandedKey);++ result.push({+ kind: "group",+ datasetKey: group.datasetKey,+ label: group.label,+ runCount: group.options.length,+ isBaselineDataset: group.datasetKey === baselineDatasetKey,+ isExpanded,+ });++ if (!isExpanded) continue;- const handleItemsChange = (items: ExperimentOption[]) => {+ for (const option of group.options) {+ result.push({+ kind: "option",+ option,+ isBaseline: option.experimentId === baselineExperimentId,+ isLatest: option.experimentId === latestExperimentId,+ });+ }+ }++ return result;+ }, [+ groups,+ expandedOverrides,+ isSearchActive,+ baselineDatasetKey,+ baselineExperimentId,+ latestExperimentId,+ ]);++ const selectedRows = useMemo<ComparisonRow[]>(+ () =>+ selectedIds.map((id) => ({+ kind: "option",+ option: availableExperimentNames.find(+ (experiment) => experiment.experimentId === id,+ ) ?? {+ // An id from the URL that no longer resolves to a run.+ experimentId: id,+ experimentName: id,+ datasetId: null,+ datasetName: null,+ startTime: null,+ },+ isBaseline: false,+ isLatest: false,+ })),+ [selectedIds, availableExperimentNames],+ );++ // Past a few chips the row turns into a horizontal scroll nobody scrolls, so+ // the tail collapses into one "+N" badge that names what it hides.+ const hiddenSelectedOptions = useMemo(+ () =>+ selectedRows+ .flatMap((row) => (row.kind === "option" ? [row.option] : []))+ .slice(MAX_VISIBLE_COMPARISON_CHIPS),+ [selectedRows],+ );++ const handleItemsChange = (items: ComparisonRow[]) => {const newIds = items- .map((item) => item.experimentId)+ .flatMap((item) =>+ item.kind === "option" ? [item.option.experimentId] : [],+ ).slice(0,Math.max(@@ -100,32 +309,19 @@ export function ExperimentComparisonSelector({onSelectedIdsChange(newIds);};- const handlePickerOpenChange = (open: boolean) => {- if (!open) return;- capture(- "experiment:comparison_picker_opened",- comparisonPickerOpenedProps({- tableName: "experiment-items",- optionCount: searchResultsExclBaseline.length,- datasetIds: searchResultsExclBaseline.map((exp) => exp.datasetId),- queryLength: searchQuery.length,- }),- );- };-const isMaxReached = selectedExperimentCount >= MAX_SELECTED_EXPERIMENTS;return (<div className="space-y-2">- <MultiSelectCombobox<ExperimentOption>- labelLeft="Experiment selection"- selectedItems={selectedExperiments}+ <MultiSelectCombobox<ComparisonRow>+ labelLeft="Compare"+ selectedItems={selectedRows}onItemsChange={handleItemsChange}- onOpenChange={handlePickerOpenChange}searchQuery={searchQuery}onSearchChange={setSearchQuery}- searchResults={searchResultsExclBaseline}+ searchResults={rows}isLoading={isLoading}+ onOpenChange={handleOpenChange}placeholder={isMaxReached? `Max ${MAX_SELECTED_EXPERIMENTS} experiments`@@ -133,52 +329,181 @@ export function ExperimentComparisonSelector({}disabled={isLoading}showSearchIcon={false}- getItemKey={(item) => item.experimentId}- renderItem={(item, isSelected, onToggle) => (- <button- type="button"- onClick={onToggle}- disabled={- !isSelected && selectedExperimentCount >= MAX_SELECTED_EXPERIMENTS- }- className="hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left disabled:cursor-not-allowed disabled:opacity-50"- >- <div className="flex h-4 w-4 items-center justify-center">- {isSelected && <Check className="text-primary h-4 w-4" />}- </div>- <div className="min-w-0 flex-1">- <p- className="truncate text-sm font-bold"- title={item.experimentName}+ dropdownClassName="bg-background absolute top-0 z-10 max-h-80 w-full overflow-y-auto rounded-md border shadow-md"+ getItemKey={rowKeyOf}+ renderItem={(row, isSelected, onToggle) => {+ if (row.kind === "preference") {+ return (+ <button+ type="button"+ onClick={() => {+ onAutoSelectEnabledChange(!isAutoSelectEnabled);+ }}+ className="text-muted-foreground hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left">- {item.experimentName}- </p>- </div>- </button>- )}- renderSelectedItem={(item, onRemove) => (- <Badge- variant="secondary"- className="flex items-center gap-1 px-2 py-0.5"- >- <span- className="max-w-24 truncate text-xs"- title={item.experimentName}- >- {item.experimentName}- </span>+ <div className="border-input flex h-4 w-4 shrink-0 items-center justify-center rounded-sm border">+ {isAutoSelectEnabled && <Check className="h-3 w-3" />}+ </div>+ <span className="text-xs">+ Auto-select a comparison experiment by default+ </span>+ </button>+ );+ }++ if (row.kind === "group") {+ return (+ <button+ type="button"+ onClick={() =>+ setExpandedOverrides((previous) => ({+ ...previous,+ [row.datasetKey]: !row.isExpanded,+ }))+ }+ className="bg-muted/40 hover:bg-muted flex w-full items-center gap-2 px-2 py-1.5 text-left"+ >+ {row.isExpanded ? (+ <ChevronDown className="h-3 w-3 shrink-0" />+ ) : (+ <ChevronRight className="h-3 w-3 shrink-0" />+ )}+ <span className="truncate text-xs font-bold" title={row.label}>+ {row.label}+ </span>+ <span className="text-muted-foreground shrink-0 text-xs">+ {row.runCount}+ </span>+ {!row.isBaselineDataset && (+ <span+ className="text-muted-foreground ml-auto shrink-0 text-xs italic"+ title="Runs on another dataset scored different items, so their values are not comparable."+ >+ other dataset+ </span>+ )}+ </button>+ );+ }++ const { option } = row;++ return (<buttontype="button"- onClick={(e) => {- e.stopPropagation();- onRemove();- }}- className="hover:bg-muted ml-0.5 rounded-full"+ onClick={row.isBaseline ? undefined : onToggle}+ disabled={row.isBaseline || (!isSelected && isMaxReached)}+ className="hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left disabled:cursor-not-allowed disabled:opacity-50">- <X className="h-3 w-3" />+ <div className="flex h-4 w-4 shrink-0 items-center justify-center">+ {isSelected && <Check className="text-primary h-4 w-4" />}+ </div>+ <span+ className="min-w-0 flex-1 truncate text-sm font-bold"+ title={option.experimentName}+ >+ {option.experimentName}+ </span>+ {row.isBaseline && (+ <Badge variant="outline" className="shrink-0 text-[10px]">+ BASELINE+ </Badge>+ )}+ {row.isLatest && (+ <Badge variant="outline" className="shrink-0 text-[10px]">+ LATEST+ </Badge>+ )}+ {option.startTime && (+ <span+ className="text-muted-foreground shrink-0 text-xs"+ title={option.startTime.toLocaleString()}+ >+ {formatRunRecency(option.startTime)}+ </span>+ )}</button>- </Badge>- )}+ );+ }}+ renderSelectedItem={(row, onRemove) => {+ if (row.kind !== "option") return null;++ const { option } = row;+ const position = selectedIds.indexOf(option.experimentId);++ if (position >= MAX_VISIBLE_COMPARISON_CHIPS) {+ // One badge stands in for the whole tail; the rest render nothing.+ return position === MAX_VISIBLE_COMPARISON_CHIPS ? (+ <Tooltip>+ <TooltipTrigger asChild>+ <Badge+ variant="secondary"+ className="shrink-0 px-2 py-0.5 text-xs"+ >+ +{hiddenSelectedOptions.length}+ </Badge>+ </TooltipTrigger>+ <TooltipContent className="max-w-[280px]">+ {hiddenSelectedOptions.map((hidden) => (+ <p key={hidden.experimentId}>+ {hidden.experimentName}+ {hidden.startTime+ ? ` · ${formatRunRecency(hidden.startTime)}`+ : ""}+ </p>+ ))}+ </TooltipContent>+ </Tooltip>+ ) : null;+ }++ // A comparison should never be on another dataset, but the URL can say+ // anything, so name the dataset rather than leaving it unexplained.+ const isOtherDataset =+ baselineDatasetKey !== null &&+ option.startTime !== null &&+ datasetKeyOf(option) !== baselineDatasetKey;++ const chipTitle = [+ option.experimentName,+ option.startTime+ ? `Started ${option.startTime.toLocaleString()}`+ : "This experiment is no longer available",+ isOtherDataset ? `Dataset: ${datasetLabelOf(option)}` : null,+ ]+ .filter(Boolean)+ .join("\n");++ return (+ <Badge+ variant="secondary"+ className="flex shrink-0 items-center gap-1 px-2 py-0.5"+ >+ <span className="max-w-40 truncate text-xs" title={chipTitle}>+ {option.experimentName}+ </span>+ {isOtherDataset && (+ <span+ className="text-muted-foreground max-w-24 truncate text-[10px]"+ title={`Dataset: ${datasetLabelOf(option)}`}+ >+ {datasetLabelOf(option)}+ </span>+ )}+ <button+ type="button"+ onClick={(e) => {+ e.stopPropagation();+ onRemove();+ }}+ className="hover:bg-muted ml-0.5 rounded-full"+ aria-label={`Remove ${option.experimentName}`}+ >+ <X className="h-3 w-3" />+ </button>+ </Badge>+ );+ }}/></div>);diff --git a/web/src/features/experiments/components/ExperimentSelectionControls.tsx b/web/src/features/experiments/components/ExperimentSelectionControls.tsxindex deb98508e3..c9ec5280f8 100644--- a/web/src/features/experiments/components/ExperimentSelectionControls.tsx+++ b/web/src/features/experiments/components/ExperimentSelectionControls.tsx@@ -1,3 +1,5 @@+import { type UrlUpdateType } from "use-query-params";+import { useExperimentComparisonAutoSelect } from "@/src/features/experiments/hooks/useExperimentComparisonAutoSelect";import { ExperimentBaselineControls } from "./ExperimentBaselineControls";import { ExperimentComparisonSelector } from "./ExperimentComparisonSelector";@@ -9,7 +11,10 @@ type ExperimentSelectionControlsProps = {selectedExperimentCount: number;onBaselineChange: (id: string) => void;onBaselineClear: () => void;- onComparisonIdsChange: (ids: string[]) => void;+ onComparisonIdsChange: (+ ids: string[],+ options?: { updateType?: UrlUpdateType },+ ) => void;};export function ExperimentSelectionControls({@@ -22,8 +27,18 @@ export function ExperimentSelectionControls({onBaselineClear,onComparisonIdsChange,}: ExperimentSelectionControlsProps) {+ // Mounted once per results page, so the default comparison is decided here+ // rather than in the state hook that every consumer of the URL calls.+ const { isAutoSelectEnabled, setIsAutoSelectEnabled } =+ useExperimentComparisonAutoSelect({+ projectId,+ baselineId,+ comparisonIds,+ onComparisonIdsChange,+ });+return (- <div className="flex w-[50dvw] min-w-0 flex-row gap-3">+ <div className="flex w-[56dvw] min-w-0 flex-row gap-3"><div className="flex min-w-0 items-center"><div className="border-input bg-muted/30 flex h-8 w-auto shrink-0 items-center rounded-l-md border px-3 text-xs">Baseline@@ -46,6 +61,8 @@ export function ExperimentSelectionControls({selectedIds={comparisonIds}selectedExperimentCount={selectedExperimentCount}onSelectedIdsChange={onComparisonIdsChange}+ isAutoSelectEnabled={isAutoSelectEnabled}+ onAutoSelectEnabledChange={setIsAutoSelectEnabled}/></div></div>diff --git a/web/src/features/experiments/constants/comparison.ts b/web/src/features/experiments/constants/comparison.tsindex 3f7c70b518..06a63fd6e7 100644--- a/web/src/features/experiments/constants/comparison.ts+++ b/web/src/features/experiments/constants/comparison.ts@@ -1 +1,22 @@export const MAX_SELECTED_EXPERIMENTS = 10;++/** Group key for runs that carry no dataset id. */+export const NO_DATASET_KEY = "__no_dataset__";++/** Group labels for a dataset we cannot name. */+export const NO_DATASET_LABEL = "No dataset";+export const UNNAMED_DATASET_LABEL = "Deleted dataset";++/**+ * Per-user preference: auto-select a comparison when a results page opens+ * without one.+ */+export const AUTO_SELECT_COMPARISON_STORAGE_KEY =+ "experiments-auto-select-comparison";++/**+ * Comparison chips rendered before the tail folds into a "+N" badge. Two full+ * names are what the header's width fits; beyond that the row would only be+ * reachable by horizontal scrolling, which nobody does.+ */+export const MAX_VISIBLE_COMPARISON_CHIPS = 2;diff --git a/web/src/features/experiments/fns/formatRunRecency.ts b/web/src/features/experiments/fns/formatRunRecency.tsnew file mode 100644index 0000000000..d5f51da432--- /dev/null+++ b/web/src/features/experiments/fns/formatRunRecency.ts@@ -0,0 +1,18 @@+/**+ * How a run's start time reads next to its name: relative inside the last day,+ * a calendar date beyond it — "20m ago", "5h ago", "Aug 21", "Aug 21, 2025".+ */+export const formatRunRecency = (startTime: Date, now = new Date()): string => {+ const minutes = (now.getTime() - startTime.getTime()) / 60_000;+ if (minutes < 1) return "just now";+ if (minutes < 60) return `${Math.floor(minutes)}m ago`;+ if (minutes < 60 * 24) return `${Math.floor(minutes / 60)}h ago`;++ return startTime.toLocaleDateString("en-US", {+ month: "short",+ day: "numeric",+ ...(startTime.getFullYear() === now.getFullYear()+ ? {}+ : { year: "numeric" }),+ });+};diff --git a/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.ts b/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.tsnew file mode 100644index 0000000000..032cfe1468--- /dev/null+++ b/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.ts@@ -0,0 +1,85 @@+import { useEffect, useRef } from "react";+import { useRouter } from "next/router";+import { type UrlUpdateType } from "use-query-params";+import useLocalStorage from "@/src/components/useLocalStorage";+import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";+import { AUTO_SELECT_COMPARISON_STORAGE_KEY } from "@/src/features/experiments/constants/comparison";++type UseExperimentComparisonAutoSelectProps = {+ projectId: string;+ baselineId?: string;+ comparisonIds: string[];+ onComparisonIdsChange: (+ ids: string[],+ options?: { updateType?: UrlUpdateType },+ ) => void;+};++/**+ * A results page opened without a comparison shows no movement at all, which is+ * where three out of four users stop. Default to the previous run on the same+ * dataset and write it into the URL like a user's own pick, so the view stays+ * shareable, with a preference for anyone who would rather choose themselves.+ */+export function useExperimentComparisonAutoSelect({+ projectId,+ baselineId,+ comparisonIds,+ onComparisonIdsChange,+}: UseExperimentComparisonAutoSelectProps) {+ const router = useRouter();+ const [isAutoSelectEnabled, setIsAutoSelectEnabled] =+ useLocalStorage<boolean>(AUTO_SELECT_COMPARISON_STORAGE_KEY, true);+ const { experimentNames, isLoading } = useExperimentNames({ projectId });++ // At most one attempt per baseline while mounted: clearing the auto-selected+ // comparison must not be undone on the next render. It cannot outlive the+ // mount — an empty selection drops `c` from the URL entirely, so a reload+ // cannot tell "cleared" from "never chosen" and defaults again. The+ // preference below is the durable way to opt out.+ const attemptedForBaselineRef = useRef<string | undefined>(undefined);++ useEffect(() => {+ if (!isAutoSelectEnabled || !baselineId || isLoading) return;+ // Query-param writes issued before the router is ready are dropped, and the+ // URL's own `c=` is not readable yet, so nothing here can be trusted.+ if (!router.isReady) return;+ if (attemptedForBaselineRef.current === baselineId) return;++ const baseline = experimentNames.find(+ (experiment) => experiment.experimentId === baselineId,+ );+ if (!baseline) return;++ attemptedForBaselineRef.current = baselineId;++ // The URL already carries a choice.+ if (comparisonIds.length > 0) return;+ // Never guess across datasets: a run on another dataset scored other items.+ if (!baseline.datasetId) return;++ // experimentNames is newest-first, so this is the run that ran before it.+ const previousRun = experimentNames.find(+ (experiment) =>+ experiment.datasetId === baseline.datasetId &&+ experiment.experimentId !== baselineId &&+ experiment.startTime.getTime() <= baseline.startTime.getTime(),+ );+ // The only run on its dataset — there is nothing to compare it to.+ if (!previousRun) return;++ onComparisonIdsChange([previousRun.experimentId], {+ updateType: "replaceIn",+ });+ }, [+ isAutoSelectEnabled,+ baselineId,+ isLoading,+ router.isReady,+ experimentNames,+ comparisonIds,+ onComparisonIdsChange,+ ]);++ return { isAutoSelectEnabled, setIsAutoSelectEnabled };+}diff --git a/web/src/features/experiments/hooks/useExperimentNames.ts b/web/src/features/experiments/hooks/useExperimentNames.tsindex 9e6c7eb164..4d0960c809 100644--- a/web/src/features/experiments/hooks/useExperimentNames.ts+++ b/web/src/features/experiments/hooks/useExperimentNames.ts@@ -1,12 +1,17 @@+import { useMemo } from "react";import { useHasProjectAccess } from "@/src/features/rbac";import { api } from "@/src/utils/api";+export type ExperimentNameOption = {+ experimentId: string;+ experimentName: string;+ datasetId: string | null;+ datasetName: string | null;+ startTime: Date;+};+type UseExperimentNamesResponse = {- experimentNames: {- experimentId: string;- experimentName: string;- datasetId: string | null;- }[];+ experimentNames: ExperimentNameOption[];isLoading: boolean;};@@ -29,9 +34,16 @@ export function useExperimentNames({{ enabled: Boolean(projectId) && hasExperimentsReadAccess },);- const sortedExperimentNames = data?.experimentNames.sort((a, b) =>- a.experimentName.localeCompare(b.experimentName),+ // Newest first: the run worth comparing against is a recent one, not the one+ // the alphabet happens to put on top. Copied before sorting so the query+ // cache is not mutated in place.+ const experimentNames = useMemo(+ () =>+ [...(data?.experimentNames ?? [])].sort(+ (a, b) => b.startTime.getTime() - a.startTime.getTime(),+ ),+ [data?.experimentNames],);- return { experimentNames: sortedExperimentNames ?? [], isLoading };+ return { experimentNames, isLoading };}diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.ts b/web/src/features/experiments/hooks/useExperimentResultsState.tsindex e94376bca9..f45f15f12a 100644--- a/web/src/features/experiments/hooks/useExperimentResultsState.ts+++ b/web/src/features/experiments/hooks/useExperimentResultsState.ts@@ -3,6 +3,7 @@ import {withDefault,ArrayParam,StringParam,+ type UrlUpdateType,} from "use-query-params";import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";@@ -37,15 +38,19 @@ export function useExperimentResultsState() {const commitSelection = (orderedIds: string[],baseline: string | undefined,+ updateType?: UrlUpdateType,) => {const ids = dedupe([baseline, ...orderedIds]).slice(0,MAX_SELECTED_EXPERIMENTS,);- setState({- baseline,- c: baseline ? ids.filter((id) => id !== baseline) : ids,- });+ setState(+ {+ baseline,+ c: baseline ? ids.filter((id) => id !== baseline) : ids,+ },+ updateType,+ );};// Selecting a baseline moves it to the front of the selection and marks it,@@ -65,8 +70,13 @@ export function useExperimentResultsState() {commitSelection([...comparisonIds, explicitBaselineId], undefined);};- const setComparisonIds = (ids: string[]) =>- commitSelection(ids, explicitBaselineId);+ // `options.updateType` carries the history semantics: a user's own pick keeps+ // the default (a Back-able step), a programmatic default passes `replaceIn` so+ // it does not mint a history entry Back would bounce off.+ const setComparisonIds = (+ ids: string[],+ options?: { updateType?: UrlUpdateType },+ ) => commitSelection(ids, explicitBaselineId, options?.updateType);const addComparisonId = (id: string) => {if (id === explicitBaselineId) return; // Can't compare baseline with itselfdiff --git a/web/src/features/experiments/hooks/useExperimentSearch.ts b/web/src/features/experiments/hooks/useExperimentSearch.tsindex 0748d19064..631fe33ac5 100644--- a/web/src/features/experiments/hooks/useExperimentSearch.ts+++ b/web/src/features/experiments/hooks/useExperimentSearch.ts@@ -1,4 +1,4 @@-import { useState, useEffect } from "react";+import { useState, useEffect, useMemo } from "react";import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";interface UseExperimentSearchProps {@@ -20,18 +20,25 @@ export function useExperimentSearch({ projectId }: UseExperimentSearchProps) {const { experimentNames, isLoading } = useExperimentNames({ projectId });- // apply search query to experiment names- const filteredExperimentNames = experimentNames?.filter((name) =>- name.experimentName- .toLowerCase()- .includes(debouncedSearchQuery.toLowerCase()),- );+ // Match the run name or its dataset, so typing a dataset name scopes the list+ // the way the dataset filter does on the experiments table.+ const filteredExperimentNames = useMemo(() => {+ const needle = debouncedSearchQuery.trim().toLowerCase();+ if (!needle) return experimentNames;++ return experimentNames.filter(+ (experiment) =>+ experiment.experimentName.toLowerCase().includes(needle) ||+ experiment.datasetName?.toLowerCase().includes(needle),+ );+ }, [experimentNames, debouncedSearchQuery]);return {searchQuery,setSearchQuery,- searchResults: filteredExperimentNames ?? [],+ searchResults: filteredExperimentNames,+ isSearchActive: debouncedSearchQuery.trim().length > 0,isLoading,- availableExperimentNames: experimentNames ?? [],+ availableExperimentNames: experimentNames,};}diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.tsindex fbf35a977e..0e5d9ff6c0 100644--- a/web/src/features/experiments/server/router.ts+++ b/web/src/features/experiments/server/router.ts@@ -900,6 +900,34 @@ export const experimentsRouter = createTRPCRouter({projectId: input.projectId,});- return { experimentNames: experiments };+ // Dataset names live in Postgres only — ClickHouse carries the id — so the+ // display name is resolved here rather than in the projection, and the+ // pickers never have to render a raw dataset id.+ const datasetIds = [+ ...new Set(+ experiments+ .map((experiment) => experiment.datasetId)+ .filter((datasetId): datasetId is string => Boolean(datasetId)),+ ),+ ];+ const datasets = datasetIds.length+ ? await ctx.prisma.dataset.findMany({+ where: { projectId: input.projectId, id: { in: datasetIds } },+ select: { id: true, name: true },+ })+ : [];+ const datasetNameById = new Map(datasets.map((d) => [d.id, d.name]));++ return {+ experimentNames: experiments.map((experiment) => ({+ experimentId: experiment.experimentId,+ experimentName: experiment.experimentName,+ startTime: experiment.startTime,+ datasetId: experiment.datasetId,+ datasetName: experiment.datasetId+ ? (datasetNameById.get(experiment.datasetId) ?? null)+ : null,+ })),+ };}),});