Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): make the comparison selectable, grouped and pre-picked (#16916)

Nikita Kabardin07af5596579f
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.ts
index 1248ee8576..84abab63fb 100644
--- a/packages/shared/src/server/repositories/experiments.ts
+++ b/packages/shared/src/server/repositories/experiments.ts
@@ -1497,16 +1497,27 @@ export const getExperimentItemsBatchIO = async (props: {
});
};
+/**
+ * Experiment options for the baseline / comparison pickers: one row per run,
+ * with the dataset it ran on and when it started so the UI can group by dataset
+ * and order by recency.
+ */
export const getExperimentNamesFromEvents = async (props: {
projectId: string;
}) => {
const queryBuilder = new EventsAggQueryBuilder({
projectId: props.projectId,
+ // Grouped by experiment id: two runs sharing a name are two runs, not one
+ // option pointing at an arbitrary id.
groupByColumn: "e.experiment_id",
- selectExpression:
- "any(e.experiment_name) as experimentName, e.experiment_id as experimentId, nullIf(any(e.experiment_dataset_id), '') as datasetId",
+ selectExpression: `any(e.experiment_name) as experimentName,
+ e.experiment_id as experimentId,
+ nullIf(any(e.experiment_dataset_id), '') as datasetId,
+ min(e.start_time) as startTime`,
})
.whereRaw("e.experiment_name IS NOT NULL AND length(e.experiment_name) > 0")
+ // Keeps the bound meaningful: the newest 1000 runs, not an arbitrary 1000.
+ .orderBy("ORDER BY min(e.start_time) DESC")
.limit(1000, 0);
const { query, params } = queryBuilder.buildWithParams();
@@ -1515,6 +1526,7 @@ export const getExperimentNamesFromEvents = async (props: {
experimentName: string;
experimentId: string;
datasetId: string | null;
+ startTime: string;
}>({
query,
params,
@@ -1522,5 +1534,10 @@ export const getExperimentNamesFromEvents = async (props: {
preferredClickhouseService: "EventsReadOnly",
});
- return res;
+ return res.map((row) => ({
+ experimentId: row.experimentId,
+ experimentName: row.experimentName,
+ datasetId: row.datasetId,
+ startTime: parseClickhouseUTCDateTimeFormat(row.startTime),
+ }));
};
diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
index bc64ed0797..7358dbcdf6 100644
--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
@@ -1355,17 +1355,21 @@ describe("Clickhouse Experiment Repository Test", () => {
projectId: isolatedProjectId,
});
+ // `startTime` (the run's first event) came with the recency ordering the
+ // comparison picker needs — the rows are otherwise unchanged.
expect(result).toEqual(
expect.arrayContaining([
{
experimentId: experimentIdA,
experimentName: sharedName,
datasetId: datasetIdA,
+ startTime: expect.any(Date),
},
{
experimentId: experimentIdB,
experimentName: sharedName,
datasetId: datasetIdB,
+ startTime: expect.any(Date),
},
]),
);
@@ -1827,4 +1831,56 @@ describe("Clickhouse Experiment Repository Test", () => {
);
});
});
+ maybe("getExperimentNamesFromEvents", () => {
+ it("should return one option per run when two runs share a name", async () => {
+ // Own project so the assertion sees only the runs created here.
+ const ownProjectId = randomUUID();
+ const sharedName = "shared-name-" + randomUUID();
+ const datasetId = randomUUID();
+ const olderExperimentId = randomUUID();
+ const newerExperimentId = randomUUID();
+ const now = new Date();
+ const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000);
+
+ const eventFor = (experimentId: string, startTime: Date) => {
+ const rootSpanId = randomUUID();
+ return createEvent({
+ id: randomUUID(),
+ span_id: rootSpanId,
+ project_id: ownProjectId,
+ trace_id: randomUUID(),
+ type: "GENERATION",
+ name: "test-generation",
+ experiment_id: experimentId,
+ experiment_name: sharedName,
+ experiment_metadata_names: [],
+ experiment_metadata_values: [],
+ experiment_dataset_id: datasetId,
+ experiment_item_id: randomUUID(),
+ experiment_item_version: null,
+ experiment_item_root_span_id: rootSpanId,
+ start_time: startTime.getTime() * 1000,
+ });
+ };
+
+ await createEventsCh([
+ eventFor(olderExperimentId, yesterday),
+ eventFor(newerExperimentId, now),
+ ]);
+
+ const options = await getExperimentNamesFromEvents({
+ projectId: ownProjectId,
+ });
+
+ expect(options).toHaveLength(2);
+ expect(options.map((option) => option.experimentId)).toEqual([
+ newerExperimentId,
+ olderExperimentId,
+ ]);
+ options.forEach((option) => {
+ expect(option.experimentName).toBe(sharedName);
+ expect(option.datasetId).toBe(datasetId);
+ });
+ });
+ });
});
diff --git a/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx b/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx
index 6927e4c927..fcfd487521 100644
--- a/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx
+++ b/web/src/features/experiments/components/ExperimentBaselineControls.clienttest.tsx
@@ -13,6 +13,8 @@ vi.mock("@/src/features/posthog-analytics/usePostHogClientCapture", () => ({
usePostHogClientCapture: () => h.capture,
}));
+// The rebuilt control groups its options by dataset and dates each run, so the
+// options carry a dataset name and a start time.
vi.mock("@/src/features/experiments/hooks/useExperimentNames", () => ({
useExperimentNames: () => ({
experimentNames: [
@@ -20,11 +22,15 @@ vi.mock("@/src/features/experiments/hooks/useExperimentNames", () => ({
experimentId: "exp-a",
experimentName: "My Experiment A",
datasetId: "ds-1",
+ datasetName: "My Dataset One",
+ startTime: new Date("2026-08-26T10:00:00Z"),
},
{
experimentId: "exp-b",
experimentName: "My Experiment B",
datasetId: "ds-2",
+ datasetName: "My Dataset Two",
+ startTime: new Date("2026-08-25T10:00:00Z"),
},
],
isLoading: false,
@@ -104,7 +110,9 @@ describe("ExperimentBaselineControls analytics", () => {
);
fireEvent.click(screen.getByRole("combobox"));
- fireEvent.click(screen.getByRole("option", { name: "My Experiment A" }));
+ // Each option now carries a relative-date badge, so its accessible name is
+ // the run name plus that badge rather than the name alone.
+ fireEvent.click(screen.getByRole("option", { name: /My Experiment A/ }));
expect(h.capture).not.toHaveBeenCalled();
expect(h.onBaselineChange).not.toHaveBeenCalled();
diff --git a/web/src/features/experiments/components/ExperimentBaselineControls.tsx b/web/src/features/experiments/components/ExperimentBaselineControls.tsx
index b68f3dbcf5..b06b387b2d 100644
--- a/web/src/features/experiments/components/ExperimentBaselineControls.tsx
+++ b/web/src/features/experiments/components/ExperimentBaselineControls.tsx
@@ -1,7 +1,17 @@
+import { useMemo } from "react";
import { Button } from "@/src/components/ui/button";
-import { Combobox } from "@/src/components/ui/combobox";
+import {
+ Combobox,
+ type ComboboxOptionGroup,
+} from "@/src/components/ui/combobox";
import { X } from "lucide-react";
import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";
+import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";
+import {
+ NO_DATASET_KEY,
+ NO_DATASET_LABEL,
+ UNNAMED_DATASET_LABEL,
+} from "@/src/features/experiments/constants/comparison";
import { cn } from "@/src/utils/tailwind";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { baselineChangedProps } from "@/src/features/experiments/lib/analytics";
@@ -21,20 +31,41 @@ export function ExperimentBaselineControls({
onBaselineChange,
onBaselineClear,
}: ExperimentBaselineControlsProps) {
- const capture = usePostHogClientCapture();
const { experimentNames, isLoading } = useExperimentNames({
projectId,
});
- const baselineOptions = experimentNames.map((exp) => ({
- value: exp.experimentId,
- label: exp.experimentName,
- }));
+ const capture = usePostHogClientCapture();
+
+ // Grouped by dataset and dated, so two runs sharing a name are two readable
+ // options rather than the same label twice.
+ const baselineOptionGroups = useMemo<ComboboxOptionGroup<string>[]>(() => {
+ const byDataset = new Map<string, ComboboxOptionGroup<string>>();
+
+ for (const experiment of experimentNames) {
+ const datasetKey = experiment.datasetId ?? NO_DATASET_KEY;
+ const group = byDataset.get(datasetKey) ?? {
+ heading:
+ experiment.datasetId === null
+ ? NO_DATASET_LABEL
+ : (experiment.datasetName ?? UNNAMED_DATASET_LABEL),
+ options: [],
+ };
+ group.options.push({
+ value: experiment.experimentId,
+ label: experiment.experimentName,
+ badge: formatRunRecency(experiment.startTime),
+ });
+ byDataset.set(datasetKey, group);
+ }
+
+ return [...byDataset.values()];
+ }, [experimentNames]);
return (
<div className="flex min-w-0 items-center">
<div className="min-w-0 flex-1">
<Combobox
- options={baselineOptions}
+ options={baselineOptionGroups}
value={baselineId}
onValueChange={(id) => {
if (id === baselineId) return;
diff --git a/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx b/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx
index 72249c4a1a..813418eb75 100644
--- a/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx
+++ b/web/src/features/experiments/components/ExperimentComparisonSelector.clienttest.tsx
@@ -6,21 +6,30 @@ const h = vi.hoisted(() => ({
capture: vi.fn(),
onSelectedIdsChange: vi.fn(),
searchQuery: "",
+ onAutoSelectEnabledChange: vi.fn(),
+ // Newest first, the order the picker's own query returns. `startTime` and
+ // `datasetName` came with the dataset grouping and recency ordering.
searchResults: [
{
experimentId: "exp-a",
experimentName: "My Experiment A",
datasetId: "ds-1",
+ datasetName: "My Dataset One",
+ startTime: new Date("2026-08-26T10:00:00Z"),
},
{
experimentId: "exp-b",
experimentName: "My Experiment B",
datasetId: "ds-1",
+ datasetName: "My Dataset One",
+ startTime: new Date("2026-08-25T10:00:00Z"),
},
{
experimentId: "exp-c",
experimentName: "My Experiment C",
datasetId: "ds-2",
+ datasetName: "My Dataset Two",
+ startTime: new Date("2026-08-24T10:00:00Z"),
},
],
}));
@@ -34,6 +43,7 @@ vi.mock("@/src/features/experiments/hooks/useExperimentSearch", () => ({
searchResults: h.searchResults,
searchQuery: h.searchQuery,
setSearchQuery: vi.fn(),
+ isSearchActive: false,
isLoading: false,
availableExperimentNames: h.searchResults,
}),
@@ -45,6 +55,7 @@ describe("ExperimentComparisonSelector analytics", () => {
beforeEach(() => {
h.capture.mockClear();
h.onSelectedIdsChange.mockClear();
+ h.onAutoSelectEnabledChange.mockClear();
h.searchQuery = "";
});
@@ -56,6 +67,8 @@ describe("ExperimentComparisonSelector analytics", () => {
selectedIds={[]}
selectedExperimentCount={1}
onSelectedIdsChange={h.onSelectedIdsChange}
+ isAutoSelectEnabled={true}
+ onAutoSelectEnabledChange={h.onAutoSelectEnabledChange}
/>,
);
@@ -86,6 +99,8 @@ describe("ExperimentComparisonSelector analytics", () => {
selectedIds={[]}
selectedExperimentCount={1}
onSelectedIdsChange={h.onSelectedIdsChange}
+ isAutoSelectEnabled={true}
+ onAutoSelectEnabledChange={h.onAutoSelectEnabledChange}
/>,
);
diff --git a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx b/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
index 2b5e545b74..10912a2620 100644
--- a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
+++ b/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
@@ -1,19 +1,71 @@
-import { useMemo } from "react";
-import { Check, X } from "lucide-react";
+import { useCallback, useMemo, useRef, useState } from "react";
+import { Check, ChevronDown, ChevronRight, X } from "lucide-react";
import { MultiSelectCombobox } from "@/src/components/ui/multi-select-combobox";
import { Badge } from "@/src/components/ui/badge";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
import { useExperimentSearch } from "@/src/features/experiments/hooks/useExperimentSearch";
-import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";
+import { type ExperimentNameOption } from "@/src/features/experiments/hooks/useExperimentNames";
+import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
comparisonChangedProps,
comparisonPickerOpenedProps,
} from "@/src/features/experiments/lib/analytics";
+import {
+ MAX_SELECTED_EXPERIMENTS,
+ MAX_VISIBLE_COMPARISON_CHIPS,
+ NO_DATASET_KEY,
+ NO_DATASET_LABEL,
+ UNNAMED_DATASET_LABEL,
+} from "@/src/features/experiments/constants/comparison";
+
+export type ExperimentOption = Omit<ExperimentNameOption, "startTime"> & {
+ /** null when an id in the URL no longer resolves to a run. */
+ startTime: Date | null;
+};
+
+/**
+ * The dropdown is a flat list of rows so the shared combobox can render it, but
+ * the rows carry the dataset grouping: a header per dataset, its runs beneath.
+ */
+type ComparisonRow =
+ | { kind: "preference" }
+ | {
+ kind: "group";
+ datasetKey: string;
+ label: string;
+ runCount: number;
+ isBaselineDataset: boolean;
+ isExpanded: boolean;
+ }
+ | {
+ kind: "option";
+ option: ExperimentOption;
+ isBaseline: boolean;
+ isLatest: boolean;
+ };
+
+const datasetKeyOf = (option: { datasetId: string | null }) =>
+ option.datasetId ?? NO_DATASET_KEY;
-export type ExperimentOption = {
- experimentId: string;
- experimentName: string;
- datasetId: string | null;
+const datasetLabelOf = (option: ExperimentOption) =>
+ option.datasetId === null
+ ? NO_DATASET_LABEL
+ : (option.datasetName ?? UNNAMED_DATASET_LABEL);
+
+const rowKeyOf = (row: ComparisonRow) => {
+ switch (row.kind) {
+ case "preference":
+ return "preference";
+ case "group":
+ return `group:${row.datasetKey}`;
+ default:
+ return `experiment:${row.option.experimentId}`;
+ }
};
type ExperimentComparisonSelectorProps = {
@@ -22,6 +74,8 @@ type ExperimentComparisonSelectorProps = {
selectedIds: string[];
selectedExperimentCount: number;
onSelectedIdsChange: (ids: string[]) => void;
+ isAutoSelectEnabled: boolean;
+ onAutoSelectEnabledChange: (isEnabled: boolean) => void;
};
export function ExperimentComparisonSelector({
@@ -30,40 +84,195 @@ export function ExperimentComparisonSelector({
selectedIds,
selectedExperimentCount,
onSelectedIdsChange,
+ isAutoSelectEnabled,
+ onAutoSelectEnabledChange,
}: ExperimentComparisonSelectorProps) {
- const capture = usePostHogClientCapture();
const {
searchResults,
searchQuery,
setSearchQuery,
+ isSearchActive,
isLoading,
availableExperimentNames,
} = useExperimentSearch({
projectId,
});
- // Filter out baseline experiment from options
- const searchResultsExclBaseline = useMemo(() => {
- return baselineExperimentId
- ? searchResults.filter((exp) => exp.experimentId !== baselineExperimentId)
- : searchResults;
- }, [baselineExperimentId, searchResults]);
-
- // Map selected IDs to full experiment objects
- const selectedExperiments = useMemo(() => {
- return selectedIds.map((id) => {
- const found = availableExperimentNames.find(
- (exp) => exp.experimentId === id,
+ const [expandedOverrides, setExpandedOverrides] = useState<
+ Record<string, boolean>
+ >({});
+ const capture = usePostHogClientCapture();
+
+ // How long is the list people choose from, and how many datasets are mixed
+ // into it? Counted over the whole available list, not the filtered rows, so
+ // it describes the picker rather than the current search.
+ const isPickerOpenRef = useRef(false);
+ const handleOpenChange = useCallback(
+ (isOpen: boolean) => {
+ // The combobox re-announces "open" on every input focus, including the
+ // refocus after picking a row; only the closed-to-open edge is an open.
+ if (!isOpen) {
+ isPickerOpenRef.current = false;
+ return;
+ }
+ if (isPickerOpenRef.current) return;
+ isPickerOpenRef.current = true;
+
+ // Counted over the whole available list rather than the filtered rows, so
+ // it describes the pool the user chooses from; the baseline is not one of
+ // its options. The query itself is never sent — only whether there is one
+ // and how long it is.
+ const options = availableExperimentNames.filter(
+ (experiment) => experiment.experimentId !== baselineExperimentId,
+ );
+ capture(
+ "experiment:comparison_picker_opened",
+ comparisonPickerOpenedProps({
+ tableName: "experiment-items",
+ optionCount: options.length,
+ datasetIds: options.map((experiment) => experiment.datasetId),
+ queryLength: searchQuery.length,
+ }),
+ );
+ },
+ [capture, availableExperimentNames, baselineExperimentId, searchQuery],
+ );
+
+ const baselineDatasetKey = useMemo(() => {
+ const baseline = availableExperimentNames.find(
+ (experiment) => experiment.experimentId === baselineExperimentId,
+ );
+ return baseline ? datasetKeyOf(baseline) : null;
+ }, [availableExperimentNames, baselineExperimentId]);
+
+ // The newest run on the baseline's dataset, taken from the full list rather
+ // than the search results so the badge does not move when filtering. Absent
+ // when the baseline is itself the newest run.
+ const latestExperimentId = useMemo(
+ () =>
+ baselineDatasetKey
+ ? availableExperimentNames.find(
+ (experiment) => datasetKeyOf(experiment) === baselineDatasetKey,
+ )?.experimentId
+ : undefined,
+ [availableExperimentNames, baselineDatasetKey],
+ );
+
+ const groups = useMemo(() => {
+ const byDataset = new Map<
+ string,
+ { datasetKey: string; label: string; options: ExperimentNameOption[] }
+ >();
+
+ // searchResults arrive newest-first, so the runs inside a group already are.
+ for (const option of searchResults) {
+ const datasetKey = datasetKeyOf(option);
+ const group = byDataset.get(datasetKey) ?? {
+ datasetKey,
+ label: datasetLabelOf(option),
+ options: [],
+ };
+ group.options.push(option);
+ byDataset.set(datasetKey, group);
+ }
+
+ // The baseline's dataset first: comparing across datasets compares
+ // different items, so it must never be what the list offers first.
+ return [...byDataset.values()].sort((a, b) => {
+ if (a.datasetKey === baselineDatasetKey) return -1;
+ if (b.datasetKey === baselineDatasetKey) return 1;
+ return (
+ (b.options[0]?.startTime.getTime() ?? 0) -
+ (a.options[0]?.startTime.getTime() ?? 0)
);
- if (found) return found;
- // If not in current results, create placeholder with ID as name
- return { experimentId: id, experimentName: id, datasetId: null };
});
- }, [selectedIds, availableExperimentNames]);
+ }, [searchResults, baselineDatasetKey]);
+
+ const rows = useMemo(() => {
+ // With no baseline yet there is no "current" dataset, so open the one
+ // holding the most recent run instead of showing nothing but headers.
+ const defaultExpandedKey = baselineDatasetKey ?? groups[0]?.datasetKey;
+ // The preference row is chrome, not a result: seeding it unconditionally
+ // would make the list never look empty, and the combobox's "no results"
+ // state — which keys off the row count — could never be reached.
+ const result: ComparisonRow[] =
+ groups.length > 0 ? [{ kind: "preference" }] : [];
+
+ for (const group of groups) {
+ // While searching, every group with a match is open: `??` falls through
+ // only on nullish, so a group the user collapsed earlier would keep its
+ // explicit `false` and swallow the rows the search just found.
+ const isExpanded = isSearchActive
+ ? true
+ : (expandedOverrides[group.datasetKey] ??
+ group.datasetKey === defaultExpandedKey);
+
+ result.push({
+ kind: "group",
+ datasetKey: group.datasetKey,
+ label: group.label,
+ runCount: group.options.length,
+ isBaselineDataset: group.datasetKey === baselineDatasetKey,
+ isExpanded,
+ });
+
+ if (!isExpanded) continue;
- const handleItemsChange = (items: ExperimentOption[]) => {
+ for (const option of group.options) {
+ result.push({
+ kind: "option",
+ option,
+ isBaseline: option.experimentId === baselineExperimentId,
+ isLatest: option.experimentId === latestExperimentId,
+ });
+ }
+ }
+
+ return result;
+ }, [
+ groups,
+ expandedOverrides,
+ isSearchActive,
+ baselineDatasetKey,
+ baselineExperimentId,
+ latestExperimentId,
+ ]);
+
+ const selectedRows = useMemo<ComparisonRow[]>(
+ () =>
+ selectedIds.map((id) => ({
+ kind: "option",
+ option: availableExperimentNames.find(
+ (experiment) => experiment.experimentId === id,
+ ) ?? {
+ // An id from the URL that no longer resolves to a run.
+ experimentId: id,
+ experimentName: id,
+ datasetId: null,
+ datasetName: null,
+ startTime: null,
+ },
+ isBaseline: false,
+ isLatest: false,
+ })),
+ [selectedIds, availableExperimentNames],
+ );
+
+ // Past a few chips the row turns into a horizontal scroll nobody scrolls, so
+ // the tail collapses into one "+N" badge that names what it hides.
+ const hiddenSelectedOptions = useMemo(
+ () =>
+ selectedRows
+ .flatMap((row) => (row.kind === "option" ? [row.option] : []))
+ .slice(MAX_VISIBLE_COMPARISON_CHIPS),
+ [selectedRows],
+ );
+
+ const handleItemsChange = (items: ComparisonRow[]) => {
const newIds = items
- .map((item) => item.experimentId)
+ .flatMap((item) =>
+ item.kind === "option" ? [item.option.experimentId] : [],
+ )
.slice(
0,
Math.max(
@@ -100,32 +309,19 @@ export function ExperimentComparisonSelector({
onSelectedIdsChange(newIds);
};
- const handlePickerOpenChange = (open: boolean) => {
- if (!open) return;
- capture(
- "experiment:comparison_picker_opened",
- comparisonPickerOpenedProps({
- tableName: "experiment-items",
- optionCount: searchResultsExclBaseline.length,
- datasetIds: searchResultsExclBaseline.map((exp) => exp.datasetId),
- queryLength: searchQuery.length,
- }),
- );
- };
-
const isMaxReached = selectedExperimentCount >= MAX_SELECTED_EXPERIMENTS;
return (
<div className="space-y-2">
- <MultiSelectCombobox<ExperimentOption>
- labelLeft="Experiment selection"
- selectedItems={selectedExperiments}
+ <MultiSelectCombobox<ComparisonRow>
+ labelLeft="Compare"
+ selectedItems={selectedRows}
onItemsChange={handleItemsChange}
- onOpenChange={handlePickerOpenChange}
searchQuery={searchQuery}
onSearchChange={setSearchQuery}
- searchResults={searchResultsExclBaseline}
+ searchResults={rows}
isLoading={isLoading}
+ onOpenChange={handleOpenChange}
placeholder={
isMaxReached
? `Max ${MAX_SELECTED_EXPERIMENTS} experiments`
@@ -133,52 +329,181 @@ export function ExperimentComparisonSelector({
}
disabled={isLoading}
showSearchIcon={false}
- getItemKey={(item) => item.experimentId}
- renderItem={(item, isSelected, onToggle) => (
- <button
- type="button"
- onClick={onToggle}
- disabled={
- !isSelected && selectedExperimentCount >= MAX_SELECTED_EXPERIMENTS
- }
- className="hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left disabled:cursor-not-allowed disabled:opacity-50"
- >
- <div className="flex h-4 w-4 items-center justify-center">
- {isSelected && <Check className="text-primary h-4 w-4" />}
- </div>
- <div className="min-w-0 flex-1">
- <p
- className="truncate text-sm font-bold"
- title={item.experimentName}
+ dropdownClassName="bg-background absolute top-0 z-10 max-h-80 w-full overflow-y-auto rounded-md border shadow-md"
+ getItemKey={rowKeyOf}
+ renderItem={(row, isSelected, onToggle) => {
+ if (row.kind === "preference") {
+ return (
+ <button
+ type="button"
+ onClick={() => {
+ onAutoSelectEnabledChange(!isAutoSelectEnabled);
+ }}
+ className="text-muted-foreground hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left"
>
- {item.experimentName}
- </p>
- </div>
- </button>
- )}
- renderSelectedItem={(item, onRemove) => (
- <Badge
- variant="secondary"
- className="flex items-center gap-1 px-2 py-0.5"
- >
- <span
- className="max-w-24 truncate text-xs"
- title={item.experimentName}
- >
- {item.experimentName}
- </span>
+ <div className="border-input flex h-4 w-4 shrink-0 items-center justify-center rounded-sm border">
+ {isAutoSelectEnabled && <Check className="h-3 w-3" />}
+ </div>
+ <span className="text-xs">
+ Auto-select a comparison experiment by default
+ </span>
+ </button>
+ );
+ }
+
+ if (row.kind === "group") {
+ return (
+ <button
+ type="button"
+ onClick={() =>
+ setExpandedOverrides((previous) => ({
+ ...previous,
+ [row.datasetKey]: !row.isExpanded,
+ }))
+ }
+ className="bg-muted/40 hover:bg-muted flex w-full items-center gap-2 px-2 py-1.5 text-left"
+ >
+ {row.isExpanded ? (
+ <ChevronDown className="h-3 w-3 shrink-0" />
+ ) : (
+ <ChevronRight className="h-3 w-3 shrink-0" />
+ )}
+ <span className="truncate text-xs font-bold" title={row.label}>
+ {row.label}
+ </span>
+ <span className="text-muted-foreground shrink-0 text-xs">
+ {row.runCount}
+ </span>
+ {!row.isBaselineDataset && (
+ <span
+ className="text-muted-foreground ml-auto shrink-0 text-xs italic"
+ title="Runs on another dataset scored different items, so their values are not comparable."
+ >
+ other dataset
+ </span>
+ )}
+ </button>
+ );
+ }
+
+ const { option } = row;
+
+ return (
<button
type="button"
- onClick={(e) => {
- e.stopPropagation();
- onRemove();
- }}
- className="hover:bg-muted ml-0.5 rounded-full"
+ onClick={row.isBaseline ? undefined : onToggle}
+ disabled={row.isBaseline || (!isSelected && isMaxReached)}
+ className="hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left disabled:cursor-not-allowed disabled:opacity-50"
>
- <X className="h-3 w-3" />
+ <div className="flex h-4 w-4 shrink-0 items-center justify-center">
+ {isSelected && <Check className="text-primary h-4 w-4" />}
+ </div>
+ <span
+ className="min-w-0 flex-1 truncate text-sm font-bold"
+ title={option.experimentName}
+ >
+ {option.experimentName}
+ </span>
+ {row.isBaseline && (
+ <Badge variant="outline" className="shrink-0 text-[10px]">
+ BASELINE
+ </Badge>
+ )}
+ {row.isLatest && (
+ <Badge variant="outline" className="shrink-0 text-[10px]">
+ LATEST
+ </Badge>
+ )}
+ {option.startTime && (
+ <span
+ className="text-muted-foreground shrink-0 text-xs"
+ title={option.startTime.toLocaleString()}
+ >
+ {formatRunRecency(option.startTime)}
+ </span>
+ )}
</button>
- </Badge>
- )}
+ );
+ }}
+ renderSelectedItem={(row, onRemove) => {
+ if (row.kind !== "option") return null;
+
+ const { option } = row;
+ const position = selectedIds.indexOf(option.experimentId);
+
+ if (position >= MAX_VISIBLE_COMPARISON_CHIPS) {
+ // One badge stands in for the whole tail; the rest render nothing.
+ return position === MAX_VISIBLE_COMPARISON_CHIPS ? (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <Badge
+ variant="secondary"
+ className="shrink-0 px-2 py-0.5 text-xs"
+ >
+ +{hiddenSelectedOptions.length}
+ </Badge>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[280px]">
+ {hiddenSelectedOptions.map((hidden) => (
+ <p key={hidden.experimentId}>
+ {hidden.experimentName}
+ {hidden.startTime
+ ? ` · ${formatRunRecency(hidden.startTime)}`
+ : ""}
+ </p>
+ ))}
+ </TooltipContent>
+ </Tooltip>
+ ) : null;
+ }
+
+ // A comparison should never be on another dataset, but the URL can say
+ // anything, so name the dataset rather than leaving it unexplained.
+ const isOtherDataset =
+ baselineDatasetKey !== null &&
+ option.startTime !== null &&
+ datasetKeyOf(option) !== baselineDatasetKey;
+
+ const chipTitle = [
+ option.experimentName,
+ option.startTime
+ ? `Started ${option.startTime.toLocaleString()}`
+ : "This experiment is no longer available",
+ isOtherDataset ? `Dataset: ${datasetLabelOf(option)}` : null,
+ ]
+ .filter(Boolean)
+ .join("\n");
+
+ return (
+ <Badge
+ variant="secondary"
+ className="flex shrink-0 items-center gap-1 px-2 py-0.5"
+ >
+ <span className="max-w-40 truncate text-xs" title={chipTitle}>
+ {option.experimentName}
+ </span>
+ {isOtherDataset && (
+ <span
+ className="text-muted-foreground max-w-24 truncate text-[10px]"
+ title={`Dataset: ${datasetLabelOf(option)}`}
+ >
+ {datasetLabelOf(option)}
+ </span>
+ )}
+ <button
+ type="button"
+ onClick={(e) => {
+ e.stopPropagation();
+ onRemove();
+ }}
+ className="hover:bg-muted ml-0.5 rounded-full"
+ aria-label={`Remove ${option.experimentName}`}
+ >
+ <X className="h-3 w-3" />
+ </button>
+ </Badge>
+ );
+ }}
/>
</div>
);
diff --git a/web/src/features/experiments/components/ExperimentSelectionControls.tsx b/web/src/features/experiments/components/ExperimentSelectionControls.tsx
index deb98508e3..c9ec5280f8 100644
--- a/web/src/features/experiments/components/ExperimentSelectionControls.tsx
+++ b/web/src/features/experiments/components/ExperimentSelectionControls.tsx
@@ -1,3 +1,5 @@
+import { type UrlUpdateType } from "use-query-params";
+import { useExperimentComparisonAutoSelect } from "@/src/features/experiments/hooks/useExperimentComparisonAutoSelect";
import { ExperimentBaselineControls } from "./ExperimentBaselineControls";
import { ExperimentComparisonSelector } from "./ExperimentComparisonSelector";
@@ -9,7 +11,10 @@ type ExperimentSelectionControlsProps = {
selectedExperimentCount: number;
onBaselineChange: (id: string) => void;
onBaselineClear: () => void;
- onComparisonIdsChange: (ids: string[]) => void;
+ onComparisonIdsChange: (
+ ids: string[],
+ options?: { updateType?: UrlUpdateType },
+ ) => void;
};
export function ExperimentSelectionControls({
@@ -22,8 +27,18 @@ export function ExperimentSelectionControls({
onBaselineClear,
onComparisonIdsChange,
}: ExperimentSelectionControlsProps) {
+ // Mounted once per results page, so the default comparison is decided here
+ // rather than in the state hook that every consumer of the URL calls.
+ const { isAutoSelectEnabled, setIsAutoSelectEnabled } =
+ useExperimentComparisonAutoSelect({
+ projectId,
+ baselineId,
+ comparisonIds,
+ onComparisonIdsChange,
+ });
+
return (
- <div className="flex w-[50dvw] min-w-0 flex-row gap-3">
+ <div className="flex w-[56dvw] min-w-0 flex-row gap-3">
<div className="flex min-w-0 items-center">
<div className="border-input bg-muted/30 flex h-8 w-auto shrink-0 items-center rounded-l-md border px-3 text-xs">
Baseline
@@ -46,6 +61,8 @@ export function ExperimentSelectionControls({
selectedIds={comparisonIds}
selectedExperimentCount={selectedExperimentCount}
onSelectedIdsChange={onComparisonIdsChange}
+ isAutoSelectEnabled={isAutoSelectEnabled}
+ onAutoSelectEnabledChange={setIsAutoSelectEnabled}
/>
</div>
</div>
diff --git a/web/src/features/experiments/constants/comparison.ts b/web/src/features/experiments/constants/comparison.ts
index 3f7c70b518..06a63fd6e7 100644
--- a/web/src/features/experiments/constants/comparison.ts
+++ b/web/src/features/experiments/constants/comparison.ts
@@ -1 +1,22 @@
export const MAX_SELECTED_EXPERIMENTS = 10;
+
+/** Group key for runs that carry no dataset id. */
+export const NO_DATASET_KEY = "__no_dataset__";
+
+/** Group labels for a dataset we cannot name. */
+export const NO_DATASET_LABEL = "No dataset";
+export const UNNAMED_DATASET_LABEL = "Deleted dataset";
+
+/**
+ * Per-user preference: auto-select a comparison when a results page opens
+ * without one.
+ */
+export const AUTO_SELECT_COMPARISON_STORAGE_KEY =
+ "experiments-auto-select-comparison";
+
+/**
+ * Comparison chips rendered before the tail folds into a "+N" badge. Two full
+ * names are what the header's width fits; beyond that the row would only be
+ * reachable by horizontal scrolling, which nobody does.
+ */
+export const MAX_VISIBLE_COMPARISON_CHIPS = 2;
diff --git a/web/src/features/experiments/fns/formatRunRecency.ts b/web/src/features/experiments/fns/formatRunRecency.ts
new file mode 100644
index 0000000000..d5f51da432
--- /dev/null
+++ b/web/src/features/experiments/fns/formatRunRecency.ts
@@ -0,0 +1,18 @@
+/**
+ * How a run's start time reads next to its name: relative inside the last day,
+ * a calendar date beyond it — "20m ago", "5h ago", "Aug 21", "Aug 21, 2025".
+ */
+export const formatRunRecency = (startTime: Date, now = new Date()): string => {
+ const minutes = (now.getTime() - startTime.getTime()) / 60_000;
+ if (minutes < 1) return "just now";
+ if (minutes < 60) return `${Math.floor(minutes)}m ago`;
+ if (minutes < 60 * 24) return `${Math.floor(minutes / 60)}h ago`;
+
+ return startTime.toLocaleDateString("en-US", {
+ month: "short",
+ day: "numeric",
+ ...(startTime.getFullYear() === now.getFullYear()
+ ? {}
+ : { year: "numeric" }),
+ });
+};
diff --git a/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.ts b/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.ts
new file mode 100644
index 0000000000..032cfe1468
--- /dev/null
+++ b/web/src/features/experiments/hooks/useExperimentComparisonAutoSelect.ts
@@ -0,0 +1,85 @@
+import { useEffect, useRef } from "react";
+import { useRouter } from "next/router";
+import { type UrlUpdateType } from "use-query-params";
+import useLocalStorage from "@/src/components/useLocalStorage";
+import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";
+import { AUTO_SELECT_COMPARISON_STORAGE_KEY } from "@/src/features/experiments/constants/comparison";
+
+type UseExperimentComparisonAutoSelectProps = {
+ projectId: string;
+ baselineId?: string;
+ comparisonIds: string[];
+ onComparisonIdsChange: (
+ ids: string[],
+ options?: { updateType?: UrlUpdateType },
+ ) => void;
+};
+
+/**
+ * A results page opened without a comparison shows no movement at all, which is
+ * where three out of four users stop. Default to the previous run on the same
+ * dataset and write it into the URL like a user's own pick, so the view stays
+ * shareable, with a preference for anyone who would rather choose themselves.
+ */
+export function useExperimentComparisonAutoSelect({
+ projectId,
+ baselineId,
+ comparisonIds,
+ onComparisonIdsChange,
+}: UseExperimentComparisonAutoSelectProps) {
+ const router = useRouter();
+ const [isAutoSelectEnabled, setIsAutoSelectEnabled] =
+ useLocalStorage<boolean>(AUTO_SELECT_COMPARISON_STORAGE_KEY, true);
+ const { experimentNames, isLoading } = useExperimentNames({ projectId });
+
+ // At most one attempt per baseline while mounted: clearing the auto-selected
+ // comparison must not be undone on the next render. It cannot outlive the
+ // mount — an empty selection drops `c` from the URL entirely, so a reload
+ // cannot tell "cleared" from "never chosen" and defaults again. The
+ // preference below is the durable way to opt out.
+ const attemptedForBaselineRef = useRef<string | undefined>(undefined);
+
+ useEffect(() => {
+ if (!isAutoSelectEnabled || !baselineId || isLoading) return;
+ // Query-param writes issued before the router is ready are dropped, and the
+ // URL's own `c=` is not readable yet, so nothing here can be trusted.
+ if (!router.isReady) return;
+ if (attemptedForBaselineRef.current === baselineId) return;
+
+ const baseline = experimentNames.find(
+ (experiment) => experiment.experimentId === baselineId,
+ );
+ if (!baseline) return;
+
+ attemptedForBaselineRef.current = baselineId;
+
+ // The URL already carries a choice.
+ if (comparisonIds.length > 0) return;
+ // Never guess across datasets: a run on another dataset scored other items.
+ if (!baseline.datasetId) return;
+
+ // experimentNames is newest-first, so this is the run that ran before it.
+ const previousRun = experimentNames.find(
+ (experiment) =>
+ experiment.datasetId === baseline.datasetId &&
+ experiment.experimentId !== baselineId &&
+ experiment.startTime.getTime() <= baseline.startTime.getTime(),
+ );
+ // The only run on its dataset — there is nothing to compare it to.
+ if (!previousRun) return;
+
+ onComparisonIdsChange([previousRun.experimentId], {
+ updateType: "replaceIn",
+ });
+ }, [
+ isAutoSelectEnabled,
+ baselineId,
+ isLoading,
+ router.isReady,
+ experimentNames,
+ comparisonIds,
+ onComparisonIdsChange,
+ ]);
+
+ return { isAutoSelectEnabled, setIsAutoSelectEnabled };
+}
diff --git a/web/src/features/experiments/hooks/useExperimentNames.ts b/web/src/features/experiments/hooks/useExperimentNames.ts
index 9e6c7eb164..4d0960c809 100644
--- a/web/src/features/experiments/hooks/useExperimentNames.ts
+++ b/web/src/features/experiments/hooks/useExperimentNames.ts
@@ -1,12 +1,17 @@
+import { useMemo } from "react";
import { useHasProjectAccess } from "@/src/features/rbac";
import { api } from "@/src/utils/api";
+export type ExperimentNameOption = {
+ experimentId: string;
+ experimentName: string;
+ datasetId: string | null;
+ datasetName: string | null;
+ startTime: Date;
+};
+
type UseExperimentNamesResponse = {
- experimentNames: {
- experimentId: string;
- experimentName: string;
- datasetId: string | null;
- }[];
+ experimentNames: ExperimentNameOption[];
isLoading: boolean;
};
@@ -29,9 +34,16 @@ export function useExperimentNames({
{ enabled: Boolean(projectId) && hasExperimentsReadAccess },
);
- const sortedExperimentNames = data?.experimentNames.sort((a, b) =>
- a.experimentName.localeCompare(b.experimentName),
+ // Newest first: the run worth comparing against is a recent one, not the one
+ // the alphabet happens to put on top. Copied before sorting so the query
+ // cache is not mutated in place.
+ const experimentNames = useMemo(
+ () =>
+ [...(data?.experimentNames ?? [])].sort(
+ (a, b) => b.startTime.getTime() - a.startTime.getTime(),
+ ),
+ [data?.experimentNames],
);
- return { experimentNames: sortedExperimentNames ?? [], isLoading };
+ return { experimentNames, isLoading };
}
diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.ts b/web/src/features/experiments/hooks/useExperimentResultsState.ts
index e94376bca9..f45f15f12a 100644
--- a/web/src/features/experiments/hooks/useExperimentResultsState.ts
+++ b/web/src/features/experiments/hooks/useExperimentResultsState.ts
@@ -3,6 +3,7 @@ import {
withDefault,
ArrayParam,
StringParam,
+ type UrlUpdateType,
} from "use-query-params";
import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";
@@ -37,15 +38,19 @@ export function useExperimentResultsState() {
const commitSelection = (
orderedIds: string[],
baseline: string | undefined,
+ updateType?: UrlUpdateType,
) => {
const ids = dedupe([baseline, ...orderedIds]).slice(
0,
MAX_SELECTED_EXPERIMENTS,
);
- setState({
- baseline,
- c: baseline ? ids.filter((id) => id !== baseline) : ids,
- });
+ setState(
+ {
+ baseline,
+ c: baseline ? ids.filter((id) => id !== baseline) : ids,
+ },
+ updateType,
+ );
};
// Selecting a baseline moves it to the front of the selection and marks it,
@@ -65,8 +70,13 @@ export function useExperimentResultsState() {
commitSelection([...comparisonIds, explicitBaselineId], undefined);
};
- const setComparisonIds = (ids: string[]) =>
- commitSelection(ids, explicitBaselineId);
+ // `options.updateType` carries the history semantics: a user's own pick keeps
+ // the default (a Back-able step), a programmatic default passes `replaceIn` so
+ // it does not mint a history entry Back would bounce off.
+ const setComparisonIds = (
+ ids: string[],
+ options?: { updateType?: UrlUpdateType },
+ ) => commitSelection(ids, explicitBaselineId, options?.updateType);
const addComparisonId = (id: string) => {
if (id === explicitBaselineId) return; // Can't compare baseline with itself
diff --git a/web/src/features/experiments/hooks/useExperimentSearch.ts b/web/src/features/experiments/hooks/useExperimentSearch.ts
index 0748d19064..631fe33ac5 100644
--- a/web/src/features/experiments/hooks/useExperimentSearch.ts
+++ b/web/src/features/experiments/hooks/useExperimentSearch.ts
@@ -1,4 +1,4 @@
-import { useState, useEffect } from "react";
+import { useState, useEffect, useMemo } from "react";
import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";
interface UseExperimentSearchProps {
@@ -20,18 +20,25 @@ export function useExperimentSearch({ projectId }: UseExperimentSearchProps) {
const { experimentNames, isLoading } = useExperimentNames({ projectId });
- // apply search query to experiment names
- const filteredExperimentNames = experimentNames?.filter((name) =>
- name.experimentName
- .toLowerCase()
- .includes(debouncedSearchQuery.toLowerCase()),
- );
+ // Match the run name or its dataset, so typing a dataset name scopes the list
+ // the way the dataset filter does on the experiments table.
+ const filteredExperimentNames = useMemo(() => {
+ const needle = debouncedSearchQuery.trim().toLowerCase();
+ if (!needle) return experimentNames;
+
+ return experimentNames.filter(
+ (experiment) =>
+ experiment.experimentName.toLowerCase().includes(needle) ||
+ experiment.datasetName?.toLowerCase().includes(needle),
+ );
+ }, [experimentNames, debouncedSearchQuery]);
return {
searchQuery,
setSearchQuery,
- searchResults: filteredExperimentNames ?? [],
+ searchResults: filteredExperimentNames,
+ isSearchActive: debouncedSearchQuery.trim().length > 0,
isLoading,
- availableExperimentNames: experimentNames ?? [],
+ availableExperimentNames: experimentNames,
};
}
diff --git a/web/src/features/experiments/server/router.ts b/web/src/features/experiments/server/router.ts
index fbf35a977e..0e5d9ff6c0 100644
--- a/web/src/features/experiments/server/router.ts
+++ b/web/src/features/experiments/server/router.ts
@@ -900,6 +900,34 @@ export const experimentsRouter = createTRPCRouter({
projectId: input.projectId,
});
- return { experimentNames: experiments };
+ // Dataset names live in Postgres only — ClickHouse carries the id — so the
+ // display name is resolved here rather than in the projection, and the
+ // pickers never have to render a raw dataset id.
+ const datasetIds = [
+ ...new Set(
+ experiments
+ .map((experiment) => experiment.datasetId)
+ .filter((datasetId): datasetId is string => Boolean(datasetId)),
+ ),
+ ];
+ const datasets = datasetIds.length
+ ? await ctx.prisma.dataset.findMany({
+ where: { projectId: input.projectId, id: { in: datasetIds } },
+ select: { id: true, name: true },
+ })
+ : [];
+ const datasetNameById = new Map(datasets.map((d) => [d.id, d.name]));
+
+ return {
+ experimentNames: experiments.map((experiment) => ({
+ experimentId: experiment.experimentId,
+ experimentName: experiment.experimentName,
+ startTime: experiment.startTime,
+ datasetId: experiment.datasetId,
+ datasetName: experiment.datasetId
+ ? (datasetNameById.get(experiment.datasetId) ?? null)
+ : null,
+ })),
+ };
}),
});