Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

18.4 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-deprecated -- Legacy UI usage retained during migration to design-system components. */
import { useCallback, useMemo, useRef, useState } from "react";
import { Check, X } from "@langfuse/icons";
import { DropdownIndicator } from "@/src/components/design-system/DropdownIndicator/DropdownIndicator";
import { MultiSelectCombobox } from "@/src/components/ui/MultiSelectCombobox";
import { Badge } from "@/src/components/design-system/Badge/Badge";
import { ExperimentRunBadge } from "./ExperimentRunBadge";
import {
  Tooltip,
  TooltipContent,
  TooltipTrigger,
} from "@/src/components/ui/tooltip";
import { useExperimentSearch } from "@/src/features/experiments/hooks/useExperimentSearch";
import { type ExperimentNameOption } from "@/src/features/experiments/hooks/useExperimentNames";
import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
import {
  autoComparisonPreferenceChangedProps,
  comparisonChangedProps,
  comparisonPickerOpenedProps,
} from "@/src/features/experiments/lib/analytics";
import {
  MAX_SELECTED_EXPERIMENTS,
  MAX_VISIBLE_COMPARISON_CHIPS,
  NO_DATASET_KEY,
  NO_DATASET_LABEL,
  UNNAMED_DATASET_LABEL,
} from "@/src/features/experiments/constants/comparison";
import { getExperimentColorStyles } from "./table/types";
import { cn } from "@/src/utils/tailwind";

export type ExperimentOption = Omit<ExperimentNameOption, "startTime"> & {
  /** null when an id in the URL no longer resolves to a run. */
  startTime: Date | null;
};

/**
 * The dropdown is a flat list of rows so the shared combobox can render it, but
 * the rows carry the dataset grouping: a header per dataset, its runs beneath.
 */
type ComparisonRow =
  | { kind: "preference" }
  | {
      kind: "group";
      datasetKey: string;
      label: string;
      runCount: number;
      isBaselineDataset: boolean;
      isExpanded: boolean;
    }
  | {
      kind: "option";
      option: ExperimentOption;
      isBaseline: boolean;
      isLatest: boolean;
    };

const datasetKeyOf = (option: { datasetId: string | null }) =>
  option.datasetId ?? NO_DATASET_KEY;

const datasetLabelOf = (option: ExperimentOption) =>
  option.datasetId === null
    ? NO_DATASET_LABEL
    : (option.datasetName ?? UNNAMED_DATASET_LABEL);

const rowKeyOf = (row: ComparisonRow) => {
  switch (row.kind) {
    case "preference":
      return "preference";
    case "group":
      return `group:${row.datasetKey}`;
    default:
      return `experiment:${row.option.experimentId}`;
  }
};

type ExperimentComparisonSelectorProps = {
  projectId: string;
  baselineExperimentId?: string;
  selectedIds: string[];
  /** The run order the table's cells colour by. See `useExperimentResultsState`. */
  colorExperimentIds: string[];
  selectedExperimentCount: number;
  onSelectedIdsChange: (ids: string[]) => void;
  isAutoSelectEnabled: boolean;
  onAutoSelectEnabledChange: (isEnabled: boolean) => void;
};

export function ExperimentComparisonSelector({
  projectId,
  baselineExperimentId,
  selectedIds,
  colorExperimentIds,
  selectedExperimentCount,
  onSelectedIdsChange,
  isAutoSelectEnabled,
  onAutoSelectEnabledChange,
}: ExperimentComparisonSelectorProps) {
  const {
    searchResults,
    searchQuery,
    setSearchQuery,
    isSearchActive,
    isLoading,
    availableExperimentNames,
  } = useExperimentSearch({
    projectId,
  });

  const [expandedOverrides, setExpandedOverrides] = useState<
    Record<string, boolean>
  >({});
  const capture = usePostHogClientCapture();

  // How long is the list people choose from, and how many datasets are mixed
  // into it? Counted over the whole available list, not the filtered rows, so
  // it describes the picker rather than the current search.
  const isPickerOpenRef = useRef(false);
  const handleOpenChange = useCallback(
    (isOpen: boolean) => {
      // The combobox re-announces "open" on every input focus, including the
      // refocus after picking a row; only the closed-to-open edge is an open.
      if (!isOpen) {
        isPickerOpenRef.current = false;
        return;
      }
      if (isPickerOpenRef.current) return;
      isPickerOpenRef.current = true;

      // Counted over the whole available list rather than the filtered rows, so
      // it describes the pool the user chooses from; the baseline is not one of
      // its options. The query itself is never sent — only whether there is one
      // and how long it is.
      const options = availableExperimentNames.filter(
        (experiment) => experiment.experimentId !== baselineExperimentId,
      );
      capture(
        "experiment:comparison_picker_opened",
        comparisonPickerOpenedProps({
          tableName: "experiment-items",
          optionCount: options.length,
          datasetIds: options.map((experiment) => experiment.datasetId),
          queryLength: searchQuery.length,
        }),
      );
    },
    [capture, availableExperimentNames, baselineExperimentId, searchQuery],
  );

  const baselineDatasetKey = useMemo(() => {
    const baseline = availableExperimentNames.find(
      (experiment) => experiment.experimentId === baselineExperimentId,
    );
    return baseline ? datasetKeyOf(baseline) : null;
  }, [availableExperimentNames, baselineExperimentId]);

  // The newest run on the baseline's dataset, taken from the full list rather
  // than the search results so the badge does not move when filtering. Absent
  // when the baseline is itself the newest run.
  const latestExperimentId = useMemo(
    () =>
      baselineDatasetKey
        ? availableExperimentNames.find(
            (experiment) => datasetKeyOf(experiment) === baselineDatasetKey,
          )?.experimentId
        : undefined,
    [availableExperimentNames, baselineDatasetKey],
  );

  const groups = useMemo(() => {
    const byDataset = new Map<
      string,
      { datasetKey: string; label: string; options: ExperimentNameOption[] }
    >();

    // searchResults arrive newest-first, so the runs inside a group already are.
    for (const option of searchResults) {
      const datasetKey = datasetKeyOf(option);
      const group = byDataset.get(datasetKey) ?? {
        datasetKey,
        label: datasetLabelOf(option),
        options: [],
      };
      group.options.push(option);
      byDataset.set(datasetKey, group);
    }

    // The baseline's dataset first: comparing across datasets compares
    // different items, so it must never be what the list offers first.
    return [...byDataset.values()].sort((a, b) => {
      if (a.datasetKey === baselineDatasetKey) return -1;
      if (b.datasetKey === baselineDatasetKey) return 1;
      return (
        (b.options[0]?.startTime.getTime() ?? 0) -
        (a.options[0]?.startTime.getTime() ?? 0)
      );
    });
  }, [searchResults, baselineDatasetKey]);

  const rows = useMemo(() => {
    // With no baseline yet there is no "current" dataset, so open the one
    // holding the most recent run instead of showing nothing but headers.
    const defaultExpandedKey = baselineDatasetKey ?? groups[0]?.datasetKey;
    // The preference row is chrome, not a result: seeding it unconditionally
    // would make the list never look empty, and the combobox's "no results"
    // state — which keys off the row count — could never be reached.
    const result: ComparisonRow[] =
      groups.length > 0 ? [{ kind: "preference" }] : [];

    for (const group of groups) {
      // While searching, every group with a match is open: `??` falls through
      // only on nullish, so a group the user collapsed earlier would keep its
      // explicit `false` and swallow the rows the search just found.
      const isExpanded = isSearchActive
        ? true
        : (expandedOverrides[group.datasetKey] ??
          group.datasetKey === defaultExpandedKey);

      result.push({
        kind: "group",
        datasetKey: group.datasetKey,
        label: group.label,
        runCount: group.options.length,
        isBaselineDataset: group.datasetKey === baselineDatasetKey,
        isExpanded,
      });

      if (!isExpanded) continue;

      for (const option of group.options) {
        result.push({
          kind: "option",
          option,
          isBaseline: option.experimentId === baselineExperimentId,
          isLatest: option.experimentId === latestExperimentId,
        });
      }
    }

    return result;
  }, [
    groups,
    expandedOverrides,
    isSearchActive,
    baselineDatasetKey,
    baselineExperimentId,
    latestExperimentId,
  ]);

  const selectedRows = useMemo<ComparisonRow[]>(
    () =>
      selectedIds.map((id) => ({
        kind: "option",
        option: availableExperimentNames.find(
          (experiment) => experiment.experimentId === id,
        ) ?? {
          // An id from the URL that no longer resolves to a run.
          experimentId: id,
          experimentName: id,
          datasetId: null,
          datasetName: null,
          startTime: null,
        },
        isBaseline: false,
        isLatest: false,
      })),
    [selectedIds, availableExperimentNames],
  );

  // Past a few chips the row turns into a horizontal scroll nobody scrolls, so
  // the tail collapses into one "+N" badge that names what it hides.
  const hiddenSelectedOptions = useMemo(
    () =>
      selectedRows
        .flatMap((row) => (row.kind === "option" ? [row.option] : []))
        .slice(MAX_VISIBLE_COMPARISON_CHIPS),
    [selectedRows],
  );

  const handleItemsChange = (items: ComparisonRow[]) => {
    const newIds = items
      .flatMap((item) =>
        item.kind === "option" ? [item.option.experimentId] : [],
      )
      .slice(
        0,
        Math.max(
          0,
          MAX_SELECTED_EXPERIMENTS -
            (selectedExperimentCount - selectedIds.length),
        ),
      );
    if (
      newIds.length === selectedIds.length &&
      newIds.every((id, index) => id === selectedIds[index])
    ) {
      return;
    }
    const baselineDatasetId = availableExperimentNames.find(
      (exp) => exp.experimentId === baselineExperimentId,
    )?.datasetId;
    capture(
      "experiment:comparison_changed",
      comparisonChangedProps({
        tableName: "experiment-items",
        comparisonCount: newIds.length,
        datasetIds: [
          baselineDatasetId,
          ...newIds.map(
            (id) =>
              availableExperimentNames.find((exp) => exp.experimentId === id)
                ?.datasetId,
          ),
        ],
        source: "picker",
      }),
    );
    onSelectedIdsChange(newIds);
  };

  const isMaxReached = selectedExperimentCount >= MAX_SELECTED_EXPERIMENTS;

  return (
    <div className="space-y-2">
      <MultiSelectCombobox<ComparisonRow>
        labelLeft="Compare"
        selectedItems={selectedRows}
        onItemsChange={handleItemsChange}
        searchQuery={searchQuery}
        onSearchChange={setSearchQuery}
        searchResults={rows}
        isLoading={isLoading}
        onOpenChange={handleOpenChange}
        placeholder={
          isMaxReached
            ? `Max ${MAX_SELECTED_EXPERIMENTS} experiments`
            : "Search experiments..."
        }
        disabled={isLoading}
        showSearchIcon={false}
        dropdownClassName="bg-background absolute top-0 z-10 max-h-80 w-full overflow-y-auto rounded-md border shadow-md"
        getItemKey={rowKeyOf}
        renderItem={(row, isSelected, onToggle) => {
          if (row.kind === "preference") {
            return (
              <button
                type="button"
                onClick={() => {
                  capture(
                    "experiment:auto_comparison_preference_changed",
                    autoComparisonPreferenceChangedProps({
                      tableName: "experiment-items",
                      isEnabled: !isAutoSelectEnabled,
                    }),
                  );
                  onAutoSelectEnabledChange(!isAutoSelectEnabled);
                }}
                className="text-muted-foreground hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left"
              >
                <div className="border-input flex h-4 w-4 shrink-0 items-center justify-center rounded-sm border">
                  {isAutoSelectEnabled && <Check size="sm" />}
                </div>
                <span className="text-xs">
                  Auto-select a comparison experiment by default
                </span>
              </button>
            );
          }

          if (row.kind === "group") {
            return (
              <button
                type="button"
                onClick={() =>
                  setExpandedOverrides((previous) => ({
                    ...previous,
                    [row.datasetKey]: !row.isExpanded,
                  }))
                }
                className="bg-muted/40 hover:bg-muted flex w-full items-center gap-2 px-2 py-1.5 text-left"
              >
                {row.isExpanded ? (
                  <DropdownIndicator size="sm" nudge />
                ) : (
                  <DropdownIndicator direction="right" size="sm" nudge />
                )}
                <span className="truncate text-xs font-bold" title={row.label}>
                  {row.label}
                </span>
                <span className="text-muted-foreground shrink-0 text-xs">
                  {row.runCount}
                </span>
                {!row.isBaselineDataset && (
                  <span
                    className="text-muted-foreground ml-auto shrink-0 text-xs italic"
                    title="Runs on another dataset scored different items, so their values are not comparable."
                  >
                    other dataset
                  </span>
                )}
              </button>
            );
          }

          const { option } = row;

          return (
            <button
              type="button"
              onClick={row.isBaseline ? undefined : onToggle}
              disabled={row.isBaseline || (!isSelected && isMaxReached)}
              className="hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left disabled:cursor-not-allowed disabled:opacity-50"
            >
              <div className="flex h-4 w-4 shrink-0 items-center justify-center">
                {isSelected && (
                  <Check size="base" __rawClassName="text-primary " />
                )}
              </div>
              <span
                className="min-w-0 flex-1 truncate text-sm font-bold"
                title={option.experimentName}
              >
                {option.experimentName}
              </span>
              {row.isBaseline && (
                <Badge color="primary" size="sm" text="BASELINE" />
              )}
              {row.isLatest && (
                <Badge color="primary" size="sm" text="LATEST" />
              )}
              {option.startTime && (
                <span
                  className="text-muted-foreground shrink-0 text-xs"
                  title={option.startTime.toLocaleString()}
                >
                  {formatRunRecency(option.startTime)}
                </span>
              )}
            </button>
          );
        }}
        renderSelectedItem={(row, onRemove) => {
          if (row.kind !== "option") return null;

          const { option } = row;
          const position = selectedIds.indexOf(option.experimentId);

          if (position >= MAX_VISIBLE_COMPARISON_CHIPS) {
            // One badge stands in for the whole tail; the rest render nothing.
            return position === MAX_VISIBLE_COMPARISON_CHIPS ? (
              <Tooltip>
                <TooltipTrigger asChild>
                  <Badge
                    color="filled"
                    text={`+${hiddenSelectedOptions.length}`}
                  />
                </TooltipTrigger>
                <TooltipContent className="max-w-[280px]">
                  {hiddenSelectedOptions.map((hidden) => (
                    <p key={hidden.experimentId}>
                      {hidden.experimentName}
                      {hidden.startTime
                        ? ` · ${formatRunRecency(hidden.startTime)}`
                        : ""}
                    </p>
                  ))}
                </TooltipContent>
              </Tooltip>
            ) : null;
          }

          // A comparison should never be on another dataset, but the URL can say
          // anything, so name the dataset rather than leaving it unexplained.
          const isOtherDataset =
            baselineDatasetKey !== null &&
            option.startTime !== null &&
            datasetKeyOf(option) !== baselineDatasetKey;

          const chipTitle = [
            option.experimentName,
            option.startTime
              ? `Started ${option.startTime.toLocaleString()}`
              : "This experiment is no longer available",
            isOtherDataset ? `Dataset: ${datasetLabelOf(option)}` : null,
          ]
            .filter(Boolean)
            .join("\n");

          // Same run, same colour as its values in the table — resolved from the
          // order the cells use, never from this chip's own position. Absent when
          // the cells show no colour at all (no baseline to compare against).
          const colorStyles = colorExperimentIds.includes(option.experimentId)
            ? getExperimentColorStyles(option.experimentId, colorExperimentIds)
            : undefined;

          return (
            <ExperimentRunBadge colorClass={colorStyles?.badgeClass}>
              {colorStyles && (
                <span
                  className={cn(
                    "block h-3 w-0.5 shrink-0 rounded-full",
                    colorStyles.markerClass,
                  )}
                />
              )}
              <span className="max-w-40 truncate text-xs" title={chipTitle}>
                {option.experimentName}
              </span>
              {isOtherDataset && (
                <span
                  className="text-muted-foreground max-w-24 truncate text-[10px]"
                  title={`Dataset: ${datasetLabelOf(option)}`}
                >
                  {datasetLabelOf(option)}
                </span>
              )}
              <button
                type="button"
                onClick={(e) => {
                  e.stopPropagation();
                  onRemove();
                }}
                className="hover:bg-muted ml-0.5 rounded-full"
                aria-label={`Remove ${option.experimentName}`}
              >
                <X size="sm" />
              </button>
            </ExperimentRunBadge>
          );
        }}
      />
    </div>
  );
}