Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.9 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import {
  useQueryParams,
  withDefault,
  ArrayParam,
  StringParam,
  type UrlUpdateType,
} from "use-query-params";
import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";
import useLocalStorage from "@/src/components/useLocalStorage";

/**
 * How the selected experiments are laid out against each other: one row per item
 * ("list"), one column per experiment ("grid"), or the transpose — scores as
 * rows and experiments as columns ("matrix").
 */
export type ExperimentResultsLayout = "grid" | "list" | "matrix";

/** What a diff cell's extra line is measured against. */
export type ExperimentDiffMode = "comparison" | "expected" | "off";

const asLayout = (value: unknown): ExperimentResultsLayout | undefined =>
  value === "grid" || value === "list" || value === "matrix"
    ? value
    : undefined;

const asDiffMode = (value: unknown): ExperimentDiffMode | undefined =>
  value === "comparison" || value === "expected" || value === "off"
    ? value
    : undefined;

// Drop null/empty entries and de-duplicate, preserving first-seen order.
const dedupe = (ids: (string | null | undefined)[]): string[] =>
  Array.from(new Set(ids.filter((id): id is string => Boolean(id))));

export function useExperimentResultsState() {
  const [state, setState] = useQueryParams({
    baseline: withDefault(StringParam, undefined),
    c: withDefault(ArrayParam, []),
    layout: withDefault(StringParam, undefined),
    diff: withDefault(StringParam, undefined),
    itemVisibility: withDefault(StringParam, "baseline-only"),
  });

  // The URL keeps the explicit baseline separate from comparison IDs, but
  // internally they form one ordered selection with the baseline first.
  const explicitBaselineId = state.baseline as string | undefined;
  const selectedExperimentIds = dedupe([
    explicitBaselineId,
    ...((state.c as (string | null)[] | undefined) ?? []),
  ]).slice(0, MAX_SELECTED_EXPERIMENTS);

  const comparisonIds = selectedExperimentIds.filter(
    (id) => id !== explicitBaselineId,
  );
  const hasBaseline = Boolean(explicitBaselineId);

  // Write one ordered selection back to the URL. The baseline is prepended so it
  // is always kept and lands first; anything past the limit is evicted from the
  // end; the baseline is stripped out of `c`.
  const commitSelection = (
    orderedIds: string[],
    baseline: string | undefined,
    updateType?: UrlUpdateType,
  ) => {
    const ids = dedupe([baseline, ...orderedIds]).slice(
      0,
      MAX_SELECTED_EXPERIMENTS,
    );
    setState(
      {
        baseline,
        c: baseline ? ids.filter((id) => id !== baseline) : ids,
      },
      updateType,
    );
  };

  // Selecting a baseline moves it to the front of the selection and marks it,
  // evicting the last selection when the total limit is reached.
  const setBaseline = (id: string | undefined) => {
    if (!id) {
      clearBaseline();
      return;
    }
    commitSelection(selectedExperimentIds, id);
  };

  // Clear the explicit baseline while preserving the selected experiments; the
  // former baseline moves to the end of the comparison list.
  const clearBaseline = () => {
    if (!explicitBaselineId) return;
    commitSelection([...comparisonIds, explicitBaselineId], undefined);
  };

  // `options.updateType` carries the history semantics: a user's own pick keeps
  // the default (a Back-able step), a programmatic default passes `replaceIn` so
  // it does not mint a history entry Back would bounce off.
  const setComparisonIds = (
    ids: string[],
    options?: { updateType?: UrlUpdateType },
  ) => commitSelection(ids, explicitBaselineId, options?.updateType);

  const addComparisonId = (id: string) => {
    if (id === explicitBaselineId) return; // Can't compare baseline with itself
    if (selectedExperimentIds.length >= MAX_SELECTED_EXPERIMENTS) return;
    if (comparisonIds.includes(id)) return;
    setComparisonIds([...comparisonIds, id]);
  };

  const removeComparisonId = (id: string) =>
    setComparisonIds(comparisonIds.filter((existingId) => existingId !== id));

  // Layout and diff mode: the URL wins so a view stays shareable, then the
  // user's remembered pick, then the one-column-per-experiment default.
  const [storedLayout, setStoredLayout] =
    useLocalStorage<ExperimentResultsLayout | null>(
      "experiment-results-compare-layout",
      null,
    );
  const [storedDiffMode, setStoredDiffMode] =
    useLocalStorage<ExperimentDiffMode | null>("experiment-results-diff", null);

  const layout: ExperimentResultsLayout =
    asLayout(state.layout) ?? storedLayout ?? "grid";

  const setLayout = (newLayout: ExperimentResultsLayout) => {
    setStoredLayout(newLayout);
    setState({ layout: newLayout });
  };

  const requestedDiffMode: ExperimentDiffMode =
    asDiffMode(state.diff) ?? storedDiffMode ?? "comparison";

  // Expected → Output is the list layout's two-line cell, and no other layout
  // draws it. A shared URL or a remembered pick that pairs it with another
  // layout would leave the menu promising a comparison the table is not
  // showing, so it reads as the baseline diff until the layout can express it.
  const diffMode: ExperimentDiffMode =
    requestedDiffMode === "expected" && layout !== "list"
      ? "comparison"
      : requestedDiffMode;

  const setDiffMode = (newDiffMode: ExperimentDiffMode) => {
    setStoredDiffMode(newDiffMode);
    // Expected → Output is the diff layout's two-line cell; picking it from the
    // side-by-side layout would otherwise appear to do nothing.
    setState({
      diff: newDiffMode,
      ...(newDiffMode === "expected" ? { layout: "list" } : {}),
    });
    if (newDiffMode === "expected") setStoredLayout("list");
  };

  // Item visibility management
  const itemVisibility =
    (state.itemVisibility as "baseline-only" | "all") ?? "baseline-only";
  const setItemVisibility = (newVisibility: "baseline-only" | "all") => {
    setState({ itemVisibility: newVisibility });
  };

  return {
    // Baseline
    baselineId: explicitBaselineId,
    hasBaseline,
    setBaseline,
    clearBaseline,

    // Comparison
    comparisonIds,
    setComparisonIds,
    addComparisonId,
    removeComparisonId,
    maxSelectedExperiments: MAX_SELECTED_EXPERIMENTS,
    canAddMore: selectedExperimentIds.length < MAX_SELECTED_EXPERIMENTS,

    // All selected experiments, preserving URL order with an explicit baseline first.
    selectedExperimentIds,
    allExperimentIds: selectedExperimentIds,

    // The list `getExperimentColorStyles` indexes into, so the cells and the run
    // pickers cannot drift into two mappings. Empty without a baseline: colour
    // separates comparisons from the run they are measured against, and with no
    // baseline there is nothing to separate.
    colorExperimentIds: hasBaseline ? selectedExperimentIds : [],

    // Layout
    layout,
    setLayout,

    // Diff mode
    diffMode,
    setDiffMode,

    // Item visibility
    itemVisibility,
    setItemVisibility,
  };
}