Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.3 KB
Raw Blame
import { fireEvent, render, screen } from "@testing-library/react";
import { useExperimentResultsState } from "./useExperimentResultsState";

const queryParamStore = new Map<string, unknown>();

vi.mock("use-query-params", () => {
  const React = require("react");

  return {
    ArrayParam: {},
    StringParam: {},
    withDefault: (_param: unknown, defaultValue: unknown) => ({
      defaultValue,
    }),
    useQueryParams: (config: Record<string, { defaultValue: unknown }>) => {
      const initialState = Object.fromEntries(
        Object.entries(config).map(([key, value]) => [
          key,
          queryParamStore.has(key)
            ? queryParamStore.get(key)
            : value.defaultValue,
        ]),
      );

      const [state, setState] = React.useState(initialState);

      const setQueryState = React.useCallback(
        (updates: Record<string, any>) => {
          setState((previous: Record<string, any>) => {
            const next = { ...previous, ...updates };

            Object.entries(updates).forEach(([key, value]) => {
              if (value === undefined || value === null) {
                queryParamStore.delete(key);
              } else {
                queryParamStore.set(key, value);
              }
            });

            return next;
          });
        },
        [],
      );

      return [state, setQueryState] as const;
    },
  };
});

function Harness() {
  const {
    baselineId,
    hasBaseline,
    comparisonIds,
    setComparisonIds,
    maxSelectedExperiments,
    clearBaseline,
    selectedExperimentIds,
    layout,
    diffMode,
  } = useExperimentResultsState();

  return (
    <div>
      <div data-testid="baseline">{baselineId ?? "null"}</div>
      <div data-testid="has-baseline">{hasBaseline ? "true" : "false"}</div>
      <div data-testid="comparisons">{comparisonIds.join(",")}</div>
      <div data-testid="selected">{selectedExperimentIds.join(",")}</div>
      <div data-testid="max-experiments">{maxSelectedExperiments}</div>
      <div data-testid="layout">{layout}</div>
      <div data-testid="diff-mode">{diffMode}</div>
      <button
        type="button"
        onClick={() =>
          setComparisonIds(
            Array.from({ length: 12 }, (_, index) => `comp-${index}`),
          )
        }
      >
        set twelve comparisons
      </button>
      <button type="button" onClick={clearBaseline}>
        clear
      </button>
    </div>
  );
}

describe("useExperimentResultsState", () => {
  beforeEach(() => {
    queryParamStore.clear();
    localStorage.clear();
  });

  it("defaults to one column per experiment with or without comparisons", () => {
    queryParamStore.set("baseline", "baseline-run");

    render(<Harness />);

    expect(screen.getByTestId("layout").textContent).toBe("grid");
    expect(screen.getByTestId("diff-mode").textContent).toBe("comparison");

    queryParamStore.set("c", ["comp-a"]);

    render(<Harness />);

    expect(screen.getAllByTestId("layout")[1].textContent).toBe("grid");
  });

  it("lets an explicit layout in the URL win over the default", () => {
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set("c", ["comp-a"]);
    queryParamStore.set("layout", "list");

    render(<Harness />);

    expect(screen.getByTestId("layout").textContent).toBe("list");
  });

  it("ignores the old layout preference after the comparison layout reset", () => {
    localStorage.setItem("experiment-results-layout", JSON.stringify("list"));
    render(<Harness />);
    expect(screen.getByTestId("layout").textContent).toBe("grid");
  });

  it("preserves a remembered layout when the URL does not specify one", () => {
    localStorage.setItem(
      "experiment-results-compare-layout",
      JSON.stringify("list"),
    );
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set("c", ["comp-a"]);

    render(<Harness />);

    expect(screen.getByTestId("layout").textContent).toBe("list");
  });

  it("derives hasBaseline correctly", () => {
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set("c", ["comp-a"]);

    render(<Harness />);

    expect(screen.getByTestId("has-baseline").textContent).toBe("true");

    queryParamStore.clear();
    queryParamStore.set("c", ["comp-a"]);

    render(<Harness />);

    expect(screen.getAllByTestId("has-baseline")[1].textContent).toBe("false");
  });

  it("keeps c-only selections as comparisons without an implicit baseline", () => {
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set("c", ["comp-a", "comp-b"]);

    render(<Harness />);

    expect(screen.getByTestId("comparisons").textContent).toBe("comp-a,comp-b");

    queryParamStore.clear();
    queryParamStore.set("c", ["comp-a", "comp-b"]);

    render(<Harness />);

    expect(screen.getAllByTestId("has-baseline")[1].textContent).toBe("false");
    expect(screen.getAllByTestId("comparisons")[1].textContent).toBe(
      "comp-a,comp-b",
    );
    expect(screen.getAllByTestId("selected")[1].textContent).toBe(
      "comp-a,comp-b",
    );
  });

  it("clears baseline by URL state only and moves it into comparisons", () => {
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set("c", ["comp-a"]);

    render(<Harness />);

    fireEvent.click(screen.getByRole("button", { name: "clear" }));

    expect(screen.getByTestId("baseline").textContent).toBe("null");
    expect(screen.getByTestId("comparisons").textContent).toBe(
      "comp-a,baseline-run",
    );
    expect(screen.getByTestId("selected").textContent).toBe(
      "comp-a,baseline-run",
    );

    expect(queryParamStore.has("baseline")).toBe(false);
    expect(queryParamStore.get("c")).toEqual(["comp-a", "baseline-run"]);
  });

  it("allows up to ten selected experiments and clamps additional selections", () => {
    render(<Harness />);

    fireEvent.click(
      screen.getByRole("button", { name: "set twelve comparisons" }),
    );

    expect(screen.getByTestId("max-experiments").textContent).toBe("10");
    expect(screen.getByTestId("selected").textContent).toBe(
      Array.from({ length: 10 }, (_, index) => `comp-${index}`).join(","),
    );
    expect(screen.getByTestId("comparisons").textContent).toBe(
      Array.from({ length: 10 }, (_, index) => `comp-${index}`).join(","),
    );
  });

  it("clamps comparison IDs loaded from the URL", () => {
    queryParamStore.set(
      "c",
      Array.from({ length: 12 }, (_, index) => `comp-${index}`),
    );

    render(<Harness />);

    expect(screen.getByTestId("selected").textContent).toBe(
      Array.from({ length: 10 }, (_, index) => `comp-${index}`).join(","),
    );
    expect(screen.getByTestId("comparisons").textContent).toBe(
      Array.from({ length: 10 }, (_, index) => `comp-${index}`).join(","),
    );
  });

  it("keeps an explicit baseline outside the ten comparison slots", () => {
    queryParamStore.set("baseline", "baseline-run");
    queryParamStore.set(
      "c",
      Array.from({ length: 12 }, (_, index) => `comp-${index}`),
    );

    render(<Harness />);

    expect(screen.getByTestId("selected").textContent).toBe(
      [
        "baseline-run",
        ...Array.from({ length: 9 }, (_, index) => `comp-${index}`),
      ].join(","),
    );
    expect(screen.getByTestId("comparisons").textContent).toBe(
      Array.from({ length: 9 }, (_, index) => `comp-${index}`).join(","),
    );
  });
});