Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.8 KB
Raw Blame
import { describe, expect, it } from "vitest";
import {
  autoComparisonPreferenceChangedProps,
  baselineChangedProps,
  chartMetricChangedProps,
  chartMetricGroup,
  comparisonChangedProps,
  comparisonPickerOpenedProps,
  diffModeChangedProps,
  isSameDataset,
  itemRegressionFilterAppliedProps,
  layoutChangedProps,
  scoreColumnGroupScope,
  scoreColumnScopeToggledProps,
  uniqueDatasetCount,
} from "./analytics";

const payloadKeys = (payload: object) => Object.keys(payload);

const CONTENT_KEYS = new Set([
  "name",
  "value",
  "query",
  "search",
  "prompt",
  "experimentName",
  "datasetName",
  "filter",
  "metricId",
  "searchQuery",
]);

function expectMetadataOnly(payload: object) {
  for (const key of payloadKeys(payload)) {
    expect(CONTENT_KEYS.has(key), `payload must not include ${key}`).toBe(
      false,
    );
  }
  expect(JSON.stringify(payload)).not.toMatch(/helpfulness|My Experiment/i);
}

describe("experiment analytics payloads", () => {
  it("treats a single known dataset as same-dataset", () => {
    expect(isSameDataset(["ds-1"])).toBe(true);
    expect(isSameDataset(["ds-1", "ds-1"])).toBe(true);
    expect(isSameDataset(["ds-1", "ds-2"])).toBe(false);
    expect(isSameDataset(["ds-1", null, "ds-1"])).toBe(true);
    expect(uniqueDatasetCount(["ds-1", null, "ds-2", "ds-1"])).toBe(2);
  });

  // The four-slot chart grid became one strip, so `chartIndex`/`slotCount` are
  // gone; the score's level and data type take their place. Same event name.
  it("maps chart metric ids to a shape without keeping the id", () => {
    expect(chartMetricGroup("base:cost")).toBe("base");
    expect(chartMetricGroup("obs-score-numeric:helpfulness")).toBe("score");
    const payload = chartMetricChangedProps({
      tableName: "experiments",
      metricId: "obs-score-numeric:helpfulness",
    });
    expect(payload).toEqual({
      isV4: true,
      tableName: "experiments",
      metricGroup: "score",
      scoreLevel: "observation",
      dataType: "numeric",
    });
    expectMetadataOnly(payload);

    expect(
      chartMetricChangedProps({
        tableName: "experiments",
        metricId: "base:cost",
      }),
    ).toEqual({
      isV4: true,
      tableName: "experiments",
      metricGroup: "base",
      scoreLevel: "none",
      dataType: "none",
    });

    // Trace-level scores only became selectable in this change.
    expect(
      chartMetricChangedProps({
        tableName: "experiments",
        metricId: "trace-score-categorical:tone",
      }),
    ).toMatchObject({ scoreLevel: "trace", dataType: "categorical" });
    expect(
      chartMetricChangedProps({
        tableName: "experiments",
        metricId: "experiment-score-numeric:accuracy",
      }),
    ).toMatchObject({ scoreLevel: "experiment", dataType: "numeric" });
  });

  it("maps score column groups to scopes and ignores unknown groups", () => {
    expect(scoreColumnGroupScope("traceItemScores")).toBe("trace");
    expect(scoreColumnGroupScope("traceScores")).toBe("trace");
    expect(scoreColumnGroupScope("observationItemScores")).toBe("observation");
    expect(scoreColumnGroupScope("observationScores")).toBe("observation");
    expect(scoreColumnGroupScope("experimentScores")).toBe("experiment");
    expect(scoreColumnGroupScope("name")).toBeNull();
    expect(
      scoreColumnScopeToggledProps({
        tableName: "experiments",
        groupId: "name",
        enabledCount: 2,
      }),
    ).toBeNull();
  });

  it("builds comparison_changed without names or ids of experiments", () => {
    const payload = comparisonChangedProps({
      tableName: "experiment-items",
      comparisonCount: 2,
      datasetIds: ["ds-a", "ds-a", "ds-b"],
      source: "picker",
    });
    expect(payload).toEqual({
      isV4: true,
      tableName: "experiment-items",
      comparisonCount: 2,
      isSameDataset: false,
      source: "picker",
    });
    expectMetadataOnly(payload);
  });

  it("counts options and datasets on picker open from lengths, not query text", () => {
    const payload = comparisonPickerOpenedProps({
      tableName: "experiment-items",
      optionCount: 12,
      datasetIds: ["ds-1", "ds-2", "ds-1"],
      queryLength: "My Experiment".length,
    });
    expect(payload).toEqual({
      isV4: true,
      tableName: "experiment-items",
      optionCount: 12,
      datasetCount: 2,
      hasSearchQuery: true,
      queryLength: 13,
    });
    expect(payload).not.toHaveProperty("query");
    expectMetadataOnly(payload);
  });

  // `charts_section_toggled` and `analytics_tab_opened` are gone with the
  // accordion and the Analytics route; the layout, diff-mode and
  // auto-comparison events replace them on the surfaces that exist now.
  it("keeps baseline, layout, diff mode and the auto preference on metadata only", () => {
    expect(
      baselineChangedProps({
        tableName: "experiment-items",
        source: "clear",
      }),
    ).toEqual({
      isV4: true,
      tableName: "experiment-items",
      source: "clear",
    });
    const layout = layoutChangedProps({
      tableName: "experiment-items",
      layout: "matrix",
      comparisonCount: 2,
    });
    expect(layout).toEqual({
      isV4: true,
      tableName: "experiment-items",
      layout: "matrix",
      comparisonCount: 2,
    });
    expectMetadataOnly(layout);
    const diff = diffModeChangedProps({
      tableName: "experiment-items",
      mode: "expected",
      comparisonCount: 0,
    });
    expect(diff).toEqual({
      isV4: true,
      tableName: "experiment-items",
      mode: "expected",
      comparisonCount: 0,
    });
    expectMetadataOnly(diff);
    expect(
      autoComparisonPreferenceChangedProps({
        tableName: "experiment-items",
        isEnabled: false,
      }),
    ).toEqual({
      isV4: true,
      tableName: "experiment-items",
      isEnabled: false,
    });
  });

  // Same event name, repointed at the score-comparison filter ("worse than the
  // comparison"). No `column`: here the filtered column is a score, and a score
  // name is user content, so the level and data type describe it instead.
  it("describes a score-comparison filter by shape, not by score name", () => {
    const payload = itemRegressionFilterAppliedProps({
      tableName: "experiment-items",
      scoreLevel: "trace",
      dataType: "NUMERIC",
      operator: "lower",
      comparisonExperimentId: "cmp-2",
      comparisonIds: ["cmp-1", "cmp-2"],
      source: "header_menu",
    });
    expect(payload).toEqual({
      isV4: true,
      tableName: "experiment-items",
      scoreLevel: "trace",
      dataType: "NUMERIC",
      operator: "lower",
      comparisonIndex: 1,
      source: "header_menu",
    });
    expect(payload).not.toBeNull();
    expectMetadataOnly(payload!);
    expect(payload).not.toHaveProperty("column");

    // An unknown data type is reported as such rather than dropped, so the
    // property is present on every event.
    expect(
      itemRegressionFilterAppliedProps({
        tableName: "experiment-items",
        scoreLevel: "observation",
        dataType: undefined,
        operator: "different",
        comparisonExperimentId: "cmp-1",
        comparisonIds: ["cmp-1"],
        source: "url",
      }),
    ).toMatchObject({ dataType: "unknown", comparisonIndex: 0 });
  });

  // A shared URL can outlive the run its filter points at. The table treats
  // that filter as inactive, so there is no applied filter to report — an
  // out-of-range index would be noise in the funnel.
  it("reports no event for a filter pointing outside the compared runs", () => {
    expect(
      itemRegressionFilterAppliedProps({
        tableName: "experiment-items",
        scoreLevel: "trace",
        dataType: "NUMERIC",
        operator: "lower",
        comparisonExperimentId: "cmp-gone",
        comparisonIds: ["cmp-1", "cmp-2"],
        source: "url",
      }),
    ).toBeNull();
  });
});