Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(analytics): instrument the rebuilt experiments UI (#16919)

Nikita Kabardin9b850abc6001
diff --git a/web/src/components/table/data-table-column-visibility-filter.clienttest.tsx b/web/src/components/table/data-table-column-visibility-filter.clienttest.tsx
index 3ba99945dc..57b5b4679a 100644
--- a/web/src/components/table/data-table-column-visibility-filter.clienttest.tsx
+++ b/web/src/components/table/data-table-column-visibility-filter.clienttest.tsx
@@ -25,6 +25,14 @@ const columns: LangfuseColumnDef<{ id: string }>[] = [
header: "Input",
enableHiding: true,
},
+ {
+ accessorKey: "traceScores",
+ header: "Trace Scores",
+ enableHiding: true,
+ columns: [
+ { accessorKey: "score-a", header: "Score A", enableHiding: true },
+ ],
+ },
];
const groupedColumns: LangfuseColumnDef<{ id: string }>[] = [
diff --git a/web/src/components/table/data-table-column-visibility-filter.tsx b/web/src/components/table/data-table-column-visibility-filter.tsx
index 7b86536442..afc12d2ebe 100644
--- a/web/src/components/table/data-table-column-visibility-filter.tsx
+++ b/web/src/components/table/data-table-column-visibility-filter.tsx
@@ -49,9 +49,18 @@ import {
import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { Separator } from "@/src/components/ui/separator";
+/**
+ * A whole column GROUP was shown or hidden at once (its "Select All" /
+ * "Deselect All"). Surfaces that care which family a group is β€” the experiments
+ * tables and their score levels β€” hook in here rather than having this generic
+ * picker know about them.
+ */
export type ColumnGroupTogglePayload = {
+ /** The group column's accessorKey. */
groupId: string;
+ /** Columns visible in the group after the toggle: all of them, or none. */
enabledCount: number;
+ /** Columns in the group. */
totalCount: number;
};
diff --git a/web/src/components/table/data-table-toolbar.clienttest.tsx b/web/src/components/table/data-table-toolbar.clienttest.tsx
index 537229c923..abaae360f2 100644
--- a/web/src/components/table/data-table-toolbar.clienttest.tsx
+++ b/web/src/components/table/data-table-toolbar.clienttest.tsx
@@ -25,6 +25,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: 500,
selectedRowIds: selectedIds(50),
@@ -43,6 +44,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: null,
selectedRowIds: selectedIds(50),
@@ -59,6 +61,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: 30,
selectedRowIds: selectedIds(30),
@@ -78,6 +81,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: null,
hasNextPage: true,
@@ -98,6 +102,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
selectAll: true,
totalCount: null,
@@ -114,6 +119,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
selectAll: true,
totalCount: 823,
@@ -131,6 +137,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: null,
hasNextPage: false,
@@ -148,6 +155,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: null,
hasNextPage: true,
@@ -165,6 +173,7 @@ describe("DataTableToolbar select-all banner gate", () => {
render(
<DataTableToolbar
columns={[]}
+ tableName="test-table"
multiSelect={baseMultiSelect({
totalCount: null,
hasNextPage: true,
diff --git a/web/src/components/table/data-table-toolbar.tsx b/web/src/components/table/data-table-toolbar.tsx
index 0cbf4ac993..aea01e0e86 100644
--- a/web/src/components/table/data-table-toolbar.tsx
+++ b/web/src/components/table/data-table-toolbar.tsx
@@ -145,15 +145,18 @@ interface DataTableToolbarProps<TData, TValue> {
};
orderByState?: OrderByState;
viewConfig?: TableViewConfig;
- /** Analytics table identity (LFE-10781) for the popover filter builder's
- * `filters:applied`/`filters:cleared` events. Tables with a `viewConfig`
- * already supply it via `viewConfig.tableName`; tables WITHOUT one (users,
- * dataset runs/items) must pass this so the event isn't labeled "unknown". */
+ /** Analytics table identity, for `filters:applied`/`filters:cleared`,
+ * `table:search_submit`, `table:row_height_switch_select` and
+ * `table:column_visibility_changed`.
+ * Tables with a `viewConfig` supply it via `viewConfig.tableName`; every table
+ * WITHOUT one must pass this β€” the `ToolbarTableIdentity` union below makes
+ * that a type error rather than an "unknown" bucket in PostHog. */
tableName?: string;
- /** Surface dimension for table:* events. Forward from the owning table β€”
- * do not rely on the default. v4 events / experiments = true; v3 traces = false. */
+ /** Whether this table reads the v4 (fast-mode) data path, at the moment of the
+ * action. The headline segmentation dimension: filtering, columns and search
+ * behave very differently across v3 and v4. Forward it from the owning table β€”
+ * the fallback is the v4 events view, not a safe default. */
isV4?: boolean;
- onColumnGroupToggle?: (payload: ColumnGroupTogglePayload) => void;
filterWithAI?: boolean;
className?: string;
rowClassName?: string;
@@ -167,8 +170,23 @@ interface DataTableToolbarProps<TData, TValue> {
* shape is validated on the experiments list; needs both controls, so a
* table that passes only one keeps its single button. */
mergeSettingsIntoPopover?: boolean;
+ /** Notified when a whole column group is shown or hidden at once, for surfaces
+ * that report their own event for it (the experiments score families). */
+ onColumnGroupToggle?: (payload: ColumnGroupTogglePayload) => void;
}
+/**
+ * Every toolbar must be able to name its own table, because four analytics
+ * events fire from inside it. One of the two sources has to be present:
+ * a `viewConfig` (whose `tableName` is the saved-view table) or an explicit
+ * `tableName`. Enforced in the type so no surface can silently report
+ * `tableName: "unknown"` β€” the failure mode the first version of these events
+ * shipped and had to fix.
+ */
+type ToolbarTableIdentity =
+ | { tableName: string }
+ | { viewConfig: TableViewConfig };
+
// Helper function to get the description for DocPopup
function getSearchDescription(
searchType: TracingSearchType[] | undefined,
@@ -233,17 +251,23 @@ export function DataTableToolbar<TData, TValue>({
viewConfig,
tableName,
isV4,
- onColumnGroupToggle,
filterWithAI = false,
viewModeToggle,
leadingControls,
mergeSettingsIntoPopover = false,
-}: DataTableToolbarProps<TData, TValue>) {
+ onColumnGroupToggle,
+}: DataTableToolbarProps<TData, TValue> & ToolbarTableIdentity) {
const [searchString, setSearchString] = useState(
searchConfig?.currentQuery ?? "",
);
const capture = usePostHogClientCapture();
+ // One definition of the two analytics dimensions for everything the toolbar
+ // emits: an explicit `tableName` wins over the view's, and `isV4`
+ // falls back to the one surface that is v4 without saying so β€” the v4 events
+ // table, which filters through the grammar bar rather than this toolbar.
+ // The "unknown" fallback is unreachable β€” `ToolbarTableIdentity` requires one
+ // of the two sources.
const analyticsTableName = tableName ?? viewConfig?.tableName ?? "unknown";
const analyticsIsV4 =
isV4 ??
@@ -470,12 +494,8 @@ export function DataTableToolbar<TData, TValue>({
columnsWithCustomSelect={columnsWithCustomSelect}
filterWithAI={filterWithAI}
// Analytics (LFE-10781): the table's own identity, so popover
- // filters:applied/cleared events aren't mislabeled "unknown". Prefer
- // an explicit `tableName` (tables without a viewConfig β€” users,
- // dataset runs/items), else the view's table. The v4 events table
- // filters via the grammar bar (it omits filterColumnDefinition here,
- // so this popover is a v3/legacy surface); derive isV4 from the
- // ObservationsEvents view for consistency + future-proofing.
+ // filters:applied/cleared events aren't mislabeled "unknown". Shares
+ // the toolbar's single definition of both dimensions.
tableName={analyticsTableName}
isV4={analyticsIsV4}
/>
diff --git a/web/src/components/table/peek/hooks/usePeekNavigation.ts b/web/src/components/table/peek/hooks/usePeekNavigation.ts
index d267416d37..ead041a3e5 100644
--- a/web/src/components/table/peek/hooks/usePeekNavigation.ts
+++ b/web/src/components/table/peek/hooks/usePeekNavigation.ts
@@ -11,7 +11,13 @@ const PEEK_PARAM = "peek";
const PEEK_VIEW_PARAM = "peekView";
interface BasePeekConfig {
- /** Analytics table identity for peek:* events. Forward from the owning table. */
+ /**
+ * Which table this peek belongs to, for the `peek:*` analytics.
+ * `routePattern` alone cannot separate two tables on one route (the dataset
+ * run items by-run / by-item views, the experiments results layouts), so the
+ * table names itself. Required on purpose: an optional dimension is one a new
+ * table forgets to pass.
+ */
tableName: string;
/** Surface dimension at the moment of the action. Do not derive from the global v4 flag. */
isV4: boolean;
@@ -86,7 +92,8 @@ export function usePeekNavigation(config: PeekConfig | PeekConfigWithExpand) {
// Every peek is opened/closed through this hook, so open/close/new-tab
// analytics live here once instead of in each consuming table. Props are
// metadata-only: `routePattern` is the Next.js route PATTERN
- // (`/project/[projectId]/traces`), never a concrete URL with ids.
+ // (`/project/[projectId]/traces`), never a concrete URL with ids, and
+ // `tableName` is the table's own analytics identity.
const routePattern = router.pathname;
const { isV4, tableName } = config;
diff --git a/web/src/components/table/use-cases/models.tsx b/web/src/components/table/use-cases/models.tsx
index c19896fd5b..02f8012f0a 100644
--- a/web/src/components/table/use-cases/models.tsx
+++ b/web/src/components/table/use-cases/models.tsx
@@ -293,6 +293,7 @@ export default function ModelTable({ projectId }: { projectId: string }) {
return (
<>
<DataTableToolbar
+ tableName="models"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/components/table/use-cases/observations.tsx b/web/src/components/table/use-cases/observations.tsx
index ca5ce50cc1..09e266d18f 100644
--- a/web/src/components/table/use-cases/observations.tsx
+++ b/web/src/components/table/use-cases/observations.tsx
@@ -1474,6 +1474,7 @@ function ObservationsDataTableToolbar({
return (
<DataTableToolbar
+ tableName="observations"
{...toolbarProps}
orderByState={orderByState}
actionButtons={[
diff --git a/web/src/components/table/use-cases/score-configs.tsx b/web/src/components/table/use-cases/score-configs.tsx
index 77eca64e6e..c812aeeabc 100644
--- a/web/src/components/table/use-cases/score-configs.tsx
+++ b/web/src/components/table/use-cases/score-configs.tsx
@@ -235,6 +235,7 @@ export function ScoreConfigsTable({ projectId }: { projectId: string }) {
return (
<>
<DataTableToolbar
+ tableName="score-configs"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/ee/features/audit-log-viewer/AuditLogsTable.tsx b/web/src/ee/features/audit-log-viewer/AuditLogsTable.tsx
index d92e1dba91..8648592aac 100644
--- a/web/src/ee/features/audit-log-viewer/AuditLogsTable.tsx
+++ b/web/src/ee/features/audit-log-viewer/AuditLogsTable.tsx
@@ -131,6 +131,7 @@ export function AuditLogsTable(props: AuditLogsTableProps) {
return (
<>
<DataTableToolbar
+ tableName="audit-logs"
columns={columns}
rowHeight={rowHeight}
setRowHeight={setRowHeight}
diff --git a/web/src/ee/features/billing/components/BillingInvoiceTable.tsx b/web/src/ee/features/billing/components/BillingInvoiceTable.tsx
index fde0b7d0a3..b74cd8762a 100644
--- a/web/src/ee/features/billing/components/BillingInvoiceTable.tsx
+++ b/web/src/ee/features/billing/components/BillingInvoiceTable.tsx
@@ -277,7 +277,7 @@ export function BillingInvoiceTable() {
<div className="flex items-center justify-between pt-4">
<h3 className="font-bold">Invoice History</h3>
</div>
- <DataTableToolbar columns={columns} />
+ <DataTableToolbar columns={columns} tableName="billing-invoices" />
<DataTable
tableName="invoices"
columns={columns}
diff --git a/web/src/ee/features/billing/components/SpendAlerts/SpendAlertsTable.tsx b/web/src/ee/features/billing/components/SpendAlerts/SpendAlertsTable.tsx
index 5878662d94..ec0f7088fb 100644
--- a/web/src/ee/features/billing/components/SpendAlerts/SpendAlertsTable.tsx
+++ b/web/src/ee/features/billing/components/SpendAlerts/SpendAlertsTable.tsx
@@ -128,7 +128,7 @@ export function SpendAlertsTable({ orgId }: SpendAlertsTableProps) {
return (
<>
- <DataTableToolbar columns={columns} />
+ <DataTableToolbar columns={columns} tableName="spend-alerts" />
<DataTable tableName="spend-alerts" columns={columns} data={data} />
{editingAlert && editingAlertData && (
diff --git a/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx b/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
index ff0d33ce3f..b337d802e7 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueueItemsTable.tsx
@@ -401,6 +401,7 @@ export function AnnotationQueueItemsTable({
return (
<>
<DataTableToolbar
+ tableName="annotation-queue-items"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
index 0ca10da50e..63646cdf12 100644
--- a/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
+++ b/web/src/features/annotation-queues/components/AnnotationQueuesTable.tsx
@@ -209,6 +209,7 @@ export function AnnotationQueuesTable({ projectId }: { projectId: string }) {
return (
<>
<DataTableToolbar
+ tableName="annotation-queues"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/automations/components/AutomationExecutionsTable.tsx b/web/src/features/automations/components/AutomationExecutionsTable.tsx
index 380148c92a..60e820d940 100644
--- a/web/src/features/automations/components/AutomationExecutionsTable.tsx
+++ b/web/src/features/automations/components/AutomationExecutionsTable.tsx
@@ -155,6 +155,7 @@ export const AutomationExecutionsTable: React.FC<
return (
<>
<DataTableToolbar
+ tableName="automation-executions"
columns={columns}
rowHeight={rowHeight}
setRowHeight={setRowHeight}
diff --git a/web/src/features/background-migrations/components/background-migrations.tsx b/web/src/features/background-migrations/components/background-migrations.tsx
index 5dbf0db78e..51fdfb7740 100644
--- a/web/src/features/background-migrations/components/background-migrations.tsx
+++ b/web/src/features/background-migrations/components/background-migrations.tsx
@@ -89,7 +89,7 @@ export default function BackgroundMigrationsTable() {
title: "Background Migrations",
}}
>
- <DataTableToolbar columns={columns} />
+ <DataTableToolbar columns={columns} tableName="background-migrations" />
<DataTable
tableName="backgroundMigrations"
columns={columns}
diff --git a/web/src/features/datasets/components/DatasetCompareRunsTable.tsx b/web/src/features/datasets/components/DatasetCompareRunsTable.tsx
index 9602a7a7bd..01c892234b 100644
--- a/web/src/features/datasets/components/DatasetCompareRunsTable.tsx
+++ b/web/src/features/datasets/components/DatasetCompareRunsTable.tsx
@@ -198,6 +198,7 @@ function DatasetCompareRunsTableInternal(props: {
return (
<>
<DataTableToolbar
+ tableName="dataset-compare-runs"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/datasets/components/DatasetRunItemsByItemTable.tsx b/web/src/features/datasets/components/DatasetRunItemsByItemTable.tsx
index 2a2851c5d0..f2251ae4ac 100644
--- a/web/src/features/datasets/components/DatasetRunItemsByItemTable.tsx
+++ b/web/src/features/datasets/components/DatasetRunItemsByItemTable.tsx
@@ -228,6 +228,7 @@ export function DatasetRunItemsByItemTable(props: {
return (
<>
<DataTableToolbar
+ tableName="dataset-run-items-by-item"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/evals/components/eval-log.tsx b/web/src/features/evals/components/eval-log.tsx
index 0c6fd384f0..eca8a4d9bc 100644
--- a/web/src/features/evals/components/eval-log.tsx
+++ b/web/src/features/evals/components/eval-log.tsx
@@ -252,6 +252,7 @@ export default function EvalLogTable({
>
<div className="flex h-full w-full flex-col">
<DataTableToolbar
+ tableName="evalLogs"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/evals/components/eval-templates-table.tsx b/web/src/features/evals/components/eval-templates-table.tsx
index 1359b8bbb5..dd74f30e8e 100644
--- a/web/src/features/evals/components/eval-templates-table.tsx
+++ b/web/src/features/evals/components/eval-templates-table.tsx
@@ -519,6 +519,7 @@ export default function EvalsTemplateTable({
<>
<div className="flex h-full w-full flex-col">
<DataTableToolbar
+ tableName="eval-templates"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/evals/components/evaluator-table.tsx b/web/src/features/evals/components/evaluator-table.tsx
index c7d0513671..63dbec9930 100644
--- a/web/src/features/evals/components/evaluator-table.tsx
+++ b/web/src/features/evals/components/evaluator-table.tsx
@@ -432,6 +432,7 @@ export default function EvaluatorTable({ projectId }: { projectId: string }) {
<div className="flex h-full w-full flex-col">
{/* Toolbar spanning full width */}
<DataTableToolbar
+ tableName="evaluators"
columns={columns}
filterState={queryFilter.filterState}
columnVisibility={columnVisibility}
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
index f54c070f90..9ee1bf456b 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase.tsx
@@ -574,6 +574,8 @@ export function SampleObservationSelectorBase(
tooltip={matchingTooltip}
trailing={
<DataTableColumnVisibilityFilter
+ tableName="evaluator-sample-observations"
+ isV4
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
index de93e5a220..964d6b3eae 100644
--- a/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorsPage.tsx
@@ -154,6 +154,7 @@ function EvaluatorsTableToolbar({
return (
<DataTableToolbar
+ tableName="evaluators-v2"
{...toolbarProps}
multiSelect={{
selectAll,
diff --git a/web/src/features/experiments/components/ExperimentBaselineControls.tsx b/web/src/features/experiments/components/ExperimentBaselineControls.tsx
index b06b387b2d..f58c94cedb 100644
--- a/web/src/features/experiments/components/ExperimentBaselineControls.tsx
+++ b/web/src/features/experiments/components/ExperimentBaselineControls.tsx
@@ -36,6 +36,12 @@ export function ExperimentBaselineControls({
});
const capture = usePostHogClientCapture();
+ // `baseline_changed` fires from the two paths through this control β€” picking a
+ // run and clearing it. Arriving on the page with a baseline already set (a row
+ // click on the list, a shared link) is navigation rather than use of this
+ // control, and is deliberately not counted; the results pageview is that
+ // denominator.
+
// Grouped by dataset and dated, so two runs sharing a name are two readable
// options rather than the same label twice.
const baselineOptionGroups = useMemo<ComboboxOptionGroup<string>[]>(() => {
diff --git a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx b/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
index 10912a2620..10716c5498 100644
--- a/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
+++ b/web/src/features/experiments/components/ExperimentComparisonSelector.tsx
@@ -12,6 +12,7 @@ import { type ExperimentNameOption } from "@/src/features/experiments/hooks/useE
import { formatRunRecency } from "@/src/features/experiments/fns/formatRunRecency";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
+ autoComparisonPreferenceChangedProps,
comparisonChangedProps,
comparisonPickerOpenedProps,
} from "@/src/features/experiments/lib/analytics";
@@ -337,6 +338,13 @@ export function ExperimentComparisonSelector({
<button
type="button"
onClick={() => {
+ capture(
+ "experiment:auto_comparison_preference_changed",
+ autoComparisonPreferenceChangedProps({
+ tableName: "experiment-items",
+ isEnabled: !isAutoSelectEnabled,
+ }),
+ );
onAutoSelectEnabledChange(!isAutoSelectEnabled);
}}
className="text-muted-foreground hover:bg-muted/50 flex w-full items-center gap-3 px-3 py-2 text-left"
diff --git a/web/src/features/experiments/components/ExperimentMetricStrip.tsx b/web/src/features/experiments/components/ExperimentMetricStrip.tsx
index fc5c05eb79..2d5db6d995 100644
--- a/web/src/features/experiments/components/ExperimentMetricStrip.tsx
+++ b/web/src/features/experiments/components/ExperimentMetricStrip.tsx
@@ -31,6 +31,8 @@ import {
metricStripTriggerClasses,
} from "@/src/components/metric-strip/MetricStripTrigger";
import { useExperimentStripMetric } from "@/src/features/experiments/hooks/useExperimentStripMetric";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { chartMetricChangedProps } from "@/src/features/experiments/lib/analytics";
/**
* Stable node: `Chart` is memoized, so a fresh element on every render would
@@ -158,9 +160,22 @@ export function ExperimentMetricStrip({
const { metricId, setMetricId, availableMetricOptions, isLoading } =
useExperimentStripMetric({ projectId, experimentIds, scoreCoverage });
+ const capture = usePostHogClientCapture();
+ // Do people move the strip off its score-first default, and onto which score
+ // LEVEL? Trace-level is where an LLM-as-judge on a dataset run writes, so the
+ // level is the interesting half. The score's NAME is user content and is
+ // never sent. Reuses `chart_metric_changed` from the chart grid this strip
+ // replaced, so the metric-choice history is continuous.
const handleMetricChange = (newMetricId: string) => {
if (newMetricId === metricId) return;
+ capture(
+ "experiment:chart_metric_changed",
+ chartMetricChangedProps({
+ tableName: "experiments",
+ metricId: newMetricId,
+ }),
+ );
setMetricId(newMetricId);
};
diff --git a/web/src/features/experiments/components/ExperimentSelectionControls.tsx b/web/src/features/experiments/components/ExperimentSelectionControls.tsx
index c9ec5280f8..4638cb08df 100644
--- a/web/src/features/experiments/components/ExperimentSelectionControls.tsx
+++ b/web/src/features/experiments/components/ExperimentSelectionControls.tsx
@@ -1,5 +1,13 @@
+import { useCallback, useEffect, useRef } from "react";
+import { useRouter } from "next/router";
import { type UrlUpdateType } from "use-query-params";
import { useExperimentComparisonAutoSelect } from "@/src/features/experiments/hooks/useExperimentComparisonAutoSelect";
+import { useExperimentNames } from "@/src/features/experiments/hooks/useExperimentNames";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import {
+ comparisonChangedProps,
+ type ExperimentComparisonSource,
+} from "@/src/features/experiments/lib/analytics";
import { ExperimentBaselineControls } from "./ExperimentBaselineControls";
import { ExperimentComparisonSelector } from "./ExperimentComparisonSelector";
@@ -27,6 +35,49 @@ export function ExperimentSelectionControls({
onBaselineClear,
onComparisonIdsChange,
}: ExperimentSelectionControlsProps) {
+ const router = useRouter();
+ const capture = usePostHogClientCapture();
+ // The dataset each run belongs to, for `isSameDataset`. The picker reports its
+ // own change from the click; the two paths reported here are the ones no
+ // control owns β€” the auto-selected default, and a selection already in the URL.
+ const { experimentNames, isLoading: isDatasetContextLoading } =
+ useExperimentNames({ projectId });
+
+ const captureComparisonChanged = useCallback(
+ ({
+ comparisonIds: ids,
+ source,
+ }: {
+ comparisonIds: string[];
+ source: ExperimentComparisonSource;
+ }) => {
+ const datasetOf = (experimentId: string) =>
+ experimentNames.find((exp) => exp.experimentId === experimentId)
+ ?.datasetId ?? null;
+ capture(
+ "experiment:comparison_changed",
+ comparisonChangedProps({
+ tableName: "experiment-items",
+ comparisonCount: ids.length,
+ datasetIds: [
+ baselineId ? datasetOf(baselineId) : null,
+ ...ids.map(datasetOf),
+ ],
+ source,
+ }),
+ );
+ },
+ [capture, experimentNames, baselineId],
+ );
+
+ const handleAutoSelect = useCallback(
+ (ids: string[], options?: { updateType?: UrlUpdateType }) => {
+ captureComparisonChanged({ comparisonIds: ids, source: "auto" });
+ onComparisonIdsChange(ids, options);
+ },
+ [captureComparisonChanged, onComparisonIdsChange],
+ );
+
// Mounted once per results page, so the default comparison is decided here
// rather than in the state hook that every consumer of the URL calls.
const { isAutoSelectEnabled, setIsAutoSelectEnabled } =
@@ -34,9 +85,33 @@ export function ExperimentSelectionControls({
projectId,
baselineId,
comparisonIds,
- onComparisonIdsChange,
+ onComparisonIdsChange: handleAutoSelect,
});
+ // A results page can arrive with a comparison already chosen β€” a shared link,
+ // a Compare from the experiments list, a restored view. That is a real "this
+ // view compares" data point but not an action, so it is reported once per
+ // page, from the URL as it first reads, and never again. Declared after the
+ // auto-select hook so that on the render where auto-select writes its pick
+ // this effect still sees the URL's own (empty) selection and stays silent β€”
+ // the auto pick reports itself as `source: "auto"`.
+ const hasReportedUrlSelection = useRef(false);
+ useEffect(() => {
+ if (hasReportedUrlSelection.current) return;
+ // `c=` is not readable before the router is ready, and the dataset ids
+ // `isSameDataset` needs are not known until the run list has loaded.
+ if (!router.isReady || isDatasetContextLoading) return;
+
+ hasReportedUrlSelection.current = true;
+ if (comparisonIds.length === 0) return;
+ captureComparisonChanged({ comparisonIds, source: "url" });
+ }, [
+ router.isReady,
+ isDatasetContextLoading,
+ comparisonIds,
+ captureComparisonChanged,
+ ]);
+
return (
<div className="flex w-[56dvw] min-w-0 flex-row gap-3">
<div className="flex min-w-0 items-center">
diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
index 8bcb4e778b..0fdcc12c02 100644
--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
@@ -32,6 +32,7 @@ import {
import { ExperimentFilterPills } from "./ExperimentFilterPills";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
+ COMPARISON_OPERATOR_PROPERTY,
itemRegressionFilterAppliedProps,
scoreColumnScopeToggledProps,
} from "@/src/features/experiments/lib/analytics";
@@ -609,31 +610,12 @@ export default function ExperimentItemsTable({
}
// Update the target for this filter
- if (toExperimentId !== _fromExperimentId) {
- const props = itemRegressionFilterAppliedProps({
- tableName: "experiment-items",
- column: _filter.column,
- operator: _filter.operator,
- toExperimentId,
- baselineId,
- comparisonIds,
- });
- if (props) {
- capture("experiment:item_regression_filter_applied", props);
- }
- }
setFilterTargets((prev) => ({
...prev,
[originalIndex]: toExperimentId,
}));
},
- [
- filterTargets,
- defaultFilterTargetExperimentId,
- baselineId,
- comparisonIds,
- capture,
- ],
+ [filterTargets, defaultFilterTargetExperimentId],
);
// Handler for removing a filter via pill
@@ -855,6 +837,58 @@ export default function ExperimentItemsTable({
removeFilter: removeScoreComparisonFilter,
} = useScoreComparisonFilters();
+ // Is the regression filter used once it exists? The question the whole
+ // rebuild is for, so it carries the score's level and type, which comparison
+ // it reads against, and whether the user picked it or arrived with it in a
+ // shared URL. No score name and no score value β€” those are user content.
+ const captureScoreComparisonFilter = useCallback(
+ ({
+ filter,
+ dataType,
+ source,
+ }: {
+ filter: ScoreComparisonFilter;
+ dataType: ScoreColumnDataType | undefined;
+ source: "header_menu" | "url";
+ }) => {
+ const props = itemRegressionFilterAppliedProps({
+ tableName: "experiment-items",
+ scoreLevel: filter.level,
+ dataType,
+ operator: COMPARISON_OPERATOR_PROPERTY[filter.operator],
+ comparisonExperimentId: filter.comparisonExperimentId,
+ comparisonIds,
+ source,
+ });
+ if (props) capture("experiment:item_regression_filter_applied", props);
+ },
+ [capture, comparisonIds],
+ );
+
+ // A results page can arrive with the filter already in the URL. Reported once
+ // per page, after the score column definitions land (they carry the data
+ // type), and never again β€” it is a shared view, not an action.
+ const hasReportedUrlScoreFilters = useRef(false);
+ useEffect(() => {
+ if (hasReportedUrlScoreFilters.current) return;
+ if (isFilterOptionsLoading) return;
+ hasReportedUrlScoreFilters.current = true;
+ for (const filter of scoreComparisonFilters) {
+ captureScoreComparisonFilter({
+ filter,
+ dataType: scoreDataTypesByKey[scoreFieldForLevel(filter.level)].get(
+ filter.scoreKey,
+ ),
+ source: "url",
+ });
+ }
+ }, [
+ isFilterOptionsLoading,
+ scoreComparisonFilters,
+ scoreDataTypesByKey,
+ captureScoreComparisonFilter,
+ ]);
+
// The runs a score can be read against, the auto-selected comparison first so
// the menu's default is the one the column header already reports.
const comparisonTargets = useMemo(() => {
@@ -957,6 +991,11 @@ export default function ExperimentItemsTable({
operator,
comparisonExperimentId,
};
+ captureScoreComparisonFilter({
+ filter: nextFilter,
+ dataType,
+ source: "header_menu",
+ });
setScoreComparisonFilter(nextFilter);
}}
onClear={() =>
@@ -1086,6 +1125,7 @@ export default function ExperimentItemsTable({
scoreComparisonFilters,
setScoreComparisonFilter,
removeScoreComparisonFilter,
+ captureScoreComparisonFilter,
runNameOf,
],
);
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsx
index 816f2173d6..1dbcad8c9f 100644
--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx
@@ -171,6 +171,8 @@ function ExperimentsMultiSelectActionMenu({
if (selectedExperimentIds.length === 0) return;
const [baseline, ...comparisons] = selectedExperimentIds;
+ // The list's own way into a comparison β€” the same events the picker and the
+ // baseline control emit, told apart by their source.
capture(
"experiment:comparison_changed",
comparisonChangedProps({
@@ -857,6 +859,10 @@ export default function ExperimentsTable({
return map;
}, [rows]);
+ // Which score family do people actually want visible, now that all of them are
+ // on by default? The column drawer's per-family Select All / Deselect All is
+ // the family-level intent; the individual checkboxes stay on
+ // `table:column_visibility_changed`, which already carries the column.
const handleColumnGroupToggle = useCallback(
({ groupId, enabledCount }: ColumnGroupTogglePayload) => {
const props = scoreColumnScopeToggledProps({
diff --git a/web/src/features/experiments/lib/analytics.clienttest.ts b/web/src/features/experiments/lib/analytics.clienttest.ts
index 5955845b2a..8ae28a2fb5 100644
--- a/web/src/features/experiments/lib/analytics.clienttest.ts
+++ b/web/src/features/experiments/lib/analytics.clienttest.ts
@@ -1,14 +1,15 @@
import { describe, expect, it } from "vitest";
import {
- analyticsTabOpenedProps,
+ autoComparisonPreferenceChangedProps,
baselineChangedProps,
chartMetricChangedProps,
chartMetricGroup,
- chartsSectionToggledProps,
comparisonChangedProps,
comparisonPickerOpenedProps,
+ diffModeChangedProps,
isSameDataset,
itemRegressionFilterAppliedProps,
+ layoutChangedProps,
scoreColumnGroupScope,
scoreColumnScopeToggledProps,
uniqueDatasetCount,
@@ -47,23 +48,50 @@ describe("experiment analytics payloads", () => {
expect(uniqueDatasetCount(["ds-1", null, "ds-2", "ds-1"])).toBe(2);
});
- it("maps chart metric ids to groups without keeping the id", () => {
+ // The four-slot chart grid became one strip, so `chartIndex`/`slotCount` are
+ // gone; the score's level and data type take their place. Same event name.
+ it("maps chart metric ids to a shape without keeping the id", () => {
expect(chartMetricGroup("base:cost")).toBe("base");
expect(chartMetricGroup("obs-score-numeric:helpfulness")).toBe("score");
const payload = chartMetricChangedProps({
tableName: "experiments",
metricId: "obs-score-numeric:helpfulness",
- chartIndex: 1,
- slotCount: 3,
});
expect(payload).toEqual({
isV4: true,
tableName: "experiments",
metricGroup: "score",
- chartIndex: 1,
- slotCount: 3,
+ scoreLevel: "observation",
+ dataType: "numeric",
});
expectMetadataOnly(payload);
+
+ expect(
+ chartMetricChangedProps({
+ tableName: "experiments",
+ metricId: "base:cost",
+ }),
+ ).toEqual({
+ isV4: true,
+ tableName: "experiments",
+ metricGroup: "base",
+ scoreLevel: "none",
+ dataType: "none",
+ });
+
+ // Trace-level scores only became selectable in this change.
+ expect(
+ chartMetricChangedProps({
+ tableName: "experiments",
+ metricId: "trace-score-categorical:tone",
+ }),
+ ).toMatchObject({ scoreLevel: "trace", dataType: "categorical" });
+ expect(
+ chartMetricChangedProps({
+ tableName: "experiments",
+ metricId: "experiment-score-numeric:accuracy",
+ }),
+ ).toMatchObject({ scoreLevel: "experiment", dataType: "numeric" });
});
it("maps score column groups to scopes and ignores unknown groups", () => {
@@ -118,7 +146,10 @@ describe("experiment analytics payloads", () => {
expectMetadataOnly(payload);
});
- it("keeps baseline, charts toggle, and analytics tab on metadata only", () => {
+ // `charts_section_toggled` and `analytics_tab_opened` are gone with the
+ // accordion and the Analytics route; the layout, diff-mode and
+ // auto-comparison events replace them on the surfaces that exist now.
+ it("keeps baseline, layout, diff mode and the auto preference on metadata only", () => {
expect(
baselineChangedProps({
tableName: "experiment-items",
@@ -129,54 +160,97 @@ describe("experiment analytics payloads", () => {
tableName: "experiment-items",
source: "clear",
});
- expect(
- chartsSectionToggledProps({
- tableName: "experiments",
- isExpanded: false,
- }),
- ).toEqual({
+ const layout = layoutChangedProps({
+ tableName: "experiment-items",
+ layout: "matrix",
+ comparisonCount: 2,
+ });
+ expect(layout).toEqual({
isV4: true,
- tableName: "experiments",
- isExpanded: false,
+ tableName: "experiment-items",
+ layout: "matrix",
+ comparisonCount: 2,
+ });
+ expectMetadataOnly(layout);
+ const diff = diffModeChangedProps({
+ tableName: "experiment-items",
+ mode: "expected",
+ comparisonCount: 0,
});
+ expect(diff).toEqual({
+ isV4: true,
+ tableName: "experiment-items",
+ mode: "expected",
+ comparisonCount: 0,
+ });
+ expectMetadataOnly(diff);
expect(
- analyticsTabOpenedProps({
+ autoComparisonPreferenceChangedProps({
tableName: "experiment-items",
- hasComparison: true,
+ isEnabled: false,
}),
).toEqual({
isV4: true,
tableName: "experiment-items",
- hasComparison: true,
+ isEnabled: false,
});
});
- it("emits regression-filter props only when retargeting to a comparison", () => {
+ // Same event name, repointed at the score-comparison filter ("worse than the
+ // comparison"). No `column`: here the filtered column is a score, and a score
+ // name is user content, so the level and data type describe it instead.
+ it("describes a score-comparison filter by shape, not by score name", () => {
+ const payload = itemRegressionFilterAppliedProps({
+ tableName: "experiment-items",
+ scoreLevel: "trace",
+ dataType: "NUMERIC",
+ operator: "lower",
+ comparisonExperimentId: "cmp-2",
+ comparisonIds: ["cmp-1", "cmp-2"],
+ source: "header_menu",
+ });
+ expect(payload).toEqual({
+ isV4: true,
+ tableName: "experiment-items",
+ scoreLevel: "trace",
+ dataType: "NUMERIC",
+ operator: "lower",
+ comparisonIndex: 1,
+ source: "header_menu",
+ });
+ expect(payload).not.toBeNull();
+ expectMetadataOnly(payload!);
+ expect(payload).not.toHaveProperty("column");
+
+ // An unknown data type is reported as such rather than dropped, so the
+ // property is present on every event.
expect(
itemRegressionFilterAppliedProps({
tableName: "experiment-items",
- column: "latency",
- operator: ">",
- toExperimentId: "baseline-1",
- baselineId: "baseline-1",
+ scoreLevel: "observation",
+ dataType: undefined,
+ operator: "different",
+ comparisonExperimentId: "cmp-1",
comparisonIds: ["cmp-1"],
+ source: "url",
}),
- ).toBeNull();
+ ).toMatchObject({ dataType: "unknown", comparisonIndex: 0 });
+ });
+
+ // A shared URL can outlive the run its filter points at. The table treats
+ // that filter as inactive, so there is no applied filter to report β€” an
+ // out-of-range index would be noise in the funnel.
+ it("reports no event for a filter pointing outside the compared runs", () => {
expect(
itemRegressionFilterAppliedProps({
tableName: "experiment-items",
- column: "latency",
- operator: ">",
- toExperimentId: "cmp-1",
- baselineId: "baseline-1",
+ scoreLevel: "trace",
+ dataType: "NUMERIC",
+ operator: "lower",
+ comparisonExperimentId: "cmp-gone",
comparisonIds: ["cmp-1", "cmp-2"],
+ source: "url",
}),
- ).toEqual({
- isV4: true,
- tableName: "experiment-items",
- column: "latency",
- comparisonIndex: 0,
- operator: ">",
- });
+ ).toBeNull();
});
});
diff --git a/web/src/features/experiments/lib/analytics.ts b/web/src/features/experiments/lib/analytics.ts
index 5f01464867..fc5fca53ea 100644
--- a/web/src/features/experiments/lib/analytics.ts
+++ b/web/src/features/experiments/lib/analytics.ts
@@ -1,3 +1,5 @@
+import { type ScoreComparisonOperator } from "@/src/features/experiments/fns/scoreComparisonFilter";
+
/**
* Experiments-UI PostHog payloads. Metadata only β€” counts, enums, booleans,
* field names. Never experiment/dataset names, score values, or item content.
@@ -7,7 +9,13 @@ type ExperimentTableName = "experiments" | "experiment-items";
const EXPERIMENT_ANALYTICS_IS_V4 = true as const;
-export type ExperimentComparisonSource = "picker" | "table-selection" | "url";
+export type ExperimentComparisonSource =
+ | "picker"
+ | "table-selection"
+ | "url"
+ /** The comparison the page selects for you when none is in the URL. Kept
+ * separate so it cannot make it look as though everyone started comparing. */
+ | "auto";
export type ExperimentBaselineSource = "picker" | "table-selection" | "clear";
export type ExperimentScoreScope = "trace" | "observation" | "experiment";
export type ExperimentChartMetricGroup = "base" | "score";
@@ -108,97 +116,180 @@ export function baselineChangedProps({
};
}
+/**
+ * `chart_metric_changed` β€” the metric behind the list's compact metric strip.
+ * Inherited from the four-slot chart grid this replaced, so the event name and
+ * its history carry over; `chartIndex`/`slotCount` are gone with the slots.
+ *
+ * The score's NAME is user content and never leaves the client β€” only which
+ * family of metric it is, at which score level, of which type. The level is the
+ * interesting half: trace-level is where an LLM-as-judge on a dataset run
+ * writes, and it only became selectable here at all in this change.
+ */
export function chartMetricChangedProps({
tableName,
metricId,
- chartIndex,
- slotCount,
}: {
tableName: ExperimentTableName;
metricId: string;
- chartIndex: number;
- slotCount: number;
}) {
return {
...experimentAnalyticsDimensions(tableName),
metricGroup: chartMetricGroup(metricId),
- chartIndex,
- slotCount,
+ ...scoreMetricShape(metricId),
};
}
-export function chartsSectionToggledProps({
+/** `<level>-score-<dataType>:<scoreName>` β€” the shape, never the name. */
+function scoreMetricShape(metricId: string): {
+ scoreLevel: "observation" | "trace" | "experiment" | "none";
+ dataType: "numeric" | "categorical" | "none";
+} {
+ const match = metricId.match(
+ /^(obs|trace|experiment)-score-(numeric|categorical):/,
+ );
+ if (!match) return { scoreLevel: "none", dataType: "none" };
+ const level =
+ match[1] === "obs" ? "observation" : (match[1] as "trace" | "experiment");
+ return { scoreLevel: level, dataType: match[2] as "numeric" | "categorical" };
+}
+
+export function scoreColumnScopeToggledProps({
tableName,
- isExpanded,
+ groupId,
+ enabledCount,
}: {
tableName: ExperimentTableName;
- isExpanded: boolean;
+ groupId: string;
+ enabledCount: number;
}) {
+ const scope = scoreColumnGroupScope(groupId);
+ if (!scope) return null;
return {
...experimentAnalyticsDimensions(tableName),
- isExpanded,
+ scope,
+ enabledCount,
};
}
-export function analyticsTabOpenedProps({
+/**
+ * `item_regression_filter_applied` β€” "keep the items whose score got worse than
+ * the comparison". Inherited from the event that measured retargeting an item
+ * filter at another run, because it answers the same question with a filter
+ * that can finally express it.
+ *
+ * `comparisonIndex` is the filter's target position among the selected
+ * comparisons, so a payload never carries an experiment id. There is no
+ * `column`: on this surface the filtered column IS a score, and a score name is
+ * user content β€” `scoreLevel` + `dataType` describe it instead.
+ */
+export function itemRegressionFilterAppliedProps({
tableName,
- hasComparison,
+ scoreLevel,
+ dataType,
+ operator,
+ comparisonExperimentId,
+ comparisonIds,
+ source,
}: {
tableName: ExperimentTableName;
- hasComparison: boolean;
+ scoreLevel: ExperimentScoreComparisonLevel;
+ dataType: string | undefined;
+ operator: ExperimentScoreComparisonOperator;
+ comparisonExperimentId: string;
+ comparisonIds: string[];
+ source: ExperimentScoreComparisonSource;
}) {
+ const comparisonIndex = comparisonIds.indexOf(comparisonExperimentId);
+ // The filter points at a run that is not among the compared ones, which a
+ // shared URL can outlive. The table treats such a filter as inactive, so
+ // there is no applied filter to report and an out-of-range index would only
+ // be noise: the caller skips the event.
+ if (comparisonIndex < 0) return null;
+
return {
...experimentAnalyticsDimensions(tableName),
- hasComparison,
+ scoreLevel,
+ dataType: dataType ?? "unknown",
+ operator,
+ comparisonIndex,
+ source,
};
}
-export function scoreColumnScopeToggledProps({
+/** Which score family the filtered column belongs to. */
+export type ExperimentScoreComparisonLevel = "observation" | "trace";
+
+/** `differs` reads as an implementation word; the tracking plan says `different`. */
+export type ExperimentScoreComparisonOperator =
+ | "lower"
+ | "higher"
+ | "different";
+
+/** The domain operator as PostHog sees it. */
+export const COMPARISON_OPERATOR_PROPERTY: Record<
+ ScoreComparisonOperator,
+ ExperimentScoreComparisonOperator
+> = {
+ lower: "lower",
+ higher: "higher",
+ differs: "different",
+};
+
+/** A deliberate pick from the score header's menu, or a shared URL. */
+export type ExperimentScoreComparisonSource = "header_menu" | "url";
+
+/**
+ * `layout_changed` β€” is the transposed score matrix adopted? Captured on the
+ * menu pick rather than on the URL state, which also changes on navigation and
+ * on a restored view.
+ */
+export function layoutChangedProps({
tableName,
- groupId,
- enabledCount,
+ layout,
+ comparisonCount,
}: {
tableName: ExperimentTableName;
- groupId: string;
- enabledCount: number;
+ layout: string;
+ comparisonCount: number;
}) {
- const scope = scoreColumnGroupScope(groupId);
- if (!scope) return null;
return {
...experimentAnalyticsDimensions(tableName),
- scope,
- enabledCount,
+ layout,
+ comparisonCount,
};
}
-export function itemRegressionFilterAppliedProps({
+/** `diff_mode_changed` β€” and is `Expected β†’ Output` used at all? */
+export function diffModeChangedProps({
tableName,
- column,
- operator,
- toExperimentId,
- baselineId,
- comparisonIds,
+ mode,
+ comparisonCount,
}: {
tableName: ExperimentTableName;
- column: string;
- operator: string;
- toExperimentId: string;
- baselineId: string | undefined;
- comparisonIds: string[];
-}): {
- isV4: true;
+ mode: string;
+ comparisonCount: number;
+}) {
+ return {
+ ...experimentAnalyticsDimensions(tableName),
+ mode,
+ comparisonCount,
+ };
+}
+
+/**
+ * `auto_comparison_preference_changed` β€” the escape hatch on the auto-selected
+ * comparison. Turning it off is the signal that the default guessed wrong.
+ */
+export function autoComparisonPreferenceChangedProps({
+ tableName,
+ isEnabled,
+}: {
tableName: ExperimentTableName;
- column: string;
- comparisonIndex: number;
- operator: string;
-} | null {
- if (!baselineId || toExperimentId === baselineId) return null;
- const comparisonIndex = comparisonIds.indexOf(toExperimentId);
- if (comparisonIndex < 0) return null;
+ isEnabled: boolean;
+}) {
return {
...experimentAnalyticsDimensions(tableName),
- column,
- comparisonIndex,
- operator,
+ isEnabled,
};
}
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 8e24764d6a..fda0cdb705 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -292,14 +292,23 @@ const events = {
// Experiments UI (v4). Metadata only β€” counts/enums/booleans/field names;
// never experiment or dataset names, score values, or item content.
// `isV4` + `tableName` on every event. `source` on comparison/baseline
- // distinguishes picker vs table-selection vs url (deep link / redirect).
+ // distinguishes picker vs table-selection vs url (deep link / redirect) vs
+ // auto β€” so the auto-selected comparison stays out of "users who compare".
+ //
+ // Two events from the original plan went away with the surfaces they
+ // measured: `analytics_tab_opened` (the Analytics route is
+ // deleted) and `charts_section_toggled` (the charts accordion is replaced by
+ // an always-on metric strip). `chart_metric_changed` now belongs to that
+ // strip and `item_regression_filter_applied` to the score-comparison filter:
+ // same question, same name, so the event history stays continuous.
experiment: [
"comparison_changed",
"comparison_picker_opened",
"baseline_changed",
+ "auto_comparison_preference_changed",
"chart_metric_changed",
- "charts_section_toggled",
- "analytics_tab_opened",
+ "layout_changed",
+ "diff_mode_changed",
"score_column_scope_toggled",
"item_regression_filter_applied",
],
diff --git a/web/src/features/prompts/components/prompts-table.tsx b/web/src/features/prompts/components/prompts-table.tsx
index 7947b42471..786152ccf8 100644
--- a/web/src/features/prompts/components/prompts-table.tsx
+++ b/web/src/features/prompts/components/prompts-table.tsx
@@ -419,6 +419,7 @@ export function PromptTable() {
/>
)}
<DataTableToolbar
+ tableName="prompts"
columns={promptColumns}
filterState={queryFilter.filterState}
columnsWithCustomSelect={["labels", "tags"]}
diff --git a/web/src/features/rbac/components/MembersTable.tsx b/web/src/features/rbac/components/MembersTable.tsx
index 3a1e98e1e3..de75fe7f38 100644
--- a/web/src/features/rbac/components/MembersTable.tsx
+++ b/web/src/features/rbac/components/MembersTable.tsx
@@ -413,6 +413,7 @@ export function MembersTable({
return (
<>
<DataTableToolbar
+ tableName="members"
columns={columns}
columnVisibility={columnVisibility}
setColumnVisibility={setColumnVisibility}
diff --git a/web/src/features/rbac/components/MembershipInvitesPage.tsx b/web/src/features/rbac/components/MembershipInvitesPage.tsx
index a46da0c60a..c93e47e08c 100644
--- a/web/src/features/rbac/components/MembershipInvitesPage.tsx
+++ b/web/src/features/rbac/components/MembershipInvitesPage.tsx
@@ -173,7 +173,7 @@ export function MembershipInvitesPage({
<>
{/* Header included in order to hide it when there are not invites yet */}
<Header title="Membership Invites" />
- <DataTableToolbar columns={columns} />
+ <DataTableToolbar columns={columns} tableName="membership-invites" />
<DataTable
tableName="membershipInvites"
columns={columns}
diff --git a/web/src/pages/project/[projectId]/experiments/results.tsx b/web/src/pages/project/[projectId]/experiments/results.tsx
index fe55ded5c0..bc93a4761a 100644
--- a/web/src/pages/project/[projectId]/experiments/results.tsx
+++ b/web/src/pages/project/[projectId]/experiments/results.tsx
@@ -8,13 +8,22 @@ import {
OverviewPanelToggle,
} from "@/src/components/layouts/overview-panel";
import useSessionStorage from "@/src/components/useSessionStorage";
-import { useExperimentResultsState } from "@/src/features/experiments/hooks/useExperimentResultsState";
-import { useEffect } from "react";
+import { useCallback, useEffect } from "react";
import { ExperimentDisplaySettings } from "@/src/features/experiments/components/ExperimentDisplaySettings";
import { useExperimentAccess } from "@/src/features/experiments/hooks/useExperimentAccess";
import Spinner from "@/src/components/design-system/Spinner/Spinner";
import { ExperimentSelectionControls } from "@/src/features/experiments/components/ExperimentSelectionControls";
import { useIoRenderModeLocalStorage } from "@/src/components/table/data-table-io-render-mode-switch";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import {
+ diffModeChangedProps,
+ layoutChangedProps,
+} from "@/src/features/experiments/lib/analytics";
+import {
+ useExperimentResultsState,
+ type ExperimentDiffMode,
+ type ExperimentResultsLayout,
+} from "@/src/features/experiments/hooks/useExperimentResultsState";
export default function ExperimentResults() {
const router = useRouter();
@@ -45,6 +54,46 @@ export default function ExperimentResults() {
false,
);
+ const capture = usePostHogClientCapture();
+
+ // Is the new score-matrix layout adopted, and diff mode β€” Expected β†’ Output
+ // in particular β€” used at all? Captured on the menu pick rather than on the
+ // URL state, which also changes on navigation, on a restored view, and (for
+ // the layout) as a side effect of choosing Expected β†’ Output.
+ const handleLayoutChange = useCallback(
+ (newLayout: ExperimentResultsLayout) => {
+ if (newLayout !== layout) {
+ capture(
+ "experiment:layout_changed",
+ layoutChangedProps({
+ tableName: "experiment-items",
+ layout: newLayout,
+ comparisonCount: comparisonIds.length,
+ }),
+ );
+ }
+ setLayout(newLayout);
+ },
+ [capture, layout, comparisonIds.length, setLayout],
+ );
+
+ const handleDiffModeChange = useCallback(
+ (newDiffMode: ExperimentDiffMode) => {
+ if (newDiffMode !== diffMode) {
+ capture(
+ "experiment:diff_mode_changed",
+ diffModeChangedProps({
+ tableName: "experiment-items",
+ mode: newDiffMode,
+ comparisonCount: comparisonIds.length,
+ }),
+ );
+ }
+ setDiffMode(newDiffMode);
+ },
+ [capture, diffMode, comparisonIds.length, setDiffMode],
+ );
+
const { isExperimentsBetaActive, isInitializing } = useExperimentAccess();
useEffect(() => {
@@ -99,9 +148,9 @@ export default function ExperimentResults() {
<>
<ExperimentDisplaySettings
layout={layout}
- onLayoutChange={setLayout}
+ onLayoutChange={handleLayoutChange}
diffMode={diffMode}
- onDiffModeChange={setDiffMode}
+ onDiffModeChange={handleDiffModeChange}
itemVisibility={itemVisibility}
onItemVisibilityChange={setItemVisibility}
hasComparisons={comparisonIds.length > 0}
diff --git a/web/src/pages/project/[projectId]/prompts/metrics.tsx b/web/src/pages/project/[projectId]/prompts/metrics.tsx
index 3d7138369f..98d1f8856e 100644
--- a/web/src/pages/project/[projectId]/prompts/metrics.tsx
+++ b/web/src/pages/project/[projectId]/prompts/metrics.tsx
@@ -410,6 +410,7 @@ export default function PromptVersionTable({
)}
<div className="gap-3">
<DataTableToolbar
+ tableName="prompt-versions"
columns={columns}
timeRange={showControlsInPageHeader ? undefined : timeRange}
setTimeRange={showControlsInPageHeader ? undefined : setTimeRange}