Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): rebuild the results grid around the score columns (#16917)

Nikita Kabardinafde5387c9a6
diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts
index 48b653a255..0260649914 100644
--- a/packages/shared/src/constants.ts
+++ b/packages/shared/src/constants.ts
@@ -7,3 +7,11 @@ export enum ModelUsageUnit {
Images = "IMAGES",
Requests = "REQUESTS",
}
+
+/**
+ * How many characters of an experiment item's input / output / expected output
+ * the experiments table reads. Shared because a consumer of those fields has to
+ * be able to tell a value that was cut off from one that is complete: a string
+ * this long may be a prefix, so two of them being equal proves nothing.
+ */
+export const EXPERIMENT_IO_TRUNCATE_LENGTH = 1000;
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.ts
index 84abab63fb..d1fb90fb1b 100644
--- a/packages/shared/src/server/repositories/experiments.ts
+++ b/packages/shared/src/server/repositories/experiments.ts
@@ -1,3 +1,4 @@
+import { EXPERIMENT_IO_TRUNCATE_LENGTH } from "../../constants";
import { matchesUiColumnMapping } from "../../tableDefinitions";
import { env } from "../../env";
import { type ScoreSourceType } from "../../domain";
@@ -1367,8 +1368,6 @@ export const getExperimentItemsFromEvents = async (
// Batch IO Queries
// ============================================================================
-const IO_TRUNCATE_LENGTH = 1000;
-
/**
* Output data for a single experiment.
*/
@@ -1390,7 +1389,7 @@ export type ExperimentItemBatchIO = {
/**
* Get batch IO data for experiment items.
* Returns input/expectedOutput from base experiment, and output from all experiments.
- * All text fields are truncated to IO_TRUNCATE_LENGTH characters.
+ * All text fields are truncated to EXPERIMENT_IO_TRUNCATE_LENGTH characters.
*/
export const getExperimentItemsBatchIO = async (props: {
projectId: string;
@@ -1436,7 +1435,7 @@ export const getExperimentItemsBatchIO = async (props: {
query,
params: {
...params,
- truncateLength: IO_TRUNCATE_LENGTH,
+ truncateLength: EXPERIMENT_IO_TRUNCATE_LENGTH,
},
tags: { projectId },
preferredClickhouseService: "EventsReadOnly",
diff --git a/web/src/components/scores-table-cell.tsx b/web/src/components/scores-table-cell.tsx
index e13cf3941b..c58427304c 100644
--- a/web/src/components/scores-table-cell.tsx
+++ b/web/src/components/scores-table-cell.tsx
@@ -54,11 +54,19 @@ export const ScoresTableCell = ({
displayFormat,
wrap = true,
hasMetadata,
+ valueTitle,
}: {
aggregate: AggregatedScoreData;
displayFormat: "smart" | "aggregate";
wrap?: boolean;
hasMetadata?: boolean;
+ /**
+ * What the value belongs to, prefixed onto its hover title β€” for a table
+ * where one column holds several rows' values and the value alone does not
+ * say whose it is (the experiment comparison). Omitted everywhere else, and
+ * the title is then the value, unchanged.
+ */
+ valueTitle?: string;
}) => {
const projectId = useProjectIdFromURL();
const [copied, setCopied] = React.useState(false);
@@ -85,7 +93,10 @@ export const ScoresTableCell = ({
COLOR_MAP.get(value),
)}
>
- <span className="truncate" title={value}>
+ <span
+ className="truncate"
+ title={valueTitle ? `${valueTitle}: ${value}` : value}
+ >
{value}
</span>
{aggregate.comment && (
@@ -94,7 +105,12 @@ export const ScoresTableCell = ({
<MessageCircleMore size={12} />
</HoverCardTrigger>
<HoverCardContent className="flex flex-col p-0 text-xs break-normal whitespace-normal">
- <div className="bg-popover sticky top-0 z-10 flex h-8 items-center justify-end px-1">
+ {/* Name what the icon opened: a bare block of text next to a
+ score does not say it is the score's comment. */}
+ <div className="bg-popover sticky top-0 z-10 flex h-8 items-center justify-between px-1">
+ <span className="text-muted-foreground pl-1.5 text-[10px] font-bold uppercase">
+ Score comment
+ </span>
<Button
onClick={handleCopy}
variant="ghost"
diff --git a/web/src/components/table/data-table.tsx b/web/src/components/table/data-table.tsx
index a614449e83..a09bdb50b9 100644
--- a/web/src/components/table/data-table.tsx
+++ b/web/src/components/table/data-table.tsx
@@ -541,21 +541,39 @@ export function DataTable<TData extends object, TValue>({
}}
>
{header.isPlaceholder ? null : (
- <div className="flex items-center select-none">
- <span
- className="truncate leading-normal"
- title={getPlainTextFromReactNode(
- flexRender(
+ <div
+ className={cn(
+ "flex select-none",
+ columnDef.headerBlock
+ ? "items-start"
+ : "items-center",
+ )}
+ >
+ {columnDef.headerBlock ? (
+ // Opted out of the single truncated line, so a
+ // header can carry more than the column's name.
+ <div className="min-w-0 flex-1 leading-normal">
+ {flexRender(
header.column.columnDef.header,
header.getContext(),
- ),
- )}
- >
- {flexRender(
- header.column.columnDef.header,
- header.getContext(),
- )}
- </span>
+ )}
+ </div>
+ ) : (
+ <span
+ className="truncate leading-normal"
+ title={getPlainTextFromReactNode(
+ flexRender(
+ header.column.columnDef.header,
+ header.getContext(),
+ ),
+ )}
+ >
+ {flexRender(
+ header.column.columnDef.header,
+ header.getContext(),
+ )}
+ </span>
+ )}
{columnDef.headerTooltip && (
<DocPopup
description={
diff --git a/web/src/components/table/types.ts b/web/src/components/table/types.ts
index 338e34cde1..4717c4b332 100644
--- a/web/src/components/table/types.ts
+++ b/web/src/components/table/types.ts
@@ -12,6 +12,12 @@ declare module "@tanstack/react-table" {
description: React.ReactNode;
href?: string;
};
+ /**
+ * Render the header as a block instead of one truncated line, for a header
+ * that carries more than the column's name (e.g. a score column's
+ * aggregate over the rows in view).
+ */
+ headerBlock?: boolean;
/**
* Plain-text name of the column, for surfaces that want a label rather than
* the rendered header β€” the column picker. Only needed when `header` is not
diff --git a/web/src/features/column-visibility/hooks/useColumnOrder.ts b/web/src/features/column-visibility/hooks/useColumnOrder.ts
index eda4e0309e..ee0f4dea4e 100644
--- a/web/src/features/column-visibility/hooks/useColumnOrder.ts
+++ b/web/src/features/column-visibility/hooks/useColumnOrder.ts
@@ -46,13 +46,23 @@ function useColumnOrder<TData>(
});
// Apply any opt-in one-time migrations (e.g. repositioning a column whose
- // default slot changed). Each runs at most once, guarded by its versionKey.
+ // default slot changed).
+ //
+ // The flag is only set once the transform has nothing left to change: it is
+ // written synchronously while `setColumnOrder` lands a render later, so
+ // marking it as soon as the transform ran would let a repeated effect run
+ // (React re-invokes mount effects in development) read the pre-migration
+ // order back out of local storage and overwrite the result with it. Waiting
+ // for a no-op pass costs one extra application of an idempotent transform
+ // and converges instead.
migrations?.forEach((migration) => {
if (hasRunMigration(migration.versionKey)) return;
const migrated = migration.apply(finalColumnOrder);
if (migrated === null) return; // deferred, retry on a later render
+ const settled =
+ JSON.stringify(migrated) === JSON.stringify(finalColumnOrder);
finalColumnOrder = migrated;
- markMigrationRun(migration.versionKey);
+ if (settled) markMigrationRun(migration.versionKey);
});
// Compare the new order with the current order to avoid unnecessary updates
diff --git a/web/src/features/column-visibility/lib/one-time-migration.ts b/web/src/features/column-visibility/lib/one-time-migration.ts
index f8e2e8701a..1f61ed4b48 100644
--- a/web/src/features/column-visibility/lib/one-time-migration.ts
+++ b/web/src/features/column-visibility/lib/one-time-migration.ts
@@ -15,11 +15,13 @@ export type OneTimeMigration<T> = {
*/
versionKey: string;
/**
- * Pure transform applied to the reconciled state. Must not mutate its input.
- * Return the input unchanged to skip (the flag is still set so it does not
- * retry on every mount), or `null` to defer β€” for state that depends on
- * asynchronously loaded columns, so the migration is retried instead of
- * being consumed against incomplete state.
+ * Pure transform applied to the reconciled state. Must not mutate its input,
+ * and must be **idempotent**: it can be applied more than once before the
+ * migration is marked as run (`useColumnOrder` only sets the flag on a pass
+ * where the transform changes nothing, so its result cannot be lost to a
+ * repeated effect run). Return the input unchanged to skip, or `null` to
+ * defer β€” for state that depends on asynchronously loaded columns, so the
+ * migration is retried instead of being consumed against incomplete state.
*/
apply: (state: T) => T | null;
};
diff --git a/web/src/features/datasets/components/DiffLabel.tsx b/web/src/features/datasets/components/DiffLabel.tsx
index a7703010f0..11f84d285f 100644
--- a/web/src/features/datasets/components/DiffLabel.tsx
+++ b/web/src/features/datasets/components/DiffLabel.tsx
@@ -22,28 +22,51 @@ export function DiffLabel({
formatValue,
className,
preferNegativeDiff = false,
+ title,
}: {
diff: NumericDiff | CategoricalDiff;
formatValue: (value: number) => string;
className?: string;
preferNegativeDiff?: boolean;
+ /**
+ * What the chip means, on hover. `+0.07` and `a β†’ b` do not say which side
+ * is which; a caller that knows the two sides should say it
+ * (`describeRunComparison`). Falls back to the chip's own text.
+ */
+ title?: string;
}) {
if (diff.type === "NUMERIC") {
return (
<Badge
size="sm"
variant={getVariant(diff.direction, preferNegativeDiff)}
- className={cn("font-bold", className)}
+ // A number must never break across the badge's line box or give up
+ // width to a sibling: both render a fragment. If it cannot sit beside
+ // the value it qualifies, the caller's row wraps it whole.
+ className={cn("shrink-0 font-bold whitespace-nowrap", className)}
+ title={title}
>
{diff.direction}
{formatValue(diff.absoluteDifference)}
</Badge>
);
}
- if (diff.isDifferent)
+ if (diff.isDifferent) {
+ /* Name the move when both sides are a single value: a categorical
+ score going pass β†’ fail is a diff, just not a number. */
+ const move = diff.from && diff.to ? `${diff.from} β†’ ${diff.to}` : "Varies";
return (
- <Badge size="sm" variant="warning" className="font-bold">
- Varies
+ <Badge
+ size="sm"
+ variant="warning"
+ // A named move can be longer than the cell it sits in, and the value it
+ // qualifies matters more than the move does β€” so shrink and ellipsise
+ // here rather than clipping mid-word, and keep the full move on hover.
+ className={cn("min-w-0 truncate font-bold", className)}
+ title={title ?? move}
+ >
+ {move}
</Badge>
);
+ }
}
diff --git a/web/src/features/datasets/lib/calculateBaselineDiff.ts b/web/src/features/datasets/lib/calculateBaselineDiff.ts
index b564d8c41e..30336a0e90 100644
--- a/web/src/features/datasets/lib/calculateBaselineDiff.ts
+++ b/web/src/features/datasets/lib/calculateBaselineDiff.ts
@@ -10,6 +10,9 @@ export type NumericDiff = {
export type CategoricalDiff = {
type: "CATEGORICAL";
isDifferent: true;
+ /** The two values, when each side carries exactly one. */
+ from?: string;
+ to?: string;
};
export type BaselineDiff = NumericDiff | CategoricalDiff | null;
@@ -69,9 +72,13 @@ export function calculateScoreDiff(
// Same value β†’ no diff
if (currentValue === baselineValue) return null;
+ // A categorical score has no distance, so the diff is the move itself.
return {
type: "CATEGORICAL",
isDifferent: true,
+ ...(current.values.length === 1 && baseline.values.length === 1
+ ? { from: baselineValue, to: currentValue }
+ : {}),
};
}
diff --git a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx b/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
index 099b5020b6..fe6898ac0d 100644
--- a/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
+++ b/web/src/features/experiments/components/ExperimentDisplaySettings.tsx
@@ -10,10 +10,16 @@ import {
import { Button } from "@/src/components/ui/button";
import { Settings2, Check } from "lucide-react";
import { type IoRenderMode } from "@/src/components/table/data-table-io-render-mode-switch";
+import {
+ type ExperimentDiffMode,
+ type ExperimentResultsLayout,
+} from "@/src/features/experiments/hooks/useExperimentResultsState";
type ExperimentDisplaySettingsProps = {
- layout: "grid" | "list";
- onLayoutChange: (layout: "grid" | "list") => void;
+ layout: ExperimentResultsLayout;
+ onLayoutChange: (layout: ExperimentResultsLayout) => void;
+ diffMode: ExperimentDiffMode;
+ onDiffModeChange: (diffMode: ExperimentDiffMode) => void;
itemVisibility: "baseline-only" | "all";
onItemVisibilityChange: (visibility: "baseline-only" | "all") => void;
hasComparisons: boolean;
@@ -22,9 +28,33 @@ type ExperimentDisplaySettingsProps = {
onIoRenderModeChange: (mode: IoRenderMode) => void;
};
+/** A menu row that reads as a radio option. */
+const OptionItem = ({
+ selected,
+ disabled,
+ onSelect,
+ children,
+}: {
+ selected: boolean;
+ disabled?: boolean;
+ onSelect: () => void;
+ children: React.ReactNode;
+}) => (
+ <DropdownMenuItem onClick={onSelect} disabled={disabled}>
+ {selected ? (
+ <Check className="mr-2 h-4 w-4 shrink-0" />
+ ) : (
+ <span className="mr-2 h-4 w-4 shrink-0" />
+ )}
+ {children}
+ </DropdownMenuItem>
+);
+
export function ExperimentDisplaySettings({
layout,
onLayoutChange,
+ diffMode,
+ onDiffModeChange,
itemVisibility,
onItemVisibilityChange,
hasComparisons,
@@ -42,56 +72,76 @@ export function ExperimentDisplaySettings({
<span className="ml-2 hidden md:inline">Display</span>
</Button>
</DropdownMenuTrigger>
- <DropdownMenuContent align="end" className="w-56">
+ <DropdownMenuContent align="end" className="w-64">
<DropdownMenuLabel>Layout</DropdownMenuLabel>
- <DropdownMenuItem onClick={() => onLayoutChange("grid")}>
- {layout === "grid" && <Check className="mr-2 h-4 w-4" />}
- {layout !== "grid" && <span className="mr-2 h-4 w-4" />}
- Grid
- </DropdownMenuItem>
- <DropdownMenuItem onClick={() => onLayoutChange("list")}>
- {layout === "list" && <Check className="mr-2 h-4 w-4" />}
- {layout !== "list" && <span className="mr-2 h-4 w-4" />}
- List
- </DropdownMenuItem>
+ <OptionItem
+ selected={layout === "list"}
+ onSelect={() => onLayoutChange("list")}
+ >
+ Diff β€” one row per item
+ </OptionItem>
+ <OptionItem
+ selected={layout === "grid"}
+ onSelect={() => onLayoutChange("grid")}
+ >
+ Side by side β€” a column per experiment
+ </OptionItem>
+
+ <DropdownMenuSeparator />
+
+ <DropdownMenuLabel>Diff</DropdownMenuLabel>
+ <OptionItem
+ selected={diffMode === "comparison"}
+ onSelect={() => onDiffModeChange("comparison")}
+ >
+ Comparison β†’ Baseline
+ </OptionItem>
+ <OptionItem
+ selected={diffMode === "expected"}
+ onSelect={() => onDiffModeChange("expected")}
+ >
+ Expected β†’ Output
+ </OptionItem>
+ <OptionItem
+ selected={diffMode === "off"}
+ onSelect={() => onDiffModeChange("off")}
+ >
+ Off β€” values only
+ </OptionItem>
<DropdownMenuSeparator />
<DropdownMenuLabel>Item Visibility</DropdownMenuLabel>
- <DropdownMenuItem
- onClick={() => onItemVisibilityChange("baseline-only")}
+ <OptionItem
+ selected={itemVisibility === "baseline-only"}
disabled={isItemVisibilityDisabled}
+ onSelect={() => onItemVisibilityChange("baseline-only")}
>
- {itemVisibility === "baseline-only" && (
- <Check className="mr-2 h-4 w-4" />
- )}
- {itemVisibility !== "baseline-only" && (
- <span className="mr-2 h-4 w-4" />
- )}
Show only items in baseline
- </DropdownMenuItem>
- <DropdownMenuItem
- onClick={() => onItemVisibilityChange("all")}
+ </OptionItem>
+ <OptionItem
+ selected={itemVisibility === "all"}
disabled={isItemVisibilityDisabled}
+ onSelect={() => onItemVisibilityChange("all")}
>
- {itemVisibility === "all" && <Check className="mr-2 h-4 w-4" />}
- {itemVisibility !== "all" && <span className="mr-2 h-4 w-4" />}
Show all items
- </DropdownMenuItem>
+ </OptionItem>
<DropdownMenuSeparator />
<DropdownMenuLabel>Format</DropdownMenuLabel>
- <DropdownMenuItem onClick={() => onIoRenderModeChange("json")}>
- {ioRenderMode === "json" && <Check className="mr-2 h-4 w-4" />}
- {ioRenderMode !== "json" && <span className="mr-2 h-4 w-4" />}
+ <OptionItem
+ selected={ioRenderMode === "json"}
+ onSelect={() => onIoRenderModeChange("json")}
+ >
JSON
- </DropdownMenuItem>
- <DropdownMenuItem onClick={() => onIoRenderModeChange("text")}>
- {ioRenderMode === "text" && <Check className="mr-2 h-4 w-4" />}
- {ioRenderMode !== "text" && <span className="mr-2 h-4 w-4" />}
+ </OptionItem>
+ <OptionItem
+ selected={ioRenderMode === "text"}
+ onSelect={() => onIoRenderModeChange("text")}
+ >
Formatted
- </DropdownMenuItem>
+ </OptionItem>
</DropdownMenuContent>
</DropdownMenu>
);
diff --git a/web/src/features/experiments/components/table/ExperimentGridCell.clienttest.tsx b/web/src/features/experiments/components/table/ExperimentGridCell.clienttest.tsx
index 8936083315..0e901112a8 100644
--- a/web/src/features/experiments/components/table/ExperimentGridCell.clienttest.tsx
+++ b/web/src/features/experiments/components/table/ExperimentGridCell.clienttest.tsx
@@ -84,19 +84,27 @@ describe("ExperimentGridCell", () => {
expect(screen.getByText("correctness")).toBeInTheDocument();
});
- it("renders the latency row when no latency value is available", () => {
+ it("keeps cost and latency on the metadata line", () => {
renderGridCell(false, { output: false });
- expect(screen.getByText("Latency")).toBeInTheDocument();
+ // Neither is recorded in this fixture, so both render the affordance.
+ expect(screen.getAllByText("not recorded")).toHaveLength(2);
});
- it("renders output in a fixed-height scroll area", () => {
+ it("keeps the ids reachable behind the metadata line instead of listing them", () => {
+ renderGridCell(false, { output: false });
+
+ expect(screen.queryByText("item-id")).not.toBeInTheDocument();
+ expect(screen.getByRole("button", { name: "IDs" })).toBeInTheDocument();
+ });
+
+ it("lets the output section take the row's spare height", () => {
renderGridCell(false, { metadata: false });
const outputContent =
screen.getByText("Output").parentElement?.nextElementSibling;
- expect(outputContent).toHaveClass("h-16", "overflow-hidden");
+ expect(outputContent).toHaveClass("min-h-16", "flex-1", "overflow-hidden");
expect(outputContent?.firstElementChild).toBe(screen.getByText("IO cell"));
});
diff --git a/web/src/features/experiments/components/table/ExperimentGridCell.tsx b/web/src/features/experiments/components/table/ExperimentGridCell.tsx
index d9b45bc3f6..5813bb5248 100644
--- a/web/src/features/experiments/components/table/ExperimentGridCell.tsx
+++ b/web/src/features/experiments/components/table/ExperimentGridCell.tsx
@@ -1,3 +1,6 @@
+/* The experiment marker's colour is data β€” which run this is, out of the N
+ selected (`getExperimentColorStyles`) β€” so it arrives as a class rather than
+ as a variant, the same way `DiffLabel` takes one. */
/* eslint-disable @repo/no-style-props */
import { ConnectedIOTableCell } from "@/src/components/table/ConnectedIOTableCell";
import { Badge } from "@/src/components/ui/badge";
@@ -44,6 +47,7 @@ import { getPlainTextFromReactNode } from "@/src/utils/react-node-plain-text";
import Link from "next/link";
import { ScoreTag, type ScoreLevel } from "@/src/components/score-tag";
import { NotRecordedMetric } from "./NotRecordedMetric";
+import { describeRunComparison } from "@/src/features/experiments/fns/describeRunComparison";
type ExperimentGridCellProps = {
projectId: string;
@@ -64,8 +68,12 @@ type ExperimentGridCellProps = {
observationScoreOrder: string[];
traceScoreOrder: string[];
isBaseline: boolean;
+ /** Whether this cell carries deltas against the baseline (the diff mode). */
+ showDiff?: boolean;
baselineScores?: ScoreAggregate;
baselineTraceScores?: ScoreAggregate;
+ /** Named in every diff chip's hover sentence, so the arrow has a direction. */
+ baselineExperimentName?: string;
isLoading?: boolean;
columnVisibility?: VisibilityState;
markerClassName?: string;
@@ -91,6 +99,11 @@ type GridCellData = {
traceScores: ScoreAggregate;
scoreDiffs?: Record<string, BaselineDiff>;
traceScoreDiffs?: Record<string, BaselineDiff>;
+ baselineScores?: ScoreAggregate;
+ baselineTraceScores?: ScoreAggregate;
+ baselineTotalCost?: number | null;
+ baselineLatencyMs?: number | null;
+ baselineExperimentName?: string;
isLoading: boolean;
};
@@ -127,7 +140,12 @@ const ScoreCommentPeek = ({
</button>
</HoverCardTrigger>
<HoverCardContent className="flex flex-col p-0 text-xs break-normal whitespace-normal">
- <div className="bg-popover sticky top-0 z-10 flex h-8 items-center justify-end px-1">
+ {/* Name what the icon opened: a bare block of text next to a score
+ does not say it is the score's comment. */}
+ <div className="bg-popover sticky top-0 z-10 flex h-8 items-center justify-between px-1">
+ <span className="text-muted-foreground pl-1.5 text-[10px] font-bold uppercase">
+ Score comment
+ </span>
<Button
onClick={handleCopy}
variant="ghost"
@@ -211,6 +229,23 @@ const ScoreMetadataPeek = ({
);
};
+/**
+ * How one score reads in a cell: a categorical score's modal value, a numeric
+ * score's average. Shared with the hover sentence beside it, which has to
+ * quote the same value the cell shows.
+ */
+const scoreValueOf = (aggregate?: AggregatedScoreData | null): string => {
+ if (!aggregate) return "-";
+ if (aggregate.type === "CATEGORICAL") {
+ if (aggregate.valueCounts && aggregate.valueCounts.length > 0) {
+ return [...aggregate.valueCounts].sort((a, b) => b.count - a.count)[0]
+ .value;
+ }
+ return aggregate.values?.[0] ?? "-";
+ }
+ return aggregate.average !== undefined ? aggregate.average.toFixed(2) : "-";
+};
+
/**
* Simple score display component for grid cells.
* Shows only the score name, with source and type on hover.
@@ -218,6 +253,8 @@ const ScoreMetadataPeek = ({
const ScoreItem = ({
scoreKey,
aggregate,
+ baselineAggregate,
+ baselineExperimentName,
diff,
projectId,
level,
@@ -225,6 +262,8 @@ const ScoreItem = ({
}: {
scoreKey: string;
aggregate: AggregatedScoreData | null;
+ baselineAggregate?: AggregatedScoreData | null;
+ baselineExperimentName?: string;
diff?: BaselineDiff | null;
projectId: string;
level: Extract<ScoreLevel, "observation" | "trace">;
@@ -233,26 +272,27 @@ const ScoreItem = ({
// Decompose the key to get name, source, and dataType
const { name, source, dataType } = decomposeAggregateScoreKey(scoreKey);
- let displayValue = "-";
- if (aggregate) {
- if (aggregate.type === "CATEGORICAL") {
- if (aggregate.valueCounts && aggregate.valueCounts.length > 0) {
- const sorted = [...aggregate.valueCounts].sort(
- (a, b) => b.count - a.count,
- );
- displayValue = sorted[0].value;
- } else if (aggregate.values && aggregate.values.length > 0) {
- displayValue = aggregate.values[0];
- }
- } else {
- displayValue =
- aggregate.average !== undefined ? aggregate.average.toFixed(2) : "-";
- }
- }
+ const displayValue = scoreValueOf(aggregate);
const hasComment = aggregate?.comment;
const hasMetadata = aggregate?.hasMetadata && aggregate?.id;
+ // `true β†’ false` and `+0.07` do not say which side is the baseline.
+ const diffTitle = diff
+ ? describeRunComparison({
+ baselineName: baselineExperimentName,
+ ...(diff.type === "CATEGORICAL"
+ ? {
+ baselineText: diff.from ?? "several values",
+ currentText: diff.to ?? "several values",
+ }
+ : {
+ baselineText: scoreValueOf(baselineAggregate),
+ currentText: scoreValueOf(aggregate),
+ }),
+ })
+ : undefined;
+
return (
<div className="flex items-center justify-between gap-1 text-xs">
<div className="flex max-w-[50%] min-w-0 items-center gap-1">
@@ -286,25 +326,39 @@ const ScoreItem = ({
</HoverCardContent>
</HoverCard>
</div>
- <div className="flex items-center gap-1">
- {displayValue === "-" ? (
- <span className="text-muted-foreground text-xs">-</span>
- ) : (
- <Badge variant="secondary" className="text-xs">
- {displayValue}
- </Badge>
- )}
- {hasComment && (
- <ScoreCommentPeek
- comment={aggregate.comment!}
- executionTraceId={aggregate.executionTraceId}
- projectId={projectId}
+ {/* The value never gives up width for the move chip beside it: a
+ clipped value reads as data, so the move wraps under it instead. */}
+ <div className="flex min-w-0 flex-wrap items-center justify-end gap-x-1 gap-y-0.5">
+ <span className="flex max-w-full shrink-0 items-center gap-1">
+ {displayValue === "-" ? (
+ <span className="text-muted-foreground text-xs">-</span>
+ ) : (
+ <Badge
+ variant="secondary"
+ className="min-w-0 truncate text-xs"
+ title={displayValue}
+ >
+ {displayValue}
+ </Badge>
+ )}
+ {hasComment && (
+ <ScoreCommentPeek
+ comment={aggregate.comment!}
+ executionTraceId={aggregate.executionTraceId}
+ projectId={projectId}
+ />
+ )}
+ {hasMetadata && (
+ <ScoreMetadataPeek scoreId={aggregate.id!} projectId={projectId} />
+ )}
+ </span>
+ {diff && (
+ <DiffLabel
+ diff={diff}
+ formatValue={(v) => v.toFixed(2)}
+ title={diffTitle}
/>
)}
- {hasMetadata && (
- <ScoreMetadataPeek scoreId={aggregate.id!} projectId={projectId} />
- )}
- {diff && <DiffLabel diff={diff} formatValue={(v) => v.toFixed(2)} />}
</div>
</div>
);
@@ -322,6 +376,12 @@ const getScoreRowDefinition = (
aggregate={
level === "trace" ? data.traceScores[scoreKey] : data.scores[scoreKey]
}
+ baselineAggregate={
+ level === "trace"
+ ? data.baselineTraceScores?.[scoreKey]
+ : data.baselineScores?.[scoreKey]
+ }
+ baselineExperimentName={data.baselineExperimentName}
diff={
level === "trace"
? data.traceScoreDiffs?.[scoreKey]
@@ -355,6 +415,212 @@ const MetadataItem = ({
</div>
);
+/** Metadata fields that used to be a line each inside the cell. */
+const METADATA_KEYS = [
+ "totalCost",
+ "latencyMs",
+ "level",
+ "itemId",
+ "observationId",
+ "traceId",
+ "startTime",
+] as const;
+
+const isMetadataFieldVisible = (
+ columnVisibility: VisibilityState,
+ key: (typeof METADATA_KEYS)[number],
+) => columnVisibility[key] !== false;
+
+const hasVisibleCellMetadata = (columnVisibility: VisibilityState) =>
+ METADATA_KEYS.some((key) => isMetadataFieldVisible(columnVisibility, key));
+
+/** The footer fields that live behind the identifiers hover card. */
+const IDENTIFIER_KEYS = [
+ "itemId",
+ "observationId",
+ "traceId",
+ "startTime",
+] as const;
+
+const hasVisibleCellIdentifiers = (columnVisibility: VisibilityState) =>
+ IDENTIFIER_KEYS.some((key) => isMetadataFieldVisible(columnVisibility, key));
+
+/**
+ * The item's identifiers, one hover away. They are the same on every row of a
+ * run and the peek view already carries them, so they don't earn a line each in
+ * a cell whose payload is the output β€” but they stay reachable here, including
+ * the link out to the execution trace.
+ */
+const CellIdentifiers = ({
+ data,
+ columnVisibility,
+}: {
+ data: GridCellData;
+ columnVisibility: VisibilityState;
+}) => {
+ const visible = (key: (typeof METADATA_KEYS)[number]) =>
+ isMetadataFieldVisible(columnVisibility, key);
+
+ const rows: Array<{ label: string; value: React.ReactNode }> = [
+ ...(visible("itemId")
+ ? [
+ {
+ label: "Item ID",
+ value: <span className="font-mono text-xs">{data.itemId}</span>,
+ },
+ ]
+ : []),
+ ...(visible("observationId")
+ ? [
+ {
+ label: "Observation",
+ value: (
+ <span className="font-mono text-xs">{data.observationId}</span>
+ ),
+ },
+ ]
+ : []),
+ ...(visible("traceId")
+ ? [
+ {
+ label: "Execution Trace",
+ value: (
+ <Link
+ href={`/project/${encodeURIComponent(data.projectId)}/traces/${encodeURIComponent(data.traceId)}?observation=${encodeURIComponent(data.observationId)}`}
+ className="text-primary inline-flex max-w-full items-center gap-1 hover:underline"
+ onClick={(event) => event.stopPropagation()}
+ >
+ <span
+ className="truncate font-mono text-xs"
+ title={data.traceId}
+ >
+ {data.traceId}
+ </span>
+ <ExternalLink className="h-3 w-3 shrink-0" />
+ </Link>
+ ),
+ },
+ ]
+ : []),
+ ...(visible("startTime")
+ ? [
+ {
+ label: "Start Time",
+ value: (
+ <span
+ className="text-xs"
+ title={
+ buildLocalIsoDatePresentation({ date: data.startTime })?.title
+ }
+ >
+ {
+ buildLocalIsoDatePresentation({ date: data.startTime })
+ ?.display
+ }
+ </span>
+ ),
+ },
+ ]
+ : []),
+ ];
+
+ return (
+ <HoverCard>
+ <HoverCardTrigger asChild>
+ <button
+ type="button"
+ className="text-muted-foreground hover:text-foreground cursor-default text-[10px] font-bold uppercase underline decoration-dotted"
+ onClick={(event) => event.stopPropagation()}
+ >
+ IDs
+ </button>
+ </HoverCardTrigger>
+ <HoverCardContent
+ side="left"
+ align="start"
+ className="flex w-auto max-w-96 flex-col gap-1 p-2"
+ >
+ {rows.map((row) => (
+ <MetadataItem key={row.label} label={row.label}>
+ {row.value}
+ </MetadataItem>
+ ))}
+ </HoverCardContent>
+ </HoverCard>
+ );
+};
+
+/**
+ * One line of per-item metadata under the scores: the two metrics, the status,
+ * and the identifiers behind a hover. Replaces the five-line labelled block
+ * that made a comparison row ~380px of chrome around a 64px output.
+ */
+const CellMetadataFooter = ({
+ data,
+ columnVisibility,
+}: {
+ data: GridCellData;
+ columnVisibility: VisibilityState;
+}) => {
+ const visible = (key: (typeof METADATA_KEYS)[number]) =>
+ isMetadataFieldVisible(columnVisibility, key);
+
+ return (
+ <div className="text-muted-foreground flex flex-wrap items-center gap-x-2 gap-y-0.5 text-xs">
+ {visible("totalCost") && (
+ // Same rule as the diff layout's metric columns: the delta wraps
+ // whole rather than being clipped mid-number.
+ <span className="inline-flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5">
+ {data.totalCost ? (
+ usdFormatter(data.totalCost, 2, 6)
+ ) : (
+ <NotRecordedMetric metric="cost" />
+ )}
+ {data.totalCostDiff && (
+ <DiffLabel
+ diff={data.totalCostDiff}
+ preferNegativeDiff
+ formatValue={(value) => usdFormatter(value, 2, 6)}
+ title={describeRunComparison({
+ baselineName: data.baselineExperimentName,
+ baselineText: usdFormatter(data.baselineTotalCost ?? 0, 2, 6),
+ currentText: usdFormatter(data.totalCost ?? 0, 2, 6),
+ verb: "cost",
+ })}
+ />
+ )}
+ </span>
+ )}
+ {visible("latencyMs") && (
+ <span className="inline-flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5">
+ {data.latencyMs != null ? (
+ latencyFormatter(data.latencyMs)
+ ) : (
+ <NotRecordedMetric metric="latency" />
+ )}
+ {data.latencyDiff && (
+ <DiffLabel
+ diff={data.latencyDiff}
+ preferNegativeDiff
+ formatValue={(value) => latencyFormatter(value)}
+ title={describeRunComparison({
+ baselineName: data.baselineExperimentName,
+ baselineText: latencyFormatter(data.baselineLatencyMs ?? 0),
+ currentText: latencyFormatter(data.latencyMs ?? 0),
+ verb: "took",
+ })}
+ />
+ )}
+ </span>
+ )}
+ {visible("level") && <span>{data.level}</span>}
+ {hasVisibleCellIdentifiers(columnVisibility) && (
+ <CellIdentifiers data={data} columnVisibility={columnVisibility} />
+ )}
+ </div>
+ );
+};
+
/**
* Renders a group section with header and content
*/
@@ -362,12 +628,23 @@ const GroupSection = ({
header,
children,
markerClassName,
+ grow = false,
}: {
header?: string;
children: React.ReactNode;
markerClassName?: string;
+ /** Take the cell's spare vertical space instead of hugging its content. */
+ grow?: boolean;
}) => (
- <div className="flex shrink-0 flex-col gap-1 px-2 py-1.5">
+ <div
+ className={cn(
+ "flex flex-col gap-1 px-2 py-1.5",
+ // `grow` takes the spare space but keeps its content's height as its
+ // base, so a cell with no spare space still shows the output rather than
+ // collapsing to its label and letting the output spill over the scores.
+ grow ? "shrink-0 grow" : "shrink-0",
+ )}
+ >
{header && (
<div className="flex items-center gap-1.5">
{markerClassName !== undefined && (
@@ -409,8 +686,10 @@ export const ExperimentGridCell = ({
observationScoreOrder,
traceScoreOrder,
isBaseline,
+ showDiff = true,
baselineScores,
baselineTraceScores,
+ baselineExperimentName,
isLoading = false,
columnVisibility = {},
markerClassName,
@@ -418,29 +697,33 @@ export const ExperimentGridCell = ({
}: ExperimentGridCellProps) => {
const scoreDiffs = useMemo(
() =>
- isBaseline || !baselineScores
+ !showDiff || isBaseline || !baselineScores
? undefined
: computeScoreDiffs(scores, baselineScores),
- [scores, baselineScores, isBaseline],
+ [scores, baselineScores, isBaseline, showDiff],
);
const traceScoreDiffs = useMemo(
() =>
- isBaseline || !baselineTraceScores
+ !showDiff || isBaseline || !baselineTraceScores
? undefined
: computeScoreDiffs(traceScores, baselineTraceScores),
- [traceScores, baselineTraceScores, isBaseline],
+ [traceScores, baselineTraceScores, isBaseline, showDiff],
);
const totalCostDiff = useMemo(
() =>
- isBaseline ? null : calculateNumericDiff(totalCost, baselineTotalCost),
- [baselineTotalCost, isBaseline, totalCost],
+ !showDiff || isBaseline
+ ? null
+ : calculateNumericDiff(totalCost, baselineTotalCost),
+ [baselineTotalCost, isBaseline, showDiff, totalCost],
);
const latencyDiff = useMemo(
() =>
- isBaseline ? null : calculateNumericDiff(latencyMs, baselineLatencyMs),
- [baselineLatencyMs, isBaseline, latencyMs],
+ !showDiff || isBaseline
+ ? null
+ : calculateNumericDiff(latencyMs, baselineLatencyMs),
+ [baselineLatencyMs, isBaseline, latencyMs, showDiff],
);
const orderedObservationKeys = useMemo(
@@ -475,6 +758,11 @@ export const ExperimentGridCell = ({
traceScores,
scoreDiffs,
traceScoreDiffs,
+ baselineScores,
+ baselineTraceScores,
+ baselineTotalCost,
+ baselineLatencyMs,
+ baselineExperimentName,
isLoading,
};
@@ -519,115 +807,22 @@ export const ExperimentGridCell = ({
: []),
],
},
- // Metadata group - itemId, observationId, level, startTime
- {
- accessorKey: "metadata",
- header: "Metadata",
- children: [
- {
- accessorKey: "itemId",
- cell: ({ data }) => (
- <MetadataItem label="Item ID">
- <span className="font-mono text-xs">{data.itemId}</span>
- </MetadataItem>
- ),
- },
- {
- accessorKey: "observationId",
- cell: ({ data }) => (
- <MetadataItem label="Observation">
- <span className="font-mono text-xs">{data.observationId}</span>
- </MetadataItem>
- ),
- },
- {
- accessorKey: "traceId",
- cell: ({ data }) => (
- <MetadataItem label="Execution Trace">
- <Link
- href={`/project/${encodeURIComponent(data.projectId)}/traces/${encodeURIComponent(data.traceId)}?observation=${encodeURIComponent(data.observationId)}`}
- className="text-primary inline-flex max-w-full items-center gap-1 hover:underline"
- onClick={(event) => event.stopPropagation()}
- >
- <span
- className="truncate font-mono text-xs"
- title={data.traceId}
- >
- {data.traceId}
- </span>
- <ExternalLink className="h-3 w-3 shrink-0" />
- </Link>
- </MetadataItem>
- ),
- },
- {
- accessorKey: "level",
- cell: ({ data }) => (
- <MetadataItem label="Status">
- <span className="text-xs">{data.level}</span>
- </MetadataItem>
- ),
- },
- {
- accessorKey: "startTime",
- cell: ({ data }) => {
- const preparedDate = buildLocalIsoDatePresentation({
- date: data.startTime,
- });
-
- return (
- <MetadataItem label="Start Time">
- <span className="text-xs" title={preparedDate?.title}>
- {preparedDate?.display}
- </span>
- </MetadataItem>
- );
+ // One compact line, replacing the five labelled metadata rows. The ids
+ // moved into its hover card; cost and latency stay visible because they
+ // are what a reader compares between runs.
+ ...(hasVisibleCellMetadata(columnVisibility)
+ ? ([
+ {
+ accessorKey: "metadata",
+ cell: ({ data }) => (
+ <CellMetadataFooter
+ data={data}
+ columnVisibility={columnVisibility}
+ />
+ ),
},
- },
- {
- accessorKey: "totalCost",
- cell: ({ data }) => (
- <MetadataItem label="Total Cost">
- <span className="inline-flex items-center gap-1 text-xs">
- {data.totalCost ? (
- usdFormatter(data.totalCost, 2, 6)
- ) : (
- <NotRecordedMetric metric="cost" />
- )}
- {data.totalCostDiff && (
- <DiffLabel
- diff={data.totalCostDiff}
- preferNegativeDiff
- formatValue={(value) => usdFormatter(value, 2, 6)}
- />
- )}
- </span>
- </MetadataItem>
- ),
- },
- {
- accessorKey: "latencyMs",
- cell: ({ data }) => (
- <MetadataItem label="Latency">
- <span className="inline-flex items-center gap-1 text-xs">
- {data.latencyMs != null ? (
- latencyFormatter(data.latencyMs)
- ) : (
- <NotRecordedMetric metric="latency" />
- )}
- {data.latencyDiff && (
- <DiffLabel
- diff={data.latencyDiff}
- preferNegativeDiff
- formatValue={(value) => latencyFormatter(value)}
- />
- )}
- </span>
- </MetadataItem>
- ),
- },
- ],
- },
+ ] satisfies CellRowDef<GridCellData>[])
+ : []),
],
[
columnVisibility,
@@ -668,15 +863,18 @@ export const ExperimentGridCell = ({
const isFirst = index === 0;
const isLast = index === sectionsToRender.length - 1;
- // Output section - special handling for IOTableCell
+ // Output section - special handling for ConnectedIOTableCell. It is the
+ // one section that grows, so a taller row shows more output rather than
+ // more chrome.
if (row.accessorKey === "output" && row.cell) {
return (
<Fragment key={row.accessorKey}>
<GroupSection
header={row.header}
markerClassName={isFirst ? markerClassName : undefined}
+ grow
>
- <div className="h-16 overflow-hidden">
+ <div className="min-h-16 flex-1 overflow-hidden">
{row.cell({ data: cellData })}
</div>
</GroupSection>
@@ -706,6 +904,21 @@ export const ExperimentGridCell = ({
);
}
+ // Sections that render one cell of their own (the metadata footer).
+ if (row.cell) {
+ return (
+ <Fragment key={row.accessorKey}>
+ <GroupSection
+ header={row.header}
+ markerClassName={isFirst ? markerClassName : undefined}
+ >
+ {row.cell({ data: cellData })}
+ </GroupSection>
+ {!isLast && <Separator />}
+ </Fragment>
+ );
+ }
+
return null;
})}
</div>
diff --git a/web/src/features/experiments/components/table/ExperimentGridView.tsx b/web/src/features/experiments/components/table/ExperimentGridView.tsx
index 62682a4e00..76356cde23 100644
--- a/web/src/features/experiments/components/table/ExperimentGridView.tsx
+++ b/web/src/features/experiments/components/table/ExperimentGridView.tsx
@@ -34,6 +34,8 @@ type ExperimentGridViewProps = {
baselineExperimentId?: string;
comparisonExperimentIds: string[];
useExperimentColors?: boolean;
+ /** Whether cells carry a delta against the baseline (the diff mode). */
+ showDiff: boolean;
/** Render I/O cells as single-line text (true) or JSON tree (false). */
singleLine: boolean;
rows: ExperimentItemsTableRow[];
@@ -68,6 +70,7 @@ export const ExperimentGridView = ({
baselineExperimentId,
comparisonExperimentIds,
useExperimentColors = true,
+ showDiff,
singleLine,
rows,
isLoading,
@@ -176,8 +179,14 @@ export const ExperimentGridView = ({
traceScoreOrder={traceScoreOrder}
showScoreLevelLabels={showScoreLevelLabels}
isBaseline={isBaseline}
+ showDiff={showDiff}
baselineScores={baselineData?.observationScores}
baselineTraceScores={baselineData?.traceScores}
+ baselineExperimentName={
+ experimentNames.find(
+ (e) => e.experimentId === baselineExperimentId,
+ )?.experimentName
+ }
columnVisibility={columnVisibility}
markerClassName={colorStyles?.markerClass}
/>
@@ -195,6 +204,7 @@ export const ExperimentGridView = ({
showScoreLevelLabels,
columnVisibility,
useExperimentColors,
+ showDiff,
singleLine,
]);
diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
index 8452713589..872c447aa9 100644
--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
@@ -20,12 +20,14 @@ import {
} from "../../config/experiment-items-filter-config";
import { type LangfuseColumnDef } from "@/src/components/table/types";
import {
+ type AggregatedScoreData,
type FilterState,
type FilterCondition,
TableViewPresetTableName,
BatchExportTableName,
ActionId,
BatchActionType,
+ EXPERIMENT_IO_TRUNCATE_LENGTH,
} from "@langfuse/shared";
import { ExperimentFilterPills } from "./ExperimentFilterPills";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
@@ -62,6 +64,7 @@ import {
getExperimentColorStyles,
} from "./types";
import { ConnectedIOTableCell } from "@/src/components/table/ConnectedIOTableCell";
+import { Badge } from "@/src/components/ui/badge";
import { type DataTablePeekViewProps } from "@/src/components/table/peek";
import { cn } from "@/src/utils/tailwind";
import { createScoreColumns } from "@/src/features/scores/hooks/useScoreColumns";
@@ -79,10 +82,18 @@ import {
type ScoreColumnDef,
} from "@/src/features/experiments/hooks/useExperimentItemsFilterOptions";
import { DiffLabel } from "@/src/features/datasets/components/DiffLabel";
+import { describeRunComparison } from "@/src/features/experiments/fns/describeRunComparison";
+import { calculateNumericDiff } from "@/src/features/datasets/lib/calculateBaselineDiff";
import { computeScoreDiffs } from "@/src/features/datasets/lib/computeScoreDiffs";
import { TablePeekViewExperimentItemDetail } from "@/src/components/table/peek/peek-experiment-item-detail";
import { NotRecordedMetric } from "./NotRecordedMetric";
-
+import {
+ summariseScoreColumn,
+ type ScoreColumnDataType,
+ type ScoreColumnSummary,
+} from "@/src/features/experiments/fns/summariseScoreColumn";
+import { ScoreColumnHeaderSummary } from "./ScoreColumnHeaderSummary";
+import { resetStaleDefaultColumnOrder } from "@/src/features/experiments/fns/experimentItemsColumnOrder";
const renderExperimentSpecificHeader = (label: string) => (
<span className="text-muted-foreground">{label}</span>
);
@@ -109,6 +120,53 @@ function toScoreColumnInput(scoreColumnDefs: ScoreColumnDef[]): Array<{
}));
}
+/**
+ * One summary per score column: the primary experiment's aggregate over the
+ * items in view, and the same for the comparison it is read against. Built once
+ * per fetch rather than per header render.
+ */
+function buildScoreColumnSummaries({
+ rows,
+ scoreField,
+ dataTypeByKey,
+ primaryExperimentId,
+ comparisonExperimentId,
+}: {
+ rows: ExperimentItemsTableRow[];
+ scoreField: "observationScores" | "traceScores";
+ dataTypeByKey: Map<string, ScoreColumnDataType>;
+ primaryExperimentId?: string;
+ comparisonExperimentId?: string;
+}): Map<string, ScoreColumnSummary> {
+ const summaries = new Map<string, ScoreColumnSummary>();
+ if (!primaryExperimentId) return summaries;
+
+ const scoresFor = (row: ExperimentItemsTableRow, experimentId?: string) =>
+ experimentId
+ ? row.experiments.find((exp) => exp.experimentId === experimentId)?.[
+ scoreField
+ ]
+ : undefined;
+
+ for (const [key, dataType] of dataTypeByKey) {
+ summaries.set(
+ key,
+ summariseScoreColumn({
+ // Every item in view is a pair, including the ones only one of the two
+ // experiments scored β€” those are counted as not comparable.
+ pairs: rows.map((row) => ({
+ baseline: scoresFor(row, primaryExperimentId)?.[key] ?? null,
+ comparison: scoresFor(row, comparisonExperimentId)?.[key] ?? null,
+ })),
+ dataType,
+ hasComparison: Boolean(comparisonExperimentId),
+ }),
+ );
+ }
+
+ return summaries;
+}
+
const getDefaultExperimentFilterTarget = (props: {
baselineId?: string;
comparisonIds: string[];
@@ -123,6 +181,20 @@ const shouldEnableExperimentPeek = (props: {
* Cell component that renders stacked values for each experiment.
* Uses CSS grid for consistent horizontal alignment across columns.
*/
+/**
+ * A single score's value, as `ScoresTableCell` renders it in the smart format
+ * β€” for the hover sentence next to it, which has to quote the same number the
+ * cell shows.
+ */
+const formatScoreAggregateValue = (
+ aggregate?: AggregatedScoreData | null,
+): string => {
+ if (!aggregate) return "nothing";
+ return aggregate.type === "NUMERIC"
+ ? aggregate.average.toFixed(4)
+ : (aggregate.values[0] ?? "nothing");
+};
+
const StackedExperimentCell = ({
experiments,
allExperimentIds,
@@ -188,10 +260,13 @@ const StackedOutputRow = ({
output,
markerClass,
singleLine,
+ chip,
}: {
output: string;
markerClass: string;
singleLine: boolean;
+ /** Rendered after the value, e.g. the expected-output verdict. */
+ chip?: React.ReactNode;
}) => {
return (
<div className="flex min-w-0 items-start">
@@ -206,10 +281,44 @@ const StackedOutputRow = ({
singleLine={singleLine}
variant="output"
/>
+ {chip}
</div>
);
};
+/**
+ * Whether an output is the expected one. Exact match after trimming β€” anything
+ * looser would be guessing at what "close enough" means for the user's data.
+ *
+ * `null` when the answer is not knowable from what the table loaded: the values
+ * arrive truncated, so two that agree up to the cut may still differ past it.
+ * A verdict that can be wrong is worse than no verdict, so the caller shows
+ * nothing instead.
+ */
+const matchesExpectedOutput = (
+ output: string | null | undefined,
+ expectedOutput: string | null | undefined,
+): boolean | null => {
+ if (!output || !expectedOutput) return null;
+ const left = output.trim();
+ const right = expectedOutput.trim();
+ if (left !== right) return false;
+ const mayBeTruncated =
+ output.length >= EXPERIMENT_IO_TRUNCATE_LENGTH ||
+ expectedOutput.length >= EXPERIMENT_IO_TRUNCATE_LENGTH;
+ return mayBeTruncated ? null : true;
+};
+
+const ExpectedMatchChip = ({ matches }: { matches: boolean }) => (
+ <Badge
+ size="sm"
+ variant={matches ? "success" : "error"}
+ className="mt-0.5 ml-1 shrink-0 font-bold"
+ >
+ {matches ? "match" : "differs"}
+ </Badge>
+);
+
/**
* Cell component that renders stacked output values for each experiment.
*/
@@ -219,31 +328,58 @@ const StackedOutputCell = ({
colorExperimentIds,
singleLine,
isLoading,
+ expectedOutput,
}: {
outputs: ExperimentOutputData[];
allExperimentIds: string[];
colorExperimentIds?: string[];
singleLine: boolean;
isLoading: boolean;
+ /**
+ * The item's expected output, shown as the cell's first line with a verdict on
+ * each output β€” the `Expected β†’ Output` diff mode. Undefined in every other
+ * mode, and for the items that simply have no expected output.
+ */
+ expectedOutput?: string | null;
}) => {
const outputsByExperimentId = useMemo(
() => new Map(outputs.map((out) => [out.experimentId, out])),
[outputs],
);
+ const showExpectedLine = Boolean(expectedOutput);
+
return (
<div
className="grid h-full min-h-0 gap-1"
style={{
- gridTemplateRows: `repeat(${Math.max(allExperimentIds.length, 1)}, minmax(0, 1fr))`,
+ gridTemplateRows: `repeat(${Math.max(allExperimentIds.length, 1) + (showExpectedLine ? 1 : 0)}, minmax(0, 1fr))`,
}}
>
+ {showExpectedLine && (
+ <div className="flex min-h-0 items-start overflow-hidden py-0.5 pr-1 pl-1.5">
+ <div className="flex min-w-0 items-start">
+ <span className="text-muted-foreground mt-0.5 mr-1 shrink-0 text-[10px] font-bold uppercase">
+ Exp
+ </span>
+ <ConnectedIOTableCell
+ isLoading={false}
+ data={expectedOutput ?? null}
+ singleLine={singleLine}
+ />
+ </div>
+ </div>
+ )}
{allExperimentIds.map((experimentId) => {
const out = outputsByExperimentId.get(experimentId);
const colorStyles = getExperimentColorStyles(
experimentId,
colorExperimentIds ?? allExperimentIds,
);
+ // null when the loaded text cannot settle the question, so no chip.
+ const expectedMatch = showExpectedLine
+ ? matchesExpectedOutput(out?.output, expectedOutput)
+ : null;
return (
<div
key={experimentId}
@@ -259,6 +395,11 @@ const StackedOutputCell = ({
output={out.output}
markerClass={colorStyles.markerClass}
singleLine={singleLine}
+ chip={
+ expectedMatch === null ? undefined : (
+ <ExpectedMatchChip matches={expectedMatch} />
+ )
+ }
/>
) : (
<span className="text-muted-foreground px-2 py-1">β€”</span>
@@ -299,9 +440,16 @@ export default function ExperimentItemsTable({
comparisonIds,
allExperimentIds,
layout,
+ diffMode,
itemVisibility,
} = useExperimentResultsState();
+ // "Off" turns the whole diff apparatus into plain values; "expected" keeps the
+ // baseline deltas on the scores (a score has no expected value to diff
+ // against) and points the output cell at the item's expected output instead.
+ const showComparisonDiff = diffMode !== "off";
+ const isExpectedDiff = diffMode === "expected";
+
const defaultFilterTargetExperimentId = getDefaultExperimentFilterTarget({
baselineId,
comparisonIds,
@@ -318,6 +466,17 @@ export default function ExperimentItemsTable({
);
}, [experimentNames, allExperimentIds]);
+ // A stacked cell holds one line per run, told apart by a colour marker only,
+ // so every value and every comparison chip in it names its run on hover.
+ const runNameOf = useCallback(
+ (experimentId?: string) =>
+ experimentId
+ ? experimentNames.find((exp) => exp.experimentId === experimentId)
+ ?.experimentName
+ : undefined,
+ [experimentNames],
+ );
+
const { selectAll, setSelectAll } = useSelectAll(
projectId,
"experiment-items",
@@ -328,9 +487,12 @@ export default function ExperimentItemsTable({
limit: "pageSize",
});
+ // Medium by default: with the ids out of the cell a row no longer needs the
+ // tallest option to show its output, and two rows per screen was the
+ // complaint. Large stays one click away for reading long outputs.
const [rowHeight, setRowHeight] = useRowHeightLocalStorage(
"experiment-items",
- "l",
+ "m",
);
const ioSingleLine = ioRenderMode === "text";
@@ -543,46 +705,6 @@ export default function ExperimentItemsTable({
expectedOutputOnPage ||
expectedOutputSeenFor === experimentSelectionKey;
- useEffect(() => {
- if (items.status === "success") {
- // Store all experiment targets for peek navigation
- setDetailPageList(
- "experiment-items",
- items?.rows?.map((item: ExperimentItemsTableRow) => {
- const baselineExp = baselineId
- ? item.experiments.find((e) => e.experimentId === baselineId)
- : item.experiments[0];
-
- // Build experiment targets map for all experiments
- const experimentTargets = Object.fromEntries(
- item.experiments.map((exp) => [
- exp.experimentId,
- {
- traceId: exp.traceId,
- observationId: exp.observationId,
- timestamp: exp.startTime.toISOString(),
- },
- ]),
- );
-
- return {
- id: item.itemId,
- params: {
- // Primary trace params (baseline, for URL compat and initial load)
- traceId: baselineExp?.traceId ?? "",
- observation: baselineExp?.observationId ?? "",
- timestamp: baselineExp?.startTime?.toISOString() ?? "",
- },
- // All experiment targets for switching between experiments.
- // Kept out of `params` so they never leak into the URL.
- meta: { experimentTargets },
- };
- }),
- );
- }
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [items.status, items.rows, baselineId]);
-
const { selectActionColumn } = TableSelectionManager<ExperimentItemsTableRow>(
{
projectId,
@@ -628,8 +750,13 @@ export default function ExperimentItemsTable({
displayFormat: "smart",
headerPrefix: "Observation",
rawKey: true,
+ valueTitle: (exp) => runNameOf(exp.experimentId),
}),
- [scoreColumnDefs.observationScoreColumns, presentScoreKeys?.observation],
+ [
+ scoreColumnDefs.observationScoreColumns,
+ presentScoreKeys?.observation,
+ runNameOf,
+ ],
);
const traceScoreColumns = useMemo(
@@ -643,84 +770,209 @@ export default function ExperimentItemsTable({
displayFormat: "smart",
prefix: "Trace",
rawKey: true,
+ valueTitle: (exp) => runNameOf(exp.experimentId),
}),
- [scoreColumnDefs.traceScoreColumns, presentScoreKeys?.trace],
+ [scoreColumnDefs.traceScoreColumns, presentScoreKeys?.trace, runNameOf],
);
// Use the shared loading state for both sidebar and columns
const isObservationScoreColumnsLoading = isFilterOptionsLoading;
const isTraceScoreColumnsLoading = isFilterOptionsLoading;
+ // The experiment a score column's header reads as "this experiment", and the
+ // one it is compared against. Without an explicit baseline the first selected
+ // run still stands in as "this experiment", but nothing is compared against
+ // it: the cells only draw a diff once there is an explicit baseline, so a
+ // header delta here would count movements no row in the table shows.
+ const primaryExperimentId = baselineId ?? allExperimentIds[0];
+ const primaryComparisonId = hasBaseline
+ ? allExperimentIds.find((id) => id !== primaryExperimentId)
+ : undefined;
+ const primaryComparisonName = useMemo(
+ () =>
+ selectedExperimentNames.find(
+ (exp) => exp.experimentId === primaryComparisonId,
+ )?.experimentName,
+ [selectedExperimentNames, primaryComparisonId],
+ );
+
+ const scoreDataTypesByKey = useMemo(() => {
+ const build = (columns: ScoreColumnDef[]) =>
+ new Map(
+ toScoreColumnInput(columns).map(({ key, dataType }) => [key, dataType]),
+ );
+ return {
+ observationScores: build(scoreColumnDefs.observationScoreColumns),
+ traceScores: build(scoreColumnDefs.traceScoreColumns),
+ };
+ }, [
+ scoreColumnDefs.observationScoreColumns,
+ scoreColumnDefs.traceScoreColumns,
+ ]);
+
+ const scoreColumnSummaries = useMemo(() => {
+ const rowsInView = items.rows ?? [];
+ return {
+ observationScores: buildScoreColumnSummaries({
+ rows: rowsInView,
+ scoreField: "observationScores",
+ dataTypeByKey: scoreDataTypesByKey.observationScores,
+ primaryExperimentId,
+ comparisonExperimentId: primaryComparisonId,
+ }),
+ traceScores: buildScoreColumnSummaries({
+ rows: rowsInView,
+ scoreField: "traceScores",
+ dataTypeByKey: scoreDataTypesByKey.traceScores,
+ primaryExperimentId,
+ comparisonExperimentId: primaryComparisonId,
+ }),
+ };
+ }, [
+ items.rows,
+ scoreDataTypesByKey,
+ primaryExperimentId,
+ primaryComparisonId,
+ ]);
+
const buildExperimentScoreColumns = useCallback(
(
scoreColumns: LangfuseColumnDef<ExperimentItemData>[],
scoreField: "observationScores" | "traceScores",
): LangfuseColumnDef<ExperimentItemsTableRow>[] =>
- scoreColumns.map((scoreCol) => ({
- ...scoreCol,
- // Override the cell renderer to show stacked scores for each experiment
- cell: ({ row }) => {
- const experiments = row.original.experiments;
- const baselineExperiment = hasBaseline
- ? experiments.find((exp) => exp.experimentId === baselineId)
- : undefined;
- const baselineScoresData = baselineExperiment?.[scoreField] ?? null;
- // todo: fix properly
- const scoreKey = scoreCol.accessorKey?.replace(`Trace-`, "");
- return (
- <StackedExperimentCell
- experiments={experiments}
- allExperimentIds={allExperimentIds}
- colorExperimentIds={colorExperimentIds}
- renderValue={(exp) => {
- const scoresData = exp[scoreField] ?? {};
- const value = scoresData[scoreKey];
-
- if (!value)
- return <span className="text-muted-foreground">-</span>;
-
- const mockRow = {
- getValue: (key: string) =>
- key === scoreField ? scoresData : undefined,
- original: exp,
- } as any;
- const scoreCell = scoreCol.cell;
- const diff =
- hasBaseline &&
- baselineId &&
- exp.experimentId !== baselineId &&
- scoreKey &&
- baselineScoresData
- ? computeScoreDiffs(scoresData, baselineScoresData)[
- scoreKey
- ]
- : null;
-
- const renderedScore =
- typeof scoreCell === "function"
- ? scoreCell({
- row: mockRow,
- getValue: mockRow.getValue,
- } as any)
- : null;
-
- return (
- <div className="flex items-center gap-1">
- {renderedScore}
- {diff && (
- <DiffLabel
- diff={diff}
- formatValue={(v) => v.toFixed(2)}
- />
- )}
- </div>
- );
- }}
- />
- );
- },
- })) as LangfuseColumnDef<ExperimentItemsTableRow>[],
- [allExperimentIds, baselineId, colorExperimentIds, hasBaseline],
+ scoreColumns.map((scoreCol) => {
+ const key = scoreCol.accessorKey?.replace(`Trace-`, "");
+ const summary = key
+ ? scoreColumnSummaries[scoreField].get(key)
+ : undefined;
+ const dataType = key
+ ? scoreDataTypesByKey[scoreField].get(key)
+ : undefined;
+ const label =
+ typeof scoreCol.header === "string"
+ ? scoreCol.header
+ : (scoreCol.accessorKey ?? "");
+
+ return {
+ ...scoreCol,
+ // The header carries the column's aggregate over the items in view, and
+ // the movement against the comparison. Keeps the plain name for the
+ // column picker.
+ ...(summary && dataType
+ ? {
+ headerBlock: true,
+ headerLabel: label,
+ header: () => (
+ <ScoreColumnHeaderSummary
+ label={label}
+ dataType={dataType}
+ summary={summary}
+ comparisonName={primaryComparisonName}
+ />
+ ),
+ }
+ : {}),
+ // Override the cell renderer to show stacked scores for each experiment
+ cell: ({ row }) => {
+ const experiments = row.original.experiments;
+ const baselineExperiment = hasBaseline
+ ? experiments.find((exp) => exp.experimentId === baselineId)
+ : undefined;
+ const baselineScoresData = baselineExperiment?.[scoreField] ?? null;
+ // todo: fix properly
+ const scoreKey = scoreCol.accessorKey?.replace(`Trace-`, "");
+ return (
+ <StackedExperimentCell
+ experiments={experiments}
+ allExperimentIds={allExperimentIds}
+ colorExperimentIds={colorExperimentIds}
+ renderValue={(exp) => {
+ const scoresData = exp[scoreField] ?? {};
+ const value = scoresData[scoreKey];
+
+ if (!value)
+ return <span className="text-muted-foreground">-</span>;
+
+ const mockRow = {
+ getValue: (key: string) =>
+ key === scoreField ? scoresData : undefined,
+ original: exp,
+ } as any;
+ const scoreCell = scoreCol.cell;
+ const diff =
+ showComparisonDiff &&
+ hasBaseline &&
+ baselineId &&
+ exp.experimentId !== baselineId &&
+ scoreKey &&
+ baselineScoresData
+ ? computeScoreDiffs(scoresData, baselineScoresData)[
+ scoreKey
+ ]
+ : null;
+
+ const renderedScore =
+ typeof scoreCell === "function"
+ ? scoreCell({
+ row: mockRow,
+ getValue: mockRow.getValue,
+ } as any)
+ : null;
+
+ // `true β†’ false` and `+0.07` do not say which side is the
+ // baseline; the hover title does.
+ const diffTitle = diff
+ ? describeRunComparison({
+ baselineName: runNameOf(baselineId),
+ ...(diff.type === "CATEGORICAL"
+ ? {
+ baselineText: diff.from ?? "several values",
+ currentText: diff.to ?? "several values",
+ }
+ : {
+ baselineText: formatScoreAggregateValue(
+ baselineScoresData?.[scoreKey ?? ""],
+ ),
+ currentText: formatScoreAggregateValue(value),
+ }),
+ })
+ : undefined;
+
+ return (
+ // The value never gives up width for the chip beside it:
+ // a clipped value reads as data, so the move wraps under
+ // it instead. `max-w-full` keeps a pathological value
+ // inside the cell, where its own truncation has a title.
+ <div className="flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5">
+ <span className="flex max-w-full shrink-0 items-center">
+ {renderedScore}
+ </span>
+ {diff && (
+ <DiffLabel
+ diff={diff}
+ formatValue={(v) => v.toFixed(2)}
+ title={diffTitle}
+ />
+ )}
+ </div>
+ );
+ }}
+ />
+ );
+ },
+ };
+ }) as LangfuseColumnDef<ExperimentItemsTableRow>[],
+ [
+ allExperimentIds,
+ baselineId,
+ colorExperimentIds,
+ hasBaseline,
+ primaryComparisonName,
+ scoreColumnSummaries,
+ scoreDataTypesByKey,
+ showComparisonDiff,
+ runNameOf,
+ ],
);
const observationExperimentScoreColumns = useMemo(
@@ -765,6 +1017,43 @@ export default function ExperimentItemsTable({
variant: "output",
}) as LangfuseColumnDef<ExperimentItemsTableRow>;
+ const baselineExperimentOf = (experiments: ExperimentItemData[]) =>
+ hasBaseline && baselineId
+ ? experiments.find((exp) => exp.experimentId === baselineId)
+ : undefined;
+
+ /** A comparison line's move against the baseline's, lower being better. */
+ const renderMetricDiff = ({
+ exp,
+ value,
+ baselineValue,
+ format,
+ verb,
+ }: {
+ exp: ExperimentItemData;
+ value?: number | null;
+ baselineValue?: number | null;
+ format: (value: number) => string;
+ verb: "cost" | "took";
+ }) => {
+ if (!showComparisonDiff || exp.experimentId === baselineId) return null;
+ const diff = calculateNumericDiff(value, baselineValue);
+ if (!diff) return null;
+ return (
+ <DiffLabel
+ diff={diff}
+ preferNegativeDiff
+ formatValue={format}
+ title={describeRunComparison({
+ baselineName: runNameOf(baselineId),
+ baselineText: format(baselineValue ?? 0),
+ currentText: format(value ?? 0),
+ verb,
+ })}
+ />
+ );
+ };
+
const columns: LangfuseColumnDef<ExperimentItemsTableRow>[] = [
...(hideControls ? [] : [selectActionColumn]),
createIdTableColumn<ExperimentItemsTableRow>({
@@ -773,65 +1062,157 @@ export default function ExperimentItemsTable({
size: 150,
enableHiding: true,
}),
+ createIOTableColumn<ExperimentItemsTableRow>({
+ accessorKey: "input",
+ header: "Input",
+ size: 300,
+ enableHiding: true,
+ getCell: (value) => (ioLoading ? { type: "loading" } : (value ?? null)),
+ singleLine: ioSingleLine,
+ }),
+ // The scores sit between the item's input and its outputs: the input says
+ // which item this is, the score headers carry the judgement, and the outputs
+ // are the drill-down a regression sends you to (peek carries it too).
{
- accessorKey: "observationId",
- id: "observationId",
- headerLabel: "Observation ID",
- header: () => renderExperimentSpecificHeader("Observation ID"),
- size: 180,
+ accessorKey: "observationScores",
+ header: "Observation Scores",
+ id: "observationScores",
+ enableHiding: true,
+ cell: () => {
+ return isObservationScoreColumnsLoading ? (
+ <Skeleton className="h-3 w-1/2" />
+ ) : null;
+ },
+ columns: observationExperimentScoreColumns,
+ },
+ {
+ accessorKey: "traceScores",
+ header: "Trace Scores",
+ id: "traceScores",
+ enableHiding: true,
+ cell: () => {
+ return isTraceScoreColumnsLoading ? (
+ <Skeleton className="h-3 w-1/2" />
+ ) : null;
+ },
+ columns: traceExperimentScoreColumns,
+ },
+ // The expected output moves inside the output cell in that diff mode, so it
+ // does not also hold a column of its own.
+ ...(showExpectedOutput && !isExpectedDiff ? [expectedOutputColumn] : []),
+ {
+ accessorKey: "output",
+ id: "output",
+ header: "Output",
+ size: 300,
+ enableHiding: true,
+ cell: ({ row }) => {
+ const outputs = row.original.outputs ?? [];
+ return (
+ <StackedOutputCell
+ outputs={outputs}
+ allExperimentIds={allExperimentIds}
+ colorExperimentIds={colorExperimentIds}
+ singleLine={ioSingleLine}
+ isLoading={ioLoading}
+ // Items with no expected output get no expected line and no
+ // verdict, rather than a diff against nothing.
+ expectedOutput={
+ isExpectedDiff
+ ? (row.original.expectedOutput ?? undefined)
+ : undefined
+ }
+ />
+ );
+ },
+ },
+ // Cost and latency read as measurements, the ids as lookups β€” both
+ // sit behind the score columns so the analysis is above the fold.
+ {
+ accessorKey: "totalCost",
+ id: "totalCost",
+ headerLabel: getExperimentItemsColumnName("totalCost"),
+ header: () =>
+ renderExperimentSpecificHeader(
+ getExperimentItemsColumnName("totalCost"),
+ ),
+ size: 120,
enableHiding: true,
cell: ({ row }) => {
const experiments = row.original.experiments;
+ const baselineCost = baselineExperimentOf(experiments)?.totalCost;
return (
<StackedExperimentCell
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) => <IdTableCell value={exp.observationId} />}
+ renderValue={(exp) => (
+ // Wraps rather than clipping: in a 120px column a six-decimal
+ // cost and its delta do not fit on one line, and half a currency
+ // value reads as data.
+ <span className="inline-flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5">
+ {exp.totalCost ? (
+ usdFormatter(exp.totalCost, 2, 6)
+ ) : (
+ <NotRecordedMetric metric="cost" />
+ )}
+ {renderMetricDiff({
+ exp,
+ value: exp.totalCost,
+ baselineValue: baselineCost,
+ format: (value) => usdFormatter(value, 2, 6),
+ verb: "cost",
+ })}
+ </span>
+ )}
/>
);
},
},
{
- accessorKey: "startTime",
- id: "startTime",
- headerLabel: getExperimentItemsColumnName("startTime"),
+ accessorKey: "latencyMs",
+ id: "latencyMs",
+ headerLabel: getExperimentItemsColumnName("latencyMs"),
header: () =>
renderExperimentSpecificHeader(
- getExperimentItemsColumnName("startTime"),
+ getExperimentItemsColumnName("latencyMs"),
),
- size: 180,
- defaultHidden: true,
+ size: 120,
enableHiding: true,
- enableSorting: true,
cell: ({ row }) => {
const experiments = row.original.experiments;
+ const baselineLatency = baselineExperimentOf(experiments)?.latencyMs;
return (
<StackedExperimentCell
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) => {
- const preparedDate = buildLocalIsoDatePresentation({
- date: exp.startTime,
- });
-
- return preparedDate ? (
- <span title={preparedDate.title}>{preparedDate.display}</span>
- ) : null;
- }}
+ renderValue={(exp) => (
+ <span className="inline-flex min-w-0 flex-wrap items-center gap-x-1 gap-y-0.5">
+ {exp.latencyMs != null ? (
+ latencyFormatter(exp.latencyMs)
+ ) : (
+ <NotRecordedMetric metric="latency" />
+ )}
+ {renderMetricDiff({
+ exp,
+ value: exp.latencyMs,
+ baselineValue: baselineLatency,
+ format: latencyFormatter,
+ verb: "took",
+ })}
+ </span>
+ )}
/>
);
},
},
{
- accessorKey: "level",
- id: "level",
- headerLabel: getExperimentItemsColumnName("level"),
- header: () =>
- renderExperimentSpecificHeader(getExperimentItemsColumnName("level")),
- size: 120,
- defaultHidden: true,
+ accessorKey: "observationId",
+ id: "observationId",
+ headerLabel: "Observation ID",
+ header: () => renderExperimentSpecificHeader("Observation ID"),
+ size: 180,
enableHiding: true,
cell: ({ row }) => {
const experiments = row.original.experiments;
@@ -840,21 +1221,23 @@ export default function ExperimentItemsTable({
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) => <span>{exp.level}</span>}
+ renderValue={(exp) => <IdTableCell value={exp.observationId} />}
/>
);
},
},
{
- accessorKey: "totalCost",
- id: "totalCost",
- headerLabel: getExperimentItemsColumnName("totalCost"),
+ accessorKey: "startTime",
+ id: "startTime",
+ headerLabel: getExperimentItemsColumnName("startTime"),
header: () =>
renderExperimentSpecificHeader(
- getExperimentItemsColumnName("totalCost"),
+ getExperimentItemsColumnName("startTime"),
),
- size: 120,
+ size: 180,
+ defaultHidden: true,
enableHiding: true,
+ enableSorting: true,
cell: ({ row }) => {
const experiments = row.original.experiments;
return (
@@ -862,26 +1245,27 @@ export default function ExperimentItemsTable({
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) =>
- exp.totalCost ? (
- <span>{usdFormatter(exp.totalCost, 2, 6)}</span>
- ) : (
- <NotRecordedMetric metric="cost" />
- )
- }
+ renderValue={(exp) => {
+ const preparedDate = buildLocalIsoDatePresentation({
+ date: exp.startTime,
+ });
+
+ return preparedDate ? (
+ <span title={preparedDate.title}>{preparedDate.display}</span>
+ ) : null;
+ }}
/>
);
},
},
{
- accessorKey: "latencyMs",
- id: "latencyMs",
- headerLabel: getExperimentItemsColumnName("latencyMs"),
+ accessorKey: "level",
+ id: "level",
+ headerLabel: getExperimentItemsColumnName("level"),
header: () =>
- renderExperimentSpecificHeader(
- getExperimentItemsColumnName("latencyMs"),
- ),
+ renderExperimentSpecificHeader(getExperimentItemsColumnName("level")),
size: 120,
+ defaultHidden: true,
enableHiding: true,
cell: ({ row }) => {
const experiments = row.original.experiments;
@@ -890,13 +1274,7 @@ export default function ExperimentItemsTable({
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) =>
- exp.latencyMs != null ? (
- <span>{latencyFormatter(exp.latencyMs)}</span>
- ) : (
- <NotRecordedMetric metric="latency" />
- )
- }
+ renderValue={(exp) => <span>{exp.level}</span>}
/>
);
},
@@ -932,58 +1310,6 @@ export default function ExperimentItemsTable({
);
},
},
- createIOTableColumn<ExperimentItemsTableRow>({
- accessorKey: "input",
- header: "Input",
- size: 300,
- enableHiding: true,
- getCell: (value) => (ioLoading ? { type: "loading" } : (value ?? null)),
- singleLine: ioSingleLine,
- }),
- ...(showExpectedOutput ? [expectedOutputColumn] : []),
- {
- accessorKey: "output",
- id: "output",
- header: "Output",
- size: 300,
- enableHiding: true,
- cell: ({ row }) => {
- const outputs = row.original.outputs ?? [];
- return (
- <StackedOutputCell
- outputs={outputs}
- allExperimentIds={allExperimentIds}
- colorExperimentIds={colorExperimentIds}
- singleLine={ioSingleLine}
- isLoading={ioLoading}
- />
- );
- },
- },
- {
- accessorKey: "observationScores",
- header: "Observation Scores",
- id: "observationScores",
- enableHiding: true,
- cell: () => {
- return isObservationScoreColumnsLoading ? (
- <Skeleton className="h-3 w-1/2" />
- ) : null;
- },
- columns: observationExperimentScoreColumns,
- },
- {
- accessorKey: "traceScores",
- header: "Trace Scores",
- id: "traceScores",
- enableHiding: true,
- cell: () => {
- return isTraceScoreColumnsLoading ? (
- <Skeleton className="h-3 w-1/2" />
- ) : null;
- },
- columns: traceExperimentScoreColumns,
- },
];
const scoreColumnIds = useMemo(
@@ -1020,9 +1346,24 @@ export default function ExperimentItemsTable({
columnVisibilityMigrations,
);
+ // the score columns moved ahead of the metrics and ids so their
+ // headers' analysis needs no horizontal scroll. A returning user has the old
+ // order persisted, so the new default only reaches him through a migration β€”
+ // and only when that stored order is still a default, not one he arranged.
+ const columnOrderMigrations = useMemo(
+ () => [
+ {
+ versionKey: `experimentItemsColumnOrder-scoresEarlier-v2-${projectId}`,
+ apply: resetStaleDefaultColumnOrder,
+ },
+ ],
+ [projectId],
+ );
+
const [columnOrder, setColumnOrder] = useColumnOrder<ExperimentItemsTableRow>(
`experimentItemsColumnOrder-${projectId}`,
columns,
+ columnOrderMigrations,
);
const peekNavigationProps = usePeekNavigation({
@@ -1085,13 +1426,51 @@ export default function ExperimentItemsTable({
}, [peekNavigationProps, canUsePeek]);
const rows: ExperimentItemsTableRow[] = useMemo(() => {
- if (items.status === "success" && items.rows) {
- // Add 'id' field for DataTable row identification (peek view requires it)
- return items.rows.map((row) => ({ ...row, id: row.itemId }));
- }
- return [];
+ if (items.status !== "success" || !items.rows) return [];
+ // Add 'id' field for DataTable row identification (peek view requires it)
+ return items.rows.map((row) => ({ ...row, id: row.itemId }));
}, [items]);
+ useEffect(() => {
+ if (items.status === "success") {
+ // Store all experiment targets for peek navigation
+ setDetailPageList(
+ "experiment-items",
+ rows.map((item: ExperimentItemsTableRow) => {
+ const baselineExp = baselineId
+ ? item.experiments.find((e) => e.experimentId === baselineId)
+ : item.experiments[0];
+
+ // Build experiment targets map for all experiments
+ const experimentTargets = Object.fromEntries(
+ item.experiments.map((exp) => [
+ exp.experimentId,
+ {
+ traceId: exp.traceId,
+ observationId: exp.observationId,
+ timestamp: exp.startTime.toISOString(),
+ },
+ ]),
+ );
+
+ return {
+ id: item.itemId,
+ params: {
+ // Primary trace params (baseline, for URL compat and initial load)
+ traceId: baselineExp?.traceId ?? "",
+ observation: baselineExp?.observationId ?? "",
+ timestamp: baselineExp?.startTime?.toISOString() ?? "",
+ },
+ // All experiment targets for switching between experiments.
+ // Kept out of `params` so they never leak into the URL.
+ meta: { experimentTargets },
+ };
+ }),
+ );
+ }
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [items.status, rows, baselineId]);
+
const pagination = useMemo(
() => ({
totalCount: totalCount ?? null,
@@ -1292,6 +1671,7 @@ export default function ExperimentItemsTable({
baselineId ? comparisonIds : allExperimentIds
}
useExperimentColors={hasBaseline}
+ showDiff={showComparisonDiff}
singleLine={ioSingleLine}
rows={rows}
isLoading={items.status === "loading" || isViewLoading}
diff --git a/web/src/features/experiments/components/table/ScoreColumnHeaderSummary.tsx b/web/src/features/experiments/components/table/ScoreColumnHeaderSummary.tsx
new file mode 100644
index 0000000000..4b7b023439
--- /dev/null
+++ b/web/src/features/experiments/components/table/ScoreColumnHeaderSummary.tsx
@@ -0,0 +1,207 @@
+import {
+ HoverCard,
+ HoverCardContent,
+ HoverCardTrigger,
+} from "@/src/components/ui/hover-card";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { DiffLabel } from "@/src/features/datasets/components/DiffLabel";
+import {
+ getScoreDataTypeExplanation,
+ splitScoreDataTypeIcon,
+} from "@/src/features/scores/lib/scoreColumns";
+import {
+ type ScoreColumnDataType,
+ type ScoreColumnSummary,
+} from "@/src/features/experiments/fns/summariseScoreColumn";
+import {
+ formatScoreColumnAggregate,
+ formatScoreValue,
+} from "@/src/features/experiments/fns/formatScoreColumnAggregate";
+
+const DIFF_LABEL_TITLES: Record<ScoreColumnDataType, string> = {
+ NUMERIC: "average",
+ BOOLEAN: "true-rate",
+ CATEGORICAL: "modal value",
+};
+
+/** The type, quietly: the marker the column already had, now explained. */
+const ScoreDataTypeMarker = ({
+ icon,
+ dataType,
+}: {
+ icon: string;
+ dataType: ScoreColumnDataType;
+}) => (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-muted-foreground shrink-0 cursor-default">
+ {icon}
+ </span>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[280px]">
+ {getScoreDataTypeExplanation(dataType)}
+ </TooltipContent>
+ </Tooltip>
+);
+
+const SummaryRow = ({
+ label,
+ value,
+}: {
+ label: string;
+ value: React.ReactNode;
+}) => (
+ <div className="flex items-baseline justify-between gap-4 text-xs">
+ <span className="text-muted-foreground">{label}</span>
+ <span className="tabular-nums">{value}</span>
+ </div>
+);
+
+/**
+ * A score column's header, carrying the analysis instead of only the column's
+ * name: this experiment's aggregate and the item count behind it, and β€” with a
+ * comparison selected β€” the comparison's aggregate, the signed delta and how
+ * many items moved which way. This is what the deleted Analytics tab was going
+ * to be, put where the eye already is.
+ */
+export const ScoreColumnHeaderSummary = ({
+ label,
+ dataType,
+ summary,
+ comparisonName,
+ filterMenu,
+}: {
+ label: string;
+ dataType: ScoreColumnDataType;
+ summary: ScoreColumnSummary;
+ comparisonName?: string;
+ /** The column's way into the score comparison filter, rendered beside the name. */
+ filterMenu?: React.ReactNode;
+}) => {
+ const { baseline, comparison, delta, movement } = summary;
+ const notComparable = movement?.notComparable ?? 0;
+ const { icon, label: nameLabel } = splitScoreDataTypeIcon(label);
+
+ return (
+ // The filter menu sits outside the hover-card trigger so its own popover is
+ // not fighting the hover card for the pointer.
+ <div className="flex min-w-0 flex-1 items-start gap-1">
+ <HoverCard>
+ <HoverCardTrigger asChild>
+ <div className="flex min-w-0 flex-1 cursor-default flex-col gap-0.5 py-0.5">
+ <span className="flex min-w-0 items-baseline gap-1">
+ {icon && <ScoreDataTypeMarker icon={icon} dataType={dataType} />}
+ <span className="truncate" title={label}>
+ {nameLabel}
+ </span>
+ </span>
+ <span className="text-muted-foreground flex flex-wrap items-center gap-x-1 text-[10px] leading-tight font-normal tabular-nums">
+ {baseline ? (
+ <>
+ <span className="text-foreground font-bold">
+ {formatScoreColumnAggregate(baseline)}
+ </span>
+ {baseline.kind !== "distribution" && (
+ <span>Β· {baseline.count}</span>
+ )}
+ </>
+ ) : (
+ <span>no values</span>
+ )}
+ </span>
+ {movement && (
+ <span className="text-muted-foreground flex flex-wrap items-center gap-x-1 text-[10px] leading-tight font-normal tabular-nums">
+ {comparison && (
+ <span>vs {formatScoreColumnAggregate(comparison)}</span>
+ )}
+ {delta !== null && delta !== 0 && (
+ <DiffLabel
+ diff={{
+ type: "NUMERIC",
+ absoluteDifference: Math.abs(delta),
+ direction: delta > 0 ? "+" : "-",
+ }}
+ formatValue={formatScoreValue}
+ />
+ )}
+ {movement.improved > 0 && (
+ <span className="text-dark-green font-bold">
+ β†—{movement.improved}
+ </span>
+ )}
+ {movement.regressed > 0 && (
+ <span className="text-dark-red font-bold">
+ β†˜{movement.regressed}
+ </span>
+ )}
+ {movement.changed > 0 && <span>↻{movement.changed}</span>}
+ {/* Items only one of the two runs scored: visible, and never
+ folded into the regression count. */}
+ {notComparable > 0 && (
+ <span className="opacity-70">{notComparable} n/a</span>
+ )}
+ </span>
+ )}
+ </div>
+ </HoverCardTrigger>
+ <HoverCardContent
+ align="start"
+ className="flex w-64 flex-col gap-1 p-3 font-normal"
+ >
+ <span className="text-xs font-bold break-all">{label}</span>
+ <span className="text-muted-foreground text-[10px]">
+ {getScoreDataTypeExplanation(dataType)}
+ </span>
+ <SummaryRow
+ label={`This experiment (${DIFF_LABEL_TITLES[dataType]})`}
+ value={
+ baseline ? formatScoreColumnAggregate(baseline) : "no values"
+ }
+ />
+ <SummaryRow label="Items scored" value={baseline?.count ?? 0} />
+ {movement && (
+ <>
+ <SummaryRow
+ label={comparisonName ? `vs ${comparisonName}` : "Comparison"}
+ value={
+ comparison
+ ? formatScoreColumnAggregate(comparison)
+ : "no values"
+ }
+ />
+ {delta !== null && (
+ <SummaryRow
+ label="Change"
+ value={`${delta > 0 ? "+" : ""}${formatScoreValue(delta)}`}
+ />
+ )}
+ {dataType === "CATEGORICAL" ? (
+ <SummaryRow label="Changed value" value={movement.changed} />
+ ) : (
+ <>
+ <SummaryRow label="Improved" value={movement.improved} />
+ <SummaryRow label="Regressed" value={movement.regressed} />
+ </>
+ )}
+ <SummaryRow label="Unchanged" value={movement.unchanged} />
+ <SummaryRow
+ label="Not comparable"
+ value={movement.notComparable}
+ />
+ <span className="text-muted-foreground text-[10px]">
+ Not comparable: only one of the two experiments has a score for
+ the item, so it counts as neither an improvement nor a
+ regression.
+ </span>
+ </>
+ )}
+ </HoverCardContent>
+ </HoverCard>
+ {filterMenu}
+ </div>
+ );
+};
diff --git a/web/src/features/experiments/fns/describeRunComparison.clienttest.ts b/web/src/features/experiments/fns/describeRunComparison.clienttest.ts
new file mode 100644
index 0000000000..15f1e3aebb
--- /dev/null
+++ b/web/src/features/experiments/fns/describeRunComparison.clienttest.ts
@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+
+import { describeRunComparison } from "./describeRunComparison";
+
+describe("describeRunComparison", () => {
+ // The direction is the whole point: `from` is the baseline, `to` is the run
+ // the chip is attached to. Inverting it would silently invert every reading.
+ it("names the baseline and calls the chip's own run 'this run'", () => {
+ expect(
+ describeRunComparison({
+ baselineName: "sonnet-research-4-steps",
+ baselineText: "true",
+ currentText: "false",
+ }),
+ ).toBe("sonnet-research-4-steps scored true Β· this run scored false");
+ });
+
+ it("uses the metric's verb", () => {
+ expect(
+ describeRunComparison({
+ baselineName: "run-a",
+ baselineText: "$0.10",
+ currentText: "$0.11",
+ verb: "cost",
+ }),
+ ).toBe("run-a cost $0.10 Β· this run cost $0.11");
+ });
+
+ it("still reads without a name", () => {
+ expect(
+ describeRunComparison({ baselineText: "0.24", currentText: "0.63" }),
+ ).toBe("The baseline scored 0.24 Β· this run scored 0.63");
+ });
+});
diff --git a/web/src/features/experiments/fns/describeRunComparison.ts b/web/src/features/experiments/fns/describeRunComparison.ts
new file mode 100644
index 0000000000..35dcfa3de1
--- /dev/null
+++ b/web/src/features/experiments/fns/describeRunComparison.ts
@@ -0,0 +1,23 @@
+/**
+ * What a comparison chip says on hover. A chip is attached to one run, so the
+ * sentence reads from that run's side: the baseline is named, this run is "this
+ * run". `a β†’ b` and `+0.07` are ambiguous on their own β€” the reader cannot tell
+ * "was, now is" from "this run vs the comparison" β€” and this is the sentence
+ * that settles it.
+ */
+export function describeRunComparison({
+ baselineName,
+ baselineText,
+ currentText,
+ verb = "scored",
+}: {
+ /** The baseline run's name; omitted while the names are still loading. */
+ baselineName?: string;
+ /** Both sides pre-formatted by the caller, so they read as the cell does. */
+ baselineText: string;
+ currentText: string;
+ /** "scored" for a score, "cost" for cost, "took" for latency. */
+ verb?: "scored" | "cost" | "took";
+}): string {
+ return `${baselineName ?? "The baseline"} ${verb} ${baselineText} Β· this run ${verb} ${currentText}`;
+}
diff --git a/web/src/features/experiments/fns/experimentItemsColumnOrder.ts b/web/src/features/experiments/fns/experimentItemsColumnOrder.ts
new file mode 100644
index 0000000000..291052de83
--- /dev/null
+++ b/web/src/features/experiments/fns/experimentItemsColumnOrder.ts
@@ -0,0 +1,55 @@
+/**
+ * The experiment items table's column order as it shipped previously:
+ * the ids and the operational metrics held the space to the left of the fold,
+ * so the score columns β€” and the aggregate analysis their headers now carry β€”
+ * only appeared after a horizontal scroll.
+ */
+const PREVIOUS_DEFAULT_ORDER = [
+ "select",
+ "itemId",
+ "observationId",
+ "startTime",
+ "level",
+ "totalCost",
+ "latencyMs",
+ "experimentId",
+ "input",
+ "expectedOutput",
+ "output",
+ "observationScores",
+ "traceScores",
+];
+
+/** A column that is always defined, so its absence means "not built yet". */
+const ALWAYS_PRESENT_ID = "traceScores";
+
+/**
+ * Whether a stored order is still a default the app handed out, rather than an
+ * arrangement of the user's own: the ids it shares with the old default appear
+ * in the old default's relative order. Columns that came or went since (a
+ * conditional column such as `expectedOutput`) are ignored, so their absence
+ * does not read as a reorder.
+ */
+const isUntouchedDefault = (order: string[]): boolean => {
+ const shared = order.filter((id) => PREVIOUS_DEFAULT_ORDER.includes(id));
+ const expected = PREVIOUS_DEFAULT_ORDER.filter((id) => order.includes(id));
+ return shared.join(" ") === expected.join(" ");
+};
+
+/**
+ * Let the score columns' new default slot reach returning users.
+ * The reconciliation in `useColumnOrder` never overrides a stored order, so
+ * without this a new default only ever reaches new users β€” but a user who
+ * arranged the columns himself is left alone.
+ */
+export const resetStaleDefaultColumnOrder = (
+ order: string[],
+): string[] | null => {
+ if (!order.includes(ALWAYS_PRESENT_ID)) return null; // defer: no columns yet
+ if (!isUntouchedDefault(order)) return order; // his own order, not ours
+
+ // Dropping the stored order makes `useColumnOrder` rebuild it from the
+ // table's current column definitions, so the new default lives in exactly one
+ // place: the column list itself.
+ return [];
+};
diff --git a/web/src/features/experiments/fns/formatScoreColumnAggregate.ts b/web/src/features/experiments/fns/formatScoreColumnAggregate.ts
new file mode 100644
index 0000000000..9d2155ce7f
--- /dev/null
+++ b/web/src/features/experiments/fns/formatScoreColumnAggregate.ts
@@ -0,0 +1,15 @@
+import { type ScoreColumnAggregate } from "./summariseScoreColumn";
+
+export const formatScoreValue = (value: number) => value.toFixed(2);
+
+/** How a score column's aggregate reads, which depends on the score's type. */
+export const formatScoreColumnAggregate = (aggregate: ScoreColumnAggregate) => {
+ if (aggregate.kind === "average")
+ return `Ø ${formatScoreValue(aggregate.value)}`;
+ if (aggregate.kind === "trueRate")
+ return `${Math.round(aggregate.value * 100)}% true`;
+ const modalCount =
+ aggregate.distribution.find((entry) => entry.value === aggregate.modalValue)
+ ?.count ?? 0;
+ return `${aggregate.modalValue} ${modalCount}/${aggregate.count}`;
+};
diff --git a/web/src/features/experiments/fns/summariseScoreColumn.clienttest.ts b/web/src/features/experiments/fns/summariseScoreColumn.clienttest.ts
new file mode 100644
index 0000000000..ea17510a63
--- /dev/null
+++ b/web/src/features/experiments/fns/summariseScoreColumn.clienttest.ts
@@ -0,0 +1,198 @@
+import { type AggregatedScoreData } from "@langfuse/shared";
+import { summariseScoreColumn } from "./summariseScoreColumn";
+
+const numeric = (value: number): AggregatedScoreData => ({
+ type: "NUMERIC",
+ values: [value],
+ average: value,
+});
+
+const categorical = (value: string): AggregatedScoreData => ({
+ type: "CATEGORICAL",
+ values: [value],
+ valueCounts: [{ value, count: 1 }],
+});
+
+/** One item several annotators scored, all picking the same value. */
+const categoricalMulti = (
+ value: string,
+ count: number,
+): AggregatedScoreData => ({
+ type: "CATEGORICAL",
+ values: Array.from({ length: count }, () => value),
+ valueCounts: [{ value, count }],
+});
+
+describe("summariseScoreColumn", () => {
+ it("averages a numeric column and counts the items behind it", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: numeric(0.2), comparison: null },
+ { baseline: numeric(0.4), comparison: null },
+ { baseline: null, comparison: null },
+ ],
+ dataType: "NUMERIC",
+ hasComparison: false,
+ });
+
+ expect(summary.baseline).toEqual({
+ kind: "average",
+ value: 0.30000000000000004,
+ count: 2,
+ });
+ expect(summary.movement).toBeNull();
+ });
+
+ it("signs the delta and splits improved from regressed", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: numeric(0.2), comparison: numeric(0.5) },
+ { baseline: numeric(0.6), comparison: numeric(0.4) },
+ { baseline: numeric(0.3), comparison: numeric(0.3) },
+ ],
+ dataType: "NUMERIC",
+ hasComparison: true,
+ });
+
+ // The header belongs to the baseline, so the delta is baseline βˆ’ comparison.
+ expect(summary.delta).toBeCloseTo(-0.03333, 4);
+ expect(summary.movement).toEqual({
+ improved: 1,
+ regressed: 1,
+ unchanged: 1,
+ changed: 0,
+ notComparable: 0,
+ });
+ });
+
+ // Pins the frame's direction so it cannot silently flip back: the run you
+ // opened scoring higher than the run it is read against is an improvement.
+ it("reads a baseline that scored higher as an improvement", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: numeric(0.6), comparison: numeric(0.4) },
+ { baseline: numeric(0.5), comparison: numeric(0.4) },
+ ],
+ dataType: "NUMERIC",
+ hasComparison: true,
+ });
+
+ expect(summary.delta).toBeCloseTo(0.15, 4);
+ expect(summary.movement).toMatchObject({ improved: 2, regressed: 0 });
+ });
+
+ it("never counts a missing score as a regression", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ // Item the comparison run never scored.
+ { baseline: numeric(0.9), comparison: null },
+ // Item this run never scored.
+ { baseline: null, comparison: numeric(0.1) },
+ // Item neither run scored.
+ { baseline: null, comparison: null },
+ { baseline: numeric(0.2), comparison: numeric(0.4) },
+ ],
+ dataType: "NUMERIC",
+ hasComparison: true,
+ });
+
+ expect(summary.movement).toEqual({
+ improved: 0,
+ regressed: 1,
+ unchanged: 0,
+ changed: 0,
+ notComparable: 3,
+ });
+ // The aggregates still describe the items each run did score.
+ expect(summary.baseline).toMatchObject({ count: 2 });
+ expect(summary.comparison).toMatchObject({ count: 2 });
+ });
+
+ it("reads a boolean column as a true-rate, false below true", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: categorical("false"), comparison: categorical("true") },
+ { baseline: categorical("true"), comparison: categorical("false") },
+ { baseline: categorical("true"), comparison: categorical("true") },
+ { baseline: categorical("true"), comparison: null },
+ ],
+ dataType: "BOOLEAN",
+ hasComparison: true,
+ });
+
+ expect(summary.baseline).toEqual({
+ kind: "trueRate",
+ value: 0.75,
+ count: 4,
+ });
+ expect(summary.comparison).toEqual({
+ kind: "trueRate",
+ value: 2 / 3,
+ count: 3,
+ });
+ expect(summary.movement).toMatchObject({
+ improved: 1,
+ regressed: 1,
+ unchanged: 1,
+ notComparable: 1,
+ });
+ });
+
+ it("reports a categorical column as a distribution, and its moves as changed", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: categorical("grounded"), comparison: categorical("weak") },
+ {
+ baseline: categorical("grounded"),
+ comparison: categorical("grounded"),
+ },
+ { baseline: categorical("weak"), comparison: categorical("grounded") },
+ ],
+ dataType: "CATEGORICAL",
+ hasComparison: true,
+ });
+
+ expect(summary.baseline).toMatchObject({
+ kind: "distribution",
+ modalValue: "grounded",
+ distribution: [
+ { value: "grounded", count: 2 },
+ { value: "weak", count: 1 },
+ ],
+ });
+ // No order, so no direction β€” and therefore no delta.
+ expect(summary.delta).toBeNull();
+ expect(summary.movement).toEqual({
+ improved: 0,
+ regressed: 0,
+ unchanged: 1,
+ changed: 2,
+ notComparable: 0,
+ });
+ });
+
+ // The modal count and the total are printed as one fraction, so they have to
+ // be the same unit. Pooling an item's raw values made a value out-count the
+ // items it was printed over.
+ it("counts a categorical column in items, not in raw values", () => {
+ const summary = summariseScoreColumn({
+ pairs: [
+ { baseline: categoricalMulti("A", 3), comparison: null },
+ { baseline: categorical("A"), comparison: null },
+ { baseline: categorical("B"), comparison: null },
+ ],
+ dataType: "CATEGORICAL",
+ hasComparison: false,
+ });
+
+ expect(summary.baseline).toEqual({
+ kind: "distribution",
+ modalValue: "A",
+ distribution: [
+ { value: "A", count: 2 },
+ { value: "B", count: 1 },
+ ],
+ count: 3,
+ });
+ });
+});
diff --git a/web/src/features/experiments/fns/summariseScoreColumn.ts b/web/src/features/experiments/fns/summariseScoreColumn.ts
new file mode 100644
index 0000000000..aa20c466ae
--- /dev/null
+++ b/web/src/features/experiments/fns/summariseScoreColumn.ts
@@ -0,0 +1,229 @@
+import { type AggregatedScoreData } from "@langfuse/shared";
+
+/** The score's declared type, which decides what an aggregate of it can mean. */
+export type ScoreColumnDataType = "NUMERIC" | "BOOLEAN" | "CATEGORICAL";
+
+/** One item's pair of aggregates for a single score column. */
+export type ScoreColumnPair = {
+ baseline: AggregatedScoreData | null;
+ comparison: AggregatedScoreData | null;
+};
+
+export type ScoreColumnAggregate =
+ /** Mean of the items' values. NUMERIC only. */
+ | { kind: "average"; value: number; count: number }
+ /** Share of `true` across the items. BOOLEAN only. */
+ | { kind: "trueRate"; value: number; count: number }
+ /** Categorical scores have no mean, so the modal value carries the column. */
+ | {
+ kind: "distribution";
+ modalValue: string;
+ distribution: Array<{ value: string; count: number }>;
+ count: number;
+ };
+
+type ScoreColumnMovement = {
+ /** Items the baseline β€” the experiment you opened β€” scored better on. */
+ improved: number;
+ /** Items the baseline scored worse on. */
+ regressed: number;
+ unchanged: number;
+ /** A different value with no order to it β€” categorical only. */
+ changed: number;
+ /** One side has no score for this item, or the two cannot be compared. */
+ notComparable: number;
+};
+
+export type ScoreColumnSummary = {
+ baseline: ScoreColumnAggregate | null;
+ comparison: ScoreColumnAggregate | null;
+ /** `baseline βˆ’ comparison` on the ordered reading of the score, else null. */
+ delta: number | null;
+ /** Null when no comparison is selected. */
+ movement: ScoreColumnMovement | null;
+};
+
+const BOOLEAN_TRUE_VALUES = new Set(["true", "True"]);
+
+const countValues = (aggregate: AggregatedScoreData) =>
+ aggregate.type === "NUMERIC"
+ ? aggregate.values.length
+ : aggregate.valueCounts.reduce((total, entry) => total + entry.count, 0);
+
+/**
+ * The score read as a number, so two items can be ordered. A boolean's ordered
+ * reading is its true-rate (false < true); a categorical has none.
+ */
+const readOrderedScoreValue = (
+ aggregate: AggregatedScoreData | null,
+ dataType: ScoreColumnDataType,
+): number | null => {
+ if (!aggregate) return null;
+ if (dataType === "CATEGORICAL") return null;
+ if (aggregate.type === "NUMERIC") return aggregate.average;
+ if (dataType !== "BOOLEAN") return null;
+
+ const total = countValues(aggregate);
+ if (total === 0) return null;
+ const trueCount = aggregate.valueCounts
+ .filter((entry) => BOOLEAN_TRUE_VALUES.has(entry.value))
+ .reduce((sum, entry) => sum + entry.count, 0);
+ return trueCount / total;
+};
+
+/**
+ * The single categorical value of an item, or null if it carries several.
+ * Distinct values, not raw ones: several annotators who all picked the same
+ * value still leave the item with one value to stand for it, while annotators
+ * who disagree leave it with none.
+ */
+const singleCategoricalValue = (
+ aggregate: AggregatedScoreData | null,
+): string | null => {
+ if (!aggregate || aggregate.type !== "CATEGORICAL") return null;
+ const distinct = new Set(aggregate.values);
+ return distinct.size === 1 ? (aggregate.values[0] ?? null) : null;
+};
+
+const aggregateAcrossItems = (
+ aggregates: AggregatedScoreData[],
+ dataType: ScoreColumnDataType,
+): ScoreColumnAggregate | null => {
+ if (aggregates.length === 0) return null;
+
+ if (dataType === "CATEGORICAL") {
+ const counts = new Map<string, number>();
+ let counted = 0;
+ for (const aggregate of aggregates) {
+ // One vote per item, so the modal count and the total it is printed over
+ // are both counts of items. Pooling each item's raw values instead let a
+ // value scored by several annotators on one item out-count the items
+ // themselves, and the header read "A 7/5".
+ const value = singleCategoricalValue(aggregate);
+ if (value === null) continue;
+ counts.set(value, (counts.get(value) ?? 0) + 1);
+ counted += 1;
+ }
+ if (counts.size === 0) return null;
+ // Most frequent first, ties by value so the header is stable across fetches.
+ const distribution = [...counts.entries()]
+ .map(([value, count]) => ({ value, count }))
+ .sort((a, b) => b.count - a.count || a.value.localeCompare(b.value));
+ return {
+ kind: "distribution",
+ modalValue: distribution[0].value,
+ distribution,
+ count: counted,
+ };
+ }
+
+ const values = aggregates
+ .map((aggregate) => readOrderedScoreValue(aggregate, dataType))
+ .filter((value): value is number => value !== null);
+ if (values.length === 0) return null;
+
+ const mean = values.reduce((sum, value) => sum + value, 0) / values.length;
+ return dataType === "BOOLEAN"
+ ? { kind: "trueRate", value: mean, count: values.length }
+ : { kind: "average", value: mean, count: values.length };
+};
+
+const emptyMovement = (): ScoreColumnMovement => ({
+ improved: 0,
+ regressed: 0,
+ unchanged: 0,
+ changed: 0,
+ notComparable: 0,
+});
+
+/**
+ * What a score column's header says about the items in view: this experiment's
+ * aggregate, the comparison's, the signed delta, and how many items moved which
+ * way. This is the analysis the deleted Analytics tab was going to carry, put
+ * where the eye already is.
+ *
+ * A delta chip describes the thing it is attached to, and this summary is the
+ * header of the experiment you opened β€” so the delta reads
+ * `baseline βˆ’ comparison` and an item counts as improved when the baseline
+ * scored better. (A stacked cell line belongs to its comparison instead, and
+ * keeps the opposite frame.)
+ *
+ * An item with no score on either side is **not comparable**, never a
+ * regression: it is counted separately and stays visible in the header's hover.
+ * Categorical scores have no order, so they only ever report changed/unchanged.
+ */
+export const summariseScoreColumn = ({
+ pairs,
+ dataType,
+ hasComparison,
+}: {
+ pairs: ScoreColumnPair[];
+ dataType: ScoreColumnDataType;
+ hasComparison: boolean;
+}): ScoreColumnSummary => {
+ const baseline = aggregateAcrossItems(
+ pairs
+ .map((pair) => pair.baseline)
+ .filter((aggregate): aggregate is AggregatedScoreData => !!aggregate),
+ dataType,
+ );
+
+ if (!hasComparison) {
+ return { baseline, comparison: null, delta: null, movement: null };
+ }
+
+ const comparison = aggregateAcrossItems(
+ pairs
+ .map((pair) => pair.comparison)
+ .filter((aggregate): aggregate is AggregatedScoreData => !!aggregate),
+ dataType,
+ );
+
+ const movement = emptyMovement();
+ for (const pair of pairs) {
+ if (!pair.baseline || !pair.comparison) {
+ movement.notComparable += 1;
+ continue;
+ }
+
+ if (dataType === "CATEGORICAL") {
+ const baselineValue = singleCategoricalValue(pair.baseline);
+ const comparisonValue = singleCategoricalValue(pair.comparison);
+ if (baselineValue === null || comparisonValue === null) {
+ movement.notComparable += 1;
+ } else if (baselineValue === comparisonValue) {
+ movement.unchanged += 1;
+ } else {
+ movement.changed += 1;
+ }
+ continue;
+ }
+
+ const baselineValue = readOrderedScoreValue(pair.baseline, dataType);
+ const comparisonValue = readOrderedScoreValue(pair.comparison, dataType);
+ if (baselineValue === null || comparisonValue === null) {
+ movement.notComparable += 1;
+ } else if (baselineValue > comparisonValue) {
+ movement.improved += 1;
+ } else if (baselineValue < comparisonValue) {
+ movement.regressed += 1;
+ } else {
+ movement.unchanged += 1;
+ }
+ }
+
+ const orderedBaseline =
+ baseline && baseline.kind !== "distribution" ? baseline.value : null;
+ const orderedComparison =
+ comparison && comparison.kind !== "distribution" ? comparison.value : null;
+
+ return {
+ baseline,
+ comparison,
+ delta:
+ orderedBaseline !== null && orderedComparison !== null
+ ? orderedBaseline - orderedComparison
+ : null,
+ movement,
+ };
+};
diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.clienttest.tsx b/web/src/features/experiments/hooks/useExperimentResultsState.clienttest.tsx
index 5e2b391888..6c2ceba5e2 100644
--- a/web/src/features/experiments/hooks/useExperimentResultsState.clienttest.tsx
+++ b/web/src/features/experiments/hooks/useExperimentResultsState.clienttest.tsx
@@ -57,6 +57,8 @@ function Harness() {
maxSelectedExperiments,
clearBaseline,
selectedExperimentIds,
+ layout,
+ diffMode,
} = useExperimentResultsState();
return (
@@ -66,6 +68,8 @@ function Harness() {
<div data-testid="comparisons">{comparisonIds.join(",")}</div>
<div data-testid="selected">{selectedExperimentIds.join(",")}</div>
<div data-testid="max-experiments">{maxSelectedExperiments}</div>
+ <div data-testid="layout">{layout}</div>
+ <div data-testid="diff-mode">{diffMode}</div>
<button
type="button"
onClick={() =>
@@ -86,6 +90,32 @@ function Harness() {
describe("useExperimentResultsState", () => {
beforeEach(() => {
queryParamStore.clear();
+ localStorage.clear();
+ });
+
+ it("defaults to the one-row-per-item diff layout once a comparison exists", () => {
+ queryParamStore.set("baseline", "baseline-run");
+
+ render(<Harness />);
+
+ expect(screen.getByTestId("layout").textContent).toBe("grid");
+ expect(screen.getByTestId("diff-mode").textContent).toBe("comparison");
+
+ queryParamStore.set("c", ["comp-a"]);
+
+ render(<Harness />);
+
+ expect(screen.getAllByTestId("layout")[1].textContent).toBe("list");
+ });
+
+ it("lets an explicit layout in the URL win over the default", () => {
+ queryParamStore.set("baseline", "baseline-run");
+ queryParamStore.set("c", ["comp-a"]);
+ queryParamStore.set("layout", "grid");
+
+ render(<Harness />);
+
+ expect(screen.getByTestId("layout").textContent).toBe("grid");
});
it("derives hasBaseline correctly", () => {
diff --git a/web/src/features/experiments/hooks/useExperimentResultsState.ts b/web/src/features/experiments/hooks/useExperimentResultsState.ts
index f45f15f12a..c488aa4894 100644
--- a/web/src/features/experiments/hooks/useExperimentResultsState.ts
+++ b/web/src/features/experiments/hooks/useExperimentResultsState.ts
@@ -6,6 +6,24 @@ import {
type UrlUpdateType,
} from "use-query-params";
import { MAX_SELECTED_EXPERIMENTS } from "@/src/features/experiments/constants/comparison";
+import useLocalStorage from "@/src/components/useLocalStorage";
+
+/**
+ * How the selected experiments are laid out against each other: one row per item
+ * ("list") or one column per experiment ("grid").
+ */
+export type ExperimentResultsLayout = "grid" | "list";
+
+/** What a diff cell's extra line is measured against. */
+export type ExperimentDiffMode = "comparison" | "expected" | "off";
+
+const asLayout = (value: unknown): ExperimentResultsLayout | undefined =>
+ value === "grid" || value === "list" ? value : undefined;
+
+const asDiffMode = (value: unknown): ExperimentDiffMode | undefined =>
+ value === "comparison" || value === "expected" || value === "off"
+ ? value
+ : undefined;
// Drop null/empty entries and de-duplicate, preserving first-seen order.
const dedupe = (ids: (string | null | undefined)[]): string[] =>
@@ -15,7 +33,8 @@ export function useExperimentResultsState() {
const [state, setState] = useQueryParams({
baseline: withDefault(StringParam, undefined),
c: withDefault(ArrayParam, []),
- layout: withDefault(StringParam, "grid"),
+ layout: withDefault(StringParam, undefined),
+ diff: withDefault(StringParam, undefined),
itemVisibility: withDefault(StringParam, "baseline-only"),
});
@@ -88,12 +107,52 @@ export function useExperimentResultsState() {
const removeComparisonId = (id: string) =>
setComparisonIds(comparisonIds.filter((existingId) => existingId !== id));
- // Layout management
- const layout = (state.layout as "grid" | "list") ?? "list";
- const setLayout = (newLayout: "grid" | "list") => {
+ // Layout and diff mode: the URL wins so a view stays shareable, then the
+ // user's remembered pick, then the default for the current selection.
+ const [storedLayout, setStoredLayout] =
+ useLocalStorage<ExperimentResultsLayout | null>(
+ "experiment-results-layout",
+ null,
+ );
+ const [storedDiffMode, setStoredDiffMode] =
+ useLocalStorage<ExperimentDiffMode | null>("experiment-results-diff", null);
+
+ // With something to compare against, one row per item beats one wide column
+ // per experiment: a three-way comparison pushes the third experiment off a
+ // 1512px screen entirely.
+ const layout: ExperimentResultsLayout =
+ asLayout(state.layout) ??
+ storedLayout ??
+ (comparisonIds.length > 0 ? "list" : "grid");
+
+ const setLayout = (newLayout: ExperimentResultsLayout) => {
+ setStoredLayout(newLayout);
setState({ layout: newLayout });
};
+ const requestedDiffMode: ExperimentDiffMode =
+ asDiffMode(state.diff) ?? storedDiffMode ?? "comparison";
+
+ // Expected β†’ Output is the list layout's two-line cell, and no other layout
+ // draws it. A shared URL or a remembered pick that pairs it with another
+ // layout would leave the menu promising a comparison the table is not
+ // showing, so it reads as the baseline diff until the layout can express it.
+ const diffMode: ExperimentDiffMode =
+ requestedDiffMode === "expected" && layout !== "list"
+ ? "comparison"
+ : requestedDiffMode;
+
+ const setDiffMode = (newDiffMode: ExperimentDiffMode) => {
+ setStoredDiffMode(newDiffMode);
+ // Expected β†’ Output is the diff layout's two-line cell; picking it from the
+ // side-by-side layout would otherwise appear to do nothing.
+ setState({
+ diff: newDiffMode,
+ ...(newDiffMode === "expected" ? { layout: "list" } : {}),
+ });
+ if (newDiffMode === "expected") setStoredLayout("list");
+ };
+
// Item visibility management
const itemVisibility =
(state.itemVisibility as "baseline-only" | "all") ?? "baseline-only";
@@ -124,6 +183,10 @@ export function useExperimentResultsState() {
layout,
setLayout,
+ // Diff mode
+ diffMode,
+ setDiffMode,
+
// Item visibility
itemVisibility,
setItemVisibility,
diff --git a/web/src/features/scores/hooks/useScoreColumns.ts b/web/src/features/scores/hooks/useScoreColumns.ts
index a2c1c09de0..358d7d539d 100644
--- a/web/src/features/scores/hooks/useScoreColumns.ts
+++ b/web/src/features/scores/hooks/useScoreColumns.ts
@@ -22,6 +22,7 @@ export function createScoreColumns<T extends Record<string, any>>({
headerPrefix,
defaultHidden,
rawKey,
+ valueTitle,
}: {
scoreColumns: Array<{
key: string;
@@ -37,6 +38,8 @@ export function createScoreColumns<T extends Record<string, any>>({
headerPrefix?: string;
defaultHidden?: boolean;
rawKey?: boolean;
+ /** See `ScoresTableCell`'s `valueTitle`: what this row's value belongs to. */
+ valueTitle?: (original: T) => string | undefined;
}): LangfuseColumnDef<T>[] {
return scoreColumns.map(({ key, name, source, dataType }) => {
const accessorKey = prefix ? `${prefix}-${key}` : key;
@@ -67,6 +70,7 @@ export function createScoreColumns<T extends Record<string, any>>({
aggregate: value,
displayFormat,
hasMetadata: value.hasMetadata ?? false,
+ valueTitle: valueTitle?.(row.original),
});
},
};
diff --git a/web/src/features/scores/lib/scoreColumns.ts b/web/src/features/scores/lib/scoreColumns.ts
index f0698a6b46..186f54d728 100644
--- a/web/src/features/scores/lib/scoreColumns.ts
+++ b/web/src/features/scores/lib/scoreColumns.ts
@@ -266,6 +266,28 @@ export const getScoreDataTypeExplanation = (
}
};
+const SCORE_DATA_TYPE_ICONS = new Set(["#", "β’Έ", "β’·", "Aa"]);
+
+/**
+ * Splits a header built by `createScoreColumns` into its data-type icon and the
+ * rest of the label, so a surface that explains the type itself does not render
+ * the bare glyph twice. A CORRECTION carries no glyph, so the label is trimmed
+ * rather than left with the gap the missing icon would have filled.
+ */
+export const splitScoreDataTypeIcon = (
+ header: string,
+): { icon?: string; label: string } => {
+ const parts = header.split(" ");
+ const iconIndex = parts.findIndex((part) => SCORE_DATA_TYPE_ICONS.has(part));
+ if (iconIndex === -1) return { label: header.trim() };
+ return {
+ icon: parts[iconIndex],
+ label: [...parts.slice(0, iconIndex), ...parts.slice(iconIndex + 1)]
+ .join(" ")
+ .trim(),
+ };
+};
+
export const getScoreDataTypeIcon = (dataType: ScoreDataTypeType): string => {
switch (dataType) {
case "NUMERIC":
diff --git a/web/src/pages/project/[projectId]/experiments/results.tsx b/web/src/pages/project/[projectId]/experiments/results.tsx
index 5ecad894ad..fe55ded5c0 100644
--- a/web/src/pages/project/[projectId]/experiments/results.tsx
+++ b/web/src/pages/project/[projectId]/experiments/results.tsx
@@ -29,6 +29,8 @@ export default function ExperimentResults() {
setComparisonIds,
layout,
setLayout,
+ diffMode,
+ setDiffMode,
itemVisibility,
setItemVisibility,
allExperimentIds,
@@ -98,6 +100,8 @@ export default function ExperimentResults() {
<ExperimentDisplaySettings
layout={layout}
onLayoutChange={setLayout}
+ diffMode={diffMode}
+ onDiffModeChange={setDiffMode}
itemVisibility={itemVisibility}
onItemVisibilityChange={setItemVisibility}
hasComparisons={comparisonIds.length > 0}