Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(experiments): show the score columns, and make a score cell readable (#16913)

Nikita Kabardinf0cf0ce774e3
diff --git a/web/src/__tests__/aggregateScores.clienttest.ts b/web/src/__tests__/aggregateScores.clienttest.ts
index dbd47ceeca..54aac168df 100644
--- a/web/src/__tests__/aggregateScores.clienttest.ts
+++ b/web/src/__tests__/aggregateScores.clienttest.ts
@@ -140,12 +140,13 @@ describe("aggregateScores", () => {
});
});
- it("should correctly aggregate multiple Categorical scores with the same key", () => {
+ it("should render Boolean scores as true/false", () => {
const scores = [
{
name: "test",
source: "API",
dataType: "BOOLEAN",
+ value: 1,
stringValue: "True",
comment: "test comment",
},
@@ -153,6 +154,7 @@ describe("aggregateScores", () => {
name: "test",
source: "API",
dataType: "BOOLEAN",
+ value: 0,
stringValue: "False",
comment: "another comment",
},
@@ -160,16 +162,30 @@ describe("aggregateScores", () => {
expect(aggregateScores(scores)).toEqual({
"test-API-BOOLEAN": {
type: "CATEGORICAL",
- values: ["True", "False"],
+ values: ["false", "true"],
valueCounts: [
- { value: "True", count: 1 },
- { value: "False", count: 1 },
+ { value: "false", count: 1 },
+ { value: "true", count: 1 },
],
comment: undefined,
},
});
});
+ it("should order a cell's values the same way regardless of input order", () => {
+ const asScores = (stringValues: string[]) =>
+ stringValues.map((stringValue) => ({
+ name: "verdict",
+ source: "API",
+ dataType: "CATEGORICAL",
+ stringValue,
+ })) as ScoreToAggregate[];
+
+ expect(aggregateScores(asScores(["pass", "fail", "pass"]))).toEqual(
+ aggregateScores(asScores(["fail", "pass", "pass"])),
+ );
+ });
+
it("should correctly aggregate scores with mixed types and the same name", () => {
const scores = [
{
@@ -213,10 +229,10 @@ describe("aggregateScores", () => {
},
"test-ANNOTATION-CATEGORICAL": {
type: "CATEGORICAL",
- values: ["good", "bad", "good"],
+ values: ["bad", "good", "good"],
valueCounts: [
- { value: "good", count: 2 },
{ value: "bad", count: 1 },
+ { value: "good", count: 2 },
],
comment: undefined,
},
diff --git a/web/src/components/scores-table-cell.tsx b/web/src/components/scores-table-cell.tsx
index 8b722af2f6..e13cf3941b 100644
--- a/web/src/components/scores-table-cell.tsx
+++ b/web/src/components/scores-table-cell.tsx
@@ -19,8 +19,12 @@ import React from "react";
import { copyTextToClipboard } from "@/src/utils/clipboard";
import { Button } from "@/src/components/ui/button";
+// Boolean scores render as `true`/`false`; the capitalised entries keep the same
+// treatment for categorical scores whose category happens to be True/False.
const COLOR_MAP = new Map([
+ ["true", "bg-light-green p-0.5 text-dark-green"],
["True", "bg-light-green p-0.5 text-dark-green"],
+ ["false", "bg-light-red p-0.5 text-dark-red"],
["False", "bg-light-red p-0.5 text-dark-red"],
]);
const COLLAPSE_CATEGORICAL_SCORES_AFTER = 2;
diff --git a/web/src/components/table/data-table-column-visibility-filter.tsx b/web/src/components/table/data-table-column-visibility-filter.tsx
index da650befed..14bc3fbb48 100644
--- a/web/src/components/table/data-table-column-visibility-filter.tsx
+++ b/web/src/components/table/data-table-column-visibility-filter.tsx
@@ -67,6 +67,18 @@ interface DataTableColumnVisibilityFilterProps<TData, TValue> {
onColumnGroupToggle?: (payload: ColumnGroupTogglePayload) => void;
}
+/**
+ * A column's name for the picker: its own label before its rendered header, so
+ * a column whose header carries more than its name still reads here.
+ */
+const getColumnLabel = <TData, TValue>(
+ column: LangfuseColumnDef<TData, TValue>,
+) =>
+ column.headerLabel ??
+ (typeof column.header === "string" && column.header
+ ? column.header
+ : column.accessorKey);
+
const calculateColumnCounts = <TData, TValue>(
columns: LangfuseColumnDef<TData, TValue>[],
columnVisibility: VisibilityState,
@@ -157,9 +169,7 @@ function ColumnVisibilityListItem<TData, TValue>({
: undefined
}
>
- {column.header && typeof column.header === "string"
- ? column.header
- : column.accessorKey}
+ {getColumnLabel(column)}
</label>
{column.headerTooltip && (
<DocPopup
@@ -227,11 +237,7 @@ function GroupVisibilityHeader<TData, TValue>({
>
<div className="flex items-center gap-2">
<Component className="h-4 w-4 opacity-50" />
- <span className="text-sm font-bold">
- {column.header && typeof column.header === "string"
- ? column.header
- : column.accessorKey}
- </span>
+ <span className="text-sm font-bold">{getColumnLabel(column)}</span>
<span className="text-muted-foreground text-xs">
({groupVisibleCount}/{groupTotalCount})
</span>
diff --git a/web/src/components/table/types.ts b/web/src/components/table/types.ts
index 47771abd4f..338e34cde1 100644
--- a/web/src/components/table/types.ts
+++ b/web/src/components/table/types.ts
@@ -12,6 +12,12 @@ declare module "@tanstack/react-table" {
description: React.ReactNode;
href?: string;
};
+ /**
+ * Plain-text name of the column, for surfaces that want a label rather than
+ * the rendered header — the column picker. Only needed when `header` is not
+ * a string.
+ */
+ headerLabel?: string;
isFixedPosition?: boolean; // if true, column cannot be reordered
isPinnedLeft?: boolean; // if true, column will be pinned to left side
isFlexWidth?: boolean; // if true, column absorbs leftover space (one per table)
diff --git a/web/src/features/column-visibility/hooks/useColumnOrder.ts b/web/src/features/column-visibility/hooks/useColumnOrder.ts
index 4c8f18ecb5..eda4e0309e 100644
--- a/web/src/features/column-visibility/hooks/useColumnOrder.ts
+++ b/web/src/features/column-visibility/hooks/useColumnOrder.ts
@@ -1,6 +1,11 @@
import { useEffect } from "react";
import { type LangfuseColumnDef } from "@/src/components/table/types";
import useLocalStorage from "@/src/components/useLocalStorage";
+import {
+ hasRunMigration,
+ markMigrationRun,
+ type OneTimeMigration,
+} from "@/src/features/column-visibility/lib/one-time-migration";
// returns deep copy of local storage object
const readStoredColumnOrder = (localStorageKey: string): string[] => {
@@ -16,59 +21,10 @@ const readStoredColumnOrder = (localStorageKey: string): string[] => {
}
};
-/**
- * A one-time, opt-in transform of the persisted column order. Use it when a
- * table changes its *default* column position: the reconciliation in
- * `useColumnOrder` only splices in column IDs that are new to the app, so it
- * never repositions a column that already exists in a returning user's stored
- * order. A migration lets a specific table reposition such a column exactly
- * once, guarded by `versionKey` so it never re-fights a user who later moves
- * the column themselves.
- *
- * Migrations are entirely opt-in (passed per-call): tables that omit them keep
- * the previous behavior byte-for-byte.
- */
-type ColumnOrderMigration = {
- /**
- * localStorage key for the one-time guard flag. Should be unique per table +
- * migration, e.g. `experimentsColumnOrder-metadataReorder-v1-${projectId}`.
- */
- versionKey: string;
- /**
- * Pure transform applied to the reconciled column order. Must not mutate its
- * input. Return the input unchanged to skip (the flag is still set so it does
- * not retry on every mount).
- */
- apply: (columnOrder: string[]) => string[];
-};
-
-const hasRunMigration = (versionKey: string): boolean => {
- if (typeof window === "undefined") {
- return true; // never run migrations server-side
- }
- try {
- return localStorage.getItem(versionKey) !== null;
- } catch (error) {
- console.warn("Error reading migration flag from local storage", error);
- return true;
- }
-};
-
-const markMigrationRun = (versionKey: string): void => {
- if (typeof window === "undefined") {
- return;
- }
- try {
- localStorage.setItem(versionKey, "1");
- } catch (error) {
- console.warn("Error writing migration flag to local storage", error);
- }
-};
-
function useColumnOrder<TData>(
localStorageKey: string,
columns: LangfuseColumnDef<TData>[],
- migrations?: ColumnOrderMigration[],
+ migrations?: OneTimeMigration<string[]>[],
) {
const [columnOrder, setColumnOrder] = useLocalStorage<string[]>(
localStorageKey,
@@ -92,10 +48,11 @@ function useColumnOrder<TData>(
// Apply any opt-in one-time migrations (e.g. repositioning a column whose
// default slot changed). Each runs at most once, guarded by its versionKey.
migrations?.forEach((migration) => {
- if (!hasRunMigration(migration.versionKey)) {
- finalColumnOrder = migration.apply(finalColumnOrder);
- markMigrationRun(migration.versionKey);
- }
+ if (hasRunMigration(migration.versionKey)) return;
+ const migrated = migration.apply(finalColumnOrder);
+ if (migrated === null) return; // deferred, retry on a later render
+ finalColumnOrder = migrated;
+ markMigrationRun(migration.versionKey);
});
// Compare the new order with the current order to avoid unnecessary updates
diff --git a/web/src/features/column-visibility/hooks/useColumnVisibility.ts b/web/src/features/column-visibility/hooks/useColumnVisibility.ts
index 62df30b4ad..c41c51ba5d 100644
--- a/web/src/features/column-visibility/hooks/useColumnVisibility.ts
+++ b/web/src/features/column-visibility/hooks/useColumnVisibility.ts
@@ -3,6 +3,11 @@ import { type LangfuseColumnDef } from "@/src/components/table/types";
import useLocalStorage from "@/src/components/useLocalStorage";
import { useEffect } from "react";
import isEqual from "lodash/isEqual";
+import {
+ hasRunMigration,
+ markMigrationRun,
+ type OneTimeMigration,
+} from "@/src/features/column-visibility/lib/one-time-migration";
// returns deep copy of local storage object
const readStoredVisibilityState = (
@@ -41,6 +46,7 @@ function setVisibility<TData>(
function useColumnVisibility<TData>(
localStorageKey: string,
columns: LangfuseColumnDef<TData>[],
+ migrations?: OneTimeMigration<VisibilityState>[],
) {
const initialVisibilityState = () => {
const storedVisibilityState = readStoredVisibilityState(localStorageKey);
@@ -55,18 +61,28 @@ function useColumnVisibility<TData>(
useLocalStorage<VisibilityState>(localStorageKey, initialVisibilityState());
useEffect(() => {
- const initialColumnVisibility = initialVisibilityState();
+ let initialColumnVisibility = initialVisibilityState();
Object.keys(initialColumnVisibility).forEach((key) => {
if (Object.hasOwn(columnVisibility, key)) {
initialColumnVisibility[key] = columnVisibility[key];
}
});
+ // Apply any opt-in one-time migrations (e.g. a column that became visible
+ // by default). Each runs at most once, guarded by its versionKey.
+ migrations?.forEach((migration) => {
+ if (hasRunMigration(migration.versionKey)) return;
+ const migrated = migration.apply(initialColumnVisibility);
+ if (migrated === null) return; // deferred, retry on a later render
+ initialColumnVisibility = migrated;
+ markMigrationRun(migration.versionKey);
+ });
+
if (!isEqual(initialColumnVisibility, columnVisibility)) {
setColumnVisibility(initialColumnVisibility);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [columnVisibility, setColumnVisibility, columns]);
+ }, [columnVisibility, setColumnVisibility, columns, migrations]);
return [columnVisibility, setColumnVisibility] as const;
}
diff --git a/web/src/features/column-visibility/lib/one-time-migration.ts b/web/src/features/column-visibility/lib/one-time-migration.ts
new file mode 100644
index 0000000000..f8e2e8701a
--- /dev/null
+++ b/web/src/features/column-visibility/lib/one-time-migration.ts
@@ -0,0 +1,48 @@
+/**
+ * A one-time, opt-in transform of persisted table state (column order or
+ * column visibility). Use it when a table changes a *default*: the
+ * reconciliation in `useColumnOrder` / `useColumnVisibility` never overrides a
+ * value a returning user already has stored, so without a migration a new
+ * default only ever reaches new users.
+ *
+ * Migrations are entirely opt-in (passed per-call): tables that omit them keep
+ * the previous behavior byte-for-byte.
+ */
+export type OneTimeMigration<T> = {
+ /**
+ * localStorage key for the one-time guard flag. Should be unique per table +
+ * migration, e.g. `experimentsColumnOrder-metadataReorder-v1-${projectId}`.
+ */
+ versionKey: string;
+ /**
+ * Pure transform applied to the reconciled state. Must not mutate its input.
+ * Return the input unchanged to skip (the flag is still set so it does not
+ * retry on every mount), or `null` to defer — for state that depends on
+ * asynchronously loaded columns, so the migration is retried instead of
+ * being consumed against incomplete state.
+ */
+ apply: (state: T) => T | null;
+};
+
+export const hasRunMigration = (versionKey: string): boolean => {
+ if (typeof window === "undefined") {
+ return true; // never run migrations server-side
+ }
+ try {
+ return localStorage.getItem(versionKey) !== null;
+ } catch (error) {
+ console.warn("Error reading migration flag from local storage", error);
+ return true;
+ }
+};
+
+export const markMigrationRun = (versionKey: string): void => {
+ if (typeof window === "undefined") {
+ return;
+ }
+ try {
+ localStorage.setItem(versionKey, "1");
+ } catch (error) {
+ console.warn("Error writing migration flag to local storage", error);
+ }
+};
diff --git a/web/src/features/experiments/components/table/ExperimentGridCell.tsx b/web/src/features/experiments/components/table/ExperimentGridCell.tsx
index 11e1d3f111..d9b45bc3f6 100644
--- a/web/src/features/experiments/components/table/ExperimentGridCell.tsx
+++ b/web/src/features/experiments/components/table/ExperimentGridCell.tsx
@@ -38,10 +38,12 @@ import { api } from "@/src/utils/api";
import { Skeleton } from "@/src/components/ui/skeleton";
import { JSONView } from "@/src/components/ui/CodeJsonViewer";
import { decomposeAggregateScoreKey } from "@/src/features/scores/lib/aggregateScores";
+import { getScoreDataTypeExplanation } from "@/src/features/scores/lib/scoreColumns";
import { cn } from "@/src/utils/tailwind";
import { getPlainTextFromReactNode } from "@/src/utils/react-node-plain-text";
import Link from "next/link";
import { ScoreTag, type ScoreLevel } from "@/src/components/score-tag";
+import { NotRecordedMetric } from "./NotRecordedMetric";
type ExperimentGridCellProps = {
projectId: string;
@@ -275,6 +277,11 @@ const ScoreItem = ({
<span className="text-muted-foreground">Type:</span>
<span className="capitalize">{dataType.toLowerCase()}</span>
</div>
+ {/* The side-by-side layout has no score column headers, so the
+ type is explained here instead. */}
+ <p className="text-muted-foreground max-w-[260px]">
+ {getScoreDataTypeExplanation(dataType)}
+ </p>
</div>
</HoverCardContent>
</HoverCard>
@@ -582,10 +589,10 @@ export const ExperimentGridCell = ({
cell: ({ data }) => (
<MetadataItem label="Total Cost">
<span className="inline-flex items-center gap-1 text-xs">
- {data.totalCost != null ? (
+ {data.totalCost ? (
usdFormatter(data.totalCost, 2, 6)
) : (
- <span className="text-muted-foreground">-</span>
+ <NotRecordedMetric metric="cost" />
)}
{data.totalCostDiff && (
<DiffLabel
@@ -606,7 +613,7 @@ export const ExperimentGridCell = ({
{data.latencyMs != null ? (
latencyFormatter(data.latencyMs)
) : (
- <span className="text-muted-foreground">-</span>
+ <NotRecordedMetric metric="latency" />
)}
{data.latencyDiff && (
<DiffLabel
diff --git a/web/src/features/experiments/components/table/ExperimentGridView.tsx b/web/src/features/experiments/components/table/ExperimentGridView.tsx
index 12cfd4ec9c..62682a4e00 100644
--- a/web/src/features/experiments/components/table/ExperimentGridView.tsx
+++ b/web/src/features/experiments/components/table/ExperimentGridView.tsx
@@ -39,6 +39,8 @@ type ExperimentGridViewProps = {
rows: ExperimentItemsTableRow[];
isLoading: boolean;
rowHeight: RowHeight;
+ /** Whether any item in view has an expected output worth a column. */
+ showExpectedOutput: boolean;
observationScoreOrder: string[];
traceScoreOrder: string[];
showScoreLevelLabels: boolean;
@@ -70,6 +72,7 @@ export const ExperimentGridView = ({
rows,
isLoading,
rowHeight,
+ showExpectedOutput,
observationScoreOrder,
traceScoreOrder,
showScoreLevelLabels,
@@ -207,17 +210,30 @@ export const ExperimentGridView = ({
getCell: (value) => (isLoading ? { type: "loading" } : (value ?? null)),
singleLine,
}),
- createIOTableColumn<ExperimentItemsTableRow>({
- accessorKey: "expectedOutput",
- header: "Expected Output",
- size: 200,
- getCell: (value) => (isLoading ? { type: "loading" } : (value ?? null)),
- singleLine,
- variant: "output",
- }),
+ // Gated: an empty expected output used to render as two literal quote
+ // characters, and a whole column of them is worse than no column.
+ ...(showExpectedOutput
+ ? [
+ createIOTableColumn<ExperimentItemsTableRow>({
+ accessorKey: "expectedOutput",
+ header: "Expected Output",
+ size: 200,
+ getCell: (value) =>
+ isLoading ? { type: "loading" } : value || undefined,
+ singleLine,
+ variant: "output",
+ }),
+ ]
+ : []),
...experimentColumns,
],
- [experimentColumns, isLoading, selectActionColumn, singleLine],
+ [
+ experimentColumns,
+ isLoading,
+ selectActionColumn,
+ showExpectedOutput,
+ singleLine,
+ ],
);
return (
diff --git a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
index 9a84023a8e..8452713589 100644
--- a/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentItemsTable.tsx
@@ -39,7 +39,10 @@ import { useRowHeightLocalStorage } from "@/src/components/table/data-table-row-
import useColumnOrder from "@/src/features/column-visibility/hooks/useColumnOrder";
import { buildLocalIsoDatePresentation } from "@/src/utils/dates";
import { usdFormatter, latencyFormatter } from "@/src/utils/numbers";
-import { type RowSelectionState } from "@tanstack/react-table";
+import {
+ type RowSelectionState,
+ type VisibilityState,
+} from "@tanstack/react-table";
import { IdTableCell } from "@/src/components/design-system/table/components/IdTableCell/IdTableCell";
import { createIdTableColumn } from "@/src/components/design-system/table/columns/createIdTableColumn";
import { createIOTableColumn } from "@/src/components/design-system/table/columns/createIOTableColumn";
@@ -62,6 +65,11 @@ import { ConnectedIOTableCell } from "@/src/components/table/ConnectedIOTableCel
import { type DataTablePeekViewProps } from "@/src/components/table/peek";
import { cn } from "@/src/utils/tailwind";
import { createScoreColumns } from "@/src/features/scores/hooks/useScoreColumns";
+import {
+ collectPresentScoreKeys,
+ revealScoreColumns,
+ withPresentScoreKeys,
+} from "@/src/features/scores/lib/scoreColumns";
import { composeAggregateScoreKey } from "@/src/features/scores/lib/aggregateScores";
import { Skeleton } from "@/src/components/ui/skeleton";
import { ExperimentCompareTable } from "./ExperimentCompareTable";
@@ -73,6 +81,7 @@ import {
import { DiffLabel } from "@/src/features/datasets/components/DiffLabel";
import { computeScoreDiffs } from "@/src/features/datasets/lib/computeScoreDiffs";
import { TablePeekViewExperimentItemDetail } from "@/src/components/table/peek/peek-experiment-item-detail";
+import { NotRecordedMetric } from "./NotRecordedMetric";
const renderExperimentSpecificHeader = (label: string) => (
<span className="text-muted-foreground">{label}</span>
@@ -508,6 +517,32 @@ export default function ExperimentItemsTable({
itemVisibility,
});
+ // Running items without an expected output is common, so don't spend a column
+ // on it when nothing has one. Kept while IO loads so it doesn't flash.
+ //
+ // Whether the column exists is a property of the runs being compared, not of
+ // the page that happens to be loaded, so a page of items that all lack an
+ // expected output must not take the column away again. Latched per selection:
+ // once any page has shown one, the column stays until the selection changes.
+ const expectedOutputOnPage = useMemo(
+ () => (items.rows ?? []).some((row) => Boolean(row.expectedOutput)),
+ [items.rows],
+ );
+ const experimentSelectionKey = useMemo(
+ () => allExperimentIds.join(","),
+ [allExperimentIds],
+ );
+ const [expectedOutputSeenFor, setExpectedOutputSeenFor] = useState<
+ string | null
+ >(null);
+ useEffect(() => {
+ if (expectedOutputOnPage) setExpectedOutputSeenFor(experimentSelectionKey);
+ }, [expectedOutputOnPage, experimentSelectionKey]);
+ const showExpectedOutput =
+ ioLoading ||
+ expectedOutputOnPage ||
+ expectedOutputSeenFor === experimentSelectionKey;
+
useEffect(() => {
if (items.status === "success") {
// Store all experiment targets for peek navigation
@@ -562,32 +597,54 @@ export default function ExperimentItemsTable({
[hasBaseline, allExperimentIds],
);
+ // A score column that is empty for every item in view is noise, so only keep
+ // the keys the items query actually returned. Undefined while items load, so
+ // columns don't disappear and come back on each fetch.
+ const presentScoreKeys = useMemo(() => {
+ if (items.status !== "success") return undefined;
+ const experimentsInView = (items.rows ?? []).flatMap(
+ (row) => row.experiments,
+ );
+ return {
+ observation: collectPresentScoreKeys(
+ experimentsInView.map((exp) => exp.observationScores),
+ ),
+ trace: collectPresentScoreKeys(
+ experimentsInView.map((exp) => exp.traceScores),
+ ),
+ };
+ }, [items]);
+
// Create score columns from the shared filter options data
// This ensures sidebar filters and column visibility use the same data source
const observationScoreColumns = useMemo(
() =>
- createScoreColumns<ExperimentItemData>(
- toScoreColumnInput(scoreColumnDefs.observationScoreColumns),
- "observationScores",
- "smart",
- undefined,
- undefined,
- true,
- ),
- [scoreColumnDefs.observationScoreColumns],
+ createScoreColumns<ExperimentItemData>({
+ scoreColumns: withPresentScoreKeys(
+ toScoreColumnInput(scoreColumnDefs.observationScoreColumns),
+ presentScoreKeys?.observation,
+ ),
+ scoreColumnKey: "observationScores",
+ displayFormat: "smart",
+ headerPrefix: "Observation",
+ rawKey: true,
+ }),
+ [scoreColumnDefs.observationScoreColumns, presentScoreKeys?.observation],
);
const traceScoreColumns = useMemo(
() =>
- createScoreColumns<ExperimentItemData>(
- toScoreColumnInput(scoreColumnDefs.traceScoreColumns),
- "traceScores",
- "smart",
- "Trace",
- true,
- true,
- ),
- [scoreColumnDefs.traceScoreColumns],
+ createScoreColumns<ExperimentItemData>({
+ scoreColumns: withPresentScoreKeys(
+ toScoreColumnInput(scoreColumnDefs.traceScoreColumns),
+ presentScoreKeys?.trace,
+ ),
+ scoreColumnKey: "traceScores",
+ displayFormat: "smart",
+ prefix: "Trace",
+ rawKey: true,
+ }),
+ [scoreColumnDefs.traceScoreColumns, presentScoreKeys?.trace],
);
// Use the shared loading state for both sidebar and columns
@@ -697,6 +754,17 @@ export default function ExperimentItemsTable({
scoreColumnDefs.observationScoreColumns.length > 0 &&
scoreColumnDefs.traceScoreColumns.length > 0;
+ const expectedOutputColumn = createIOTableColumn<ExperimentItemsTableRow>({
+ accessorKey: "expectedOutput",
+ header: "Expected Output",
+ size: 300,
+ enableHiding: true,
+ // An empty expected output used to render as two literal quote characters.
+ getCell: (value) => (ioLoading ? { type: "loading" } : value || undefined),
+ singleLine: ioSingleLine,
+ variant: "output",
+ }) as LangfuseColumnDef<ExperimentItemsTableRow>;
+
const columns: LangfuseColumnDef<ExperimentItemsTableRow>[] = [
...(hideControls ? [] : [selectActionColumn]),
createIdTableColumn<ExperimentItemsTableRow>({
@@ -708,6 +776,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "observationId",
id: "observationId",
+ headerLabel: "Observation ID",
header: () => renderExperimentSpecificHeader("Observation ID"),
size: 180,
enableHiding: true,
@@ -726,6 +795,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "startTime",
id: "startTime",
+ headerLabel: getExperimentItemsColumnName("startTime"),
header: () =>
renderExperimentSpecificHeader(
getExperimentItemsColumnName("startTime"),
@@ -757,6 +827,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "level",
id: "level",
+ headerLabel: getExperimentItemsColumnName("level"),
header: () =>
renderExperimentSpecificHeader(getExperimentItemsColumnName("level")),
size: 120,
@@ -777,6 +848,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "totalCost",
id: "totalCost",
+ headerLabel: getExperimentItemsColumnName("totalCost"),
header: () =>
renderExperimentSpecificHeader(
getExperimentItemsColumnName("totalCost"),
@@ -790,15 +862,13 @@ export default function ExperimentItemsTable({
experiments={experiments}
allExperimentIds={allExperimentIds}
colorExperimentIds={colorExperimentIds}
- renderValue={(exp) => (
- <span>
- {exp.totalCost != null ? (
- usdFormatter(exp.totalCost, 2, 6)
- ) : (
- <span className="text-muted-foreground">-</span>
- )}
- </span>
- )}
+ renderValue={(exp) =>
+ exp.totalCost ? (
+ <span>{usdFormatter(exp.totalCost, 2, 6)}</span>
+ ) : (
+ <NotRecordedMetric metric="cost" />
+ )
+ }
/>
);
},
@@ -806,6 +876,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "latencyMs",
id: "latencyMs",
+ headerLabel: getExperimentItemsColumnName("latencyMs"),
header: () =>
renderExperimentSpecificHeader(
getExperimentItemsColumnName("latencyMs"),
@@ -822,7 +893,9 @@ export default function ExperimentItemsTable({
renderValue={(exp) =>
exp.latencyMs != null ? (
<span>{latencyFormatter(exp.latencyMs)}</span>
- ) : undefined
+ ) : (
+ <NotRecordedMetric metric="latency" />
+ )
}
/>
);
@@ -831,6 +904,7 @@ export default function ExperimentItemsTable({
{
accessorKey: "experimentId",
id: "experimentId",
+ headerLabel: "Experiment",
header: () => renderExperimentSpecificHeader("Experiment"),
size: 150,
defaultHidden: true,
@@ -866,15 +940,7 @@ export default function ExperimentItemsTable({
getCell: (value) => (ioLoading ? { type: "loading" } : (value ?? null)),
singleLine: ioSingleLine,
}),
- createIOTableColumn<ExperimentItemsTableRow>({
- accessorKey: "expectedOutput",
- header: "Expected Output",
- size: 300,
- enableHiding: true,
- getCell: (value) => (ioLoading ? { type: "loading" } : (value ?? "")),
- singleLine: ioSingleLine,
- variant: "output",
- }),
+ ...(showExpectedOutput ? [expectedOutputColumn] : []),
{
accessorKey: "output",
id: "output",
@@ -899,7 +965,6 @@ export default function ExperimentItemsTable({
header: "Observation Scores",
id: "observationScores",
enableHiding: true,
- defaultHidden: true,
cell: () => {
return isObservationScoreColumnsLoading ? (
<Skeleton className="h-3 w-1/2" />
@@ -912,7 +977,6 @@ export default function ExperimentItemsTable({
header: "Trace Scores",
id: "traceScores",
enableHiding: true,
- defaultHidden: true,
cell: () => {
return isTraceScoreColumnsLoading ? (
<Skeleton className="h-3 w-1/2" />
@@ -922,10 +986,38 @@ export default function ExperimentItemsTable({
},
];
+ const scoreColumnIds = useMemo(
+ () =>
+ [...observationScoreColumns, ...traceScoreColumns].map(
+ (column) => column.accessorKey,
+ ),
+ [observationScoreColumns, traceScoreColumns],
+ );
+
+ // Score columns are now visible by default. A returning user has `false`
+ // persisted for every one of them from the previous default, so this one-time
+ // migration reaches them too — see `revealScoreColumns` for how a user who
+ // picked their own score columns is left alone. It is consumed once and for
+ // good, so it waits for the score columns to be known rather than running
+ // against an empty or half-loaded set.
+ const columnVisibilityMigrations = useMemo(
+ () => [
+ {
+ versionKey: `experimentItemsColumnVisibility-scoresVisible-v1-${projectId}`,
+ apply: (visibility: VisibilityState) =>
+ isFilterOptionsLoading
+ ? null
+ : revealScoreColumns(visibility, scoreColumnIds),
+ },
+ ],
+ [projectId, scoreColumnIds, isFilterOptionsLoading],
+ );
+
const [columnVisibility, setColumnVisibilityState] =
useColumnVisibility<ExperimentItemsTableRow>(
`experimentItemsColumnVisibility-${projectId}`,
columns,
+ columnVisibilityMigrations,
);
const [columnOrder, setColumnOrder] = useColumnOrder<ExperimentItemsTableRow>(
@@ -1204,6 +1296,7 @@ export default function ExperimentItemsTable({
rows={rows}
isLoading={items.status === "loading" || isViewLoading}
rowHeight={rowHeight}
+ showExpectedOutput={showExpectedOutput}
pagination={pagination}
observationScoreOrder={observationScoreOrder}
traceScoreOrder={traceScoreOrder}
diff --git a/web/src/features/experiments/components/table/ExperimentsTable.tsx b/web/src/features/experiments/components/table/ExperimentsTable.tsx
index 439509822c..28270bae43 100644
--- a/web/src/features/experiments/components/table/ExperimentsTable.tsx
+++ b/web/src/features/experiments/components/table/ExperimentsTable.tsx
@@ -45,6 +45,7 @@ import Link from "next/link";
import { TableActionMenu } from "@/src/features/table/components/TableActionMenu";
import { type TableAction } from "@/src/features/table/types";
import { Badge } from "@/src/components/ui/badge";
+import { type VisibilityState } from "@tanstack/react-table";
import { useStore } from "zustand";
import { createIdTableColumn } from "@/src/components/design-system/table/columns/createIdTableColumn";
import { Skeleton } from "@/src/components/ui/skeleton";
@@ -53,7 +54,11 @@ import { useTableViewManager } from "@/src/components/table/table-view-presets/h
import { useRouter } from "next/router";
import { TableSelectionManager } from "@/src/features/table/components/TableSelectionManager";
import { useScoreColumns } from "@/src/features/scores/hooks/useScoreColumns";
-import { scoreFilters } from "@/src/features/scores/lib/scoreColumns";
+import {
+ collectPresentScoreKeys,
+ revealScoreColumns,
+ scoreFilters,
+} from "@/src/features/scores/lib/scoreColumns";
import useColumnVisibility from "@/src/features/column-visibility/hooks/useColumnVisibility";
import { useExperimentsTableData } from "../../hooks/useExperimentsTableData";
import { type ExperimentsTableRow, type ExperimentsTableProps } from "./types";
@@ -68,6 +73,7 @@ import {
type ExperimentsTableStore,
} from "@/src/features/experiments/store/experimentsTableStore";
import { useExperimentsTableSelectionSync } from "@/src/features/experiments/hooks/useExperimentsTableSelectionSync";
+import { createExperimentMetricColumn } from "./createExperimentMetricColumn";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import {
baselineChangedProps,
@@ -424,12 +430,28 @@ export default function ExperimentsTable({
);
// Use the custom hook for experiments data fetching
- const { experiments, totalCount, dataUpdatedAt } = useExperimentsTableData({
- projectId,
- filterState,
- orderByState,
- paginationState,
- });
+ const { experiments, totalCount, dataUpdatedAt, metricsLoading } =
+ useExperimentsTableData({
+ projectId,
+ filterState,
+ orderByState,
+ paginationState,
+ });
+
+ // A score column that is empty for every experiment in view is noise, so only
+ // create columns for the keys the metrics query actually returned. Undefined
+ // while metrics load, so columns don't disappear and come back on each fetch.
+ const presentScoreKeys = useMemo(() => {
+ if (metricsLoading || experiments.status !== "success") return undefined;
+ const rows = experiments.rows ?? [];
+ return {
+ traceItem: collectPresentScoreKeys(rows.map((r) => r.traceItemScores)),
+ observationItem: collectPresentScoreKeys(
+ rows.map((r) => r.observationItemScores),
+ ),
+ experiment: collectPresentScoreKeys(rows.map((r) => r.experimentScores)),
+ };
+ }, [experiments, metricsLoading]);
useEffect(() => {
if (experiments.status === "success") {
@@ -461,7 +483,7 @@ export default function ExperimentsTable({
: [],
prefix: "Trace",
isFilterDataPending: experiments.status === "loading",
- defaultHidden: true,
+ presentKeys: presentScoreKeys?.traceItem,
});
// Observation-level item scores (scores on observations, observation_id IS NOT NULL)
@@ -472,6 +494,7 @@ export default function ExperimentsTable({
rawKey: true,
displayFormat: "aggregate",
scoreColumnKey: "observationItemScores",
+ headerPrefix: "Observation",
projectId,
filter:
experiments.rows && experiments.rows.length > 0
@@ -480,6 +503,7 @@ export default function ExperimentsTable({
})
: [],
isFilterDataPending: experiments.status === "loading",
+ presentKeys: presentScoreKeys?.observationItem,
});
// Experiment-level scores (direct dataset_run_id match)
@@ -498,6 +522,7 @@ export default function ExperimentsTable({
rawKey: true,
prefix: "Experiment",
isFilterDataPending: experiments.status === "loading",
+ presentKeys: presentScoreKeys?.experiment,
});
const { selectActionColumn } = TableSelectionManager<ExperimentsTableRow>({
@@ -521,6 +546,8 @@ export default function ExperimentsTable({
header: getExperimentsColumnName("description"),
size: 300,
enableHiding: true,
+ // Off by default: 300px of mostly boilerplate ahead of the score columns.
+ defaultHidden: true,
getCell: (value) => value || undefined,
singleLine: rowHeight === "s",
}),
@@ -533,6 +560,7 @@ export default function ExperimentsTable({
header: getExperimentsColumnName("metadata"),
size: 100,
enableHiding: true,
+ defaultHidden: true,
singleLine: rowHeight === "s",
}),
createNumberTableColumn<ExperimentsTableRow>({
@@ -633,7 +661,8 @@ export default function ExperimentsTable({
);
},
},
- createNumberTableColumn<ExperimentsTableRow>({
+ createExperimentMetricColumn<ExperimentsTableRow>({
+ metric: "latency",
accessorKey: "latencyAvg",
header: getExperimentsColumnName("latencyAvg"),
size: 100,
@@ -642,20 +671,22 @@ export default function ExperimentsTable({
description: "Average duration of the root span per experiment item.",
},
formatter: (value) => `${numberFormatter(value / 1000, 4)}s`,
+ metricsLoading,
}),
- createNumberTableColumn<ExperimentsTableRow>({
+ createExperimentMetricColumn<ExperimentsTableRow>({
+ metric: "cost",
accessorKey: "totalCost",
header: getExperimentsColumnName("totalCost"),
size: 100,
enableHiding: true,
formatter: (value) => `$${numberFormatter(value, 6)}`,
+ metricsLoading,
}),
{
accessorKey: "traceItemScores",
- header: "Trace Item Scores",
+ header: "Trace Scores",
id: "traceItemScores",
enableHiding: true,
- defaultHidden: true,
cell: () => {
return isTraceItemScoreLoading ? (
<Skeleton className="h-3 w-1/2" />
@@ -665,10 +696,9 @@ export default function ExperimentsTable({
},
{
accessorKey: "observationItemScores",
- header: "Observation Item Scores",
+ header: "Observation Scores",
id: "observationItemScores",
enableHiding: true,
- defaultHidden: true,
cell: () => {
return isObservationItemScoreLoading ? (
<Skeleton className="h-3 w-1/2" />
@@ -678,10 +708,9 @@ export default function ExperimentsTable({
},
{
accessorKey: "experimentScores",
- header: "Experiment-Level Scores",
+ header: "Experiment Scores",
id: "experimentScores",
enableHiding: true,
- defaultHidden: true,
cell: () => {
return isExperimentScoreColumnLoading ? (
<Skeleton className="h-3 w-1/2" />
@@ -691,10 +720,51 @@ export default function ExperimentsTable({
},
];
+ const scoreColumnIds = useMemo(
+ () =>
+ [
+ ...traceItemScoreColumns,
+ ...observationItemScoreColumns,
+ ...experimentScoreColumns,
+ ].map((column) => column.accessorKey),
+ [
+ traceItemScoreColumns,
+ observationItemScoreColumns,
+ experimentScoreColumns,
+ ],
+ );
+
+ // Each score level loads from its own query, so the union above is partial
+ // until all three have settled. The migration below is consumed once and for
+ // good, so running it early would reveal whichever level answered first and
+ // leave the other two hidden permanently.
+ const areScoreColumnsSettled =
+ !isTraceItemScoreLoading &&
+ !isObservationItemScoreLoading &&
+ !isExperimentScoreColumnLoading;
+
+ // Score columns are now visible by default. A returning user has `false`
+ // persisted for every one of them from the previous default, so this one-time
+ // migration reaches them too — see `revealScoreColumns` for how a user who
+ // picked their own score columns is left alone.
+ const columnVisibilityMigrations = useMemo(
+ () => [
+ {
+ versionKey: `experimentsColumnVisibility-scoresVisible-v1-${projectId}`,
+ apply: (visibility: VisibilityState) =>
+ areScoreColumnsSettled
+ ? revealScoreColumns(visibility, scoreColumnIds)
+ : null,
+ },
+ ],
+ [projectId, scoreColumnIds, areScoreColumnsSettled],
+ );
+
const [columnVisibility, setColumnVisibilityState] =
useColumnVisibility<ExperimentsTableRow>(
`experimentsColumnVisibility-${projectId}`,
columns,
+ columnVisibilityMigrations,
);
// One-time migration for LFE-10460 on the localStorage replay path:
diff --git a/web/src/features/experiments/components/table/NotRecordedMetric.tsx b/web/src/features/experiments/components/table/NotRecordedMetric.tsx
new file mode 100644
index 0000000000..045a18170f
--- /dev/null
+++ b/web/src/features/experiments/components/table/NotRecordedMetric.tsx
@@ -0,0 +1,30 @@
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+
+const EXPLANATION: Record<NotRecordedMetricProps["metric"], string> = {
+ cost: "No cost was recorded for this run. Cost is derived from the token usage and model pricing its calls reported.",
+ latency:
+ "No latency was recorded for this run. Latency is derived from the start and end times its spans reported.",
+};
+
+type NotRecordedMetricProps = { metric: "cost" | "latency" };
+
+/**
+ * A bare `0` or `-` for cost or latency reads as broken instrumentation, so name
+ * what is missing instead.
+ */
+export const NotRecordedMetric = ({ metric }: NotRecordedMetricProps) => (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <span className="text-muted-foreground/70 cursor-default">
+ not recorded
+ </span>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[280px]">
+ {EXPLANATION[metric]}
+ </TooltipContent>
+ </Tooltip>
+);
diff --git a/web/src/features/experiments/components/table/createExperimentMetricColumn.tsx b/web/src/features/experiments/components/table/createExperimentMetricColumn.tsx
new file mode 100644
index 0000000000..3d11b6c6b1
--- /dev/null
+++ b/web/src/features/experiments/components/table/createExperimentMetricColumn.tsx
@@ -0,0 +1,54 @@
+import { type RowData } from "@tanstack/react-table";
+
+import {
+ createTableColumn,
+ type TableColumnOptions,
+} from "@/src/components/design-system/table/columns/utils/createTableColumn";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { NotRecordedMetric } from "./NotRecordedMetric";
+
+type ExperimentMetric = "cost" | "latency";
+
+/**
+ * A cost of exactly 0 means the run's calls reported no usage or pricing, which
+ * reads as free rather than as missing. A latency of 0 is a real measurement.
+ */
+const isRecorded = (
+ metric: ExperimentMetric,
+ value: number | null | undefined,
+): value is number =>
+ value !== null && value !== undefined && (metric !== "cost" || value !== 0);
+
+/**
+ * A cost or latency column for an experiments table. The metrics load in a
+ * second query that resolves after the rows, so the cell has three states: the
+ * skeleton while that query is in flight, `NotRecordedMetric` once it lands
+ * without a value, and the formatted number otherwise.
+ */
+export function createExperimentMetricColumn<TData extends RowData>({
+ formatter,
+ metric,
+ metricsLoading,
+ ...options
+}: TableColumnOptions<TData, number> & {
+ formatter: (value: number) => string;
+ metric: ExperimentMetric;
+ metricsLoading: boolean;
+}) {
+ const loadingCell = <Skeleton className="h-4 w-1/2" />;
+
+ return createTableColumn<TData, number>({
+ ...options,
+ loadingCell,
+ renderCell: (value) => {
+ // Rows render before their metrics, so without this the cell would first
+ // claim the metric was not recorded and then correct itself.
+ if (metricsLoading) return loadingCell;
+
+ if (!isRecorded(metric, value))
+ return <NotRecordedMetric metric={metric} />;
+
+ return <span>{formatter(value)}</span>;
+ },
+ });
+}
diff --git a/web/src/features/scores/hooks/useScoreColumns.clienttest.ts b/web/src/features/scores/hooks/useScoreColumns.clienttest.ts
index 7167f4accc..39b2599c8c 100644
--- a/web/src/features/scores/hooks/useScoreColumns.clienttest.ts
+++ b/web/src/features/scores/hooks/useScoreColumns.clienttest.ts
@@ -8,13 +8,14 @@ const column = {
dataType: "NUMERIC" as const,
};
-const build = (prefix?: string) =>
- createScoreColumns<{ scores: Record<string, unknown> }>(
- [column],
- "scores",
- "aggregate",
+const build = (prefix?: string, headerPrefix?: string) =>
+ createScoreColumns<{ scores: Record<string, unknown> }>({
+ scoreColumns: [column],
+ scoreColumnKey: "scores",
+ displayFormat: "aggregate",
prefix,
- )[0];
+ headerPrefix,
+ })[0];
describe("createScoreColumns headers", () => {
it("leads with the score name and trails the level", () => {
@@ -30,4 +31,11 @@ describe("createScoreColumns headers", () => {
it("keeps the level out of the column id, so persisted layouts survive", () => {
expect(build("Trace").id).toBe("Trace-persona_fit-api-NUMERIC");
});
+
+ it("renames the level in the header without moving the column id", () => {
+ const renamed = build("Trace", "Trace Item");
+
+ expect(renamed.header).toBe("# persona_fit (api) · Trace Item");
+ expect(renamed.id).toBe("Trace-persona_fit-api-NUMERIC");
+ });
});
diff --git a/web/src/features/scores/hooks/useScoreColumns.ts b/web/src/features/scores/hooks/useScoreColumns.ts
index 7994312616..a2c1c09de0 100644
--- a/web/src/features/scores/hooks/useScoreColumns.ts
+++ b/web/src/features/scores/hooks/useScoreColumns.ts
@@ -8,31 +8,47 @@ import {
import { type LangfuseColumnDef } from "@/src/components/table/types";
import { ScoresTableCell } from "@/src/components/scores-table-cell";
import { toOrderedScoresList } from "@/src/features/scores/lib/helpers";
-import { getScoreDataTypeIcon } from "@/src/features/scores/lib/scoreColumns";
+import {
+ getScoreDataTypeIcon,
+ withPresentScoreKeys,
+} from "@/src/features/scores/lib/scoreColumns";
// Simple score column creation - exported for reuse
-export function createScoreColumns<T extends Record<string, any>>(
+export function createScoreColumns<T extends Record<string, any>>({
+ scoreColumns,
+ scoreColumnKey,
+ displayFormat,
+ prefix,
+ headerPrefix,
+ defaultHidden,
+ rawKey,
+}: {
scoreColumns: Array<{
key: string;
name: string;
source: string;
dataType: ScoreDataTypeType;
- }>,
- scoreColumnKey: keyof T & string,
- displayFormat: "smart" | "aggregate",
- prefix?: string,
- defaultHidden?: boolean,
- rawKey?: boolean,
-): LangfuseColumnDef<T>[] {
+ }>;
+ scoreColumnKey: keyof T & string;
+ displayFormat: "smart" | "aggregate";
+ /** Prefix for the column id/accessor, keeping two score levels apart. */
+ prefix?: string;
+ /** Score level shown in the header. Defaults to `prefix`. */
+ headerPrefix?: string;
+ defaultHidden?: boolean;
+ rawKey?: boolean;
+}): LangfuseColumnDef<T>[] {
return scoreColumns.map(({ key, name, source, dataType }) => {
const accessorKey = prefix ? `${prefix}-${key}` : key;
// The score's name identifies the column, so it comes first. A score
// column is narrow and truncates from the right, so a leading level ate
// the name and left every column of a level looking alike. The level
// still trails the name, where the hover title and the column picker
- // pick it up in full.
+ // pick it up in full. The header level is decoupled from the id prefix so
+ // the level can be renamed without orphaning persisted visibility/order.
+ const level = headerPrefix ?? prefix;
const label = `${getScoreDataTypeIcon(dataType)} ${name} (${source.toLowerCase()})`;
- const header = prefix ? `${label} · ${prefix}` : label;
+ const header = level ? `${label} · ${level}` : label;
return {
accessorKey,
@@ -71,10 +87,12 @@ export function useScoreColumns<T extends Record<string, any>>({
fromTimestamp,
toTimestamp,
prefix,
+ headerPrefix,
isFilterDataPending = false,
displayFormat = "smart",
defaultHidden,
rawKey = false,
+ presentKeys,
}: {
projectId: string;
scoreColumnKey: keyof T & string;
@@ -82,10 +100,18 @@ export function useScoreColumns<T extends Record<string, any>>({
fromTimestamp?: Date;
toTimestamp?: Date;
prefix?: string;
+ /** Score level shown in the header. Defaults to `prefix`. */
+ headerPrefix?: string;
isFilterDataPending?: boolean;
displayFormat?: "smart" | "aggregate";
defaultHidden?: boolean;
rawKey?: boolean;
+ /**
+ * Restricts the columns to the score keys that actually carry a value in the
+ * current result set. Leave undefined to create a column per discovered
+ * score, and while that data is still loading.
+ */
+ presentKeys?: ReadonlySet<string>;
}) {
const scoreColumnsQuery = api.scores.getScoreColumns.useQuery(
{
@@ -102,21 +128,27 @@ export function useScoreColumns<T extends Record<string, any>>({
const scoreColumns = useMemo(() => {
if (!scoreColumnsQuery.data?.scoreColumns) return [];
- return createScoreColumns<T>(
- toOrderedScoresList(scoreColumnsQuery.data.scoreColumns),
+ return createScoreColumns<T>({
+ scoreColumns: withPresentScoreKeys(
+ toOrderedScoresList(scoreColumnsQuery.data.scoreColumns),
+ presentKeys,
+ ),
scoreColumnKey,
displayFormat,
prefix,
+ headerPrefix,
defaultHidden,
rawKey,
- );
+ });
}, [
scoreColumnsQuery.data?.scoreColumns,
scoreColumnKey,
prefix,
+ headerPrefix,
displayFormat,
defaultHidden,
rawKey,
+ presentKeys,
]);
return {
diff --git a/web/src/features/scores/lib/aggregateScores.ts b/web/src/features/scores/lib/aggregateScores.ts
index d2b7460ca7..e08a839873 100644
--- a/web/src/features/scores/lib/aggregateScores.ts
+++ b/web/src/features/scores/lib/aggregateScores.ts
@@ -60,6 +60,16 @@ export type ScoreToAggregate =
hasMetadata?: boolean;
});
+/**
+ * Display value of a boolean score. They are stored as 0/1 with "True"/"False"
+ * string values, but users read and write them as booleans.
+ */
+export const toBooleanScoreValue = (score: {
+ value?: number | null;
+ stringValue?: string | null;
+}): string =>
+ score.value === 1 || score.stringValue === "True" ? "true" : "false";
+
/**
* Maps score data types to aggregate types for processing.
* Boolean scores are treated as categorical since they share the same
@@ -113,7 +123,15 @@ export const aggregateScores = <T extends ScoreToAggregate>(
timestamp: values.length === 1 ? scores[0].timestamp : undefined,
};
} else {
- const values = scores.map((score) => score.stringValue ?? "n/a");
+ const isBoolean = scores[0].dataType === "BOOLEAN";
+ // Sorted by value: the score rows arrive in event-timestamp order, so an
+ // unsorted cell lists "true, false" in one row and "false, true" in the
+ // next, which makes a score column impossible to scan.
+ const values = scores
+ .map((score) =>
+ isBoolean ? toBooleanScoreValue(score) : (score.stringValue ?? "n/a"),
+ )
+ .sort((a, b) => a.localeCompare(b));
if (!Boolean(values.length)) return acc;
const valueCounts = values.reduce(
(acc, value) => {
diff --git a/web/src/features/scores/lib/mergeScoresWithCache.ts b/web/src/features/scores/lib/mergeScoresWithCache.ts
index 772e4ca3e5..499a9987f2 100644
--- a/web/src/features/scores/lib/mergeScoresWithCache.ts
+++ b/web/src/features/scores/lib/mergeScoresWithCache.ts
@@ -1,7 +1,10 @@
import { type ScoreDomain, type ScoreAggregate } from "@langfuse/shared";
import { type CachedScore } from "@/src/features/scores/contexts/ScoreCacheContext";
import { type AnnotationScore } from "@/src/features/scores/types";
-import { composeAggregateScoreKey } from "@/src/features/scores/lib/aggregateScores";
+import {
+ composeAggregateScoreKey,
+ toBooleanScoreValue,
+} from "@/src/features/scores/lib/aggregateScores";
import { type WithStringifiedMetadata } from "@/src/utils/clientSideDomainTypes";
/**
@@ -94,12 +97,16 @@ export function mergeAggregatesWithCache(
id: cached.id,
};
} else {
+ const value =
+ cached.dataType === "BOOLEAN"
+ ? toBooleanScoreValue(cached)
+ : (cached.stringValue as string);
merged[key] = {
type: "CATEGORICAL",
- values: [cached.stringValue as string],
+ values: [value],
valueCounts: [
{
- value: cached.stringValue as string,
+ value,
count: 1,
},
],
diff --git a/web/src/features/scores/lib/scoreColumns.clienttest.ts b/web/src/features/scores/lib/scoreColumns.clienttest.ts
new file mode 100644
index 0000000000..b84751037e
--- /dev/null
+++ b/web/src/features/scores/lib/scoreColumns.clienttest.ts
@@ -0,0 +1,53 @@
+// @vitest-environment node
+
+import { revealScoreColumns } from "@/src/features/scores/lib/scoreColumns";
+
+describe("revealScoreColumns", () => {
+ it("defers while no score column is known yet", () => {
+ expect(revealScoreColumns({ itemCount: true }, [])).toBeNull();
+ });
+
+ it("reveals score columns the stored state holds but the page does not", () => {
+ // `offPage` is persisted `false` from a previous session and is not among
+ // the ids the current page produced. The migration runs once, so missing it
+ // here would hide it for good.
+ expect(
+ revealScoreColumns(
+ {
+ itemCount: true,
+ "Trace-groundedness-EVAL-NUMERIC": false,
+ "Trace-offPage-API-BOOLEAN": false,
+ "accuracy-ANNOTATION-CATEGORICAL": false,
+ },
+ ["Trace-groundedness-EVAL-NUMERIC"],
+ ),
+ ).toEqual({
+ itemCount: true,
+ "Trace-groundedness-EVAL-NUMERIC": true,
+ "Trace-offPage-API-BOOLEAN": true,
+ "accuracy-ANNOTATION-CATEGORICAL": true,
+ });
+ });
+
+ it("leaves a layout alone once any score column is enabled", () => {
+ const curated = {
+ "Trace-groundedness-EVAL-NUMERIC": false,
+ "Trace-offPage-API-BOOLEAN": true,
+ };
+ expect(
+ revealScoreColumns(curated, ["Trace-groundedness-EVAL-NUMERIC"]),
+ ).toBe(curated);
+ });
+
+ it("does not mistake an ordinary column id for a score column", () => {
+ expect(
+ revealScoreColumns({ startTime: false, "expected-output": false }, [
+ "Trace-groundedness-EVAL-NUMERIC",
+ ]),
+ ).toEqual({
+ startTime: false,
+ "expected-output": false,
+ "Trace-groundedness-EVAL-NUMERIC": true,
+ });
+ });
+});
diff --git a/web/src/features/scores/lib/scoreColumns.ts b/web/src/features/scores/lib/scoreColumns.ts
index e5fdac3f7b..f0698a6b46 100644
--- a/web/src/features/scores/lib/scoreColumns.ts
+++ b/web/src/features/scores/lib/scoreColumns.ts
@@ -1,4 +1,7 @@
+import { type VisibilityState } from "@tanstack/react-table";
import {
+ ScoreDataTypeArray,
+ ScoreSourceArray,
type ScoreAggregate,
type FilterCondition,
type ScoreDataTypeType,
@@ -158,6 +161,111 @@ export const addPrefixToScoreKeys = (
return prefixed;
};
+/**
+ * Aggregate keys that carry a value somewhere in the current result set. Used
+ * to drop score columns that would render empty for every visible row.
+ */
+export const collectPresentScoreKeys = (
+ aggregates: (ScoreAggregate | null | undefined)[],
+): Set<string> => {
+ const presentKeys = new Set<string>();
+ for (const aggregate of aggregates) {
+ if (!aggregate) continue;
+ for (const key of Object.keys(aggregate)) presentKeys.add(key);
+ }
+ return presentKeys;
+};
+
+/**
+ * Keeps only the score columns that have a value in the current result set.
+ * Pass `undefined` while the score data is still loading, so columns are not
+ * dropped and re-added on every fetch.
+ */
+export const withPresentScoreKeys = <T extends { key: string }>(
+ scoreColumns: T[],
+ presentKeys: ReadonlySet<string> | undefined,
+): T[] =>
+ presentKeys
+ ? scoreColumns.filter(({ key }) => presentKeys.has(key))
+ : scoreColumns;
+
+/**
+ * Recognises a persisted score-column id without knowing which level prefixes
+ * a table uses. Score columns are keyed `<name>-<SOURCE>-<DATATYPE>`, some
+ * levels behind a prefix (`Trace-…`). Score names have their `-` and `.`
+ * normalised to `_`, and both trailing segments come from closed enums, so the
+ * suffix identifies a score column while ordinary column ids (`itemCount`,
+ * `startTime`) never match.
+ */
+const isScoreColumnId = (columnId: string): boolean => {
+ const segments = columnId.split("-");
+ if (segments.length < 3) return false;
+ const [source, dataType] = segments.slice(-2);
+ return (
+ (ScoreSourceArray as readonly string[]).includes(source) &&
+ (ScoreDataTypeArray as readonly string[]).includes(dataType)
+ );
+};
+
+/**
+ * One-time transform that opts a returning user into score columns being
+ * visible by default. Their stored visibility state already holds `false` for
+ * every score column from the previous default, so changing `defaultHidden`
+ * alone would only ever reach new users. Only acts on that stale default: as
+ * soon as one score column is enabled the user has made their own choice and
+ * their layout is left untouched. Returns `null` while no score column is known
+ * yet (they arrive with the score-column query) so the migration is retried
+ * instead of being consumed against an empty column set.
+ *
+ * `scoreColumnIds` only covers the page in view, while this runs once and for
+ * good, so every score column the stored state already holds is revealed too.
+ * Otherwise a score first met on a later page would keep its stale `false`
+ * forever. A column the user has never seen carries no stored entry and picks
+ * up the new default on its own. Hiding score columns that are empty for the
+ * rows in view is a separate, per-result-set decision — `withPresentScoreKeys`
+ * makes it, and widening the reveal does not disturb it.
+ */
+export const revealScoreColumns = (
+ visibility: VisibilityState,
+ scoreColumnIds: string[],
+): VisibilityState | null => {
+ if (scoreColumnIds.length === 0) return null;
+ const revealedIds = Array.from(
+ new Set([
+ ...scoreColumnIds,
+ ...Object.keys(visibility).filter(isScoreColumnId),
+ ]),
+ );
+ if (revealedIds.some((id) => visibility[id])) return visibility;
+ return {
+ ...visibility,
+ ...Object.fromEntries(revealedIds.map((id) => [id, true])),
+ };
+};
+
+/**
+ * What a score's data type means for how its values read. Spelled out because a
+ * numeric score whose values happen to be 0 and 1 is otherwise indistinguishable
+ * from a boolean one, and a user read her own data wrong because of it.
+ */
+export const getScoreDataTypeExplanation = (
+ dataType: ScoreDataTypeType,
+): string => {
+ switch (dataType) {
+ case "BOOLEAN":
+ return "Boolean score. Stored as true / false and summarised as the share of items that are true.";
+ case "CATEGORICAL":
+ return "Categorical score. Its values have no order, so there is no average — the most frequent value stands for the group instead.";
+ case "TEXT":
+ return "Text score. Free-form, so it is neither averaged nor ordered.";
+ case "CORRECTION":
+ return "Correction. A human-supplied replacement for a value rather than a measurement.";
+ case "NUMERIC":
+ default:
+ return "Numeric score. Averaged across the items. A numeric score that only ever holds 0 or 1 is still numeric — it is not a boolean, and 1 does not mean true.";
+ }
+};
+
export const getScoreDataTypeIcon = (dataType: ScoreDataTypeType): string => {
switch (dataType) {
case "NUMERIC":