Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): refine decision model evaluator UI (#17795)

Hassieb Pakzad4971d438ec48
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
index 2ba0fb516d..30a6126d55 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
@@ -1,3 +1,5 @@
+import { expect, within } from "storybook/test";
+
import preview from "../../../../../../../../.storybook/preview";
import { DecisionModelResultView } from "./DecisionModelResultView";
@@ -94,3 +96,57 @@ export const ScoreAtEdges = meta.story({
],
},
});
+
+/** Maximum-length scales keep descriptions available without rendering them inline. */
+export const TenLevels = meta.story({
+ name: "(Test) Ten Levels",
+ args: {
+ results: [
+ {
+ questionId: "q1",
+ type: "score",
+ scoreName: "technical_depth",
+ instructions: "How technical is the answer in `output`?",
+ score: 6.4,
+ levels: [
+ "No technical detail",
+ "Uses basic terminology",
+ "Explains one simple concept",
+ "Connects multiple concepts",
+ "Includes implementation detail",
+ "Discusses tradeoffs",
+ "Explains system behavior",
+ "Covers architecture and constraints",
+ "Provides deep technical analysis",
+ "Expert-level treatment with edge cases",
+ ],
+ probabilities: {
+ "0": 0,
+ "1": 0,
+ "2": 0.01,
+ "3": 0.03,
+ "4": 0.08,
+ "5": 0.16,
+ "6": 0.3,
+ "7": 0.25,
+ "8": 0.12,
+ "9": 0.05,
+ },
+ confidence: 0.72,
+ },
+ ],
+ },
+ play: async ({ canvasElement }) => {
+ const canvas = within(canvasElement);
+ const page = within(canvasElement.ownerDocument.body);
+ const levelDescription = canvas.getByRole("button", {
+ name: "Level 0 description",
+ });
+
+ levelDescription.focus();
+
+ await expect(await page.findByRole("tooltip")).toHaveTextContent(
+ "No technical detail",
+ );
+ },
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
index bce806e500..0bc230fa0b 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
@@ -1,4 +1,12 @@
+import { DECISION_MODEL_LIMITS } from "@langfuse/shared";
+import { InfoIcon } from "lucide-react";
+
import { Badge } from "@/src/components/ui/badge";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
import { QUESTION_TYPE_COPY } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionTypeSelector/QuestionTypeSelector";
import { cn } from "@/src/utils/tailwind";
@@ -42,6 +50,27 @@ function ConfidenceBadge({ confidence }: { confidence: number }) {
);
}
+function LevelDescriptionTooltip({
+ index,
+ description,
+}: {
+ index: number;
+ description: string;
+}) {
+ return (
+ <Tooltip>
+ <TooltipTrigger
+ type="button"
+ className="text-muted-foreground focus-visible:ring-ring cursor-help rounded-sm focus-visible:ring-2 focus-visible:outline-none"
+ aria-label={`Level ${index} description`}
+ >
+ <InfoIcon className="h-3.5 w-3.5" aria-hidden="true" />
+ </TooltipTrigger>
+ <TooltipContent className="max-w-xs">{description}</TooltipContent>
+ </Tooltip>
+ );
+}
+
function ProbabilityBars({
entries,
highlight,
@@ -103,6 +132,7 @@ function ScoreDistribution({
const top = Math.max(levels.length - 1, 1);
const nearest = Math.min(Math.max(Math.round(score), 0), levels.length - 1);
const position = Math.min(Math.max(score / top, 0), 1) * 100;
+ const showDescriptions = levels.length < DECISION_MODEL_LIMITS.maxScoreLevels;
return (
<div className="flex flex-col gap-2">
<div className="relative pt-4">
@@ -136,12 +166,21 @@ function ScoreDistribution({
index === nearest ? "" : "text-muted-foreground",
)}
>
- <span className="font-mono">
- {index} · {percent(probabilities[String(index)] ?? 0)}
- </span>
- <span className="line-clamp-2" title={description}>
- {description}
+ <span className="flex items-center gap-0.5 font-mono">
+ {index}
+ {!showDescriptions ? (
+ <LevelDescriptionTooltip
+ index={index}
+ description={description}
+ />
+ ) : null}
+ · {percent(probabilities[String(index)] ?? 0)}
</span>
+ {showDescriptions ? (
+ <span className="line-clamp-2" title={description}>
+ {description}
+ </span>
+ ) : null}
</li>
))}
</ol>
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
index 4240cf5365..7b26b655eb 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
@@ -101,8 +101,8 @@ export function ScoreLevelsEditor({
Add level
</Button>
<span className="text-muted-foreground text-xs">
- Score = 0 … {Math.max(levels.length - 1, 0)}, e.g. 1.3 sits between
- levels 1 and 2.
+ The score is the probability-weighted average of the level numbers, so
+ it can be a decimal from 0 to {Math.max(levels.length - 1, 0)}.
</span>
</div>
{error ? <p className="text-destructive text-xs">{error}</p> : null}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx
index 96d8e0823e..e770f05e53 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx
@@ -8,6 +8,7 @@ const meta = preview.meta({ component: NameStep });
export const Default = meta.story({
args: {
step: 3,
+ variant: "default",
open: true,
onOpenChange: fn(),
name: "Factual correctness",
@@ -22,6 +23,7 @@ export const Default = meta.story({
export const SuggestingMetadata = meta.story({
args: {
step: 3,
+ variant: "default",
open: true,
onOpenChange: fn(),
name: "",
@@ -32,3 +34,18 @@ export const SuggestingMetadata = meta.story({
descriptionAIAssistance: { state: "generating" },
},
});
+
+export const DecisionModel = meta.story({
+ args: {
+ step: 3,
+ variant: "decisionModel",
+ open: true,
+ onOpenChange: fn(),
+ name: "Technical depth",
+ onNameChange: fn(),
+ description: "Measures how technically detailed each question is.",
+ onDescriptionChange: fn(),
+ nameAIAssistance: { state: "idle", onGenerate: fn() },
+ descriptionAIAssistance: { state: "idle", onGenerate: fn() },
+ },
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx
index c30058ab06..1d6e486857 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx
@@ -5,8 +5,20 @@ import { Label } from "@/src/components/ui/label";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Stepper } from "@/src/features/evals/v2/components/Stepper/Stepper";
+const COPY = {
+ default: {
+ description:
+ "Give the evaluator a clear name (it's also used as the score name) and explain when it should be used.",
+ },
+ decisionModel: {
+ description:
+ "Give the evaluator a clear name and explain when it should be used.",
+ },
+} as const;
+
export function NameStep({
step,
+ variant,
open,
onOpenChange,
name,
@@ -17,6 +29,7 @@ export function NameStep({
descriptionAIAssistance,
}: {
step: number;
+ variant: keyof typeof COPY;
open: boolean;
onOpenChange: (open: boolean) => void;
name: string;
@@ -32,7 +45,7 @@ export function NameStep({
<Stepper
number={step}
title="Name evaluator"
- description="Give the evaluator a clear name (it's also used as the score name) and explain when it should be used."
+ description={COPY[variant].description}
open={open}
onOpenChange={onOpenChange}
>
@@ -40,10 +53,12 @@ export function NameStep({
<div className="space-y-2">
<Label htmlFor="evaluator-name" className="flex items-center gap-1.5">
Name
- <InfoTooltip label="About evaluator names">
- The evaluator name is also used as the score name for the scores
- it produces.
- </InfoTooltip>
+ {variant === "default" ? (
+ <InfoTooltip label="About evaluator names">
+ The evaluator name is also used as the score name for the scores
+ it produces.
+ </InfoTooltip>
+ ) : null}
</Label>
<AIAssistedInput
id="evaluator-name"
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
index d741d018b6..75d389c22d 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx
@@ -4,6 +4,7 @@ import { useShallow } from "zustand/react/shallow";
import type { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";
import { NameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep";
+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
export function NameStepContainer({
@@ -22,9 +23,13 @@ export function NameStepContainer({
const state = useStore(
store,
useShallow((state) => {
- const step = state.type === "LLM_AS_JUDGE" ? 3 : 2;
+ const step = getEvaluatorNameStep(state.type);
return {
step,
+ variant:
+ state.type === "DECISION_MODEL"
+ ? ("decisionModel" as const)
+ : ("default" as const),
open: Boolean(state.openSteps[step]),
name: state.name,
description: state.description,
@@ -36,6 +41,7 @@ export function NameStepContainer({
return (
<NameStep
step={state.step}
+ variant={state.variant}
open={state.open}
onOpenChange={(open) => onStepOpenChange(state.step, open)}
name={state.name}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.ts
new file mode 100644
index 0000000000..ee2a2c16f0
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.ts
@@ -0,0 +1,5 @@
+import type { EvalTemplateType } from "@langfuse/shared";
+
+export function getEvaluatorNameStep(type: EvalTemplateType): number {
+ return type === "LLM_AS_JUDGE" || type === "DECISION_MODEL" ? 3 : 2;
+}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
index 5020aaa498..486d1c6fb7 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";
import {
applyEvaluatorSuggestion,
getEvaluatorVersionDefinition,
@@ -18,6 +19,16 @@ describe("shouldOfferRuleAttachment", () => {
});
});
+describe("getEvaluatorNameStep", () => {
+ it.each([
+ ["CODE", 2],
+ ["LLM_AS_JUDGE", 3],
+ ["DECISION_MODEL", 3],
+ ] as const)("%s evaluators use step %i", (type, step) => {
+ expect(getEvaluatorNameStep(type)).toBe(step);
+ });
+});
+
describe("getEvaluatorVersionDefinition", () => {
it("rebuilds an LLM definition from a saved version", () => {
const outputDefinition = {
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index cdab9b7c57..5d5c52784b 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -24,6 +24,7 @@ import { EvaluatorVersionHistorySheet } from "../components/Evaluators/Evaluator
import type { EvaluatorVersion } from "../components/Evaluators/EvaluatorVersionHistorySheet/types";
import { EvaluatorVersionConflictDialog } from "../components/Evaluators/EvaluatorVersionConflictDialog/EvaluatorVersionConflictDialog";
import { EvaluatorSetupEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor";
+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";
import { EvaluatorSetupFooter } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter";
import { SampleObservationSelectorContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer";
import { EvaluatorTestPanelContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer";
@@ -453,7 +454,7 @@ export function EvaluatorSetupPage(
const setStepOpen = (step: number, open: boolean) => {
const state = evaluatorSetupStore.getState();
state.actions.setStepOpen(step, open);
- const isNameStep = step === (state.type === "LLM_AS_JUDGE" ? 3 : 2);
+ const isNameStep = step === getEvaluatorNameStep(state.type);
if (
nameAIAssistanceAvailable &&
open &&