🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): refine decision model evaluator UI (#17795)
Hassieb Pakzad4971d438ec48
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsxindex 2ba0fb516d..30a6126d55 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx@@ -1,3 +1,5 @@+import { expect, within } from "storybook/test";+import preview from "../../../../../../../../.storybook/preview";import { DecisionModelResultView } from "./DecisionModelResultView";@@ -94,3 +96,57 @@ export const ScoreAtEdges = meta.story({],},});++/** Maximum-length scales keep descriptions available without rendering them inline. */+export const TenLevels = meta.story({+ name: "(Test) Ten Levels",+ args: {+ results: [+ {+ questionId: "q1",+ type: "score",+ scoreName: "technical_depth",+ instructions: "How technical is the answer in `output`?",+ score: 6.4,+ levels: [+ "No technical detail",+ "Uses basic terminology",+ "Explains one simple concept",+ "Connects multiple concepts",+ "Includes implementation detail",+ "Discusses tradeoffs",+ "Explains system behavior",+ "Covers architecture and constraints",+ "Provides deep technical analysis",+ "Expert-level treatment with edge cases",+ ],+ probabilities: {+ "0": 0,+ "1": 0,+ "2": 0.01,+ "3": 0.03,+ "4": 0.08,+ "5": 0.16,+ "6": 0.3,+ "7": 0.25,+ "8": 0.12,+ "9": 0.05,+ },+ confidence: 0.72,+ },+ ],+ },+ play: async ({ canvasElement }) => {+ const canvas = within(canvasElement);+ const page = within(canvasElement.ownerDocument.body);+ const levelDescription = canvas.getByRole("button", {+ name: "Level 0 description",+ });++ levelDescription.focus();++ await expect(await page.findByRole("tooltip")).toHaveTextContent(+ "No technical detail",+ );+ },+});diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsxindex bce806e500..0bc230fa0b 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx@@ -1,4 +1,12 @@+import { DECISION_MODEL_LIMITS } from "@langfuse/shared";+import { InfoIcon } from "lucide-react";+import { Badge } from "@/src/components/ui/badge";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/src/components/ui/tooltip";import { QUESTION_TYPE_COPY } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionTypeSelector/QuestionTypeSelector";import { cn } from "@/src/utils/tailwind";@@ -42,6 +50,27 @@ function ConfidenceBadge({ confidence }: { confidence: number }) {);}+function LevelDescriptionTooltip({+ index,+ description,+}: {+ index: number;+ description: string;+}) {+ return (+ <Tooltip>+ <TooltipTrigger+ type="button"+ className="text-muted-foreground focus-visible:ring-ring cursor-help rounded-sm focus-visible:ring-2 focus-visible:outline-none"+ aria-label={`Level ${index} description`}+ >+ <InfoIcon className="h-3.5 w-3.5" aria-hidden="true" />+ </TooltipTrigger>+ <TooltipContent className="max-w-xs">{description}</TooltipContent>+ </Tooltip>+ );+}+function ProbabilityBars({entries,highlight,@@ -103,6 +132,7 @@ function ScoreDistribution({const top = Math.max(levels.length - 1, 1);const nearest = Math.min(Math.max(Math.round(score), 0), levels.length - 1);const position = Math.min(Math.max(score / top, 0), 1) * 100;+ const showDescriptions = levels.length < DECISION_MODEL_LIMITS.maxScoreLevels;return (<div className="flex flex-col gap-2"><div className="relative pt-4">@@ -136,12 +166,21 @@ function ScoreDistribution({index === nearest ? "" : "text-muted-foreground",)}>- <span className="font-mono">- {index} · {percent(probabilities[String(index)] ?? 0)}- </span>- <span className="line-clamp-2" title={description}>- {description}+ <span className="flex items-center gap-0.5 font-mono">+ {index}+ {!showDescriptions ? (+ <LevelDescriptionTooltip+ index={index}+ description={description}+ />+ ) : null}+ · {percent(probabilities[String(index)] ?? 0)}</span>+ {showDescriptions ? (+ <span className="line-clamp-2" title={description}>+ {description}+ </span>+ ) : null}</li>))}</ol>diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsxindex 4240cf5365..7b26b655eb 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx@@ -101,8 +101,8 @@ export function ScoreLevelsEditor({Add level</Button><span className="text-muted-foreground text-xs">- Score = 0 … {Math.max(levels.length - 1, 0)}, e.g. 1.3 sits between- levels 1 and 2.+ The score is the probability-weighted average of the level numbers, so+ it can be a decimal from 0 to {Math.max(levels.length - 1, 0)}.</span></div>{error ? <p className="text-destructive text-xs">{error}</p> : null}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsxindex 96d8e0823e..e770f05e53 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.stories.tsx@@ -8,6 +8,7 @@ const meta = preview.meta({ component: NameStep });export const Default = meta.story({args: {step: 3,+ variant: "default",open: true,onOpenChange: fn(),name: "Factual correctness",@@ -22,6 +23,7 @@ export const Default = meta.story({export const SuggestingMetadata = meta.story({args: {step: 3,+ variant: "default",open: true,onOpenChange: fn(),name: "",@@ -32,3 +34,18 @@ export const SuggestingMetadata = meta.story({descriptionAIAssistance: { state: "generating" },},});++export const DecisionModel = meta.story({+ args: {+ step: 3,+ variant: "decisionModel",+ open: true,+ onOpenChange: fn(),+ name: "Technical depth",+ onNameChange: fn(),+ description: "Measures how technically detailed each question is.",+ onDescriptionChange: fn(),+ nameAIAssistance: { state: "idle", onGenerate: fn() },+ descriptionAIAssistance: { state: "idle", onGenerate: fn() },+ },+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsxindex c30058ab06..1d6e486857 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep.tsx@@ -5,8 +5,20 @@ import { Label } from "@/src/components/ui/label";import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";import { Stepper } from "@/src/features/evals/v2/components/Stepper/Stepper";+const COPY = {+ default: {+ description:+ "Give the evaluator a clear name (it's also used as the score name) and explain when it should be used.",+ },+ decisionModel: {+ description:+ "Give the evaluator a clear name and explain when it should be used.",+ },+} as const;+export function NameStep({step,+ variant,open,onOpenChange,name,@@ -17,6 +29,7 @@ export function NameStep({descriptionAIAssistance,}: {step: number;+ variant: keyof typeof COPY;open: boolean;onOpenChange: (open: boolean) => void;name: string;@@ -32,7 +45,7 @@ export function NameStep({<Steppernumber={step}title="Name evaluator"- description="Give the evaluator a clear name (it's also used as the score name) and explain when it should be used."+ description={COPY[variant].description}open={open}onOpenChange={onOpenChange}>@@ -40,10 +53,12 @@ export function NameStep({<div className="space-y-2"><Label htmlFor="evaluator-name" className="flex items-center gap-1.5">Name- <InfoTooltip label="About evaluator names">- The evaluator name is also used as the score name for the scores- it produces.- </InfoTooltip>+ {variant === "default" ? (+ <InfoTooltip label="About evaluator names">+ The evaluator name is also used as the score name for the scores+ it produces.+ </InfoTooltip>+ ) : null}</Label><AIAssistedInputid="evaluator-name"diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsxindex d741d018b6..75d389c22d 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/components/NameStepContainer/NameStepContainer.tsx@@ -4,6 +4,7 @@ import { useShallow } from "zustand/react/shallow";import type { AIAssistedInput } from "@/src/components/ui/ai-assisted-input";import { NameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/NameStep/NameStep";+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";export function NameStepContainer({@@ -22,9 +23,13 @@ export function NameStepContainer({const state = useStore(store,useShallow((state) => {- const step = state.type === "LLM_AS_JUDGE" ? 3 : 2;+ const step = getEvaluatorNameStep(state.type);return {step,+ variant:+ state.type === "DECISION_MODEL"+ ? ("decisionModel" as const)+ : ("default" as const),open: Boolean(state.openSteps[step]),name: state.name,description: state.description,@@ -36,6 +41,7 @@ export function NameStepContainer({return (<NameStepstep={state.step}+ variant={state.variant}open={state.open}onOpenChange={(open) => onStepOpenChange(state.step, open)}name={state.name}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.tsnew file mode 100644index 0000000000..ee2a2c16f0--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps.ts@@ -0,0 +1,5 @@+import type { EvalTemplateType } from "@langfuse/shared";++export function getEvaluatorNameStep(type: EvalTemplateType): number {+ return type === "LLM_AS_JUDGE" || type === "DECISION_MODEL" ? 3 : 2;+}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsxindex 5020aaa498..486d1c6fb7 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx@@ -1,6 +1,7 @@import { describe, expect, it, vi } from "vitest";import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";import {applyEvaluatorSuggestion,getEvaluatorVersionDefinition,@@ -18,6 +19,16 @@ describe("shouldOfferRuleAttachment", () => {});});+describe("getEvaluatorNameStep", () => {+ it.each([+ ["CODE", 2],+ ["LLM_AS_JUDGE", 3],+ ["DECISION_MODEL", 3],+ ] as const)("%s evaluators use step %i", (type, step) => {+ expect(getEvaluatorNameStep(type)).toBe(step);+ });+});+describe("getEvaluatorVersionDefinition", () => {it("rebuilds an LLM definition from a saved version", () => {const outputDefinition = {diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex cdab9b7c57..5d5c52784b 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -24,6 +24,7 @@ import { EvaluatorVersionHistorySheet } from "../components/Evaluators/Evaluatorimport type { EvaluatorVersion } from "../components/Evaluators/EvaluatorVersionHistorySheet/types";import { EvaluatorVersionConflictDialog } from "../components/Evaluators/EvaluatorVersionConflictDialog/EvaluatorVersionConflictDialog";import { EvaluatorSetupEditor } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor";+import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/evaluatorSetupSteps";import { EvaluatorSetupFooter } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter";import { SampleObservationSelectorContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer";import { EvaluatorTestPanelContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer";@@ -453,7 +454,7 @@ export function EvaluatorSetupPage(const setStepOpen = (step: number, open: boolean) => {const state = evaluatorSetupStore.getState();state.actions.setStepOpen(step, open);- const isNameStep = step === (state.type === "LLM_AS_JUDGE" ? 3 : 2);+ const isNameStep = step === getEvaluatorNameStep(state.type);if (nameAIAssistanceAvailable &&open &&