🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): allow restoring evaluator versions (#16674)
Tobias Wochinger17cd98225408
diff --git a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsxindex 60d0afe9c3..9cb2497f5b 100644--- a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx+++ b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx@@ -36,7 +36,7 @@ export function CollapsibleCard({title={triggerTitle}><ChevronDown className="text-muted-foreground h-3.5 w-3.5 shrink-0 -rotate-90 transition-transform group-data-[state=open]:rotate-0" />- <span className="flex min-w-0 flex-1 items-center gap-2">+ <span className="flex min-w-0 flex-1 items-baseline gap-2">{header}</span></button>diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsxnew file mode 100644index 0000000000..70860bf9e4--- /dev/null+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsx@@ -0,0 +1,55 @@+import type { ReactNode } from "react";+import { act, render, screen } from "@testing-library/react";++import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { EvaluatorTestPanelContainer } from "./EvaluatorTestPanelContainer";++vi.mock(+ "@/src/features/evals/v2/components/EvaluatorTestPanel/EvaluatorTestPanel",+ () => ({+ EvaluatorTestPanel: ({ testSection }: { testSection: ReactNode }) =>+ testSection,+ }),+);++vi.mock(+ "@/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer",+ () => ({+ TestSectionContainer: ({ hasValidModel }: { hasValidModel: boolean }) => (+ <div>{hasValidModel ? "Model available" : "Model missing"}</div>+ ),+ }),+);++describe("EvaluatorTestPanelContainer", () => {+ it("subscribes to model validity without requiring it from the page", () => {+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ mode: "create",+ });++ render(+ <EvaluatorTestPanelContainer+ projectId="project-1"+ store={store}+ sampleSelector={null}+ testResult={null}+ testPending={false}+ rawResultOpen={false}+ onRawResultOpenChange={vi.fn()}+ onRunTest={vi.fn()}+ onOpenExecutionTrace={vi.fn()}+ />,+ );++ expect(screen.getByText("Model missing")).toBeInTheDocument();++ act(() => {+ store+ .getState()+ .actions.selectModel({ provider: "OpenAI", model: "gpt-4.1-mini" });+ });++ expect(screen.getByText("Model available")).toBeInTheDocument();+ });+});diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsxindex c96a8b5f00..28e798401b 100644--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx@@ -3,12 +3,14 @@ import { useStore } from "zustand";import { EvaluatorTestPanel } from "@/src/features/evals/v2/components/EvaluatorTestPanel/EvaluatorTestPanel";import { TestSectionContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer";-import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import {+ selectHasValidModel,+ type EvaluatorSetupStore,+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";export function EvaluatorTestPanelContainer({projectId,store,- hasValidModel,sampleSelector,testResult,testPending,@@ -19,7 +21,6 @@ export function EvaluatorTestPanelContainer({}: {projectId: string;store: EvaluatorSetupStore;- hasValidModel: boolean;sampleSelector: ReactNode;testResult: unknown;testPending: boolean;@@ -29,6 +30,7 @@ export function EvaluatorTestPanelContainer({onOpenExecutionTrace: (traceId: string) => void;}) {const open = useStore(store, (state) => state.testPanelOpen);+ const hasValidModel = useStore(store, selectHasValidModel);const actions = store.getState().actions;return (diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsxnew file mode 100644index 0000000000..6004a155ce--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx@@ -0,0 +1,90 @@+import { fireEvent, render, screen } from "@testing-library/react";+import { EvalTemplateTypeEnum } from "@langfuse/shared";++import { TooltipProvider } from "@/src/components/ui/tooltip";+import { EvaluatorVersionHistorySheet } from "./EvaluatorVersionHistorySheet";+import type { EvaluatorVersion } from "./types";++const currentVersion = {+ id: "version-2",+ version: 2,+ createdAt: new Date(),+ type: EvalTemplateTypeEnum.LLM_AS_JUDGE,+ sourceCode: null,+ sourceCodeLanguage: null,+ prompt: "Current prompt",+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: null,+ vars: [],+ variableMapping: null,+ outputDefinition: null,+ createdByUser: { name: "Ada Lovelace", email: "[email protected]" },+} satisfies EvaluatorVersion;++const oldVersion = {+ ...currentVersion,+ id: "version-1",+ version: 1,+ prompt: "Old prompt",+};++describe("EvaluatorVersionHistorySheet", () => {+ it("keeps version interactions local and confirms before restoring", () => {+ const onRestoreVersion = vi.fn();+ const onOpenChange = vi.fn();+ const onVersionExpansionChange = vi.fn();++ render(+ <TooltipProvider>+ <EvaluatorVersionHistorySheet+ open+ onOpenChange={onOpenChange}+ evaluatorName="Answer quality"+ versions={[currentVersion, oldVersion]}+ currentVersionId={currentVersion.id}+ defaultModel={null}+ onVersionExpansionChange={onVersionExpansionChange}+ onRestoreVersion={onRestoreVersion}+ isLoading={false}+ hasMore={false}+ isLoadingMore={false}+ onLoadMore={vi.fn()}+ />+ </TooltipProvider>,+ );++ expect(screen.getByText("Version 1").parentElement).toHaveClass(+ "items-baseline",+ );+ expect(+ screen.queryByRole("button", { name: "Restore version 2" }),+ ).not.toBeInTheDocument();++ const oldVersionTrigger = screen.getByTitle(+ "Show the definition saved as version 1",+ );+ fireEvent.click(oldVersionTrigger);+ expect(oldVersionTrigger).toHaveAttribute("aria-expanded", "true");+ expect(onVersionExpansionChange).toHaveBeenCalledWith(oldVersion.id);++ fireEvent.click(screen.getByRole("button", { name: "Restore version 1" }));++ expect(onRestoreVersion).not.toHaveBeenCalled();+ expect(+ screen.getByText(+ `This will replace the current evaluator definition with version 1. It won't be saved until you click "Save changes".`,+ ),+ ).toBeInTheDocument();++ fireEvent.click(screen.getByRole("button", { name: "Cancel" }));+ expect(onOpenChange).not.toHaveBeenCalled();+ expect(screen.getByText("Evaluator versions")).toBeInTheDocument();++ fireEvent.click(screen.getByRole("button", { name: "Restore version 1" }));+ fireEvent.click(screen.getByRole("button", { name: "Restore version" }));++ expect(onRestoreVersion).toHaveBeenCalledWith(oldVersion);+ expect(onOpenChange).toHaveBeenCalledWith(false);+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsxindex 4121a3e855..8f15f71fef 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx@@ -15,6 +15,9 @@ const version = {prompt: "Judge this response.",provider: "openai",model: "gpt-4.1-mini",+ modelParams: null,+ vars: [],+ variableMapping: null,outputDefinition: null,createdByUser: { name: "Ada Lovelace", email: "[email protected]" },} satisfies EvaluatorVersion;@@ -28,8 +31,8 @@ const defaultArgs = {versions: [version, { ...version, id: "version-1", version: 1 }],currentVersionId: version.id,defaultModel: { provider: "OpenAI", model: "gpt-4.1-mini" },- expandedVersionId: null,- onExpandedVersionChange: fn(),+ onVersionExpansionChange: fn(),+ onRestoreVersion: fn(),isLoading: false,hasMore: false,isLoadingMore: false,@@ -41,5 +44,5 @@ export const VersionList = meta.story({});export const ExpandedVersion = meta.story({- args: { ...defaultArgs, expandedVersionId: version.id },+ args: { ...defaultArgs, defaultExpandedVersionId: version.id },});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsxindex 6ba9ff0daa..4853d47a52 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx@@ -1,4 +1,4 @@-import { useCallback } from "react";+import { useCallback, useState } from "react";import {Sheet,SheetContent,@@ -8,6 +8,7 @@ import {} from "@/src/components/ui/sheet";import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import Spinner from "@/src/components/design-system/Spinner/Spinner";+import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";import { EvaluatorVersionHistoryList } from "./components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList";import type { EvaluatorVersion } from "./types";@@ -18,8 +19,9 @@ export function EvaluatorVersionHistorySheet({versions,currentVersionId,defaultModel,- expandedVersionId,- onExpandedVersionChange,+ defaultExpandedVersionId = null,+ onVersionExpansionChange,+ onRestoreVersion,isLoading,hasMore,isLoadingMore,@@ -31,13 +33,19 @@ export function EvaluatorVersionHistorySheet({versions: EvaluatorVersion[];currentVersionId: string;defaultModel: JudgeModel | null;- expandedVersionId: string | null;- onExpandedVersionChange: (versionId: string | null) => void;+ defaultExpandedVersionId?: string | null;+ onVersionExpansionChange: (versionId: string | null) => void;+ onRestoreVersion: (version: EvaluatorVersion) => void;isLoading: boolean;hasMore: boolean;isLoadingMore: boolean;onLoadMore: () => void;}) {+ const [versionToRestore, setVersionToRestore] =+ useState<EvaluatorVersion | null>(null);+ const [expandedVersionId, setExpandedVersionId] = useState<string | null>(+ defaultExpandedVersionId,+ );const loadMoreSentinelRef = useCallback((sentinel: HTMLDivElement | null) => {if (!sentinel?.parentElement) return;@@ -61,13 +69,19 @@ export function EvaluatorVersionHistorySheet({const shouldObserveLoadMore = open && !isLoading && hasMore && !isLoadingMore;return (- <Sheet open={open} modal={false} onOpenChange={onOpenChange}>+ <Sheet+ open={open}+ modal={false}+ onOpenChange={(nextOpen) => {+ if (!nextOpen && versionToRestore) return;+ onOpenChange(nextOpen);+ }}+ ><SheetContent className="flex flex-col gap-5 overflow-y-auto sm:max-w-2xl"><SheetHeader><SheetTitle>Evaluator versions</SheetTitle><SheetDescription>- Saved definition versions for {evaluatorName}. Version history is- read-only.+ Review or restore saved definition versions for {evaluatorName}.</SheetDescription></SheetHeader><EvaluatorVersionHistoryList@@ -75,7 +89,11 @@ export function EvaluatorVersionHistorySheet({currentVersionId={currentVersionId}defaultModel={defaultModel}expandedVersionId={expandedVersionId}- onExpandedVersionChange={onExpandedVersionChange}+ onExpandedVersionChange={(versionId) => {+ setExpandedVersionId(versionId);+ onVersionExpansionChange(versionId);+ }}+ onRestoreVersion={setVersionToRestore}isLoading={isLoading}/>{hasMore || isLoadingMore ? (@@ -92,6 +110,22 @@ export function EvaluatorVersionHistorySheet({</div>) : null}</SheetContent>+ <ConfirmDialog+ open={versionToRestore !== null}+ onOpenChange={(open) => {+ if (!open) setVersionToRestore(null);+ }}+ title={`Restore version ${versionToRestore?.version}?`}+ description={`This will replace the current evaluator definition with version ${versionToRestore?.version}. It won't be saved until you click "Save changes".`}+ confirmLabel="Restore version"+ confirmVariant="default"+ onConfirm={() => {+ if (!versionToRestore) return;+ onRestoreVersion(versionToRestore);+ setVersionToRestore(null);+ onOpenChange(false);+ }}+ /></Sheet>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsxindex 04baa8fd0a..acfa117b8a 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx@@ -16,6 +16,9 @@ const version = {prompt: "Judge whether the response answers {{query}}.",provider: "openai",model: "gpt-4.1-mini",+ modelParams: null,+ vars: [],+ variableMapping: null,outputDefinition: null,createdByUser: { name: "Ada Lovelace", email: "[email protected]" },} satisfies EvaluatorVersion;@@ -54,6 +57,7 @@ const defaultArgs = {defaultModel: { provider: "OpenAI", model: "gpt-4.1-mini" },expandedVersionId: null,onExpandedVersionChange: fn(),+ onRestoreVersion: fn(),isLoading: false,};diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsxindex 60d3debbd8..9d1577e00c 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx@@ -1,7 +1,9 @@import { EvalTemplateTypeEnum } from "@langfuse/shared";import { formatDistanceToNowStrict } from "date-fns";+import { RotateCcw } from "lucide-react";import { Badge } from "@/src/components/ui/badge";+import { Button } from "@/src/components/ui/button";import { Skeleton } from "@/src/components/ui/skeleton";import { CollapsibleCard } from "@/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard";import {@@ -44,6 +46,7 @@ export function EvaluatorVersionHistoryList({defaultModel,expandedVersionId,onExpandedVersionChange,+ onRestoreVersion,isLoading,}: {versions: EvaluatorVersion[];@@ -52,6 +55,7 @@ export function EvaluatorVersionHistoryList({/** The version whose definition is open; null collapses them all. */expandedVersionId: string | null;onExpandedVersionChange: (versionId: string | null) => void;+ onRestoreVersion: (version: EvaluatorVersion) => void;isLoading: boolean;}) {if (isLoading) {@@ -118,7 +122,22 @@ export function EvaluatorVersionHistoryList({</span></>}- actions={null}+ actions={+ version.id === currentVersionId ? null : (+ <span className="flex shrink-0 items-center pr-1">+ <Button+ type="button"+ variant="ghost"+ size="icon-xs"+ aria-label={`Restore version ${version.version}`}+ title={`Restore version ${version.version}`}+ onClick={() => onRestoreVersion(version)}+ >+ <RotateCcw className="h-3.5 w-3.5" />+ </Button>+ </span>+ )+ }><div className="p-3"><EvaluatorDefinitionViewdiff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.tsindex 490f5cb417..6f12d7ffd0 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts@@ -1,6 +1,7 @@import type {EvalTemplateSourceCodeLanguage,EvalTemplateType,+ ModelConfig,} from "@langfuse/shared";export type EvaluatorVersion = {@@ -13,6 +14,9 @@ export type EvaluatorVersion = {prompt: string | null;provider: string | null;model: string | null;+ modelParams: ModelConfig | null;+ vars: string[];+ variableMapping: unknown;outputDefinition: unknown;createdByUser: { name: string | null; email: string | null } | null;};diff --git a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsxnew file mode 100644index 0000000000..e37ba9fd2a--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsx@@ -0,0 +1,42 @@+import { render } from "@testing-library/react";++import { PromptVariableEditor } from "./PromptVariableEditor";++describe("PromptVariableEditor", () => {+ it("keeps the interpolated preview aligned and theme-aware", () => {+ const { container } = render(+ <PromptVariableEditor+ value="Question: {{input}}"+ onChange={vi.fn()}+ previewEnabled+ preview={{+ status: "ready",+ fragments: [+ { type: "text", text: "Question: " },+ { type: "variable", name: "input", value: "What is Langfuse?" },+ ],+ }}+ />,+ );++ const preview = container.querySelector("pre");+ expect(preview).toHaveClass(+ "bg-muted/50",+ "px-3",+ "py-2",+ "min-h-[140px]",+ "max-h-[50dvh]",+ );++ const interpolatedValue = container.querySelector('[title="{{input}}"]');+ expect(interpolatedValue).toHaveClass(+ "bg-primary-accent/10",+ "dark:bg-accent-light-blue",+ "dark:text-accent-dark-blue",+ );+ expect(interpolatedValue).not.toHaveClass(+ "bg-accent-light-blue",+ "text-accent-dark-blue",+ );+ });+});diff --git a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsxindex fbf0adcb81..88d33cd80a 100644--- a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx@@ -41,7 +41,7 @@ const promptFontTheme = EditorView.theme({// Tailwind class cannot outrank.const readOnlySurfaceTheme = EditorView.theme({"&.cm-editor, &.cm-editor .cm-gutters": {- backgroundColor: "hsl(var(--muted))",+ backgroundColor: "hsl(var(--muted) / 0.5)",},});@@ -235,18 +235,18 @@ export function PromptVariableEditor({{previewEnabled && preview ? (preview.status === "unavailable" ? (- <p className="bg-card text-muted-foreground rounded-b-md border p-3 text-sm">+ <p className="bg-muted/50 text-muted-foreground min-h-[140px] rounded-b-md border px-3 py-2 text-sm">{preview.message}</p>) : (- <pre className="bg-card text-card-foreground max-h-[60dvh] overflow-y-auto rounded-b-md border p-3 font-sans text-sm whitespace-pre-wrap">+ <pre className="bg-muted/50 text-card-foreground max-h-[50dvh] min-h-[140px] overflow-y-auto rounded-b-md border px-3 py-2 font-sans text-sm whitespace-pre-wrap">{preview.fragments.map((fragment, index) => (<Fragment key={index}>{fragment.type === "text" ? (fragment.text) : (<span- className="bg-primary-accent/10 rounded px-0.5"+ className="bg-primary-accent/10 dark:bg-accent-light-blue dark:text-accent-dark-blue rounded px-0.5"title={`{{${fragment.name}}}`}>{fragment.value}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsxindex 0068f1a834..c1b8335021 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx@@ -1,6 +1,10 @@import { describe, expect, it, vi } from "vitest";++import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";import {applyEvaluatorSuggestion,+ getEvaluatorVersionDefinition,+ restoreEvaluatorVersion,shouldOfferRuleAttachment,} from "./EvaluatorSetupPage";@@ -14,6 +18,87 @@ describe("shouldOfferRuleAttachment", () => {});});+describe("getEvaluatorVersionDefinition", () => {+ it("rebuilds an LLM definition from a saved version", () => {+ const outputDefinition = {+ version: 2 as const,+ dataType: "NUMERIC" as const,+ score: { description: "Answer quality", minValue: 0, maxValue: 1 },+ reasoning: { description: "Explain the score" },+ };++ expect(+ getEvaluatorVersionDefinition({+ id: "version-1",+ version: 1,+ createdAt: new Date(),+ createdByUser: null,+ type: "LLM_AS_JUDGE",+ prompt: "Judge {{output}}",+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0.2 },+ vars: ["output"],+ variableMapping: [+ {+ templateVariable: "output",+ selectedColumnId: "output",+ jsonSelector: null,+ },+ ],+ outputDefinition,+ sourceCode: null,+ sourceCodeLanguage: null,+ }),+ ).toMatchObject({+ type: "LLM_AS_JUDGE",+ prompt: "Judge {{output}}",+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0.2 },+ vars: ["output"],+ outputDefinition,+ });+ });+});++describe("restoreEvaluatorVersion", () => {+ it("clears stale test state after loading the saved definition", () => {+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ mode: "create",+ });+ const resetTestState = vi.fn();++ restoreEvaluatorVersion({+ store,+ version: {+ id: "version-1",+ version: 1,+ createdAt: new Date(),+ createdByUser: null,+ type: "CODE",+ sourceCode: "return { score: 1 };",+ sourceCodeLanguage: "TYPESCRIPT",+ prompt: null,+ provider: null,+ model: null,+ modelParams: null,+ vars: [],+ variableMapping: null,+ outputDefinition: null,+ },+ resetTestState,+ });++ expect(store.getState()).toMatchObject({+ type: "CODE",+ sourceCode: "return { score: 1 };",+ });+ expect(resetTestState).toHaveBeenCalledOnce();+ });+});+describe("applyEvaluatorSuggestion", () => {it("reports when generation returns no suggestion", () => {const setSuggestion = vi.fn();diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex f893bdca98..828c870742 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -1,4 +1,4 @@-import { useMemo, useRef, useState } from "react";+import { useEffect, useMemo, useRef, useState } from "react";import { useRouter } from "next/router";import { TRPCClientError } from "@trpc/client";import { History, Trash2 } from "lucide-react";@@ -35,7 +35,10 @@ import { detailPageListKeys } from "@/src/features/navigate-detail-pages/contextimport { TableHeaderControls } from "@/src/components/table/table-header-controls";import { useTableDateRange } from "@/src/hooks/useTableDateRange";import { toAbsoluteTimeRange } from "@/src/utils/date-range-utils";-import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import {+ createEvaluatorSetupStore,+ type EvaluatorSetupStore,+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";import type { EvaluatorSetupDraft } from "@/src/features/evals/v2/types/templateGallery";import { EvaluatorRuleRelationships } from "@/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships";import { DefaultModelChangeConfirmationDialog } from "@/src/features/evals/v2/components/Evaluators/ProjectDefaultModel/DefaultModelChangeConfirmationDialog";@@ -81,6 +84,51 @@ export function applyEvaluatorSuggestion(return true;}+export function getEvaluatorVersionDefinition(+ version: EvaluatorVersion,+): EvaluatorDefinition {+ if (version.type === "CODE") {+ return {+ type: version.type,+ sourceCode: version.sourceCode ?? "",+ sourceCodeLanguage: version.sourceCodeLanguage ?? "TYPESCRIPT",+ };+ }++ type LlmEvaluatorDefinition = Extract<+ EvaluatorDefinition,+ { type: "LLM_AS_JUDGE" }+ >;++ return {+ type: version.type,+ prompt: version.prompt ?? "",+ provider: version.provider,+ model: version.model,+ modelParams: version.modelParams,+ vars: version.vars,+ variableMapping:+ version.variableMapping as LlmEvaluatorDefinition["variableMapping"],+ outputDefinition:+ version.outputDefinition as LlmEvaluatorDefinition["outputDefinition"],+ };+}++export function restoreEvaluatorVersion({+ store,+ version,+ resetTestState,+}: {+ store: EvaluatorSetupStore;+ version: EvaluatorVersion;+ resetTestState: () => void;+}) {+ store+ .getState()+ .actions.applyDefinition(getEvaluatorVersionDefinition(version));+ resetTestState();+}+export function EvaluatorSetupPage(props:| {@@ -115,14 +163,24 @@ export function EvaluatorSetupPage(projectId,scope: "evalTemplate:CUD",});+ const projectDefaultModel = useProjectDefaultModel({+ projectId,+ source: "editor",+ });const [evaluatorSetupStore] = useState(() =>createEvaluatorSetupStore({initialEvaluator: initialEvaluator ?? initialDraft,initialSampleFilter: initialEvaluator?.sampleFilter,initialType: props.mode === "create" ? props.initialType : undefined,+ defaultModel: projectDefaultModel.defaultModel,mode: props.mode,}),);+ useEffect(() => {+ evaluatorSetupStore+ .getState()+ .actions.setDefaultModel(projectDefaultModel.defaultModel);+ }, [evaluatorSetupStore, projectDefaultModel.defaultModel]);const codeDraft = useStore(evaluatorSetupStore,useShallow((state) => ({@@ -147,14 +205,6 @@ export function EvaluatorSetupPage(evaluatorSetupStore,(state) => state.testPanelOpen,);- const judgeModelSelection = useStore(- evaluatorSetupStore,- useShallow((state) => ({- type: state.type,- mode: state.modelMode,- selectedModel: state.selectedModel,- })),- );const [testResult, setTestResult] = useState<unknown>(null);const [hasCompletedTestCall, setHasCompletedTestCall] = useState(false);const [lastTestRunCostUsd, setLastTestRunCostUsd] = useState<number | null>(@@ -163,9 +213,6 @@ export function EvaluatorSetupPage(const [rawResultOpen, setRawResultOpen] = useState(false);const hasRequestedName = useRef(false);const [historyOpen, setHistoryOpen] = useState(false);- const [expandedVersionId, setExpandedVersionId] = useState<string | null>(- null,- );const [deleteOpen, setDeleteOpen] = useState(false);const [discardOpen, setDiscardOpen] = useState(false);const [versionConflictOpen, setVersionConflictOpen] = useState(false);@@ -220,22 +267,13 @@ export function EvaluatorSetupPage(prompt: version.prompt,provider: version.provider,model: version.model,+ modelParams: version.modelParams as EvaluatorVersion["modelParams"],+ vars: version.vars,+ variableMapping: version.variableMapping,outputDefinition: version.outputDefinition,createdByUser: version.createdByUser,}));- const projectDefaultModel = useProjectDefaultModel({- projectId,- source: "editor",- });- const hasValidModel =- judgeModelSelection.type !== "LLM_AS_JUDGE" ||- Boolean(- judgeModelSelection.mode === "custom"- ? judgeModelSelection.selectedModel- : projectDefaultModel.defaultModel,- );-const create = api.evalsV2.create.useMutation();const update = api.evalsV2.update.useMutation();const reactivate = api.evalsV2.reactivate.useMutation({@@ -585,7 +623,6 @@ export function EvaluatorSetupPage(<EvaluatorTestPanelContainerprojectId={projectId}store={evaluatorSetupStore}- hasValidModel={hasValidModel}sampleSelector={<SampleObservationSelectorContainerstore={evaluatorSetupStore}@@ -734,13 +771,26 @@ export function EvaluatorSetupPage(versions={versions}currentVersionId={versions[0]?.id ?? ""}defaultModel={projectDefaultModel.defaultModel}- expandedVersionId={expandedVersionId}- onExpandedVersionChange={(versionId) => {+ onVersionExpansionChange={(versionId) => {capture("evaluators:version_history_interaction", {action:versionId === null ? "collapse_version" : "expand_version",});- setExpandedVersionId(versionId);+ }}+ onRestoreVersion={(version) => {+ restoreEvaluatorVersion({+ store: evaluatorSetupStore,+ version,+ resetTestState: () => {+ setTestResult(null);+ setHasCompletedTestCall(false);+ setLastTestRunCostUsd(null);+ setRawResultOpen(false);+ },+ });+ capture("evaluators:version_history_interaction", {+ action: "restore_version",+ });}}isLoading={versionHistory.isPending}hasMore={versionHistory.hasNextPage}diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.tsindex 963b754493..1d0b571d07 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts@@ -3,7 +3,10 @@ import { describe, expect, it } from "vitest";import { getDefaultCodeEvalSource } from "@/src/features/evals/utils/code-eval-template-starter-examples";import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";-import { createEvaluatorSetupStore } from "./evaluatorSetupStore";+import {+ createEvaluatorSetupStore,+ selectHasValidModel,+} from "./evaluatorSetupStore";describe("createEvaluatorSetupStore", () => {it("starts new evaluator descriptions empty", () => {@@ -186,6 +189,29 @@ describe("createEvaluatorSetupStore", () => {]);});+ it("derives whether any usable model is available", () => {+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ mode: "create",+ });++ expect(selectHasValidModel(store.getState())).toBe(false);++ store+ .getState()+ .actions.setDefaultModel({ provider: "OpenAI", model: "gpt-4.1-mini" });+ expect(selectHasValidModel(store.getState())).toBe(true);++ store.getState().actions.setDefaultModel(null);+ store+ .getState()+ .actions.selectModel({ provider: "OpenAI", model: "gpt-4.1" });+ expect(selectHasValidModel(store.getState())).toBe(true);++ store.getState().actions.setType("CODE");+ expect(selectHasValidModel(store.getState())).toBe(true);+ });+it("keeps configured parameters for the selected model and resets them when the model changes", () => {const store = createEvaluatorSetupStore({initialEvaluator: null,@@ -203,4 +229,82 @@ describe("createEvaluatorSetupStore", () => {actions.selectModel({ provider: "OpenAI", model: "gpt-4.1" });expect(store.getState().modelParams).toBeNull();});++ it("loads an old LLM definition without replacing evaluator metadata", () => {+ const store = createEvaluatorSetupStore({+ initialEvaluator: {+ name: "Answer quality",+ description: "Checks answer quality",+ definition: {+ type: "CODE",+ sourceCode: "return { score: 1 };",+ sourceCodeLanguage: "TYPESCRIPT",+ },+ },+ mode: "edit",+ });+ const sampleFilter = store.getState().sampleFilter;++ store.getState().actions.applyDefinition({+ type: "LLM_AS_JUDGE",+ prompt: "Judge {{output}}",+ provider: "openai",+ model: "gpt-4.1-mini",+ modelParams: { temperature: 0.2 },+ vars: ["output"],+ variableMapping: [+ {+ templateVariable: "output",+ selectedColumnId: "output",+ jsonSelector: null,+ },+ ],+ outputDefinition: {+ version: 2,+ dataType: "NUMERIC",+ score: {+ description: "Answer quality",+ minValue: 0,+ maxValue: 1,+ },+ reasoning: { description: "Explain the score" },+ },+ });++ expect(store.getState()).toMatchObject({+ type: "LLM_AS_JUDGE",+ name: "Answer quality",+ description: "Checks answer quality",+ prompt: "Judge {{output}}",+ modelMode: "custom",+ selectedModel: { provider: "openai", model: "gpt-4.1-mini" },+ modelParams: { temperature: 0.2 },+ variableFields: {+ output: { selectedColumnId: "output", jsonSelector: null },+ },+ });+ expect(store.getState().sampleFilter).toBe(sampleFilter);+ });++ it("loads an old code definition", () => {+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ mode: "create",+ });++ store.getState().actions.applyDefinition({+ type: "CODE",+ sourceCode: "def evaluate(ctx):\n return []",+ sourceCodeLanguage: "PYTHON",+ });++ expect(store.getState()).toMatchObject({+ type: "CODE",+ sourceCode: "def evaluate(ctx):\n return []",+ sourceCodeLanguage: "PYTHON",+ sourceCodeDrafts: {+ PYTHON: "def evaluate(ctx):\n return []",+ },+ });+ });});diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.tsindex 4a661be9e8..a577786a6d 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts@@ -69,6 +69,7 @@ type EvaluatorSetupStoreActions = {setActiveMapping: (activeMapping: ActiveVariableMapping) => void;setModelPickerOpen: (modelPickerOpen: boolean) => void;setModelMode: (modelMode: "default" | "custom") => void;+ setDefaultModel: (defaultModel: JudgeModel | null) => void;selectModel: (selectedModel: JudgeModel) => void;configureModel: (selectedModel: JudgeModel, modelParams: ModelConfig) => void;setSelectedObservation: (@@ -77,6 +78,7 @@ type EvaluatorSetupStoreActions = {setSampleFilter: (sampleFilter: FilterState) => void;setPromptPreviewEnabled: (promptPreviewEnabled: boolean) => void;setTestPanelOpen: (testPanelOpen: boolean) => void;+ applyDefinition: (definition: EvaluatorDefinition) => void;};export type EvaluatorSetupStoreState = {@@ -94,6 +96,7 @@ export type EvaluatorSetupStoreState = {activeMapping: ActiveVariableMapping;modelPickerOpen: boolean;modelMode: "default" | "custom";+ defaultModel: JudgeModel | null;selectedModel: JudgeModel | null;modelParams: ModelConfig | null;selectedObservation: SampleObservation | null;@@ -105,10 +108,17 @@ export type EvaluatorSetupStoreState = {export type EvaluatorSetupStore = StoreApi<EvaluatorSetupStoreState>;+export const selectHasValidModel = (state: EvaluatorSetupStoreState) =>+ state.type !== EvalTemplateTypeEnum.LLM_AS_JUDGE ||+ Boolean(+ state.modelMode === "custom" ? state.selectedModel : state.defaultModel,+ );+export function createEvaluatorSetupStore({initialEvaluator,initialSampleFilter,initialType,+ defaultModel = null,}: {initialEvaluator: {name: string;@@ -117,6 +127,7 @@ export function createEvaluatorSetupStore({} | null;initialSampleFilter?: FilterState;initialType?: EvalTemplateType;+ defaultModel?: JudgeModel | null;mode: "create" | "edit";}): EvaluatorSetupStore {const initialDefinition = initialEvaluator?.definition;@@ -159,6 +170,7 @@ export function createEvaluatorSetupStore({initialDefinition?.type === "LLM_AS_JUDGE" && initialDefinition.model? "custom": "default",+ defaultModel,selectedModel:initialDefinition?.type === "LLM_AS_JUDGE" &&initialDefinition.provider &&@@ -217,6 +229,7 @@ export function createEvaluatorSetupStore({setActiveMapping: (activeMapping) => set({ activeMapping }),setModelPickerOpen: (modelPickerOpen) => set({ modelPickerOpen }),setModelMode: (modelMode) => set({ modelMode }),+ setDefaultModel: (defaultModel) => set({ defaultModel }),selectModel: (selectedModel) =>set((state) => ({selectedModel,@@ -235,6 +248,40 @@ export function createEvaluatorSetupStore({setPromptPreviewEnabled: (promptPreviewEnabled) =>set({ promptPreviewEnabled }),setTestPanelOpen: (testPanelOpen) => set({ testPanelOpen }),+ applyDefinition: (definition) =>+ set((state) => {+ if (definition.type === EvalTemplateTypeEnum.CODE) {+ return {+ type: definition.type,+ sourceCode: definition.sourceCode,+ sourceCodeLanguage: definition.sourceCodeLanguage,+ sourceCodeDrafts: {+ ...state.sourceCodeDrafts,+ [definition.sourceCodeLanguage]: definition.sourceCode,+ },+ activeMapping: null,+ };+ }++ const selectedModel =+ definition.provider && definition.model+ ? {+ provider: definition.provider,+ model: definition.model,+ }+ : null;++ return {+ type: definition.type,+ prompt: definition.prompt,+ scoreOutput: toScoreOutputFormState(definition.outputDefinition),+ variableFields: buildInitialVariableFields(definition),+ activeMapping: null,+ modelMode: selectedModel ? "custom" : "default",+ selectedModel,+ modelParams: selectedModel ? definition.modelParams : null,+ };+ }),},}));}