Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): allow restoring evaluator versions (#16674)

Tobias Wochinger17cd98225408
diff --git a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
index 60d0afe9c3..9cb2497f5b 100644
--- a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
+++ b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
@@ -36,7 +36,7 @@ export function CollapsibleCard({
title={triggerTitle}
>
<ChevronDown className="text-muted-foreground h-3.5 w-3.5 shrink-0 -rotate-90 transition-transform group-data-[state=open]:rotate-0" />
- <span className="flex min-w-0 flex-1 items-center gap-2">
+ <span className="flex min-w-0 flex-1 items-baseline gap-2">
{header}
</span>
</button>
diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsx
new file mode 100644
index 0000000000..70860bf9e4
--- /dev/null
+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.clienttest.tsx
@@ -0,0 +1,55 @@
+import type { ReactNode } from "react";
+import { act, render, screen } from "@testing-library/react";
+
+import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { EvaluatorTestPanelContainer } from "./EvaluatorTestPanelContainer";
+
+vi.mock(
+ "@/src/features/evals/v2/components/EvaluatorTestPanel/EvaluatorTestPanel",
+ () => ({
+ EvaluatorTestPanel: ({ testSection }: { testSection: ReactNode }) =>
+ testSection,
+ }),
+);
+
+vi.mock(
+ "@/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer",
+ () => ({
+ TestSectionContainer: ({ hasValidModel }: { hasValidModel: boolean }) => (
+ <div>{hasValidModel ? "Model available" : "Model missing"}</div>
+ ),
+ }),
+);
+
+describe("EvaluatorTestPanelContainer", () => {
+ it("subscribes to model validity without requiring it from the page", () => {
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ mode: "create",
+ });
+
+ render(
+ <EvaluatorTestPanelContainer
+ projectId="project-1"
+ store={store}
+ sampleSelector={null}
+ testResult={null}
+ testPending={false}
+ rawResultOpen={false}
+ onRawResultOpenChange={vi.fn()}
+ onRunTest={vi.fn()}
+ onOpenExecutionTrace={vi.fn()}
+ />,
+ );
+
+ expect(screen.getByText("Model missing")).toBeInTheDocument();
+
+ act(() => {
+ store
+ .getState()
+ .actions.selectModel({ provider: "OpenAI", model: "gpt-4.1-mini" });
+ });
+
+ expect(screen.getByText("Model available")).toBeInTheDocument();
+ });
+});
diff --git a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx
index c96a8b5f00..28e798401b 100644
--- a/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer.tsx
@@ -3,12 +3,14 @@ import { useStore } from "zustand";
import { EvaluatorTestPanel } from "@/src/features/evals/v2/components/EvaluatorTestPanel/EvaluatorTestPanel";
import { TestSectionContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/TestSection/components/TestSectionContainer/TestSectionContainer";
-import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import {
+ selectHasValidModel,
+ type EvaluatorSetupStore,
+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
export function EvaluatorTestPanelContainer({
projectId,
store,
- hasValidModel,
sampleSelector,
testResult,
testPending,
@@ -19,7 +21,6 @@ export function EvaluatorTestPanelContainer({
}: {
projectId: string;
store: EvaluatorSetupStore;
- hasValidModel: boolean;
sampleSelector: ReactNode;
testResult: unknown;
testPending: boolean;
@@ -29,6 +30,7 @@ export function EvaluatorTestPanelContainer({
onOpenExecutionTrace: (traceId: string) => void;
}) {
const open = useStore(store, (state) => state.testPanelOpen);
+ const hasValidModel = useStore(store, selectHasValidModel);
const actions = store.getState().actions;
return (
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx
new file mode 100644
index 0000000000..6004a155ce
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx
@@ -0,0 +1,90 @@
+import { fireEvent, render, screen } from "@testing-library/react";
+import { EvalTemplateTypeEnum } from "@langfuse/shared";
+
+import { TooltipProvider } from "@/src/components/ui/tooltip";
+import { EvaluatorVersionHistorySheet } from "./EvaluatorVersionHistorySheet";
+import type { EvaluatorVersion } from "./types";
+
+const currentVersion = {
+ id: "version-2",
+ version: 2,
+ createdAt: new Date(),
+ type: EvalTemplateTypeEnum.LLM_AS_JUDGE,
+ sourceCode: null,
+ sourceCodeLanguage: null,
+ prompt: "Current prompt",
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: null,
+ vars: [],
+ variableMapping: null,
+ outputDefinition: null,
+ createdByUser: { name: "Ada Lovelace", email: "[email protected]" },
+} satisfies EvaluatorVersion;
+
+const oldVersion = {
+ ...currentVersion,
+ id: "version-1",
+ version: 1,
+ prompt: "Old prompt",
+};
+
+describe("EvaluatorVersionHistorySheet", () => {
+ it("keeps version interactions local and confirms before restoring", () => {
+ const onRestoreVersion = vi.fn();
+ const onOpenChange = vi.fn();
+ const onVersionExpansionChange = vi.fn();
+
+ render(
+ <TooltipProvider>
+ <EvaluatorVersionHistorySheet
+ open
+ onOpenChange={onOpenChange}
+ evaluatorName="Answer quality"
+ versions={[currentVersion, oldVersion]}
+ currentVersionId={currentVersion.id}
+ defaultModel={null}
+ onVersionExpansionChange={onVersionExpansionChange}
+ onRestoreVersion={onRestoreVersion}
+ isLoading={false}
+ hasMore={false}
+ isLoadingMore={false}
+ onLoadMore={vi.fn()}
+ />
+ </TooltipProvider>,
+ );
+
+ expect(screen.getByText("Version 1").parentElement).toHaveClass(
+ "items-baseline",
+ );
+ expect(
+ screen.queryByRole("button", { name: "Restore version 2" }),
+ ).not.toBeInTheDocument();
+
+ const oldVersionTrigger = screen.getByTitle(
+ "Show the definition saved as version 1",
+ );
+ fireEvent.click(oldVersionTrigger);
+ expect(oldVersionTrigger).toHaveAttribute("aria-expanded", "true");
+ expect(onVersionExpansionChange).toHaveBeenCalledWith(oldVersion.id);
+
+ fireEvent.click(screen.getByRole("button", { name: "Restore version 1" }));
+
+ expect(onRestoreVersion).not.toHaveBeenCalled();
+ expect(
+ screen.getByText(
+ `This will replace the current evaluator definition with version 1. It won't be saved until you click "Save changes".`,
+ ),
+ ).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole("button", { name: "Cancel" }));
+ expect(onOpenChange).not.toHaveBeenCalled();
+ expect(screen.getByText("Evaluator versions")).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole("button", { name: "Restore version 1" }));
+ fireEvent.click(screen.getByRole("button", { name: "Restore version" }));
+
+ expect(onRestoreVersion).toHaveBeenCalledWith(oldVersion);
+ expect(onOpenChange).toHaveBeenCalledWith(false);
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx
index 4121a3e855..8f15f71fef 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.stories.tsx
@@ -15,6 +15,9 @@ const version = {
prompt: "Judge this response.",
provider: "openai",
model: "gpt-4.1-mini",
+ modelParams: null,
+ vars: [],
+ variableMapping: null,
outputDefinition: null,
createdByUser: { name: "Ada Lovelace", email: "[email protected]" },
} satisfies EvaluatorVersion;
@@ -28,8 +31,8 @@ const defaultArgs = {
versions: [version, { ...version, id: "version-1", version: 1 }],
currentVersionId: version.id,
defaultModel: { provider: "OpenAI", model: "gpt-4.1-mini" },
- expandedVersionId: null,
- onExpandedVersionChange: fn(),
+ onVersionExpansionChange: fn(),
+ onRestoreVersion: fn(),
isLoading: false,
hasMore: false,
isLoadingMore: false,
@@ -41,5 +44,5 @@ export const VersionList = meta.story({
});
export const ExpandedVersion = meta.story({
- args: { ...defaultArgs, expandedVersionId: version.id },
+ args: { ...defaultArgs, defaultExpandedVersionId: version.id },
});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx
index 6ba9ff0daa..4853d47a52 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.tsx
@@ -1,4 +1,4 @@
-import { useCallback } from "react";
+import { useCallback, useState } from "react";
import {
Sheet,
SheetContent,
@@ -8,6 +8,7 @@ import {
} from "@/src/components/ui/sheet";
import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import Spinner from "@/src/components/design-system/Spinner/Spinner";
+import { ConfirmDialog } from "@/src/components/ui/confirm-dialog";
import { EvaluatorVersionHistoryList } from "./components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList";
import type { EvaluatorVersion } from "./types";
@@ -18,8 +19,9 @@ export function EvaluatorVersionHistorySheet({
versions,
currentVersionId,
defaultModel,
- expandedVersionId,
- onExpandedVersionChange,
+ defaultExpandedVersionId = null,
+ onVersionExpansionChange,
+ onRestoreVersion,
isLoading,
hasMore,
isLoadingMore,
@@ -31,13 +33,19 @@ export function EvaluatorVersionHistorySheet({
versions: EvaluatorVersion[];
currentVersionId: string;
defaultModel: JudgeModel | null;
- expandedVersionId: string | null;
- onExpandedVersionChange: (versionId: string | null) => void;
+ defaultExpandedVersionId?: string | null;
+ onVersionExpansionChange: (versionId: string | null) => void;
+ onRestoreVersion: (version: EvaluatorVersion) => void;
isLoading: boolean;
hasMore: boolean;
isLoadingMore: boolean;
onLoadMore: () => void;
}) {
+ const [versionToRestore, setVersionToRestore] =
+ useState<EvaluatorVersion | null>(null);
+ const [expandedVersionId, setExpandedVersionId] = useState<string | null>(
+ defaultExpandedVersionId,
+ );
const loadMoreSentinelRef = useCallback(
(sentinel: HTMLDivElement | null) => {
if (!sentinel?.parentElement) return;
@@ -61,13 +69,19 @@ export function EvaluatorVersionHistorySheet({
const shouldObserveLoadMore = open && !isLoading && hasMore && !isLoadingMore;
return (
- <Sheet open={open} modal={false} onOpenChange={onOpenChange}>
+ <Sheet
+ open={open}
+ modal={false}
+ onOpenChange={(nextOpen) => {
+ if (!nextOpen && versionToRestore) return;
+ onOpenChange(nextOpen);
+ }}
+ >
<SheetContent className="flex flex-col gap-5 overflow-y-auto sm:max-w-2xl">
<SheetHeader>
<SheetTitle>Evaluator versions</SheetTitle>
<SheetDescription>
- Saved definition versions for {evaluatorName}. Version history is
- read-only.
+ Review or restore saved definition versions for {evaluatorName}.
</SheetDescription>
</SheetHeader>
<EvaluatorVersionHistoryList
@@ -75,7 +89,11 @@ export function EvaluatorVersionHistorySheet({
currentVersionId={currentVersionId}
defaultModel={defaultModel}
expandedVersionId={expandedVersionId}
- onExpandedVersionChange={onExpandedVersionChange}
+ onExpandedVersionChange={(versionId) => {
+ setExpandedVersionId(versionId);
+ onVersionExpansionChange(versionId);
+ }}
+ onRestoreVersion={setVersionToRestore}
isLoading={isLoading}
/>
{hasMore || isLoadingMore ? (
@@ -92,6 +110,22 @@ export function EvaluatorVersionHistorySheet({
</div>
) : null}
</SheetContent>
+ <ConfirmDialog
+ open={versionToRestore !== null}
+ onOpenChange={(open) => {
+ if (!open) setVersionToRestore(null);
+ }}
+ title={`Restore version ${versionToRestore?.version}?`}
+ description={`This will replace the current evaluator definition with version ${versionToRestore?.version}. It won't be saved until you click "Save changes".`}
+ confirmLabel="Restore version"
+ confirmVariant="default"
+ onConfirm={() => {
+ if (!versionToRestore) return;
+ onRestoreVersion(versionToRestore);
+ setVersionToRestore(null);
+ onOpenChange(false);
+ }}
+ />
</Sheet>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx
index 04baa8fd0a..acfa117b8a 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.stories.tsx
@@ -16,6 +16,9 @@ const version = {
prompt: "Judge whether the response answers {{query}}.",
provider: "openai",
model: "gpt-4.1-mini",
+ modelParams: null,
+ vars: [],
+ variableMapping: null,
outputDefinition: null,
createdByUser: { name: "Ada Lovelace", email: "[email protected]" },
} satisfies EvaluatorVersion;
@@ -54,6 +57,7 @@ const defaultArgs = {
defaultModel: { provider: "OpenAI", model: "gpt-4.1-mini" },
expandedVersionId: null,
onExpandedVersionChange: fn(),
+ onRestoreVersion: fn(),
isLoading: false,
};
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx
index 60d3debbd8..9d1577e00c 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/components/EvaluatorVersionHistoryList/EvaluatorVersionHistoryList.tsx
@@ -1,7 +1,9 @@
import { EvalTemplateTypeEnum } from "@langfuse/shared";
import { formatDistanceToNowStrict } from "date-fns";
+import { RotateCcw } from "lucide-react";
import { Badge } from "@/src/components/ui/badge";
+import { Button } from "@/src/components/ui/button";
import { Skeleton } from "@/src/components/ui/skeleton";
import { CollapsibleCard } from "@/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard";
import {
@@ -44,6 +46,7 @@ export function EvaluatorVersionHistoryList({
defaultModel,
expandedVersionId,
onExpandedVersionChange,
+ onRestoreVersion,
isLoading,
}: {
versions: EvaluatorVersion[];
@@ -52,6 +55,7 @@ export function EvaluatorVersionHistoryList({
/** The version whose definition is open; null collapses them all. */
expandedVersionId: string | null;
onExpandedVersionChange: (versionId: string | null) => void;
+ onRestoreVersion: (version: EvaluatorVersion) => void;
isLoading: boolean;
}) {
if (isLoading) {
@@ -118,7 +122,22 @@ export function EvaluatorVersionHistoryList({
</span>
</>
}
- actions={null}
+ actions={
+ version.id === currentVersionId ? null : (
+ <span className="flex shrink-0 items-center pr-1">
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ aria-label={`Restore version ${version.version}`}
+ title={`Restore version ${version.version}`}
+ onClick={() => onRestoreVersion(version)}
+ >
+ <RotateCcw className="h-3.5 w-3.5" />
+ </Button>
+ </span>
+ )
+ }
>
<div className="p-3">
<EvaluatorDefinitionView
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts
index 490f5cb417..6f12d7ffd0 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/types.ts
@@ -1,6 +1,7 @@
import type {
EvalTemplateSourceCodeLanguage,
EvalTemplateType,
+ ModelConfig,
} from "@langfuse/shared";
export type EvaluatorVersion = {
@@ -13,6 +14,9 @@ export type EvaluatorVersion = {
prompt: string | null;
provider: string | null;
model: string | null;
+ modelParams: ModelConfig | null;
+ vars: string[];
+ variableMapping: unknown;
outputDefinition: unknown;
createdByUser: { name: string | null; email: string | null } | null;
};
diff --git a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsx
new file mode 100644
index 0000000000..e37ba9fd2a
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.clienttest.tsx
@@ -0,0 +1,42 @@
+import { render } from "@testing-library/react";
+
+import { PromptVariableEditor } from "./PromptVariableEditor";
+
+describe("PromptVariableEditor", () => {
+ it("keeps the interpolated preview aligned and theme-aware", () => {
+ const { container } = render(
+ <PromptVariableEditor
+ value="Question: {{input}}"
+ onChange={vi.fn()}
+ previewEnabled
+ preview={{
+ status: "ready",
+ fragments: [
+ { type: "text", text: "Question: " },
+ { type: "variable", name: "input", value: "What is Langfuse?" },
+ ],
+ }}
+ />,
+ );
+
+ const preview = container.querySelector("pre");
+ expect(preview).toHaveClass(
+ "bg-muted/50",
+ "px-3",
+ "py-2",
+ "min-h-[140px]",
+ "max-h-[50dvh]",
+ );
+
+ const interpolatedValue = container.querySelector('[title="{{input}}"]');
+ expect(interpolatedValue).toHaveClass(
+ "bg-primary-accent/10",
+ "dark:bg-accent-light-blue",
+ "dark:text-accent-dark-blue",
+ );
+ expect(interpolatedValue).not.toHaveClass(
+ "bg-accent-light-blue",
+ "text-accent-dark-blue",
+ );
+ });
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx
index fbf0adcb81..88d33cd80a 100644
--- a/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Judges/PromptVariableEditor/PromptVariableEditor.tsx
@@ -41,7 +41,7 @@ const promptFontTheme = EditorView.theme({
// Tailwind class cannot outrank.
const readOnlySurfaceTheme = EditorView.theme({
"&.cm-editor, &.cm-editor .cm-gutters": {
- backgroundColor: "hsl(var(--muted))",
+ backgroundColor: "hsl(var(--muted) / 0.5)",
},
});
@@ -235,18 +235,18 @@ export function PromptVariableEditor({
{previewEnabled && preview ? (
preview.status === "unavailable" ? (
- <p className="bg-card text-muted-foreground rounded-b-md border p-3 text-sm">
+ <p className="bg-muted/50 text-muted-foreground min-h-[140px] rounded-b-md border px-3 py-2 text-sm">
{preview.message}
</p>
) : (
- <pre className="bg-card text-card-foreground max-h-[60dvh] overflow-y-auto rounded-b-md border p-3 font-sans text-sm whitespace-pre-wrap">
+ <pre className="bg-muted/50 text-card-foreground max-h-[50dvh] min-h-[140px] overflow-y-auto rounded-b-md border px-3 py-2 font-sans text-sm whitespace-pre-wrap">
{preview.fragments.map((fragment, index) => (
<Fragment key={index}>
{fragment.type === "text" ? (
fragment.text
) : (
<span
- className="bg-primary-accent/10 rounded px-0.5"
+ className="bg-primary-accent/10 dark:bg-accent-light-blue dark:text-accent-dark-blue rounded px-0.5"
title={`{{${fragment.name}}}`}
>
{fragment.value}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
index 0068f1a834..c1b8335021 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.clienttest.tsx
@@ -1,6 +1,10 @@
import { describe, expect, it, vi } from "vitest";
+
+import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
import {
applyEvaluatorSuggestion,
+ getEvaluatorVersionDefinition,
+ restoreEvaluatorVersion,
shouldOfferRuleAttachment,
} from "./EvaluatorSetupPage";
@@ -14,6 +18,87 @@ describe("shouldOfferRuleAttachment", () => {
});
});
+describe("getEvaluatorVersionDefinition", () => {
+ it("rebuilds an LLM definition from a saved version", () => {
+ const outputDefinition = {
+ version: 2 as const,
+ dataType: "NUMERIC" as const,
+ score: { description: "Answer quality", minValue: 0, maxValue: 1 },
+ reasoning: { description: "Explain the score" },
+ };
+
+ expect(
+ getEvaluatorVersionDefinition({
+ id: "version-1",
+ version: 1,
+ createdAt: new Date(),
+ createdByUser: null,
+ type: "LLM_AS_JUDGE",
+ prompt: "Judge {{output}}",
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0.2 },
+ vars: ["output"],
+ variableMapping: [
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ jsonSelector: null,
+ },
+ ],
+ outputDefinition,
+ sourceCode: null,
+ sourceCodeLanguage: null,
+ }),
+ ).toMatchObject({
+ type: "LLM_AS_JUDGE",
+ prompt: "Judge {{output}}",
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0.2 },
+ vars: ["output"],
+ outputDefinition,
+ });
+ });
+});
+
+describe("restoreEvaluatorVersion", () => {
+ it("clears stale test state after loading the saved definition", () => {
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ mode: "create",
+ });
+ const resetTestState = vi.fn();
+
+ restoreEvaluatorVersion({
+ store,
+ version: {
+ id: "version-1",
+ version: 1,
+ createdAt: new Date(),
+ createdByUser: null,
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ sourceCodeLanguage: "TYPESCRIPT",
+ prompt: null,
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: [],
+ variableMapping: null,
+ outputDefinition: null,
+ },
+ resetTestState,
+ });
+
+ expect(store.getState()).toMatchObject({
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ });
+ expect(resetTestState).toHaveBeenCalledOnce();
+ });
+});
+
describe("applyEvaluatorSuggestion", () => {
it("reports when generation returns no suggestion", () => {
const setSuggestion = vi.fn();
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index f893bdca98..828c870742 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -1,4 +1,4 @@
-import { useMemo, useRef, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
import { useRouter } from "next/router";
import { TRPCClientError } from "@trpc/client";
import { History, Trash2 } from "lucide-react";
@@ -35,7 +35,10 @@ import { detailPageListKeys } from "@/src/features/navigate-detail-pages/context
import { TableHeaderControls } from "@/src/components/table/table-header-controls";
import { useTableDateRange } from "@/src/hooks/useTableDateRange";
import { toAbsoluteTimeRange } from "@/src/utils/date-range-utils";
-import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import {
+ createEvaluatorSetupStore,
+ type EvaluatorSetupStore,
+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
import type { EvaluatorSetupDraft } from "@/src/features/evals/v2/types/templateGallery";
import { EvaluatorRuleRelationships } from "@/src/features/evals/v2/components/Rules/EvaluatorRuleRelationships/EvaluatorRuleRelationships";
import { DefaultModelChangeConfirmationDialog } from "@/src/features/evals/v2/components/Evaluators/ProjectDefaultModel/DefaultModelChangeConfirmationDialog";
@@ -81,6 +84,51 @@ export function applyEvaluatorSuggestion(
return true;
}
+export function getEvaluatorVersionDefinition(
+ version: EvaluatorVersion,
+): EvaluatorDefinition {
+ if (version.type === "CODE") {
+ return {
+ type: version.type,
+ sourceCode: version.sourceCode ?? "",
+ sourceCodeLanguage: version.sourceCodeLanguage ?? "TYPESCRIPT",
+ };
+ }
+
+ type LlmEvaluatorDefinition = Extract<
+ EvaluatorDefinition,
+ { type: "LLM_AS_JUDGE" }
+ >;
+
+ return {
+ type: version.type,
+ prompt: version.prompt ?? "",
+ provider: version.provider,
+ model: version.model,
+ modelParams: version.modelParams,
+ vars: version.vars,
+ variableMapping:
+ version.variableMapping as LlmEvaluatorDefinition["variableMapping"],
+ outputDefinition:
+ version.outputDefinition as LlmEvaluatorDefinition["outputDefinition"],
+ };
+}
+
+export function restoreEvaluatorVersion({
+ store,
+ version,
+ resetTestState,
+}: {
+ store: EvaluatorSetupStore;
+ version: EvaluatorVersion;
+ resetTestState: () => void;
+}) {
+ store
+ .getState()
+ .actions.applyDefinition(getEvaluatorVersionDefinition(version));
+ resetTestState();
+}
+
export function EvaluatorSetupPage(
props:
| {
@@ -115,14 +163,24 @@ export function EvaluatorSetupPage(
projectId,
scope: "evalTemplate:CUD",
});
+ const projectDefaultModel = useProjectDefaultModel({
+ projectId,
+ source: "editor",
+ });
const [evaluatorSetupStore] = useState(() =>
createEvaluatorSetupStore({
initialEvaluator: initialEvaluator ?? initialDraft,
initialSampleFilter: initialEvaluator?.sampleFilter,
initialType: props.mode === "create" ? props.initialType : undefined,
+ defaultModel: projectDefaultModel.defaultModel,
mode: props.mode,
}),
);
+ useEffect(() => {
+ evaluatorSetupStore
+ .getState()
+ .actions.setDefaultModel(projectDefaultModel.defaultModel);
+ }, [evaluatorSetupStore, projectDefaultModel.defaultModel]);
const codeDraft = useStore(
evaluatorSetupStore,
useShallow((state) => ({
@@ -147,14 +205,6 @@ export function EvaluatorSetupPage(
evaluatorSetupStore,
(state) => state.testPanelOpen,
);
- const judgeModelSelection = useStore(
- evaluatorSetupStore,
- useShallow((state) => ({
- type: state.type,
- mode: state.modelMode,
- selectedModel: state.selectedModel,
- })),
- );
const [testResult, setTestResult] = useState<unknown>(null);
const [hasCompletedTestCall, setHasCompletedTestCall] = useState(false);
const [lastTestRunCostUsd, setLastTestRunCostUsd] = useState<number | null>(
@@ -163,9 +213,6 @@ export function EvaluatorSetupPage(
const [rawResultOpen, setRawResultOpen] = useState(false);
const hasRequestedName = useRef(false);
const [historyOpen, setHistoryOpen] = useState(false);
- const [expandedVersionId, setExpandedVersionId] = useState<string | null>(
- null,
- );
const [deleteOpen, setDeleteOpen] = useState(false);
const [discardOpen, setDiscardOpen] = useState(false);
const [versionConflictOpen, setVersionConflictOpen] = useState(false);
@@ -220,22 +267,13 @@ export function EvaluatorSetupPage(
prompt: version.prompt,
provider: version.provider,
model: version.model,
+ modelParams: version.modelParams as EvaluatorVersion["modelParams"],
+ vars: version.vars,
+ variableMapping: version.variableMapping,
outputDefinition: version.outputDefinition,
createdByUser: version.createdByUser,
}));
- const projectDefaultModel = useProjectDefaultModel({
- projectId,
- source: "editor",
- });
- const hasValidModel =
- judgeModelSelection.type !== "LLM_AS_JUDGE" ||
- Boolean(
- judgeModelSelection.mode === "custom"
- ? judgeModelSelection.selectedModel
- : projectDefaultModel.defaultModel,
- );
-
const create = api.evalsV2.create.useMutation();
const update = api.evalsV2.update.useMutation();
const reactivate = api.evalsV2.reactivate.useMutation({
@@ -585,7 +623,6 @@ export function EvaluatorSetupPage(
<EvaluatorTestPanelContainer
projectId={projectId}
store={evaluatorSetupStore}
- hasValidModel={hasValidModel}
sampleSelector={
<SampleObservationSelectorContainer
store={evaluatorSetupStore}
@@ -734,13 +771,26 @@ export function EvaluatorSetupPage(
versions={versions}
currentVersionId={versions[0]?.id ?? ""}
defaultModel={projectDefaultModel.defaultModel}
- expandedVersionId={expandedVersionId}
- onExpandedVersionChange={(versionId) => {
+ onVersionExpansionChange={(versionId) => {
capture("evaluators:version_history_interaction", {
action:
versionId === null ? "collapse_version" : "expand_version",
});
- setExpandedVersionId(versionId);
+ }}
+ onRestoreVersion={(version) => {
+ restoreEvaluatorVersion({
+ store: evaluatorSetupStore,
+ version,
+ resetTestState: () => {
+ setTestResult(null);
+ setHasCompletedTestCall(false);
+ setLastTestRunCostUsd(null);
+ setRawResultOpen(false);
+ },
+ });
+ capture("evaluators:version_history_interaction", {
+ action: "restore_version",
+ });
}}
isLoading={versionHistory.isPending}
hasMore={versionHistory.hasNextPage}
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
index 963b754493..1d0b571d07 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.clienttest.ts
@@ -3,7 +3,10 @@ import { describe, expect, it } from "vitest";
import { getDefaultCodeEvalSource } from "@/src/features/evals/utils/code-eval-template-starter-examples";
import { EXPERIMENTS_AND_EVALS_EXCLUSION_FILTERS } from "@/src/features/evals/v2/constants/experimentAndEvalFilters";
-import { createEvaluatorSetupStore } from "./evaluatorSetupStore";
+import {
+ createEvaluatorSetupStore,
+ selectHasValidModel,
+} from "./evaluatorSetupStore";
describe("createEvaluatorSetupStore", () => {
it("starts new evaluator descriptions empty", () => {
@@ -186,6 +189,29 @@ describe("createEvaluatorSetupStore", () => {
]);
});
+ it("derives whether any usable model is available", () => {
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ mode: "create",
+ });
+
+ expect(selectHasValidModel(store.getState())).toBe(false);
+
+ store
+ .getState()
+ .actions.setDefaultModel({ provider: "OpenAI", model: "gpt-4.1-mini" });
+ expect(selectHasValidModel(store.getState())).toBe(true);
+
+ store.getState().actions.setDefaultModel(null);
+ store
+ .getState()
+ .actions.selectModel({ provider: "OpenAI", model: "gpt-4.1" });
+ expect(selectHasValidModel(store.getState())).toBe(true);
+
+ store.getState().actions.setType("CODE");
+ expect(selectHasValidModel(store.getState())).toBe(true);
+ });
+
it("keeps configured parameters for the selected model and resets them when the model changes", () => {
const store = createEvaluatorSetupStore({
initialEvaluator: null,
@@ -203,4 +229,82 @@ describe("createEvaluatorSetupStore", () => {
actions.selectModel({ provider: "OpenAI", model: "gpt-4.1" });
expect(store.getState().modelParams).toBeNull();
});
+
+ it("loads an old LLM definition without replacing evaluator metadata", () => {
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: {
+ name: "Answer quality",
+ description: "Checks answer quality",
+ definition: {
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ sourceCodeLanguage: "TYPESCRIPT",
+ },
+ },
+ mode: "edit",
+ });
+ const sampleFilter = store.getState().sampleFilter;
+
+ store.getState().actions.applyDefinition({
+ type: "LLM_AS_JUDGE",
+ prompt: "Judge {{output}}",
+ provider: "openai",
+ model: "gpt-4.1-mini",
+ modelParams: { temperature: 0.2 },
+ vars: ["output"],
+ variableMapping: [
+ {
+ templateVariable: "output",
+ selectedColumnId: "output",
+ jsonSelector: null,
+ },
+ ],
+ outputDefinition: {
+ version: 2,
+ dataType: "NUMERIC",
+ score: {
+ description: "Answer quality",
+ minValue: 0,
+ maxValue: 1,
+ },
+ reasoning: { description: "Explain the score" },
+ },
+ });
+
+ expect(store.getState()).toMatchObject({
+ type: "LLM_AS_JUDGE",
+ name: "Answer quality",
+ description: "Checks answer quality",
+ prompt: "Judge {{output}}",
+ modelMode: "custom",
+ selectedModel: { provider: "openai", model: "gpt-4.1-mini" },
+ modelParams: { temperature: 0.2 },
+ variableFields: {
+ output: { selectedColumnId: "output", jsonSelector: null },
+ },
+ });
+ expect(store.getState().sampleFilter).toBe(sampleFilter);
+ });
+
+ it("loads an old code definition", () => {
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ mode: "create",
+ });
+
+ store.getState().actions.applyDefinition({
+ type: "CODE",
+ sourceCode: "def evaluate(ctx):\n return []",
+ sourceCodeLanguage: "PYTHON",
+ });
+
+ expect(store.getState()).toMatchObject({
+ type: "CODE",
+ sourceCode: "def evaluate(ctx):\n return []",
+ sourceCodeLanguage: "PYTHON",
+ sourceCodeDrafts: {
+ PYTHON: "def evaluate(ctx):\n return []",
+ },
+ });
+ });
});
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
index 4a661be9e8..a577786a6d 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
@@ -69,6 +69,7 @@ type EvaluatorSetupStoreActions = {
setActiveMapping: (activeMapping: ActiveVariableMapping) => void;
setModelPickerOpen: (modelPickerOpen: boolean) => void;
setModelMode: (modelMode: "default" | "custom") => void;
+ setDefaultModel: (defaultModel: JudgeModel | null) => void;
selectModel: (selectedModel: JudgeModel) => void;
configureModel: (selectedModel: JudgeModel, modelParams: ModelConfig) => void;
setSelectedObservation: (
@@ -77,6 +78,7 @@ type EvaluatorSetupStoreActions = {
setSampleFilter: (sampleFilter: FilterState) => void;
setPromptPreviewEnabled: (promptPreviewEnabled: boolean) => void;
setTestPanelOpen: (testPanelOpen: boolean) => void;
+ applyDefinition: (definition: EvaluatorDefinition) => void;
};
export type EvaluatorSetupStoreState = {
@@ -94,6 +96,7 @@ export type EvaluatorSetupStoreState = {
activeMapping: ActiveVariableMapping;
modelPickerOpen: boolean;
modelMode: "default" | "custom";
+ defaultModel: JudgeModel | null;
selectedModel: JudgeModel | null;
modelParams: ModelConfig | null;
selectedObservation: SampleObservation | null;
@@ -105,10 +108,17 @@ export type EvaluatorSetupStoreState = {
export type EvaluatorSetupStore = StoreApi<EvaluatorSetupStoreState>;
+export const selectHasValidModel = (state: EvaluatorSetupStoreState) =>
+ state.type !== EvalTemplateTypeEnum.LLM_AS_JUDGE ||
+ Boolean(
+ state.modelMode === "custom" ? state.selectedModel : state.defaultModel,
+ );
+
export function createEvaluatorSetupStore({
initialEvaluator,
initialSampleFilter,
initialType,
+ defaultModel = null,
}: {
initialEvaluator: {
name: string;
@@ -117,6 +127,7 @@ export function createEvaluatorSetupStore({
} | null;
initialSampleFilter?: FilterState;
initialType?: EvalTemplateType;
+ defaultModel?: JudgeModel | null;
mode: "create" | "edit";
}): EvaluatorSetupStore {
const initialDefinition = initialEvaluator?.definition;
@@ -159,6 +170,7 @@ export function createEvaluatorSetupStore({
initialDefinition?.type === "LLM_AS_JUDGE" && initialDefinition.model
? "custom"
: "default",
+ defaultModel,
selectedModel:
initialDefinition?.type === "LLM_AS_JUDGE" &&
initialDefinition.provider &&
@@ -217,6 +229,7 @@ export function createEvaluatorSetupStore({
setActiveMapping: (activeMapping) => set({ activeMapping }),
setModelPickerOpen: (modelPickerOpen) => set({ modelPickerOpen }),
setModelMode: (modelMode) => set({ modelMode }),
+ setDefaultModel: (defaultModel) => set({ defaultModel }),
selectModel: (selectedModel) =>
set((state) => ({
selectedModel,
@@ -235,6 +248,40 @@ export function createEvaluatorSetupStore({
setPromptPreviewEnabled: (promptPreviewEnabled) =>
set({ promptPreviewEnabled }),
setTestPanelOpen: (testPanelOpen) => set({ testPanelOpen }),
+ applyDefinition: (definition) =>
+ set((state) => {
+ if (definition.type === EvalTemplateTypeEnum.CODE) {
+ return {
+ type: definition.type,
+ sourceCode: definition.sourceCode,
+ sourceCodeLanguage: definition.sourceCodeLanguage,
+ sourceCodeDrafts: {
+ ...state.sourceCodeDrafts,
+ [definition.sourceCodeLanguage]: definition.sourceCode,
+ },
+ activeMapping: null,
+ };
+ }
+
+ const selectedModel =
+ definition.provider && definition.model
+ ? {
+ provider: definition.provider,
+ model: definition.model,
+ }
+ : null;
+
+ return {
+ type: definition.type,
+ prompt: definition.prompt,
+ scoreOutput: toScoreOutputFormState(definition.outputDefinition),
+ variableFields: buildInitialVariableFields(definition),
+ activeMapping: null,
+ modelMode: selectedModel ? "custom" : "default",
+ selectedModel,
+ modelParams: selectedModel ? definition.modelParams : null,
+ };
+ }),
},
}));
}