Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): reflect evaluator model availability (#18143)

Tobias Wochingerd418eae05ab9
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx
index 0b4bd354fa..c10a96dc4e 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx
@@ -17,6 +17,7 @@ export function EvaluatorSetupEditor({
defaultModel,
providerGroups,
providerAdapters,
+ connectionsPending,
canSetProjectDefault,
nameAIAssistance,
descriptionAIAssistance,
@@ -31,6 +32,7 @@ export function EvaluatorSetupEditor({
defaultModel: JudgeModel | null;
providerGroups: Array<[string, string[]]>;
providerAdapters: Record<string, LLMAdapter>;
+ connectionsPending: boolean;
canSetProjectDefault: boolean;
nameAIAssistance: ComponentProps<typeof AIAssistedInput>["aiAssistance"];
descriptionAIAssistance: ComponentProps<
@@ -52,6 +54,7 @@ export function EvaluatorSetupEditor({
defaultModel={defaultModel}
providerGroups={providerGroups}
providerAdapters={providerAdapters}
+ connectionsPending={connectionsPending}
canSetProjectDefault={canSetProjectDefault}
onStepOpenChange={onStepOpenChange}
onConfigureProviders={onConfigureProviders}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
index fcd335ffc5..c1e0532d2a 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
@@ -23,6 +23,7 @@ export function DefinitionStepContainer({
defaultModel,
providerGroups,
providerAdapters,
+ connectionsPending,
canSetProjectDefault,
onStepOpenChange,
onConfigureProviders,
@@ -35,6 +36,7 @@ export function DefinitionStepContainer({
defaultModel: JudgeModel | null;
providerGroups: Array<[string, string[]]>;
providerAdapters: Record<string, LLMAdapter>;
+ connectionsPending: boolean;
canSetProjectDefault: boolean;
onStepOpenChange: (step: number, open: boolean) => void;
onConfigureProviders: () => void;
@@ -80,6 +82,7 @@ export function DefinitionStepContainer({
defaultModel={defaultModel}
providerGroups={providerGroups}
providerAdapters={providerAdapters}
+ connectionsPending={connectionsPending}
canSetProjectDefault={canSetProjectDefault}
onConfigureProviders={onConfigureProviders}
onSetProjectDefault={onSetProjectDefault}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
index 450993f1b5..4ae1c84750 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx
@@ -10,7 +10,10 @@ import {
} from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";
import { JudgeModelConfigurationDialog } from "@/src/features/evals/v2/components/Evaluators/JudgeModelConfigurationDialog/JudgeModelConfigurationDialog";
import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";
-import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
+import {
+ isJudgeModelAvailable,
+ type JudgeModel,
+} from "@/src/features/evals/v2/judgeModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
@@ -20,6 +23,7 @@ export function ModelSelector({
defaultModel,
providerGroups,
providerAdapters,
+ connectionsPending,
canSetProjectDefault,
onConfigureProviders,
onSetProjectDefault,
@@ -29,6 +33,7 @@ export function ModelSelector({
defaultModel: JudgeModel | null;
providerGroups: Array<[string, string[]]>;
providerAdapters: Record<string, LLMAdapter>;
+ connectionsPending: boolean;
canSetProjectDefault: boolean;
onConfigureProviders: () => void;
onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;
@@ -63,6 +68,14 @@ export function ModelSelector({
modelParams: state.modelParams ?? {},
}
: null;
+ const effectiveModel =
+ state.mode === "default" ? defaultModel : state.selectedModel;
+ const modelAvailability =
+ !connectionsPending &&
+ effectiveModel &&
+ !isJudgeModelAvailable(effectiveModel, providerGroups)
+ ? "missing"
+ : "available";
return (
<>
@@ -106,6 +119,7 @@ export function ModelSelector({
mode={state.mode}
defaultModel={defaultModel}
selectedModel={state.selectedModel}
+ modelAvailability={modelAvailability}
disabled={false}
/>
</PopoverTrigger>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
index a3cbdd1939..2d53d7e67d 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
@@ -3,7 +3,11 @@ import { useShallow } from "zustand/react/shallow";
import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";
import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";
import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";
-import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import {
+ selectHasValidModel,
+ type EvaluatorSetupStore,
+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { ConfirmationDialogController } from "@/src/components/design-system/ConfirmationDialogController/ConfirmationDialogController";
import { EvaluatorSetupFooterView } from "./EvaluatorSetupFooterView";
export function EvaluatorSetupFooter({
@@ -31,6 +35,7 @@ export function EvaluatorSetupFooter({
promptMessagesReason,
scoreOutputReason,
nameMissing,
+ hasValidModel,
} = useStore(
store,
useShallow((state) => {
@@ -57,6 +62,7 @@ export function EvaluatorSetupFooter({
? getScoreOutputValidation(state.scoreOutput).reason
: null,
nameMissing: !state.name.trim(),
+ hasValidModel: selectHasValidModel(state),
};
}),
);
@@ -104,8 +110,23 @@ export function EvaluatorSetupFooter({
}
return (
- <EvaluatorSetupFooterView mode="create" {...sharedProps}>
- Next: attach a rule to run this evaluator on incoming observations.
- </EvaluatorSetupFooterView>
+ <ConfirmationDialogController
+ title="Create evaluator without a model?"
+ text="This evaluator won't be able to run until a model is configured. Do you want to create it anyway?"
+ confirmLabel="Create anyway"
+ variant="default"
+ loading={isSaving}
+ onConfirm={onSave}
+ >
+ {({ openDialog }) => (
+ <EvaluatorSetupFooterView
+ mode="create"
+ {...sharedProps}
+ onSave={hasValidModel ? onSave : openDialog}
+ >
+ Next: attach a rule to run this evaluator on incoming observations.
+ </EvaluatorSetupFooterView>
+ )}
+ </ConfirmationDialogController>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
index 6973895645..1e3f17a4bc 100644
--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
@@ -69,6 +69,7 @@ type JudgeModelPickerTriggerProps = Omit<
selectedModel: JudgeModel | null;
disabled: boolean;
missingDefaultLabel?: string;
+ modelAvailability?: "available" | "missing";
borderVariant?: "default" | "contrast";
};
@@ -82,6 +83,7 @@ export const JudgeModelPickerTrigger = forwardRef<
defaultModel,
selectedModel,
missingDefaultLabel,
+ modelAvailability = "available",
// Handled here rather than by `Button`, which swaps its children for the
// spinner. That would drop the model label mid-mutation and collapse this
// content-width trigger, shifting everything next to it in the header.
@@ -102,6 +104,18 @@ export const JudgeModelPickerTrigger = forwardRef<
defaultModel != null &&
selectedModel.provider === defaultModel.provider &&
selectedModel.model === defaultModel.model;
+ const modelConnectionMissing =
+ modelAvailability === "missing" &&
+ Boolean(mode === "default" ? defaultModel : selectedModel);
+ const connectionWarning = modelConnectionMissing ? (
+ <>
+ <TriangleAlert
+ aria-hidden="true"
+ className="text-dark-yellow h-3.5 w-3.5 shrink-0"
+ />
+ <span className="sr-only">Model connection missing.</span>
+ </>
+ ) : null;
return (
<Button
@@ -123,6 +137,7 @@ export const JudgeModelPickerTrigger = forwardRef<
{mode === "default" ? (
defaultModel ? (
<span className="flex min-w-0 items-center gap-2">
+ {connectionWarning}
<span
className="truncate"
title={`${defaultModel.provider} / ${defaultModel.model}`}
@@ -143,6 +158,7 @@ export const JudgeModelPickerTrigger = forwardRef<
)
) : (
<span className="flex min-w-0 items-center gap-2">
+ {connectionWarning}
<span className="truncate" title={customSelectionLabel}>
{customSelectionLabel}
</span>
diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx
index c1a7e9c24d..59bb3f89cc 100644
--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx
@@ -23,6 +23,17 @@ export const CustomModel = meta.story({
},
});
+/** The saved model references a provider connection that was deleted. */
+export const MissingConnection = meta.story({
+ args: {
+ mode: "custom",
+ defaultModel: projectDefault,
+ selectedModel: { provider: "Anthropic", model: "claude-sonnet-4" },
+ modelAvailability: "missing",
+ disabled: false,
+ },
+});
+
export const CustomModelMatchingProjectDefault = meta.story({
args: {
mode: "custom",
diff --git a/web/src/features/evals/v2/judgeModel.ts b/web/src/features/evals/v2/judgeModel.ts
index 86addae32b..8cb7ec6b97 100644
--- a/web/src/features/evals/v2/judgeModel.ts
+++ b/web/src/features/evals/v2/judgeModel.ts
@@ -3,6 +3,19 @@ import type { LLMAdapter } from "@langfuse/shared";
/** Provider and model pair identifying an LLM-as-a-judge model. */
export type JudgeModel = { provider: string; model: string };
+export function isJudgeModelAvailable(
+ model: JudgeModel | null,
+ providerGroups: Array<[string, string[]]>,
+) {
+ return Boolean(
+ model &&
+ providerGroups.some(
+ ([provider, models]) =>
+ provider === model.provider && models.includes(model.model),
+ ),
+ );
+}
+
type JudgeModelConnection = {
provider: string;
adapter: LLMAdapter;
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index f50e8891c2..7c0bb5e926 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -6,6 +6,7 @@ import { TRPCClientError } from "@trpc/client";
import { History, Trash2 } from "lucide-react";
import {
observationVariableMappingList,
+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate,
type EvaluatorBlockReason,
type EvalTemplateType,
type FilterState,
@@ -69,6 +70,7 @@ import {
import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";
import { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";
import { EvalOnboardingAnalyticsProvider } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";
+import { isJudgeModelAvailable } from "@/src/features/evals/v2/judgeModel";
import type { SampleObservation } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase";
type InitialEvaluator = {
@@ -241,6 +243,35 @@ export function EvaluatorSetupPage(
.getState()
.actions.setDefaultModel(projectDefaultModel.defaultModel);
}, [evaluatorSetupStore, projectDefaultModel.defaultModel]);
+ const modelDraft = useStore(
+ evaluatorSetupStore,
+ useShallow((state) => ({
+ type: state.type,
+ modelMode: state.modelMode,
+ defaultModel: state.defaultModel,
+ selectedModel: state.selectedModel,
+ hasChangedModelSelection: state.hasChangedModelSelection,
+ })),
+ );
+ const effectiveDraftModel =
+ modelDraft.type === "CODE"
+ ? null
+ : modelDraft.type === "LLM_AS_JUDGE" && modelDraft.modelMode === "default"
+ ? modelDraft.defaultModel
+ : modelDraft.selectedModel;
+ const draftResolvesEvaluatorBlock = Boolean(
+ initialEvaluator?.blockedAt &&
+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate(
+ initialEvaluator.blockReason,
+ ) &&
+ !projectDefaultModel.connectionsPending &&
+ isJudgeModelAvailable(
+ effectiveDraftModel,
+ projectDefaultModel.providerGroups,
+ ) &&
+ (modelDraft.hasChangedModelSelection ||
+ initialEvaluator.blockReason === "DEFAULT_EVAL_MODEL_MISSING"),
+ );
const codeDraft = useStore(
evaluatorSetupStore,
useShallow((state) => ({
@@ -572,6 +603,18 @@ export function EvaluatorSetupPage(
description: "Your evaluator changes are saved.",
});
initialSnapshot.current = getCurrentSnapshot(state);
+ utils.evalsV2.get.setData(
+ { projectId, evaluatorId: evaluator.id },
+ (current) =>
+ current
+ ? {
+ ...current,
+ blockedAt: evaluator.blockedAt,
+ blockReason: evaluator.blockReason,
+ blockMessage: evaluator.blockMessage,
+ }
+ : current,
+ );
await utils.evalsV2.filterOptions.invalidate({ projectId });
await router.push(`/project/${projectId}/evals/${evaluator.id}`);
return;
@@ -696,6 +739,7 @@ export function EvaluatorSetupPage(
defaultModel={projectDefaultModel.defaultModel}
providerGroups={projectDefaultModel.providerGroups}
providerAdapters={projectDefaultModel.providerAdapters}
+ connectionsPending={projectDefaultModel.connectionsPending}
canSetProjectDefault={projectDefaultModel.canUpdate}
onConfigureProviders={() => {
onboardingAnalytics?.track(
@@ -834,7 +878,7 @@ export function EvaluatorSetupPage(
timeRange={timeRange}
setTimeRange={setTimeRange}
/>
- {initialEvaluator?.blockedAt ? (
+ {initialEvaluator?.blockedAt && !draftResolvesEvaluatorBlock ? (
<div className="mx-3 mt-3">
<EvaluatorBlockedBanner
projectId={projectId}
diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
index b448402b79..c0083b84e3 100644
--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts
@@ -151,6 +151,7 @@ export type EvaluatorSetupStoreState = {
defaultModel: JudgeModel | null;
selectedModel: JudgeModel | null;
modelParams: ModelConfig | null;
+ hasChangedModelSelection: boolean;
selectedObservation: SampleObservation | null;
sampleFilter: FilterState;
promptPreviewEnabled: boolean;
@@ -246,6 +247,7 @@ export function createEvaluatorSetupStore({
initialDefinition?.type === "LLM_AS_JUDGE"
? initialDefinition.modelParams
: null,
+ hasChangedModelSelection: false,
selectedObservation: null,
sampleFilter: initialSampleFilter ?? [
...DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,
@@ -460,12 +462,14 @@ export function createEvaluatorSetupStore({
})),
setActiveMapping: (activeMapping) => set({ activeMapping }),
setModelPickerOpen: (modelPickerOpen) => set({ modelPickerOpen }),
- setModelMode: (modelMode) => set({ modelMode }),
+ setModelMode: (modelMode) =>
+ set({ modelMode, hasChangedModelSelection: true }),
setDefaultModel: (defaultModel) => set({ defaultModel }),
selectModel: (selectedModel) =>
set((state) => ({
selectedModel,
modelMode: "custom",
+ hasChangedModelSelection: true,
modelParams:
state.selectedModel?.provider === selectedModel.provider &&
state.selectedModel.model === selectedModel.model
@@ -473,7 +477,12 @@ export function createEvaluatorSetupStore({
: null,
})),
configureModel: (selectedModel, modelParams) =>
- set({ selectedModel, modelParams, modelMode: "custom" }),
+ set({
+ selectedModel,
+ modelParams,
+ modelMode: "custom",
+ hasChangedModelSelection: true,
+ }),
setSelectedObservation: (selectedObservation) =>
set({ selectedObservation }),
setSampleFilter: (sampleFilter) => set({ sampleFilter }),
@@ -515,6 +524,7 @@ export function createEvaluatorSetupStore({
modelMode: "custom",
selectedModel,
modelParams: null,
+ hasChangedModelSelection: true,
};
}
@@ -530,6 +540,7 @@ export function createEvaluatorSetupStore({
modelMode: selectedModel ? "custom" : "default",
selectedModel,
modelParams: selectedModel ? definition.modelParams : null,
+ hasChangedModelSelection: true,
};
}),
},