🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): reflect evaluator model availability (#18143)
Tobias Wochingerd418eae05ab9
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsxindex 0b4bd354fa..c10a96dc4e 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/EvaluatorSetupEditor.tsx@@ -17,6 +17,7 @@ export function EvaluatorSetupEditor({defaultModel,providerGroups,providerAdapters,+ connectionsPending,canSetProjectDefault,nameAIAssistance,descriptionAIAssistance,@@ -31,6 +32,7 @@ export function EvaluatorSetupEditor({defaultModel: JudgeModel | null;providerGroups: Array<[string, string[]]>;providerAdapters: Record<string, LLMAdapter>;+ connectionsPending: boolean;canSetProjectDefault: boolean;nameAIAssistance: ComponentProps<typeof AIAssistedInput>["aiAssistance"];descriptionAIAssistance: ComponentProps<@@ -52,6 +54,7 @@ export function EvaluatorSetupEditor({defaultModel={defaultModel}providerGroups={providerGroups}providerAdapters={providerAdapters}+ connectionsPending={connectionsPending}canSetProjectDefault={canSetProjectDefault}onStepOpenChange={onStepOpenChange}onConfigureProviders={onConfigureProviders}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsxindex fcd335ffc5..c1e0532d2a 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx@@ -23,6 +23,7 @@ export function DefinitionStepContainer({defaultModel,providerGroups,providerAdapters,+ connectionsPending,canSetProjectDefault,onStepOpenChange,onConfigureProviders,@@ -35,6 +36,7 @@ export function DefinitionStepContainer({defaultModel: JudgeModel | null;providerGroups: Array<[string, string[]]>;providerAdapters: Record<string, LLMAdapter>;+ connectionsPending: boolean;canSetProjectDefault: boolean;onStepOpenChange: (step: number, open: boolean) => void;onConfigureProviders: () => void;@@ -80,6 +82,7 @@ export function DefinitionStepContainer({defaultModel={defaultModel}providerGroups={providerGroups}providerAdapters={providerAdapters}+ connectionsPending={connectionsPending}canSetProjectDefault={canSetProjectDefault}onConfigureProviders={onConfigureProviders}onSetProjectDefault={onSetProjectDefault}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsxindex 450993f1b5..4ae1c84750 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/ModelSelector/ModelSelector.tsx@@ -10,7 +10,10 @@ import {} from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";import { JudgeModelConfigurationDialog } from "@/src/features/evals/v2/components/Evaluators/JudgeModelConfigurationDialog/JudgeModelConfigurationDialog";import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";-import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";+import {+ isJudgeModelAvailable,+ type JudgeModel,+} from "@/src/features/evals/v2/judgeModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";import { useEvalOnboardingAnalytics } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";@@ -20,6 +23,7 @@ export function ModelSelector({defaultModel,providerGroups,providerAdapters,+ connectionsPending,canSetProjectDefault,onConfigureProviders,onSetProjectDefault,@@ -29,6 +33,7 @@ export function ModelSelector({defaultModel: JudgeModel | null;providerGroups: Array<[string, string[]]>;providerAdapters: Record<string, LLMAdapter>;+ connectionsPending: boolean;canSetProjectDefault: boolean;onConfigureProviders: () => void;onSetProjectDefault: (model: ProjectDefaultModelConfig) => void;@@ -63,6 +68,14 @@ export function ModelSelector({modelParams: state.modelParams ?? {},}: null;+ const effectiveModel =+ state.mode === "default" ? defaultModel : state.selectedModel;+ const modelAvailability =+ !connectionsPending &&+ effectiveModel &&+ !isJudgeModelAvailable(effectiveModel, providerGroups)+ ? "missing"+ : "available";return (<>@@ -106,6 +119,7 @@ export function ModelSelector({mode={state.mode}defaultModel={defaultModel}selectedModel={state.selectedModel}+ modelAvailability={modelAvailability}disabled={false}/></PopoverTrigger>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsxindex a3cbdd1939..2d53d7e67d 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx@@ -3,7 +3,11 @@ import { useShallow } from "zustand/react/shallow";import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";-import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import {+ selectHasValidModel,+ type EvaluatorSetupStore,+} from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { ConfirmationDialogController } from "@/src/components/design-system/ConfirmationDialogController/ConfirmationDialogController";import { EvaluatorSetupFooterView } from "./EvaluatorSetupFooterView";export function EvaluatorSetupFooter({@@ -31,6 +35,7 @@ export function EvaluatorSetupFooter({promptMessagesReason,scoreOutputReason,nameMissing,+ hasValidModel,} = useStore(store,useShallow((state) => {@@ -57,6 +62,7 @@ export function EvaluatorSetupFooter({? getScoreOutputValidation(state.scoreOutput).reason: null,nameMissing: !state.name.trim(),+ hasValidModel: selectHasValidModel(state),};}),);@@ -104,8 +110,23 @@ export function EvaluatorSetupFooter({}return (- <EvaluatorSetupFooterView mode="create" {...sharedProps}>- Next: attach a rule to run this evaluator on incoming observations.- </EvaluatorSetupFooterView>+ <ConfirmationDialogController+ title="Create evaluator without a model?"+ text="This evaluator won't be able to run until a model is configured. Do you want to create it anyway?"+ confirmLabel="Create anyway"+ variant="default"+ loading={isSaving}+ onConfirm={onSave}+ >+ {({ openDialog }) => (+ <EvaluatorSetupFooterView+ mode="create"+ {...sharedProps}+ onSave={hasValidModel ? onSave : openDialog}+ >+ Next: attach a rule to run this evaluator on incoming observations.+ </EvaluatorSetupFooterView>+ )}+ </ConfirmationDialogController>);}diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsxindex 6973895645..1e3f17a4bc 100644--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx@@ -69,6 +69,7 @@ type JudgeModelPickerTriggerProps = Omit<selectedModel: JudgeModel | null;disabled: boolean;missingDefaultLabel?: string;+ modelAvailability?: "available" | "missing";borderVariant?: "default" | "contrast";};@@ -82,6 +83,7 @@ export const JudgeModelPickerTrigger = forwardRef<defaultModel,selectedModel,missingDefaultLabel,+ modelAvailability = "available",// Handled here rather than by `Button`, which swaps its children for the// spinner. That would drop the model label mid-mutation and collapse this// content-width trigger, shifting everything next to it in the header.@@ -102,6 +104,18 @@ export const JudgeModelPickerTrigger = forwardRef<defaultModel != null &&selectedModel.provider === defaultModel.provider &&selectedModel.model === defaultModel.model;+ const modelConnectionMissing =+ modelAvailability === "missing" &&+ Boolean(mode === "default" ? defaultModel : selectedModel);+ const connectionWarning = modelConnectionMissing ? (+ <>+ <TriangleAlert+ aria-hidden="true"+ className="text-dark-yellow h-3.5 w-3.5 shrink-0"+ />+ <span className="sr-only">Model connection missing.</span>+ </>+ ) : null;return (<Button@@ -123,6 +137,7 @@ export const JudgeModelPickerTrigger = forwardRef<{mode === "default" ? (defaultModel ? (<span className="flex min-w-0 items-center gap-2">+ {connectionWarning}<spanclassName="truncate"title={`${defaultModel.provider} / ${defaultModel.model}`}@@ -143,6 +158,7 @@ export const JudgeModelPickerTrigger = forwardRef<)) : (<span className="flex min-w-0 items-center gap-2">+ {connectionWarning}<span className="truncate" title={customSelectionLabel}>{customSelectionLabel}</span>diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsxindex c1a7e9c24d..59bb3f89cc 100644--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPickerTrigger.stories.tsx@@ -23,6 +23,17 @@ export const CustomModel = meta.story({},});+/** The saved model references a provider connection that was deleted. */+export const MissingConnection = meta.story({+ args: {+ mode: "custom",+ defaultModel: projectDefault,+ selectedModel: { provider: "Anthropic", model: "claude-sonnet-4" },+ modelAvailability: "missing",+ disabled: false,+ },+});+export const CustomModelMatchingProjectDefault = meta.story({args: {mode: "custom",diff --git a/web/src/features/evals/v2/judgeModel.ts b/web/src/features/evals/v2/judgeModel.tsindex 86addae32b..8cb7ec6b97 100644--- a/web/src/features/evals/v2/judgeModel.ts+++ b/web/src/features/evals/v2/judgeModel.ts@@ -3,6 +3,19 @@ import type { LLMAdapter } from "@langfuse/shared";/** Provider and model pair identifying an LLM-as-a-judge model. */export type JudgeModel = { provider: string; model: string };+export function isJudgeModelAvailable(+ model: JudgeModel | null,+ providerGroups: Array<[string, string[]]>,+) {+ return Boolean(+ model &&+ providerGroups.some(+ ([provider, models]) =>+ provider === model.provider && models.includes(model.model),+ ),+ );+}+type JudgeModelConnection = {provider: string;adapter: LLMAdapter;diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex f50e8891c2..7c0bb5e926 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -6,6 +6,7 @@ import { TRPCClientError } from "@trpc/client";import { History, Trash2 } from "lucide-react";import {observationVariableMappingList,+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate,type EvaluatorBlockReason,type EvalTemplateType,type FilterState,@@ -69,6 +70,7 @@ import {import { getFilterAnalyticsProperties } from "@/src/features/evals/v2/fns/getFilterAnalyticsProperties";import { createEvalOnboardingAnalytics } from "@/src/features/evals/v2/fns/createEvalOnboardingAnalytics";import { EvalOnboardingAnalyticsProvider } from "@/src/features/evals/v2/contexts/EvalOnboardingAnalyticsContext";+import { isJudgeModelAvailable } from "@/src/features/evals/v2/judgeModel";import type { SampleObservation } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/SampleObservationSelectorBase";type InitialEvaluator = {@@ -241,6 +243,35 @@ export function EvaluatorSetupPage(.getState().actions.setDefaultModel(projectDefaultModel.defaultModel);}, [evaluatorSetupStore, projectDefaultModel.defaultModel]);+ const modelDraft = useStore(+ evaluatorSetupStore,+ useShallow((state) => ({+ type: state.type,+ modelMode: state.modelMode,+ defaultModel: state.defaultModel,+ selectedModel: state.selectedModel,+ hasChangedModelSelection: state.hasChangedModelSelection,+ })),+ );+ const effectiveDraftModel =+ modelDraft.type === "CODE"+ ? null+ : modelDraft.type === "LLM_AS_JUDGE" && modelDraft.modelMode === "default"+ ? modelDraft.defaultModel+ : modelDraft.selectedModel;+ const draftResolvesEvaluatorBlock = Boolean(+ initialEvaluator?.blockedAt &&+ isEvaluatorBlockReasonRecoverableByDefinitionUpdate(+ initialEvaluator.blockReason,+ ) &&+ !projectDefaultModel.connectionsPending &&+ isJudgeModelAvailable(+ effectiveDraftModel,+ projectDefaultModel.providerGroups,+ ) &&+ (modelDraft.hasChangedModelSelection ||+ initialEvaluator.blockReason === "DEFAULT_EVAL_MODEL_MISSING"),+ );const codeDraft = useStore(evaluatorSetupStore,useShallow((state) => ({@@ -572,6 +603,18 @@ export function EvaluatorSetupPage(description: "Your evaluator changes are saved.",});initialSnapshot.current = getCurrentSnapshot(state);+ utils.evalsV2.get.setData(+ { projectId, evaluatorId: evaluator.id },+ (current) =>+ current+ ? {+ ...current,+ blockedAt: evaluator.blockedAt,+ blockReason: evaluator.blockReason,+ blockMessage: evaluator.blockMessage,+ }+ : current,+ );await utils.evalsV2.filterOptions.invalidate({ projectId });await router.push(`/project/${projectId}/evals/${evaluator.id}`);return;@@ -696,6 +739,7 @@ export function EvaluatorSetupPage(defaultModel={projectDefaultModel.defaultModel}providerGroups={projectDefaultModel.providerGroups}providerAdapters={projectDefaultModel.providerAdapters}+ connectionsPending={projectDefaultModel.connectionsPending}canSetProjectDefault={projectDefaultModel.canUpdate}onConfigureProviders={() => {onboardingAnalytics?.track(@@ -834,7 +878,7 @@ export function EvaluatorSetupPage(timeRange={timeRange}setTimeRange={setTimeRange}/>- {initialEvaluator?.blockedAt ? (+ {initialEvaluator?.blockedAt && !draftResolvesEvaluatorBlock ? (<div className="mx-3 mt-3"><EvaluatorBlockedBannerprojectId={projectId}diff --git a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.tsindex b448402b79..c0083b84e3 100644--- a/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts+++ b/web/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore.ts@@ -151,6 +151,7 @@ export type EvaluatorSetupStoreState = {defaultModel: JudgeModel | null;selectedModel: JudgeModel | null;modelParams: ModelConfig | null;+ hasChangedModelSelection: boolean;selectedObservation: SampleObservation | null;sampleFilter: FilterState;promptPreviewEnabled: boolean;@@ -246,6 +247,7 @@ export function createEvaluatorSetupStore({initialDefinition?.type === "LLM_AS_JUDGE"? initialDefinition.modelParams: null,+ hasChangedModelSelection: false,selectedObservation: null,sampleFilter: initialSampleFilter ?? [...DEFAULT_OBSERVATION_FILTER_WHEN_REMAPPING,@@ -460,12 +462,14 @@ export function createEvaluatorSetupStore({})),setActiveMapping: (activeMapping) => set({ activeMapping }),setModelPickerOpen: (modelPickerOpen) => set({ modelPickerOpen }),- setModelMode: (modelMode) => set({ modelMode }),+ setModelMode: (modelMode) =>+ set({ modelMode, hasChangedModelSelection: true }),setDefaultModel: (defaultModel) => set({ defaultModel }),selectModel: (selectedModel) =>set((state) => ({selectedModel,modelMode: "custom",+ hasChangedModelSelection: true,modelParams:state.selectedModel?.provider === selectedModel.provider &&state.selectedModel.model === selectedModel.model@@ -473,7 +477,12 @@ export function createEvaluatorSetupStore({: null,})),configureModel: (selectedModel, modelParams) =>- set({ selectedModel, modelParams, modelMode: "custom" }),+ set({+ selectedModel,+ modelParams,+ modelMode: "custom",+ hasChangedModelSelection: true,+ }),setSelectedObservation: (selectedObservation) =>set({ selectedObservation }),setSampleFilter: (sampleFilter) => set({ sampleFilter }),@@ -515,6 +524,7 @@ export function createEvaluatorSetupStore({modelMode: "custom",selectedModel,modelParams: null,+ hasChangedModelSelection: true,};}@@ -530,6 +540,7 @@ export function createEvaluatorSetupStore({modelMode: selectedModel ? "custom" : "default",selectedModel,modelParams: selectedModel ? definition.modelParams : null,+ hasChangedModelSelection: true,};}),},