🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): keep the current decision model selectable (#17905)
David Trainac924ada110e5
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsxnew file mode 100644index 0000000000..4684af8475--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx@@ -0,0 +1,67 @@+import { render, screen } from "@testing-library/react";++import { LayerProvider } from "@/src/context/LayerContext/LayerContext";+import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { DecisionModelSelector } from "./DecisionModelSelector";++const connection = vi.hoisted(() => ({+ provider: "jev",+ adapter: "typesafe",+ withDefaultModels: true,+ customModels: [] as string[],+}));++vi.mock("@/src/utils/api", () => ({+ api: {+ llmApiKey: {+ all: {+ useQuery: () => ({ isSuccess: true, data: { data: [connection] } }),+ },+ },+ },+}));++describe("DecisionModelSelector", () => {+ beforeAll(() => {+ vi.stubGlobal(+ "ResizeObserver",+ class {+ observe() {}+ unobserve() {}+ disconnect() {}+ },+ );+ });++ it.each([+ ["lists only default models", true],+ ["lists no models", false],+ ])(+ "shows the evaluator's model when its connection %s",+ (_, withDefaultModels) => {+ connection.withDefaultModels = withDefaultModels;+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ initialType: "DECISION_MODEL",+ mode: "edit",+ });+ store.getState().actions.selectModel({+ provider: "jev",+ model: "jev-1.13.0",+ });++ render(+ <DecisionModelSelector+ projectId="project"+ store={store}+ onConfigureProviders={vi.fn()}+ />,+ { wrapper: LayerProvider },+ );++ expect(+ screen.getByRole("combobox", { name: "Decision model" }),+ ).toHaveTextContent("jev: jev-1.13.0");+ },+ );+});diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsxindex 2eaecd8c35..a807585b35 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx@@ -8,6 +8,11 @@ import { api } from "@/src/utils/api";const SEPARATOR = "::";+const toOption = (provider: string, model: string) => ({+ value: `${provider}${SEPARATOR}${model}`,+ label: `${provider}: ${model}`,+});+/*** Picks the decision-model connection and model version. Decision models have* no project default and no sampling parameters, so this replaces the judge@@ -29,17 +34,24 @@ export function DecisionModelSelector({includeDecisionModels: true,});- const options = (connections.data?.data ?? [])+ const connectionOptions = (connections.data?.data ?? []).filter((connection) => isDecisionModelAdapter(connection.adapter)).flatMap((connection) => {const models = connection.withDefaultModels? [...connection.customModels, ...supportedModels[connection.adapter]]: connection.customModels;- return models.map((model) => ({- value: `${connection.provider}${SEPARATOR}${model}`,- label: `${connection.provider}: ${model}`,- }));+ return models.map((model) => toOption(connection.provider, model));});+ const selectedOption = selectedModel+ ? toOption(selectedModel.provider, selectedModel.model)+ : null;+ // Radix renders a blank trigger for a value without a matching option, e.g.+ // a model the connection no longer lists.+ const options =+ selectedOption &&+ !connectionOptions.some((option) => option.value === selectedOption.value)+ ? [...connectionOptions, selectedOption]+ : connectionOptions;if (connections.isSuccess && options.length === 0) {return (@@ -54,11 +66,7 @@ export function DecisionModelSelector({<SelectInputaria-label="Decision model"placeholder="Select a decision model"- value={- selectedModel- ? `${selectedModel.provider}${SEPARATOR}${selectedModel.model}`- : ""- }+ value={selectedOption?.value ?? ""}options={options}onValueChange={(value) => {const separatorIndex = value.indexOf(SEPARATOR);