Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): keep the current decision model selectable (#17905)

David Trainac924ada110e5
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx
new file mode 100644
index 0000000000..4684af8475
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx
@@ -0,0 +1,67 @@
+import { render, screen } from "@testing-library/react";
+
+import { LayerProvider } from "@/src/context/LayerContext/LayerContext";
+import { createEvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { DecisionModelSelector } from "./DecisionModelSelector";
+
+const connection = vi.hoisted(() => ({
+ provider: "jev",
+ adapter: "typesafe",
+ withDefaultModels: true,
+ customModels: [] as string[],
+}));
+
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ llmApiKey: {
+ all: {
+ useQuery: () => ({ isSuccess: true, data: { data: [connection] } }),
+ },
+ },
+ },
+}));
+
+describe("DecisionModelSelector", () => {
+ beforeAll(() => {
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ });
+
+ it.each([
+ ["lists only default models", true],
+ ["lists no models", false],
+ ])(
+ "shows the evaluator's model when its connection %s",
+ (_, withDefaultModels) => {
+ connection.withDefaultModels = withDefaultModels;
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ initialType: "DECISION_MODEL",
+ mode: "edit",
+ });
+ store.getState().actions.selectModel({
+ provider: "jev",
+ model: "jev-1.13.0",
+ });
+
+ render(
+ <DecisionModelSelector
+ projectId="project"
+ store={store}
+ onConfigureProviders={vi.fn()}
+ />,
+ { wrapper: LayerProvider },
+ );
+
+ expect(
+ screen.getByRole("combobox", { name: "Decision model" }),
+ ).toHaveTextContent("jev: jev-1.13.0");
+ },
+ );
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
index 2eaecd8c35..a807585b35 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
@@ -8,6 +8,11 @@ import { api } from "@/src/utils/api";
const SEPARATOR = "::";
+const toOption = (provider: string, model: string) => ({
+ value: `${provider}${SEPARATOR}${model}`,
+ label: `${provider}: ${model}`,
+});
+
/**
* Picks the decision-model connection and model version. Decision models have
* no project default and no sampling parameters, so this replaces the judge
@@ -29,17 +34,24 @@ export function DecisionModelSelector({
includeDecisionModels: true,
});
- const options = (connections.data?.data ?? [])
+ const connectionOptions = (connections.data?.data ?? [])
.filter((connection) => isDecisionModelAdapter(connection.adapter))
.flatMap((connection) => {
const models = connection.withDefaultModels
? [...connection.customModels, ...supportedModels[connection.adapter]]
: connection.customModels;
- return models.map((model) => ({
- value: `${connection.provider}${SEPARATOR}${model}`,
- label: `${connection.provider}: ${model}`,
- }));
+ return models.map((model) => toOption(connection.provider, model));
});
+ const selectedOption = selectedModel
+ ? toOption(selectedModel.provider, selectedModel.model)
+ : null;
+ // Radix renders a blank trigger for a value without a matching option, e.g.
+ // a model the connection no longer lists.
+ const options =
+ selectedOption &&
+ !connectionOptions.some((option) => option.value === selectedOption.value)
+ ? [...connectionOptions, selectedOption]
+ : connectionOptions;
if (connections.isSuccess && options.length === 0) {
return (
@@ -54,11 +66,7 @@ export function DecisionModelSelector({
<SelectInput
aria-label="Decision model"
placeholder="Select a decision model"
- value={
- selectedModel
- ? `${selectedModel.provider}${SEPARATOR}${selectedModel.model}`
- : ""
- }
+ value={selectedOption?.value ?? ""}
options={options}
onValueChange={(value) => {
const separatorIndex = value.indexOf(SEPARATOR);