🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): support OpenAI decision models (#18389)
Tobias Wochingerc539db55570a
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 1391c43a3d..662d3fc6b2 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -199,7 +199,7 @@"@ai-sdk/azure": "^4.0.5","@ai-sdk/google": "^4.0.6","@ai-sdk/google-vertex": "^5.0.8",- "@ai-sdk/openai": "^4.0.0",+ "@ai-sdk/openai": "^4.0.86","@ai-sdk/openai-compatible": "^3.0.3","@ai-sdk/typesafe-ai": "^3.0.4","@aws-sdk/client-cloudwatch": "catalog:",@@ -223,7 +223,7 @@"@slack/web-api": "^7.15.0","@smithy/node-http-handler": "4.5.1","@types/bcryptjs": "^2.4.6",- "ai": "^7.0.0",+ "ai": "^7.0.130","ajv": "^8.18.0","ajv-formats": "^3.0.1","bcryptjs": "^2.4.3",diff --git a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.tsindex 97f5afeee1..2e75f3f1d0 100644--- a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts+++ b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts@@ -37,6 +37,7 @@ const questions: DecisionModelQuestions = DecisionModelQuestionsSchema.parse([scoreName: "refund_requested",type: "noul",instructions: "Does `question` request a refund?",+ criteria: { true: "Asks for money back", false: "Does not" },},]);@@ -91,19 +92,27 @@ describe("executeDecisionModelEvaluator", () => {readiness: {type: "choice",instructions: "Is `reply` ready to send as an answer to `question`?",- criteria: {- ready: "Answers and states the next step",- needs_revision: null,- },+ choices: [+ {+ value: "ready",+ description: "Answers and states the next step",+ },+ { value: "needs_revision" },+ ],},frustration: {type: "score",instructions: "How frustrated is the customer in `question`?",- criteria: ["Calm", "Frustrated but civil", "Very angry"],+ levels: [+ { description: "Calm" },+ { description: "Frustrated but civil" },+ { description: "Very angry" },+ ],},refund: {- type: "boolean",+ type: "predicate",instructions: "Does `question` request a refund?",+ criteria: { true: "Asks for money back", false: "Does not" },},},});@@ -172,6 +181,25 @@ describe("executeDecisionModelEvaluator", () => {]);});+ it("writes OpenAI score details under metadata.openai", async () => {+ const { client } = createClient({ ...evaluation, model: "gpt-6-luna" });++ const { scores } = await executeDecisionModelEvaluator({+ variables: [{ var: "question", value: "refund?" }],+ questions,+ client,+ });++ expect(scores.map((score) => Object.keys(score.metadata ?? {}))).toEqual([+ ["openai"],+ ["openai"],+ ["openai"],+ ]);+ expect(scores[1]?.metadata).toMatchObject({+ openai: { model: "gpt-6-luna", questionId: "frustration", type: "score" },+ });+ });+it("fails permanently when the state is empty", async () => {const { client, evaluate } = createClient(evaluation);diff --git a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.tsindex 7bf4dcdf37..3eee28abe6 100644--- a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts+++ b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts@@ -12,26 +12,33 @@ import {type InternalTraceEventInput,} from "../llm/internalTraceEvents";import {+ isOpenAIDecisionModel,LangfuseInternalTraceEnvironment,+ LLMAdapter,type InternalTraceWriteInput,} from "../llm/types";import type { CodeEvalScoreWithName } from "./codeEvalDispatcherTypes";import type { ExtractedVariable } from "./extractObservationVariables";+/** OpenAI Decisions question shape. TypeSafe inference projects this down. */export type DecisionModelRequestQuestion =| {type: "choice";instructions: DecisionModelEntry;- criteria: Record<string, DecisionModelEntry | null>;+ choices: Array<{+ value: string;+ description?: DecisionModelEntry | null;+ }>;}| {type: "score";instructions: DecisionModelEntry;- criteria: DecisionModelEntry[];+ levels: Array<{ description: DecisionModelEntry }>;}| {- type: "boolean";+ type: "predicate";instructions: DecisionModelEntry;+ /** Yes/no notes from a saved TypeSafe question. Each SDK attaches them. */criteria?: {true?: DecisionModelEntry | null;false?: DecisionModelEntry | null;@@ -97,28 +104,32 @@ export function toDecisionModelRequestQuestion(return {type: "choice",instructions: question.instructions,- criteria: Object.fromEntries(- question.options.map((option) => [- option.value,- option.description ?? null,- ]),- ),+ choices: question.options.map((option) => ({+ value: option.value,+ ...(option.description != null+ ? { description: option.description }+ : {}),+ })),};case DecisionModelQuestionType.SCORE:return {type: "score",instructions: question.instructions,- criteria: question.levels.map((level) => level.description),+ levels: question.levels,};case DecisionModelQuestionType.NOUL:return {- type: "boolean",+ type: "predicate",instructions: question.instructions,- ...(question.criteria+ ...(question.criteria?.true != null || question.criteria?.false != null? {criteria: {- true: question.criteria.true ?? null,- false: question.criteria.false ?? null,+ ...(question.criteria.true != null+ ? { true: question.criteria.true }+ : {}),+ ...(question.criteria.false != null+ ? { false: question.criteria.false }+ : {}),},}: {}),@@ -184,13 +195,15 @@ export function formatDecisionModelComment(params: {? question.levels.length - 1: 0,);- const levelDescription =+ const level =question.type === DecisionModelQuestionType.SCORE- ? question.levels[nearestLevel]?.description+ ? question.levels[nearestLevel]: undefined;+ const levelName =+ typeof level?.description === "string" ? level.description : undefined;parts.push(- typeof levelDescription === "string"- ? `${formatNumber(answer.score)} ≈ level ${nearestLevel} "${levelDescription}"`+ levelName != null+ ? `${formatNumber(answer.score)} ≈ level ${nearestLevel} "${levelName}"`: `${formatNumber(answer.score)} ≈ level ${nearestLevel}`,);if (answer.confidence !== null) {@@ -202,6 +215,11 @@ export function formatDecisionModelComment(params: {return parts.join("; ");}+/** Score details are namespaced by the API that produced them. */+function decisionScoreMetadataKey(model: string) {+ return isOpenAIDecisionModel(model) ? LLMAdapter.OpenAI : LLMAdapter.TypeSafe;+}+function toScoreMetadata(params: {question: DecisionModelQuestion;answer: DecisionModelAnswer;@@ -209,10 +227,11 @@ function toScoreMetadata(params: {}): Record<string, unknown> {const { question, answer, model } = params;const base = { questionId: question.id, type: question.type, model };+ const key = decisionScoreMetadataKey(model);switch (answer.type) {case "choice":return {- typesafe: {+ [key]: {...base,choice: answer.choice,confidence: answer.confidence,@@ -221,7 +240,7 @@ function toScoreMetadata(params: {};case "score":return {- typesafe: {+ [key]: {...base,confidence: answer.confidence,probabilities: answer.probabilities,@@ -230,14 +249,16 @@ function toScoreMetadata(params: {? Object.fromEntries(question.levels.map((level, index) => [String(index),- level.description,+ typeof level.description === "string"+ ? level.description+ : JSON.stringify(level.description),]),): undefined,},};case "boolean":- return { typesafe: base };+ return { [key]: base };}}diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 097234cbc8..9ed589de76 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -39,6 +39,7 @@ export * from "./evals/extractObservationVariables";export * from "./evals/classifyEvaluatorLlmError";export * from "./evals/llmEvaluatorExecution";export * from "./evals/decisionModelEvaluatorExecution";+export * from "./llm/createDecisionModelClient";export * from "./llm/typesafe/typeSafeDecisionModelClient";export * from "./utils/traceId";export * from "./auth/apiKeyCache";diff --git a/packages/shared/src/server/llm/createDecisionModelClient.ts b/packages/shared/src/server/llm/createDecisionModelClient.tsnew file mode 100644index 0000000000..4c1a0b2820--- /dev/null+++ b/packages/shared/src/server/llm/createDecisionModelClient.ts@@ -0,0 +1,32 @@+import {+ DecisionModelEvaluatorError,+ type DecisionModelClient,+} from "../evals/decisionModelEvaluatorExecution";+import {+ isAllowedDecisionModel,+ LLMAdapter,+ OPENAI_DECISION_MODEL_IDS,+} from "./types";+import { createOpenAIDecisionModelClient } from "./openai/openAIDecisionModelClient";+import { createTypeSafeDecisionModelClient } from "./typesafe/typeSafeDecisionModelClient";++export function createDecisionModelClient(params: {+ adapter: string;+ apiKey: string;+ model: string;+ baseURL?: string | null;+ extraHeaders?: Record<string, string>;+ fetchImpl?: typeof fetch;+}): DecisionModelClient {+ if (!isAllowedDecisionModel(params.adapter, params.model)) {+ throw new DecisionModelEvaluatorError(+ params.adapter === LLMAdapter.OpenAI+ ? `Model "${params.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`+ : `Decision-model adapter is not supported: ${params.adapter}`,+ );+ }+ if (params.adapter === LLMAdapter.OpenAI) {+ return createOpenAIDecisionModelClient(params);+ }+ return createTypeSafeDecisionModelClient(params);+}diff --git a/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.ts b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.tsnew file mode 100644index 0000000000..631febea5e--- /dev/null+++ b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.ts@@ -0,0 +1,220 @@+import { describe, expect, it, vi } from "vitest";+import type { DecisionModelRequest } from "../../evals/decisionModelEvaluatorExecution";+import { createOpenAIDecisionModelClient } from "./openAIDecisionModelClient";++function jsonResponse(body: unknown, status = 200) {+ return new Response(JSON.stringify(body), {+ status,+ headers: { "Content-Type": "application/json" },+ });+}++const request: DecisionModelRequest = {+ state: { input: "I was charged twice.", output: "Refund issued." },+ questions: {+ department: {+ type: "choice",+ instructions: "Which team should handle `input`?",+ choices: [+ { value: "billing", description: "Payments and refunds" },+ { value: "other" },+ ],+ },+ severity: {+ type: "score",+ instructions: "How severe is the issue?",+ levels: [{ description: "Appearance only" }, { description: "Blocked" }],+ },+ refund: {+ type: "predicate",+ instructions: "Does `input` request a refund?",+ criteria: { true: "Asks for money back", false: { note: "money" } },+ },+ },+};++describe("createOpenAIDecisionModelClient", () => {+ it("posts text input and maps predicate, choice, and score answers", async () => {+ const fetchImpl = vi.fn().mockResolvedValue(+ jsonResponse({+ model: "gpt-6-luna",+ answers: [+ {+ type: "choice",+ name: "department",+ choice: "billing",+ confidence: 0.8,+ probabilities: [+ { value: "billing", probability: 0.9 },+ { value: "other", probability: 0.1 },+ ],+ },+ {+ type: "score",+ name: "severity",+ score: 0.4,+ confidence: 0.7,+ probabilities: [+ { value: 0, probability: 0.6 },+ { value: 1, probability: 0.4 },+ ],+ },+ { type: "predicate", name: "refund", probability: 0.95 },+ ],+ usage: { input_tokens: 120, output_tokens: 3 },+ }),+ );++ const client = createOpenAIDecisionModelClient({+ apiKey: "sk-test",+ model: "gpt-6-luna",+ fetchImpl,+ });+ const evaluation = await client.evaluate(request);++ const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit];+ expect(url).toBe("https://api.openai.com/v1/decisions");+ expect(new Headers(init.headers).get("authorization")).toBe(+ "Bearer sk-test",+ );+ expect(JSON.parse(init.body as string)).toEqual({+ model: "gpt-6-luna",+ input: JSON.stringify(request.state),+ questions: [+ {+ type: "choice",+ name: "department",+ instructions: "Which team should handle `input`?",+ choices: [+ { value: "billing", description: "Payments and refunds" },+ { value: "other" },+ ],+ },+ {+ type: "score",+ name: "severity",+ instructions: "How severe is the issue?",+ levels: [+ { label: "0", description: "Appearance only" },+ { label: "1", description: "Blocked" },+ ],+ },+ {+ type: "predicate",+ name: "refund",+ instructions:+ 'Does `input` request a refund?\n\nCriteria for true:\nAsks for money back\n\nCriteria for false:\n{"note":"money"}',+ },+ ],+ });+ expect(evaluation).toEqual({+ model: "gpt-6-luna",+ answers: {+ department: {+ type: "choice",+ choice: "billing",+ probabilities: { billing: 0.9, other: 0.1 },+ confidence: 0.8,+ },+ severity: {+ type: "score",+ score: 0.4,+ probabilities: { "0": 0.6, "1": 0.4 },+ confidence: 0.7,+ },+ refund: { type: "boolean", probability: 0.95 },+ },+ usage: { inputTokens: 120, outputTokens: 3 },+ });+ });++ it.each([+ [401, false],+ [500, true],+ ])(+ "surfaces HTTP %s as an AI SDK call error (retryable: %s)",+ async (status, isRetryable) => {+ const fetchImpl = vi+ .fn()+ .mockImplementation(() =>+ jsonResponse(+ { error: { message: "nope", type: "invalid_request_error" } },+ status,+ ),+ );+ const client = createOpenAIDecisionModelClient({+ apiKey: "sk-bad",+ model: "gpt-6-luna",+ fetchImpl,+ });++ await expect(client.evaluate(request)).rejects.toMatchObject({+ name: "AI_APICallError",+ statusCode: status,+ isRetryable,+ });+ },+ );++ it("fails the call when any question is refused", async () => {+ const fetchImpl = vi.fn().mockImplementation(() =>+ jsonResponse({+ model: "gpt-6-luna",+ answers: [+ {+ type: "choice",+ name: "department",+ choice: "billing",+ probabilities: [+ { value: "billing", probability: 1 },+ { value: "other", probability: 0 },+ ],+ },+ {+ type: "score",+ name: "severity",+ score: 0,+ probabilities: [+ { value: 0, probability: 1 },+ { value: 1, probability: 0 },+ ],+ },+ { type: "refusal", name: "refund" },+ ],+ }),+ );+ const client = createOpenAIDecisionModelClient({+ apiKey: "sk-test",+ model: "gpt-6-luna",+ fetchImpl,+ });++ await expect(client.evaluate(request)).rejects.toMatchObject({+ name: "DecisionModelEvaluatorError",+ message: "OpenAI declined to answer: refund.",+ });+ });++ it("appends /decisions to a custom base URL", async () => {+ const fetchImpl = vi.fn().mockResolvedValue(+ jsonResponse({+ model: "gpt-6-luna",+ answers: [{ type: "predicate", name: "refund", probability: 0.5 }],+ }),+ );+ const client = createOpenAIDecisionModelClient({+ apiKey: "sk-test",+ model: "gpt-6-luna",+ baseURL: "https://llm-proxy.example.com/v1/",+ fetchImpl,+ });++ await client.evaluate({+ state: request.state,+ questions: { refund: request.questions.refund },+ });++ const [url] = fetchImpl.mock.calls[0] as [string, RequestInit];+ expect(url).toBe("https://llm-proxy.example.com/v1/decisions");+ });+});diff --git a/packages/shared/src/server/llm/openai/openAIDecisionModelClient.ts b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.tsnew file mode 100644index 0000000000..5177d19877--- /dev/null+++ b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.ts@@ -0,0 +1,167 @@+import { createOpenAI } from "@ai-sdk/openai";+import {+ experimental_decide as decide,+ Experimental_DecisionRefusalError,+ type Experimental_DecisionQuestion,+} from "ai";+import {+ DecisionModelEvaluatorError,+ type DecisionModelAnswer,+ type DecisionModelClient,+ type DecisionModelEvaluation,+ type DecisionModelRequest,+ type DecisionModelRequestQuestion,+} from "../../evals/decisionModelEvaluatorExecution";+import type { DecisionModelEntry } from "../../../features/evals/decisionModel";+import { createSecureLlmFetch } from "../secureLlmFetch";++/** Same text conversion the OpenAI SDK applies before POST /v1/decisions. */+function toDecisionText(entry: DecisionModelEntry): string {+ return typeof entry === "string" ? entry : JSON.stringify(entry);+}++/** The SDK's score question has descriptions only. It sends index labels. */+function toSdkQuestion(+ question: DecisionModelRequestQuestion,+): Experimental_DecisionQuestion {+ switch (question.type) {+ case "choice":+ return {+ type: "choice",+ instructions: toDecisionText(question.instructions),+ criteria: Object.fromEntries(+ question.choices.map((choice) => [+ choice.value,+ choice.description == null+ ? null+ : toDecisionText(choice.description),+ ]),+ ),+ };+ case "score":+ return {+ type: "score",+ instructions: toDecisionText(question.instructions),+ criteria: question.levels.map((level) =>+ toDecisionText(level.description),+ ),+ };+ case "predicate": {+ const criteria = {+ ...(question.criteria?.true != null+ ? { true: toDecisionText(question.criteria.true) }+ : {}),+ ...(question.criteria?.false != null+ ? { false: toDecisionText(question.criteria.false) }+ : {}),+ };+ return {+ type: "boolean",+ instructions: toDecisionText(question.instructions),+ ...(Object.keys(criteria).length > 0 ? { criteria } : {}),+ };+ }+ }+}++function readConfidence(providerMetadata: unknown): Record<string, number> {+ if (typeof providerMetadata !== "object" || providerMetadata === null) {+ return {};+ }+ const openai = (providerMetadata as Record<string, unknown>).openai;+ if (typeof openai !== "object" || openai === null) return {};+ const confidence = (openai as Record<string, unknown>).confidence;+ if (typeof confidence !== "object" || confidence === null) return {};+ return Object.fromEntries(+ Object.entries(confidence as Record<string, unknown>).flatMap(+ ([id, value]) =>+ typeof value === "number" && value >= 0 && value <= 1+ ? [[id, value]]+ : [],+ ),+ );+}++export function createOpenAIDecisionModelClient(params: {+ apiKey: string;+ model: string;+ baseURL?: string | null;+ extraHeaders?: Record<string, string>;+ fetchImpl?: typeof fetch;+}): DecisionModelClient {+ const provider = createOpenAI({+ apiKey: params.apiKey,+ baseURL: params.baseURL ?? undefined,+ headers: params.extraHeaders,+ fetch:+ params.fetchImpl ??+ createSecureLlmFetch({+ logContext: "OpenAI decision model",+ additionalSensitiveHeaders: Object.keys(params.extraHeaders ?? {}),+ }),+ });++ return {+ evaluate: async (request: DecisionModelRequest) => {+ const questions: Record<string, Experimental_DecisionQuestion> =+ Object.fromEntries(+ Object.entries(request.questions).map(([id, question]) => [+ id,+ toSdkQuestion(question),+ ]),+ );+ let result;+ try {+ result = await decide({+ model: provider.decisionModel(params.model),+ state: JSON.stringify(request.state),+ questions,+ maxRetries: 0,+ });+ } catch (error) {+ if (Experimental_DecisionRefusalError.isInstance(error)) {+ throw new DecisionModelEvaluatorError(+ `OpenAI declined to answer: ${error.questionIds.join(", ")}.`,+ );+ }+ throw error;+ }++ const confidenceById = readConfidence(result.providerMetadata);+ const answers: Record<string, DecisionModelAnswer> = {};+ for (const [id, answer] of Object.entries(result.answers)) {+ switch (answer.type) {+ case "choice":+ answers[id] = {+ type: "choice",+ choice: answer.choice,+ probabilities: answer.probabilities ?? { [answer.choice]: 1 },+ confidence: confidenceById[id] ?? null,+ };+ break;+ case "score":+ answers[id] = {+ type: "score",+ score: answer.score,+ probabilities: answer.probabilities ?? {},+ confidence: confidenceById[id] ?? null,+ };+ break;+ case "boolean":+ answers[id] = { type: "boolean", probability: answer.probability };+ break;+ }+ }++ const evaluation: DecisionModelEvaluation = {+ model: result.response.modelId,+ answers,+ usage: {+ inputTokens: result.usage.inputTokens ?? null,+ outputTokens: result.usage.outputTokens ?? null,+ },+ };+ return evaluation;+ },+ };+}diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex a38684c0fe..de479c968a 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -258,10 +258,31 @@ export const DECISION_MODEL_ADAPTERS: readonly LLMAdapter[] = [LLMAdapter.TypeSafe,];+/** OpenAI models the Decisions API accepts. */+export const OPENAI_DECISION_MODEL_IDS: readonly string[] = ["gpt-6-luna"];++export function isOpenAIDecisionModel(model: string): boolean {+ return OPENAI_DECISION_MODEL_IDS.includes(model);+}+export function isDecisionModelAdapter(adapter: string): boolean {return DECISION_MODEL_ADAPTERS.includes(adapter as LLMAdapter);}+/** Adapters that can answer a decision-model evaluator. OpenAI stays a text adapter too. */+export function supportsDecisionModels(adapter: string): boolean {+ return adapter === LLMAdapter.TypeSafe || adapter === LLMAdapter.OpenAI;+}++export function isAllowedDecisionModel(+ adapter: string,+ model: string,+): boolean {+ if (adapter === LLMAdapter.TypeSafe) return model.length > 0;+ if (adapter === LLMAdapter.OpenAI) return isOpenAIDecisionModel(model);+ return false;+}+// Some providers require at least one user message. The persisted-message// conversion boundary turns a lone message into a user message for them.export const PROVIDERS_WITH_REQUIRED_USER_MESSAGE: readonly LLMAdapter[] = [diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.tsindex a68682db9f..21b01a44e2 100644--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts@@ -25,15 +25,19 @@ const request: DecisionModelRequest = {readiness: {type: "choice",instructions: "Is `reply` ready to send?",- criteria: { ready: null, needs_revision: null },+ choices: [{ value: "ready" }, { value: "needs_revision" }],},frustration: {type: "score",instructions: "How frustrated is the customer?",- criteria: ["Calm", "Frustrated", "Angry"],+ levels: [+ { description: "Calm" },+ { description: "Frustrated" },+ { description: "Angry" },+ ],},refund: {- type: "boolean",+ type: "predicate",instructions: "Does `question` request a refund?",},},@@ -77,14 +81,25 @@ describe("createTypeSafeDecisionModelClient", () => {expect(new Headers(init.headers).get("authorization")).toBe("Bearer sk-test",);- // The SDK sends TypeSafe's native `noul` type for boolean questions.+ // Score levels become a description list. Predicates go out as TypeSafe's noul type.expect(JSON.parse(init.body as string)).toEqual({model: "jev-latest",state: request.state,questions: {- readiness: request.questions.readiness,- frustration: request.questions.frustration,- refund: { ...request.questions.refund, type: "noul" },+ readiness: {+ type: "choice",+ instructions: "Is `reply` ready to send?",+ criteria: { ready: null, needs_revision: null },+ },+ frustration: {+ type: "score",+ instructions: "How frustrated is the customer?",+ criteria: ["Calm", "Frustrated", "Angry"],+ },+ refund: {+ type: "noul",+ instructions: "Does `question` request a refund?",+ },},});diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.tsindex da80d48d01..5a17a25923 100644--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts@@ -1,16 +1,53 @@import { createTypeSafeAi } from "@ai-sdk/typesafe-ai";import {- experimental_evaluate as evaluate,- type Experimental_EvaluationQuestion as EvaluationQuestion,+ experimental_decide as decide,+ type Experimental_DecisionQuestion,type JSONValue,} from "ai";import type {DecisionModelAnswer,DecisionModelClient,DecisionModelEvaluation,+ DecisionModelRequestQuestion,} from "../../evals/decisionModelEvaluatorExecution";import { createSecureLlmFetch } from "../secureLlmFetch";+/** TypeSafe reads a description per level, in order. */+function toTypeSafeQuestion(question: DecisionModelRequestQuestion) {+ switch (question.type) {+ case "choice":+ return {+ type: "choice" as const,+ instructions: question.instructions,+ criteria: Object.fromEntries(+ question.choices.map((choice) => [+ choice.value,+ choice.description ?? null,+ ]),+ ),+ };+ case "score":+ return {+ type: "score" as const,+ instructions: question.instructions,+ criteria: question.levels.map((level) => level.description),+ };+ case "predicate":+ return {+ type: "boolean" as const,+ instructions: question.instructions,+ ...(question.criteria+ ? {+ criteria: {+ true: question.criteria.true ?? null,+ false: question.criteria.false ?? null,+ },+ }+ : {}),+ };+ }+}+export function createTypeSafeDecisionModelClient(params: {apiKey: string;model: string;@@ -33,10 +70,15 @@ export function createTypeSafeDecisionModelClient(params: {return {evaluate: async (request) => {- const result = await evaluate({+ const result = await decide({model,state: request.state as Record<string, JSONValue>,- questions: request.questions as Record<string, EvaluationQuestion>,+ questions: Object.fromEntries(+ Object.entries(request.questions).map(([id, question]) => [+ id,+ toTypeSafeQuestion(question),+ ]),+ ) as Record<string, Experimental_DecisionQuestion>,maxRetries: 1,});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 29891f81a0..6e845181a4 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -297,8 +297,8 @@ importers:specifier: ^5.0.8version: 5.0.8([email protected])([email protected])'@ai-sdk/openai':- specifier: ^4.0.0- version: 4.0.78([email protected])+ specifier: ^4.0.86+ version: 4.0.86([email protected])'@ai-sdk/openai-compatible':specifier: ^3.0.3version: 3.0.3([email protected])@@ -369,8 +369,8 @@ importers:specifier: ^2.4.6version: 2.4.6ai:- specifier: ^7.0.0- version: 7.0.107([email protected])+ specifier: ^7.0.130+ version: 7.0.130([email protected])ajv:specifier: ^8.18.0version: 8.20.0@@ -1379,8 +1379,8 @@ packages:peerDependencies:zod: 4.3.6- '@ai-sdk/[email protected]':- resolution: {integrity: sha512-6wJmX/D5OO7tvDJbtFdXdBuRt8AkCnW+FLK+W35UHe/tHgolAa4hclXtJcMSYnlTDy/t5DNRgdesVaoMXtwzow==}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-NprSdFzyVnbEp2SDWtIgsCFIUHFCehOD9S6FQkF/dLrCmnke8w4SOK3KcmUUiEbwtVzlUF8prkaELp+ohcQaQw==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6@@ -1439,6 +1439,12 @@ packages:peerDependencies:zod: 4.3.6+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-sbUHayMvjAYV6ELydYXFTr5UBX5qyYmRELqZR3tV0dieUhgUYd8dED9bwEgqZJNJU6V8GGLOlaEpL1ltbNXA/g==}+ engines: {node: '>=22'}+ peerDependencies:+ zod: 4.3.6+'@ai-sdk/[email protected]':resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==}engines: {node: '>=18'}@@ -1481,6 +1487,12 @@ packages:peerDependencies:zod: 4.3.6+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-F7/U1YpVXEfSxRHbxScm0PVdr9lN6GaUWHnt8ws2UjuGeuxWHqCkV25dGpHAkpDHkuIXoC+aak/OqSylKSJmHw==}+ engines: {node: '>=22'}+ peerDependencies:+ zod: 4.3.6+'@ai-sdk/[email protected]':resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}engines: {node: '>=18'}@@ -1521,6 +1533,10 @@ packages:resolution: {integrity: sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ==}engines: {node: '>=22'}+ '@ai-sdk/[email protected]':+ resolution: {integrity: sha512-iztFqJTdxNN0ZzqZAAnVPMhRMaL1bfQFa3GIPAffWQ0+dP2FXIkyMD8kHeEI7EpB/C6TR45cqOkrS7YZZ1sATA==}+ engines: {node: '>=22'}+'@ai-sdk/[email protected]':resolution: {integrity: sha512-LQvqtf6cVla5iyH/LYs9rvl63NDWRRqggz9ilGNmjixhi5TuONAWo+SC6+xT46izn/bpyx9hmXNdMLg3ZSSW0A==}engines: {node: '>=22'}@@ -7319,8 +7335,8 @@ packages:peerDependencies:zod: 4.3.6- resolution: {integrity: sha512-PVYQ3W9kR8oYiGQxbg4aFIlhQzKskZm8UV+4E0zB2uKOzFr8ju3bXQncOOZtLsOgJY8vdpgyu3cXtomsfcdQYg==}+ resolution: {integrity: sha512-QkBkCogl46jXUg0SrSsdqQlOI2ZVMCjfnbBQS8t3gE21U/SO17oNMVFyjmcC9z20wOoYUA+2O2vsso0maoS7dw==}engines: {node: '>=22'}peerDependencies:zod: 4.3.6@@ -12803,10 +12819,10 @@ snapshots:'@vercel/oidc': 3.2.0zod: 4.3.6- '@ai-sdk/[email protected]([email protected])':+ '@ai-sdk/[email protected]([email protected])':dependencies:- '@ai-sdk/provider': 4.0.17- '@ai-sdk/provider-utils': 5.0.45([email protected])+ '@ai-sdk/provider': 4.0.24+ '@ai-sdk/provider-utils': 5.0.56([email protected])'@vercel/oidc': 3.2.0zod: 4.3.6@@ -12876,6 +12892,12 @@ snapshots:'@ai-sdk/provider-utils': 5.0.49([email protected])zod: 4.3.6+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 4.0.24+ '@ai-sdk/provider-utils': 5.0.56([email protected])+ zod: 4.3.6+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 1.1.3@@ -12931,6 +12953,15 @@ snapshots:undici: 7.29.1zod: 4.3.6+ '@ai-sdk/[email protected]([email protected])':+ dependencies:+ '@ai-sdk/provider': 4.0.24+ '@standard-schema/spec': 1.1.0+ '@workflow/serde': 4.1.0+ eventsource-parser: 3.1.0+ undici: 7.29.1+ zod: 4.3.6+'@ai-sdk/[email protected]':dependencies:json-schema: 0.4.0@@ -12971,6 +13002,10 @@ snapshots:dependencies:json-schema: 0.4.0+ '@ai-sdk/[email protected]':+ dependencies:+ json-schema: 0.4.0+'@ai-sdk/[email protected]([email protected])':dependencies:'@ai-sdk/provider': 4.0.17@@ -19139,11 +19174,11 @@ snapshots:'@opentelemetry/api': 1.9.1zod: 4.3.6dependencies:- '@ai-sdk/gateway': 4.0.87([email protected])- '@ai-sdk/provider': 4.0.17- '@ai-sdk/provider-utils': 5.0.45([email protected])+ '@ai-sdk/gateway': 4.0.106([email protected])+ '@ai-sdk/provider': 4.0.24+ '@ai-sdk/provider-utils': 5.0.56([email protected])zod: 4.3.6diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex f1aec88a81..c5394f1f38 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -201,6 +201,12 @@ minimumReleaseAgeExclude:- "@next/[email protected]"- "@next/[email protected]"- "@next/[email protected]"+ # AI SDK Decisions API (`experimental_decide`); eligible after 2026-10-12 01:47 UTC.+ - "@ai-sdk/[email protected]"+ - "@ai-sdk/[email protected]"+ - "@ai-sdk/[email protected]"+ - "@ai-sdk/[email protected]"# Turbo and lint companions; eligible after 2026-10-06 04:01 UTC.diff --git a/web/src/__tests__/server/llm-api-key.servertest.ts b/web/src/__tests__/server/llm-api-key.servertest.tsindex 080dad7e96..a501e956b3 100644--- a/web/src/__tests__/server/llm-api-key.servertest.ts+++ b/web/src/__tests__/server/llm-api-key.servertest.ts@@ -375,6 +375,87 @@ describe("llmApiKey.all RPC", () => {});});+ it("should treat a decision-model connection as working when the probe is answered", async () => {+ const fetchMock = vi.fn().mockResolvedValue(+ new Response(+ JSON.stringify({+ model: "jev-1.13.0",+ answers: {+ kind: {+ type: "choice",+ choice: "greeting",+ confidence: 0.9,+ probabilities: { greeting: 0.9, other: 0.1 },+ },+ },+ }),+ { status: 200, headers: { "Content-Type": "application/json" } },+ ),+ );+ vi.stubGlobal("fetch", fetchMock);++ try {+ const result = await caller.llmApiKey.test({+ projectId,+ secretKey: "sk-typesafe",+ provider: "jev",+ adapter: LLMAdapter.TypeSafe,+ baseURL: "https://example.com/typesafe/v1",+ extraHeaders: { "x-team": "evals" },+ });++ expect(result).toEqual({ success: true });+ expect(mockGenerateLLMText).not.toHaveBeenCalled();+ expect(fetchMock).toHaveBeenCalledTimes(1);+ const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];+ expect(url).toBe("https://example.com/typesafe/v1/systemone");+ expect(new Headers(init.headers).get("authorization")).toBe(+ "Bearer sk-typesafe",+ );+ expect(new Headers(init.headers).get("x-team")).toBe("evals");+ expect(JSON.parse(init.body as string)).toEqual({+ model: "jev-latest",+ state: { message: "Hello, is anyone there?" },+ questions: {+ kind: {+ type: "choice",+ instructions: "What kind of message is `message`?",+ criteria: { greeting: null, other: null },+ },+ },+ });+ } finally {+ vi.unstubAllGlobals();+ }+ });++ it("should report the upstream error when a decision-model connection test fails", async () => {+ const fetchMock = vi.fn().mockResolvedValue(+ new Response(JSON.stringify({ error: { message: "invalid api key" } }), {+ status: 401,+ headers: { "Content-Type": "application/json" },+ }),+ );+ vi.stubGlobal("fetch", fetchMock);++ try {+ const result = await caller.llmApiKey.test({+ projectId,+ secretKey: "sk-typesafe",+ provider: "jev",+ adapter: LLMAdapter.TypeSafe,+ baseURL: "https://example.com/typesafe/v1",+ });++ expect(result.success).toBe(false);+ expect(result.error).toEqual(expect.any(String));+ expect(result.error!.length).toBeGreaterThan(0);+ expect(mockGenerateLLMText).not.toHaveBeenCalled();+ } finally {+ vi.unstubAllGlobals();+ }+ });+it("should block decision-model connections with an internal custom base URL", async () => {await expect(caller.llmApiKey.create({diff --git a/web/src/components/TextActionButton/TextActionButton.stories.tsx b/web/src/components/TextActionButton/TextActionButton.stories.tsxnew file mode 100644index 0000000000..22667490d5--- /dev/null+++ b/web/src/components/TextActionButton/TextActionButton.stories.tsx@@ -0,0 +1,24 @@+import { fn } from "storybook/test";++import preview from "../../../.storybook/preview";+import { TextActionButton } from "./TextActionButton";++const meta = preview.meta({+ component: TextActionButton,+ args: { text: "Add message", onClick: fn() },+});++export const Default = meta.story({});++export const Disabled = meta.story({+ args: { text: "Add option", disabled: true },+});++export const Fill = meta.story({+ parameters: { controls: { disable: true } },+ render: () => (+ <div className="w-64 border border-dashed">+ <TextActionButton text="Add message" width="fill" onClick={fn()} />+ </div>+ ),+});diff --git a/web/src/components/TextActionButton/TextActionButton.tsx b/web/src/components/TextActionButton/TextActionButton.tsxnew file mode 100644index 0000000000..9d7b04a5cc--- /dev/null+++ b/web/src/components/TextActionButton/TextActionButton.tsx@@ -0,0 +1,49 @@+import { Plus, type LucideIcon } from "lucide-react";+import {+ type ComponentPropsWithoutRef,+ type MouseEventHandler,+ type Ref,+} from "react";++import { cn } from "@/src/utils/tailwind";++/** Underlined text action with a leading icon. */+export function TextActionButton({+ text,+ icon: Icon = Plus,+ width = "content",+ type = "button",+ disabled,+ onClick,+ ref,+ ...props+}: {+ text: string;+ icon?: LucideIcon;+ width?: "content" | "fill";+ ref?: Ref<HTMLButtonElement>;+} & Omit<+ ComponentPropsWithoutRef<"button">,+ "className" | "children" | "type" | "disabled" | "onClick"+> & {+ type?: "button" | "submit";+ disabled?: boolean;+ onClick?: MouseEventHandler<HTMLButtonElement>;+ }) {+ return (+ <button+ ref={ref}+ type={type}+ disabled={disabled}+ onClick={onClick}+ {...props}+ className={cn(+ "text-foreground hover:text-foreground inline-flex h-6 items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline disabled:cursor-not-allowed disabled:opacity-50",+ width === "fill" && "w-full justify-start",+ )}+ >+ <Icon className="icon-base text-icon-foreground shrink-0" aria-hidden />+ {text}+ </button>+ );+}diff --git a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsxindex 2ff554ccc7..064fc31065 100644--- a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx+++ b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx@@ -32,11 +32,11 @@ export function CollapsibleCard({<><ChevronDownclassName={cn(- "text-foreground-tertiary icon-base shrink-0 translate-y-px transition-transform",+ "text-foreground-tertiary icon-base shrink-0 transition-transform",open ? "rotate-0" : "-rotate-90",)}/>- <span className="flex min-w-0 flex-1 items-baseline gap-2">{header}</span>+ <span className="flex min-w-0 flex-1 items-center gap-2">{header}</span></>);diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsxindex 4549ffad27..2861aace19 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx@@ -178,7 +178,7 @@ export function EvaluatorGalleryView({type="button"variant="secondary"className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"- title="Ask TypeSafe Jev typed questions and get calibrated answers in one call. Experimental."+ title="Ask Jev or the OpenAI Decisions API typed questions in one call. Experimental."onClick={() =>onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)}diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsxindex 030aaf75a6..870fa68e42 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx@@ -19,9 +19,8 @@ export function EvaluatorGalleryDecisionModelBanner({<div className="min-w-0 flex-1"><span className="font-bold">New: decision-model evaluators.</span>{" "}<span className="text-muted-foreground">- Ask TypeSafe Jev typed questions about each observation and get- calibrated answers in one call, at a fraction of an LLM judge's- cost. Experimental.+ Ask Jev or the OpenAI Decisions API typed questions about each+ observation in one call. Experimental.</span></div></div>diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsxindex 2c30ecbf16..4f7bd2a0b3 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx@@ -1,6 +1,7 @@import { DECISION_MODEL_LIMITS } from "@langfuse/shared";-import { Plus, Trash2 } from "lucide-react";+import { Trash2 } from "lucide-react";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import { Button } from "@/src/components/ui/button";import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";import { Input } from "@/src/components/ui/input";@@ -36,16 +37,20 @@ export function ChoiceOptionsEditor({return (<div className="flex flex-col gap-2">- <Label className="flex items-center gap-1.5">+ <Label className="flex items-baseline gap-1.5">Options- <InfoTooltip label="About options">- One label per option. A short description sharpens the boundary- between options; add an “other” option when the list may not cover- every input.- </InfoTooltip>- <span className="text-muted-foreground text-xs font-normal">- {options.length} of {DECISION_MODEL_LIMITS.maxChoiceOptions}+ <span className="inline-flex self-center">+ <InfoTooltip label="About options">+ One label per option. A short description sharpens the boundary+ between options; add an “other” option when the list may not cover+ every input.+ </InfoTooltip></span>+ {options.length >= 240 ? (+ <span className="text-muted-foreground text-xs leading-none font-normal">+ {options.length} of {DECISION_MODEL_LIMITS.maxChoiceOptions}+ </span>+ ) : null}</Label><SortableListitems={options}@@ -87,18 +92,11 @@ export function ChoiceOptionsEditor({</div>)}/>- <div>- <Button- type="button"- variant="outline"- size="sm"- disabled={options.length >= DECISION_MODEL_LIMITS.maxChoiceOptions}- onClick={() => onChange([...options, { value: "", description: "" }])}- >- <Plus className="icon-base text-icon-foreground mr-1" />- Add option- </Button>- </div>+ <TextActionButton+ text="Add option"+ disabled={options.length >= DECISION_MODEL_LIMITS.maxChoiceOptions}+ onClick={() => onChange([...options, { value: "", description: "" }])}+ />{error ? <p className="text-destructive text-xs">{error}</p> : null}</div>);diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsxindex 97476870ff..aaee96eba7 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx@@ -10,6 +10,7 @@ import { Label } from "@/src/components/ui/label";import { CollapsibleCard } from "@/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard";import { ChoiceOptionsEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor";import { NoulCriteriaEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/NoulCriteriaEditor/NoulCriteriaEditor";+import { OpenAIQuestionInstructions } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions";import { QuestionInstructionsField } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionInstructionsField/QuestionInstructionsField";import {QUESTION_TYPE_COPY,@@ -32,6 +33,8 @@ export type DecisionModelQuestionCardProps = {onChange: (question: DecisionModelQuestionDraft) => void;onRemove: (() => void) | null;errors?: DecisionModelQuestionDraftErrors;+ /** OpenAI questions are plain text. Jev questions name state fields in backticks. */+ plainInstructions?: boolean;};function writesLine(question: DecisionModelQuestionDraft) {@@ -95,6 +98,7 @@ export function DecisionModelQuestionCard({onChange,onRemove,errors = {},+ plainInstructions = false,}: DecisionModelQuestionCardProps) {const id = useId();const copy = QUESTION_TYPE_COPY[question.type];@@ -109,14 +113,14 @@ export function DecisionModelQuestionCard({triggerTitle={expanded ? "Collapse question" : "Expand question"}header={<span className="flex min-w-0 items-center gap-2">- <span className="text-muted-foreground shrink-0 font-mono text-xs">+ <span className="text-muted-foreground shrink-0 font-mono text-xs leading-none">{index + 1}</span><copy.icon className="icon-base shrink-0" aria-label={copy.label} /><Badgevariant="secondary"className={cn(- "shrink-0 font-mono",+ "shrink-0 pb-0 font-mono",!question.scoreName.trim() && "text-dark-yellow",)}>@@ -124,7 +128,7 @@ export function DecisionModelQuestionCard({</Badge><spanclassName={cn(- "truncate",+ "truncate leading-none",!question.instructions.trim() && "text-muted-foreground italic",)}title={summary}@@ -172,23 +176,42 @@ export function DecisionModelQuestionCard({/></div>- <QuestionInstructionsField- value={question.instructions}- onChange={(instructions) =>- onChange({- ...question,- instructions,- scoreName:- question.scoreName === "" ||- question.scoreName === suggestScoreName(question.instructions)- ? suggestScoreName(instructions)- : question.scoreName,- })- }- stateKeys={stateKeys}- placeholder={copy.example}- error={errors.instructions}- />+ {plainInstructions ? (+ <OpenAIQuestionInstructions+ value={question.instructions}+ onChange={(instructions) =>+ onChange({+ ...question,+ instructions,+ scoreName:+ question.scoreName === "" ||+ question.scoreName === suggestScoreName(question.instructions)+ ? suggestScoreName(instructions)+ : question.scoreName,+ })+ }+ placeholder={copy.example}+ error={errors.instructions}+ />+ ) : (+ <QuestionInstructionsField+ value={question.instructions}+ onChange={(instructions) =>+ onChange({+ ...question,+ instructions,+ scoreName:+ question.scoreName === "" ||+ question.scoreName === suggestScoreName(question.instructions)+ ? suggestScoreName(instructions)+ : question.scoreName,+ })+ }+ stateKeys={stateKeys}+ placeholder={copy.example}+ error={errors.instructions}+ />+ )}<CriteriaEditorquestion={question}diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsxindex 6762edcc05..2503b792fe 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx@@ -8,6 +8,7 @@ import { moveItem } from "@/src/features/evals/v2/fns/moveItem";import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";import {DecisionModelQuestionList,+ OPENAI_QUESTION_EXAMPLES,QUESTION_EXAMPLES,} from "./DecisionModelQuestionList";@@ -61,7 +62,10 @@ function InteractiveQuestionList(args: StoryArgs) {args.onAdd();}}onAddExample={(type) => {- const question = { id: nextId(), ...QUESTION_EXAMPLES[type] };+ const question = {+ id: nextId(),+ ...(args.examples ?? QUESTION_EXAMPLES)[type],+ };setQuestions((current) => [...current, question]);setExpandedId(question.id);args.onAddExample(type);@@ -97,6 +101,19 @@ export const Empty = meta.story({questions: [],expandedId: null,stateKeys: STATE_KEYS,+ examples: QUESTION_EXAMPLES,+ },+ render: (args) => <InteractiveQuestionList {...args} />,+});++export const EmptyOpenAI = meta.story({+ args: {+ ...callbacks,+ questions: [],+ expandedId: null,+ stateKeys: ["input"],+ plainInstructions: true,+ examples: OPENAI_QUESTION_EXAMPLES,},render: (args) => <InteractiveQuestionList {...args} />,});diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsxindex cb2d3fd69b..711de7626e 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx@@ -2,8 +2,9 @@ import {DECISION_MODEL_LIMITS,DecisionModelQuestionType,} from "@langfuse/shared";-import { Plus, Sparkles } from "lucide-react";+import { Sparkles } from "lucide-react";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import { Button } from "@/src/components/ui/button";import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";import { Label } from "@/src/components/ui/label";@@ -48,7 +49,9 @@ export const QUESTION_EXAMPLES: Record<levels: [{ description: "Calm, just stating facts" },{ description: "Frustrated but civil" },- { description: "Very angry, strong language or threatening to leave" },+ {+ description: "Very angry, strong language or threatening to leave",+ },],criteria: { true: "", false: "" },},@@ -62,6 +65,25 @@ export const QUESTION_EXAMPLES: Record<},};+/** OpenAI questions cannot name state fields, so these examples stay plain. */+export const OPENAI_QUESTION_EXAMPLES: Record<+ DecisionModelQuestionType,+ Omit<DecisionModelQuestionDraft, "id">+> = {+ [DecisionModelQuestionType.CHOICE]: {+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.CHOICE],+ instructions: "Is the reply ready to send to the customer?",+ },+ [DecisionModelQuestionType.SCORE]: {+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.SCORE],+ instructions: "How frustrated is the customer?",+ },+ [DecisionModelQuestionType.NOUL]: {+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.NOUL],+ instructions: "Does the customer request a refund?",+ },+};+/*** The ordered questions of a decision-model evaluator. Every question is* answered in the same call, so adding one costs only its own tokens; the@@ -78,6 +100,8 @@ export function DecisionModelQuestionList({onAddExample,onRemove,onReorder,+ plainInstructions = false,+ examples = QUESTION_EXAMPLES,}: {questions: DecisionModelQuestionDraft[];expandedId: string | null;@@ -89,19 +113,28 @@ export function DecisionModelQuestionList({onAddExample: (type: DecisionModelQuestionType) => void;onRemove: (id: string) => void;onReorder: (fromIndex: number, toIndex: number) => void;+ plainInstructions?: boolean;+ examples?: Record<+ DecisionModelQuestionType,+ Omit<DecisionModelQuestionDraft, "id">+ >;}) {return (<div className="flex flex-col gap-3">- <Label className="flex items-center gap-1.5">+ <Label className="flex items-baseline gap-1.5">Questions- <InfoTooltip label="About questions">- Each question is one snap judgment about the state and writes one- score. All questions are answered in a single model call, so a second- or tenth question costs only its own tokens.- </InfoTooltip>- <span className="text-muted-foreground text-xs font-normal">- {questions.length} of {DECISION_MODEL_LIMITS.maxQuestions}+ <span className="inline-flex self-center">+ <InfoTooltip label="About questions">+ Each question is one snap judgment about the state and writes one+ score. All questions are answered in a single model call, so a+ second or tenth question costs only its own tokens.+ </InfoTooltip></span>+ {questions.length > 40 ? (+ <span className="text-muted-foreground text-xs leading-none font-normal">+ {questions.length} of {DECISION_MODEL_LIMITS.maxQuestions}+ </span>+ ) : null}</Label>{questions.length === 0 ? (@@ -122,8 +155,7 @@ export function DecisionModelQuestionList({onClick={() => onAddExample(type)}><Sparkles className="icon-base text-icon-foreground mr-1" />- {copy.label}: “- {QUESTION_EXAMPLES[type].instructions.replace(/`/g, "")}”+ {copy.label}: “{examples[type].instructions}”</Button>);})}@@ -152,20 +184,18 @@ export function DecisionModelQuestionList({questions.length > 1 ? () => onRemove(question.id) : null}errors={errorsById[question.id]}+ plainInstructions={plainInstructions}/>)}/>)}- <div className="flex flex-wrap items-center gap-3">- <Button type="button" variant="outline" size="sm" onClick={onAdd}>- <Plus className="icon-base text-icon-foreground mr-1" />- Add question- </Button>- <span className="text-muted-foreground text-xs">+ <div className="flex items-center gap-1.5">+ <TextActionButton text="Add question" onClick={onAdd} />+ <InfoTooltip label="About extra questions">Answered together in one call; extra questions add only their owntokens.- </span>+ </InfoTooltip></div></div>);diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsxindex f835b1f4c7..790759f00f 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx@@ -37,7 +37,9 @@ function Criteria({ question }: { question: DecisionModelQuestion }) {<span className="text-muted-foreground shrink-0 font-mono">{index}</span>- <span>{entryText(level.description)}</span>+ {level.description ? (+ <span>{entryText(level.description)}</span>+ ) : null}</li>))}</ol>diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsxindex 30a6126d55..d85ec06f11 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx@@ -65,7 +65,7 @@ export const LowConfidence = meta.story({},});-/** Expected values at both ends of the scale keep their labels inside the track. */+/** Scores at the ends of the scale highlight that level's bar. */export const ScoreAtEdges = meta.story({args: {results: [@@ -97,7 +97,7 @@ export const ScoreAtEdges = meta.story({},});-/** Maximum-length scales keep descriptions available without rendering them inline. */+/** A long scale lists every level as its own bar. */export const TenLevels = meta.story({name: "(Test) Ten Levels",args: {@@ -138,15 +138,9 @@ export const TenLevels = meta.story({},play: async ({ canvasElement }) => {const canvas = within(canvasElement);- const page = within(canvasElement.ownerDocument.body);- const levelDescription = canvas.getByRole("button", {- name: "Level 0 description",- });-- levelDescription.focus();-- await expect(await page.findByRole("tooltip")).toHaveTextContent(- "No technical detail",- );+ await expect(canvas.getByText("No technical detail")).toBeInTheDocument();+ await expect(+ canvas.getByText("Explains system behavior"),+ ).toBeInTheDocument();},});diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsxindex 65801a86a1..45814186d6 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx@@ -1,12 +1,4 @@-import { DECISION_MODEL_LIMITS } from "@langfuse/shared";-import { InfoIcon } from "lucide-react";-import { Badge } from "@/src/components/ui/badge";-import {- Tooltip,- TooltipContent,- TooltipTrigger,-} from "@/src/components/ui/tooltip";import { QUESTION_TYPE_COPY } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionTypeSelector/QuestionTypeSelector";import { cn } from "@/src/utils/tailwind";@@ -50,27 +42,6 @@ function ConfidenceBadge({ confidence }: { confidence: number }) {);}-function LevelDescriptionTooltip({- index,- description,-}: {- index: number;- description: string;-}) {- return (- <Tooltip>- <TooltipTrigger- type="button"- className="text-muted-foreground focus-visible:ring-ring cursor-help rounded-sm focus-visible:ring-2 focus-visible:outline-none"- aria-label={`Level ${index} description`}- >- <InfoIcon className="icon-base" aria-hidden="true" />- </TooltipTrigger>- <TooltipContent className="max-w-xs">{description}</TooltipContent>- </Tooltip>- );-}-function ProbabilityBars({entries,highlight,@@ -120,74 +91,6 @@ function ProbabilityBars({);}-function ScoreDistribution({- score,- levels,- probabilities,-}: {- score: number;- levels: string[];- probabilities: Record<string, number>;-}) {- const top = Math.max(levels.length - 1, 1);- const nearest = Math.min(Math.max(Math.round(score), 0), levels.length - 1);- const position = Math.min(Math.max(score / top, 0), 1) * 100;- const showDescriptions = levels.length < DECISION_MODEL_LIMITS.maxScoreLevels;- return (- <div className="flex flex-col gap-2">- <div className="relative pt-4">- <div className="bg-muted h-2 rounded-full" />- <span- className="bg-primary-accent absolute top-3 h-4 w-0.5 -translate-x-1/2 rounded"- style={{ left: `${position}%` }}- aria-hidden="true"- />- <span- className="text-primary-accent absolute top-0 font-mono text-xs font-bold"- style={{- left: `${position}%`,- transform: `translateX(-${position}%)`,- }}- >- {score.toFixed(2)}- </span>- </div>- <ol- className="grid gap-1"- style={{- gridTemplateColumns: `repeat(${levels.length}, minmax(0, 1fr))`,- }}- >- {levels.map((description, index) => (- <li- key={index}- className={cn(- "flex flex-col gap-0.5 text-xs",- index === nearest ? "" : "text-muted-foreground",- )}- >- <span className="flex items-center gap-0.5 font-mono">- {index}- {!showDescriptions ? (- <LevelDescriptionTooltip- index={index}- description={description}- />- ) : null}- · {percent(probabilities[String(index)] ?? 0)}- </span>- {showDescriptions ? (- <span className="line-clamp-2" title={description}>- {description}- </span>- ) : null}- </li>- ))}- </ol>- </div>- );-}-function NoulGauge({ probability }: { probability: number }) {const leaning = probability >= 0.5 ? "yes" : "no";return (@@ -213,12 +116,12 @@ function NoulGauge({ probability }: { probability: number }) {);}-function resultValue(result: DecisionModelQuestionResult) {+function resultValue(+ result: Exclude<DecisionModelQuestionResult, { type: "score" }>,+) {switch (result.type) {case "choice":return result.choice;- case "score":- return result.score.toFixed(2);case "noul":return result.probability.toFixed(2);}@@ -235,14 +138,23 @@ function ResultVisual({ result }: { result: DecisionModelQuestionResult }) {highlight={result.choice}/>);- case "score":+ case "score": {+ const nearest = Math.min(+ Math.max(Math.round(result.score), 0),+ Math.max(result.levels.length - 1, 0),+ );return (- <ScoreDistribution- score={result.score}- levels={result.levels}- probabilities={result.probabilities}+ <ProbabilityBars+ entries={result.levels+ .map((description, index) => ({+ label: description || String(index),+ value: result.probabilities[String(index)] ?? 0,+ }))+ .sort((left, right) => right.value - left.value)}+ highlight={result.levels[nearest] || String(nearest)}/>);+ }case "noul":return <NoulGauge probability={result.probability} />;}@@ -269,7 +181,9 @@ function ResultRow({ result }: { result: DecisionModelQuestionResult }) {{result.type !== "noul" && result.confidence !== null ? (<ConfidenceBadge confidence={result.confidence} />) : null}- <Badge className="font-mono">{resultValue(result)}</Badge>+ {result.type === "score" ? null : (+ <Badge className="font-mono">{resultValue(result)}</Badge>+ )}</span></div><ResultVisual result={result} />diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsxindex 03f350fff4..c6ecc10eac 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx@@ -1,52 +1,17 @@-import { useMemo, useState } from "react";-import {- DecisionModelStateKeySchema,- deepParseJsonIterative,-} from "@langfuse/shared";-import { ChevronDown, Plus, TriangleAlert } from "lucide-react";--import { Button } from "@/src/components/ui/button";-import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";+import { DecisionModelStateKeySchema } from "@langfuse/shared";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";+import { DecisionModelStatePreview } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview";import { EditableVariableMapping } from "@/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping";-import { extractVariableMappingValue } from "@/src/features/evals/v2/fns/variableMapping/extractVariableMappingValue";import type {ActiveVariableMapping,VariableFieldState,} from "@/src/features/evals/v2/types/variableMapping";-import { cn } from "@/src/utils/tailwind";export type DecisionModelStateField = {key: string;fieldState: VariableFieldState;};-/** TypeSafe accepts ~32k for state plus the longest question; warn well below. */-const STATE_SIZE_WARNING_CHARS = 24_000;--/**- * Builds the JSON object the model receives from the mapped fields and the- * sample observation, mirroring the worker's state builder.- */-function buildStatePreview(- fields: DecisionModelStateField[],- sourceObject: Record<string, unknown> | null,-): Record<string, unknown> | null {- if (!sourceObject) return null;- const state: Record<string, unknown> = {};- for (const field of fields) {- const columnId = field.fieldState.selectedColumnId;- if (!columnId) continue;- const { value } = extractVariableMappingValue(- sourceObject,- columnId,- field.fieldState.jsonSelector ?? undefined,- );- if (value === null || value === undefined || value === "") continue;- state[field.key] = deepParseJsonIterative(value) ?? value;- }- return state;-}-/** Mirrors the shared state-key rule plus uniqueness within this evaluator. */function validateStateKey(next: string,@@ -92,14 +57,6 @@ export function DecisionModelStateEditor({hasMatchingObservations: boolean;sourceUnavailableMessage?: string;}) {- const [previewOpen, setPreviewOpen] = useState(false);- const statePreview = useMemo(- () => buildStatePreview(fields, sourceObject),- [fields, sourceObject],- );- const stateSize = statePreview ? JSON.stringify(statePreview).length : 0;- const tooLarge = stateSize > STATE_SIZE_WARNING_CHARS;-return (<div className="flex flex-col gap-3"><EditableVariableMapping@@ -121,63 +78,9 @@ export function DecisionModelStateEditor({sourceUnavailableMessage={sourceUnavailableMessage}/>- <div>- <Button type="button" variant="outline" size="sm" onClick={onAddField}>- <Plus className="icon-base text-icon-foreground mr-1" />- Add field- </Button>- </div>+ <TextActionButton text="Add field" onClick={onAddField} />- <div className="rounded-md border">- <button- type="button"- className="hover:bg-muted/50 flex w-full items-center justify-between gap-2 px-3 py-2 text-left text-sm"- aria-expanded={previewOpen}- onClick={() => setPreviewOpen((open) => !open)}- >- <span className="flex items-center gap-2">- <span className="font-bold">State sent to the model</span>- {statePreview ? (- <span- className={cn(- "text-muted-foreground font-mono text-xs",- tooLarge && "text-dark-yellow flex items-center gap-1",- )}- >- {tooLarge ? <TriangleAlert className="icon-base" /> : null}- {Object.keys(statePreview).length} field- {Object.keys(statePreview).length === 1 ? "" : "s"} ·{" "}- {stateSize.toLocaleString()} chars- {tooLarge ? " · trim fields, ~32k limit" : ""}- </span>- ) : (- <span className="text-muted-foreground text-xs">- select a sample observation to preview- </span>- )}- </span>- <ChevronDown- className={cn(- "text-foreground-tertiary icon-base shrink-0 transition-transform",- previewOpen ? "rotate-180" : "rotate-0",- )}- />- </button>- {previewOpen && statePreview ? (- <div className="border-t">- <PrettyJsonView- json={statePreview}- currentView="pretty"- isLoading={false}- showNullValues={true}- stickyTopLevelKey={false}- showObservationTypeBadge={false}- scrollable={true}- className="max-h-80 [&_.border]:border-0 [&_.rounded-sm]:rounded-none"- />- </div>- ) : null}- </div>+ <DecisionModelStatePreview fields={fields} sourceObject={sourceObject} /></div>);}diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsxnew file mode 100644index 0000000000..fea8fcdf8a--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsx@@ -0,0 +1,112 @@+import { useMemo, useState } from "react";+import { deepParseJsonIterative } from "@langfuse/shared";+import { ChevronDown, TriangleAlert } from "lucide-react";++import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";+import { extractVariableMappingValue } from "@/src/features/evals/v2/fns/variableMapping/extractVariableMappingValue";+import type { VariableFieldState } from "@/src/features/evals/v2/types/variableMapping";+import { cn } from "@/src/utils/tailwind";++type PreviewField = {+ key: string;+ fieldState: VariableFieldState;+};++/** TypeSafe accepts ~32k for state plus the longest question; warn well below. */+const STATE_SIZE_WARNING_CHARS = 24_000;++function buildStatePreview(+ fields: PreviewField[],+ sourceObject: Record<string, unknown> | null,+): Record<string, unknown> | null {+ if (!sourceObject) return null;+ const state: Record<string, unknown> = {};+ for (const field of fields) {+ const columnId = field.fieldState.selectedColumnId;+ if (!columnId) continue;+ const { value } = extractVariableMappingValue(+ sourceObject,+ columnId,+ field.fieldState.jsonSelector ?? undefined,+ );+ if (value === null || value === undefined || value === "") continue;+ state[field.key] = deepParseJsonIterative(value) ?? value;+ }+ return state;+}++/** The JSON object that will be sent, assembled from the mapped fields. */+export function DecisionModelStatePreview({+ fields,+ sourceObject,+ questions,+ title = "State sent to the model",+}: {+ fields: PreviewField[];+ sourceObject: Record<string, unknown> | null;+ /** Included beside the state. OpenAI sends these on the same request. */+ questions?: unknown[];+ title?: string;+}) {+ const [open, setOpen] = useState(false);+ const statePreview = useMemo(() => {+ const state = buildStatePreview(fields, sourceObject);+ if (!state) return null;+ return questions ? { ...state, questions } : state;+ }, [fields, questions, sourceObject]);+ const stateSize = statePreview ? JSON.stringify(statePreview).length : 0;+ const tooLarge = stateSize > STATE_SIZE_WARNING_CHARS;++ return (+ <div className="rounded-md border">+ <button+ type="button"+ className="hover:bg-muted/50 flex w-full items-center justify-between gap-2 px-3 py-2 text-left text-sm"+ aria-expanded={open}+ onClick={() => setOpen((current) => !current)}+ >+ <span className="flex items-center gap-2">+ <span className="font-bold">{title}</span>+ {statePreview ? (+ <span+ className={cn(+ "text-muted-foreground font-mono text-xs",+ tooLarge && "text-dark-yellow flex items-center gap-1",+ )}+ >+ {tooLarge ? <TriangleAlert className="icon-base" /> : null}+ {Object.keys(statePreview).length} field+ {Object.keys(statePreview).length === 1 ? "" : "s"} ·{" "}+ {stateSize.toLocaleString()} chars+ {tooLarge ? " · trim fields, ~32k limit" : ""}+ </span>+ ) : (+ <span className="text-muted-foreground text-xs">+ select a sample observation to preview+ </span>+ )}+ </span>+ <ChevronDown+ className={cn(+ "text-foreground-tertiary icon-base shrink-0 transition-transform",+ open ? "rotate-180" : "rotate-0",+ )}+ />+ </button>+ {open && statePreview ? (+ <div className="border-t">+ <PrettyJsonView+ json={statePreview}+ currentView="pretty"+ isLoading={false}+ showNullValues={true}+ stickyTopLevelKey={false}+ showObservationTypeBadge={false}+ scrollable={true}+ className="max-h-80 [&_.border]:border-0 [&_.rounded-sm]:rounded-none"+ />+ </div>+ ) : null}+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsxnew file mode 100644index 0000000000..a5cc8ada05--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsx@@ -0,0 +1,86 @@+import { useState } from "react";+import { DecisionModelQuestionType } from "@langfuse/shared";+import { fn } from "storybook/test";++import preview from "../../../../../../../../.storybook/preview";+import type {+ ActiveVariableMapping,+ VariableFieldState,+} from "@/src/features/evals/v2/types/variableMapping";+import { OpenAIDecisionModelStateEditor } from "./OpenAIDecisionModelStateEditor";++const QUESTIONS = [+ {+ id: "severity",+ type: DecisionModelQuestionType.SCORE,+ scoreName: "severity",+ instructions: "How severe is the issue in the input?",+ options: [+ { value: "", description: "" },+ { value: "", description: "" },+ ],+ levels: [+ { description: "Appearance only" },+ { description: "Cannot continue" },+ ],+ criteria: { true: "", false: "" },+ },+];++const meta = preview.meta({ component: OpenAIDecisionModelStateEditor });++const SAMPLE = {+ input: "I was charged twice. Please refund the extra charge.",+ output: "I've issued the refund.",+};++export const MappedInput = meta.story({+ args: {+ field: {+ key: "input",+ fieldState: { selectedColumnId: "input", jsonSelector: null },+ },+ activeMapping: null,+ onActiveMappingChange: fn(),+ onChangeField: fn(),+ questions: QUESTIONS,+ sourceObject: SAMPLE,+ hasMatchingObservations: true,+ },+ render: (args) => {+ const [fieldState, setFieldState] = useState<VariableFieldState>(+ args.field.fieldState,+ );+ const [activeMapping, setActiveMapping] =+ useState<ActiveVariableMapping>(null);+ return (+ <OpenAIDecisionModelStateEditor+ {...args}+ field={{ key: "input", fieldState }}+ activeMapping={activeMapping}+ onActiveMappingChange={setActiveMapping}+ onChangeField={(next) => {+ setFieldState(next);+ args.onChangeField(next);+ }}+ />+ );+ },+});++export const NoSample = meta.story({+ args: {+ field: {+ key: "input",+ fieldState: { selectedColumnId: "input", jsonSelector: null },+ },+ activeMapping: null,+ onActiveMappingChange: fn(),+ onChangeField: fn(),+ questions: QUESTIONS,+ sourceObject: null,+ hasMatchingObservations: false,+ sourceUnavailableMessage:+ "Select a sample observation in the test panel to preview mapped values.",+ },+});diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsxnew file mode 100644index 0000000000..27718510bb--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsx@@ -0,0 +1,55 @@+import { DecisionModelStatePreview } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview";+import type { DecisionModelStateField } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor";+import { previewOpenAIDecisionQuestions } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";+import { EditableVariableMapping } from "@/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping";+import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";+import type {+ ActiveVariableMapping,+ VariableFieldState,+} from "@/src/features/evals/v2/types/variableMapping";++/**+ * OpenAI decisions take one `input` plus the questions. The question text does+ * not name fields, so this mapping cannot be renamed or extended. The preview+ * is that input beside the questions posted with it.+ */+export function OpenAIDecisionModelStateEditor({+ field,+ questions,+ activeMapping,+ onActiveMappingChange,+ onChangeField,+ sourceObject,+ hasMatchingObservations,+ sourceUnavailableMessage,+}: {+ field: DecisionModelStateField;+ questions: DecisionModelQuestionDraft[];+ activeMapping: ActiveVariableMapping;+ onActiveMappingChange: (activeMapping: ActiveVariableMapping) => void;+ onChangeField: (fieldState: VariableFieldState) => void;+ sourceObject: Record<string, unknown> | null;+ hasMatchingObservations: boolean;+ sourceUnavailableMessage?: string;+}) {+ return (+ <div className="flex flex-col gap-3">+ <EditableVariableMapping+ mappings={[{ variable: "input", fieldState: field.fieldState }]}+ variableDisplay="stateKey"+ activeMapping={activeMapping}+ onActiveMappingChange={onActiveMappingChange}+ onChangeField={(_key, fieldState) => onChangeField(fieldState)}+ sourceObject={sourceObject}+ hasMatchingObservations={hasMatchingObservations}+ sourceUnavailableMessage={sourceUnavailableMessage}+ />+ <DecisionModelStatePreview+ fields={[field]}+ sourceObject={sourceObject}+ questions={previewOpenAIDecisionQuestions(questions)}+ title="Sent to the model"+ />+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsxnew file mode 100644index 0000000000..3a8350305e--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsx@@ -0,0 +1,43 @@+import { useState } from "react";+import { fn } from "storybook/test";++import preview from "../../../../../../../../.storybook/preview";+import { OpenAIQuestionInstructions } from "./OpenAIQuestionInstructions";++const meta = preview.meta({ component: OpenAIQuestionInstructions });++export const Default = meta.story({+ args: {+ value: "Which team should handle this ticket?",+ onChange: fn(),+ },+ render: (args) => {+ const [value, setValue] = useState(args.value);+ return (+ <OpenAIQuestionInstructions+ {...args}+ value={value}+ onChange={(next) => {+ setValue(next);+ args.onChange(next);+ }}+ />+ );+ },+});++export const Empty = meta.story({+ args: {+ value: "",+ placeholder: "Which team should handle this ticket?",+ onChange: fn(),+ },+});++export const Error = meta.story({+ args: {+ value: "",+ error: "Every question needs instructions.",+ onChange: fn(),+ },+});diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsxnew file mode 100644index 0000000000..a57f5bfc5e--- /dev/null+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsx@@ -0,0 +1,46 @@+import { useId } from "react";++import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";+import { Label } from "@/src/components/ui/label";+import { Textarea } from "@/src/components/ui/textarea";+import { cn } from "@/src/utils/tailwind";++/**+ * Plain question text for OpenAI decisions. The API has no variable syntax;+ * the mapped `input` object is the only evidence the model receives.+ */+export function OpenAIQuestionInstructions({+ value,+ onChange,+ placeholder,+ error,+}: {+ value: string;+ onChange: (value: string) => void;+ placeholder?: string;+ error?: string;+}) {+ const id = useId();++ return (+ <div className="flex flex-col gap-1.5">+ <Label htmlFor={id} className="flex items-center gap-1.5">+ Question+ <InfoTooltip label="About questions">+ Ask for one snap judgment. This text is sent as written. The model+ reads the input object, so the question does not insert fields.+ </InfoTooltip>+ </Label>+ <Textarea+ id={id}+ value={value}+ onChange={(event) => onChange(event.target.value)}+ placeholder={placeholder}+ rows={3}+ aria-invalid={Boolean(error)}+ className={cn(error && "border-destructive")}+ />+ {error ? <p className="text-destructive text-xs">{error}</p> : null}+ </div>+ );+}diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsxindex 58dd22164e..207f0dccda 100644--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx@@ -1,6 +1,7 @@import { DECISION_MODEL_LIMITS } from "@langfuse/shared";-import { Plus, Trash2 } from "lucide-react";+import { Trash2 } from "lucide-react";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import { Button } from "@/src/components/ui/button";import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";import { Input } from "@/src/components/ui/input";@@ -19,9 +20,8 @@ function levelPlaceholder(index: number, count: number) {}/**- * Ordered levels of a Score question, low to high. The level number is its- * position; the model only sees the descriptions, so each one has to describe- * a recognisable situation rather than a degree.+ * Ordered score levels, low to high. The position is the level number. The+ * description is what the model reads. The stored score is that position.*/export function ScoreLevelsEditor({levels,@@ -32,23 +32,27 @@ export function ScoreLevelsEditor({onChange: (levels: ScoreLevelDraft[]) => void;error?: string;}) {- const update = (index: number, description: string) =>- onChange(levels.map((level, i) => (i === index ? { description } : level)));+ const update = (index: number, patch: Partial<ScoreLevelDraft>) =>+ onChange(+ levels.map((level, i) => (i === index ? { ...level, ...patch } : level)),+ );const remove = (index: number) =>onChange(levels.filter((_, i) => i !== index));const canRemove = levels.length > DECISION_MODEL_LIMITS.minScoreLevels;return (<div className="flex flex-col gap-2">- <Label className="flex items-center gap-1.5">+ <Label className="flex items-baseline gap-1.5">Levels, low to high- <InfoTooltip label="About levels">- Each level is one point on the scale. The model judges every level on- its own against the state and returns a position between them, so- describe situations (“broken, but a workaround exists”), not degrees- (“moderately severe”). Two to ten levels; drag to reorder.- </InfoTooltip>- <span className="text-muted-foreground text-xs font-normal">+ <span className="inline-flex self-center">+ <InfoTooltip label="About levels">+ Each level is one point on the scale. The model judges every level+ on its own against the state and returns a position between them, so+ describe situations (“broken, but a workaround exists”), not degrees+ (“moderately severe”). Two to ten levels; drag to reorder.+ </InfoTooltip>+ </span>+ <span className="text-muted-foreground text-xs leading-none font-normal">{levels.length} of {DECISION_MODEL_LIMITS.maxScoreLevels}</span></Label>@@ -67,7 +71,9 @@ export function ScoreLevelsEditor({</span><Inputvalue={level.description}- onChange={(event) => update(index, event.target.value)}+ onChange={(event) =>+ update(index, { description: event.target.value })+ }placeholder={levelPlaceholder(index, levels.length)}aria-label={`Level ${index} description`}/>@@ -89,21 +95,16 @@ export function ScoreLevelsEditor({</div>)}/>- <div className="flex items-center gap-3">- <Button- type="button"- variant="outline"- size="sm"+ <div className="flex items-center gap-1.5">+ <TextActionButton+ text="Add level"disabled={levels.length >= DECISION_MODEL_LIMITS.maxScoreLevels}onClick={() => onChange([...levels, { description: "" }])}- >- <Plus className="icon-base text-icon-foreground mr-1" />- Add level- </Button>- <span className="text-muted-foreground text-xs">+ />+ <InfoTooltip label="About the score">The score is the probability-weighted average of the level numbers, soit can be a decimal from 0 to {Math.max(levels.length - 1, 0)}.- </span>+ </InfoTooltip></div>{error ? <p className="text-destructive text-xs">{error}</p> : null}</div>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsxindex ba942b224a..044b794806 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx@@ -4,18 +4,24 @@ import { useShallow } from "zustand/react/shallow";import {DecisionModelQuestionList,+ OPENAI_QUESTION_EXAMPLES,QUESTION_EXAMPLES,} from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList";import {createEmptyQuestion,getQuestionDraftErrors,+ usesPlainDecisionInstructions,} from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";+import { preferredDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";+import { api } from "@/src/utils/api";import { safeRandomUUID } from "@/src/utils/safe-random-uuid";export function DecisionModelQuestionsEditor({+ projectId,store,}: {+ projectId: string;store: EvaluatorSetupStore;}) {const state = useStore(@@ -24,9 +30,23 @@ export function DecisionModelQuestionsEditor({questions: state.questions,expandedQuestionId: state.expandedQuestionId,stateKeys: state.stateKeys,+ selectedModel: state.selectedModel,actions: state.actions,})),);+ const connections = api.llmApiKey.all.useQuery({+ projectId,+ includeDecisionModels: true,+ });+ const model =+ state.selectedModel ?? preferredDecisionModel(connections.data?.data ?? []);+ const adapter = connections.data?.data.find(+ (connection) => connection.provider === model?.provider,+ )?.adapter;+ const openaiDecision = usesPlainDecisionInstructions({+ model: model?.model,+ adapter,+ });const errorsById = useMemo(() => {const errors = getQuestionDraftErrors(state.questions);for (const question of state.questions) {@@ -37,6 +57,10 @@ export function DecisionModelQuestionsEditor({return errors;}, [state.questions]);+ const examples = openaiDecision+ ? OPENAI_QUESTION_EXAMPLES+ : QUESTION_EXAMPLES;+return (<DecisionModelQuestionListquestions={state.questions}@@ -46,14 +70,16 @@ export function DecisionModelQuestionsEditor({onExpandedChange={state.actions.setExpandedQuestionId}onChange={state.actions.setQuestion}onAdd={() => state.actions.addQuestion(createEmptyQuestion())}+ examples={examples}onAddExample={(type) =>state.actions.addQuestion({id: safeRandomUUID(),- ...QUESTION_EXAMPLES[type],+ ...examples[type],})}onRemove={state.actions.removeQuestion}onReorder={state.actions.reorderQuestion}+ plainInstructions={openaiDecision}/>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsxindex 2ec3947d37..98e94d91bf 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx@@ -34,6 +34,8 @@ describe("DecisionModelSelector", () => {});beforeEach(() => {+ connection.provider = "jev";+ connection.adapter = "typesafe";connection.withDefaultModels = true;connection.customModels = [];});@@ -55,9 +57,31 @@ describe("DecisionModelSelector", () => {{ wrapper: LayerProvider },);- return screen.getByRole("combobox", { name: "Decision model" });+ return screen.getByRole("button");};+ it("selects gpt-6-luna when nothing is selected and the connection offers it", () => {+ connection.provider = "openai";+ connection.adapter = "openai";+ const store = createEvaluatorSetupStore({+ initialEvaluator: null,+ initialType: "DECISION_MODEL",+ mode: "create",+ });++ render(+ <DecisionModelSelector+ projectId="project"+ store={store}+ onConfigureProviders={vi.fn()}+ />,+ { wrapper: LayerProvider },+ );++ expect(screen.getByRole("button")).toHaveTextContent("openai / gpt-6-luna");+ expect(store.getState().selectedModel).toBeNull();+ });+it.each([["lists only default models", true],["lists no models", false],@@ -67,7 +91,7 @@ describe("DecisionModelSelector", () => {connection.withDefaultModels = withDefaultModels;expect(renderWithSelectedModel("jev-1.13.0")).toHaveTextContent(- "jev: jev-1.13.0",+ "jev / jev-1.13.0",);},);@@ -84,7 +108,7 @@ describe("DecisionModelSelector", () => {connection.customModels = customModels;expect(renderWithSelectedModel("jev-latest").textContent).toBe(- "jev: jev-latest",+ "jev / jev-latest",);},);diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsxindex 1b33c28cab..f202bb734c 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx@@ -1,22 +1,27 @@-import { isDecisionModelAdapter, supportedModels } from "@langfuse/shared";+import { useState } from "react";+import {+ isAllowedDecisionModel,+ isOpenAIDecisionModel,+ OPENAI_DECISION_MODEL_IDS,+} from "@langfuse/shared";import { useStore } from "zustand";import { Button } from "@/src/components/ui/button";-import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";+import { PopoverTrigger } from "@/src/components/ui/popover";+import {+ JudgeModelPicker,+ JudgeModelPickerTrigger,+} from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";+import {+ decisionModelsByProvider,+ preferredDecisionModel,+} from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";import { api } from "@/src/utils/api";-const SEPARATOR = "::";--const toOption = (provider: string, model: string) => ({- value: `${provider}${SEPARATOR}${model}`,- label: `${provider}: ${model}`,-});-/**- * Picks the decision-model connection and model version. Decision models have- * no project default and no sampling parameters, so this replaces the judge- * model picker with a flat list of `provider :: model` options.+ * Same content-width model menu as an LLM-as-a-judge, limited to decision+ * models. No project default and no sampling parameters.*/export function DecisionModelSelector({projectId,@@ -27,6 +32,7 @@ export function DecisionModelSelector({store: EvaluatorSetupStore;onConfigureProviders: () => void;}) {+ const [open, setOpen] = useState(false);const selectedModel = useStore(store, (state) => state.selectedModel);const selectModel = store.getState().actions.selectModel;const connections = api.llmApiKey.all.useQuery({@@ -34,54 +40,74 @@ export function DecisionModelSelector({includeDecisionModels: true,});- const connectionOptions = (connections.data?.data ?? [])- .filter((connection) => isDecisionModelAdapter(connection.adapter))- .flatMap((connection) => {- const models = Array.from(- new Set([- ...connection.customModels,- ...(connection.withDefaultModels- ? supportedModels[connection.adapter]- : []),- ]),- );- return models.map((model) => toOption(connection.provider, model));- });- const selectedOption = selectedModel- ? toOption(selectedModel.provider, selectedModel.model)- : null;- // Radix renders a blank trigger for a value without a matching option, e.g.- // a model the connection no longer lists.- const options =- selectedOption &&- !connectionOptions.some((option) => option.value === selectedOption.value)- ? [...connectionOptions, selectedOption]- : connectionOptions;+ const rows = connections.data?.data ?? [];+ const modelsByProvider = decisionModelsByProvider(rows);++ const selectedConnection = rows.find(+ (connection) => connection.provider === selectedModel?.provider,+ );+ // A saved TypeSafe version can outlive the connection's current list. An+ // OpenAI model outside the decision-model list must not be offered again.+ const selectedIsAllowed =+ selectedModel != null &&+ (selectedConnection == null ||+ isAllowedDecisionModel(selectedConnection.adapter, selectedModel.model));+ if (selectedModel && selectedIsAllowed) {+ const listed = modelsByProvider.get(selectedModel.provider) ?? [];+ if (!listed.includes(selectedModel.model)) {+ modelsByProvider.set(selectedModel.provider, [+ ...listed,+ selectedModel.model,+ ]);+ }+ }++ const providerGroups = Array.from(modelsByProvider.entries()).sort(+ (left, right) =>+ Number(right[1].some(isOpenAIDecisionModel)) -+ Number(left[1].some(isOpenAIDecisionModel)),+ );+ const unsupported = selectedModel != null && !selectedIsAllowed;+ // Shown until the user picks a model. The store stays untouched.+ const displayedModel = selectedModel ?? preferredDecisionModel(rows);- if (connections.isSuccess && options.length === 0) {+ if (+ connections.isSuccess &&+ providerGroups.length === 0 &&+ selectedModel == null+ ) {return (<Button type="button" variant="outline" onClick={onConfigureProviders}>- Add a TypeSafe connection+ Add an OpenAI or TypeSafe connection</Button>);}return (- <div className="max-w-xs min-w-56">- <SelectInput- aria-label="Decision model"- placeholder="Select a decision model"- value={selectedOption?.value ?? ""}- options={options}- onValueChange={(value) => {- const separatorIndex = value.indexOf(SEPARATOR);- if (separatorIndex === -1) return;- selectModel({- provider: value.slice(0, separatorIndex),- model: value.slice(separatorIndex + SEPARATOR.length),- });- }}- />+ <div className="flex w-fit max-w-full flex-col items-start gap-1.5">+ <JudgeModelPicker+ purpose="decision"+ open={open}+ onOpenChange={setOpen}+ providerGroups={providerGroups}+ selectedModel={displayedModel}+ onSelect={selectModel}+ onConfigureProviders={onConfigureProviders}+ >+ <PopoverTrigger asChild>+ <JudgeModelPickerTrigger+ mode="custom"+ selectedModel={displayedModel}+ disabled={false}+ />+ </PopoverTrigger>+ </JudgeModelPicker>+ {unsupported ? (+ <p className="text-destructive text-xs">+ {selectedModel.model} is not supported for decision models. Choose{" "}+ {OPENAI_DECISION_MODEL_IDS.join(", ")}.+ </p>+ ) : null}</div>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsxindex 39e2771ee8..dc3125f13b 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx@@ -120,7 +120,9 @@ export function DefinitionStepContainer({onConfigureProviders={onConfigureProviders}/>}- questionsEditor={<DecisionModelQuestionsEditor store={store} />}+ questionsEditor={+ <DecisionModelQuestionsEditor projectId={projectId} store={store} />+ }/>);}diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsxindex 3e963e4ac8..18203f1f2d 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx@@ -6,7 +6,6 @@ import {Copy,GripVertical,MoreVertical,- Plus,Trash2,TriangleAlert,} from "lucide-react";@@ -33,6 +32,7 @@ import { useStore } from "zustand";import { useShallow } from "zustand/react/shallow";import { Badge } from "@/src/components/ui/badge";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import { Button } from "@/src/components/ui/button";import {Tooltip,@@ -189,11 +189,9 @@ export function PromptEditorContent({/>))}</SortableContext>- <Button- type="button"- variant="ghost"- size="sm"- className="text-foreground hover:text-foreground h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"+ <TextActionButton+ text="Add message"+ width="fill"onClick={() => {state.actions.setPromptPreviewEnabled(false);state.actions.addPromptMessage();@@ -201,10 +199,7 @@ export function PromptEditorContent({modification: "message_added",});}}- >- <Plus className="icon-base text-icon-foreground shrink-0" />- Add message- </Button>+ /></div><DragOverlay dropAnimation={null}>{activeMessage ? (diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsxindex 7573ad1650..f913b9526b 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx@@ -11,7 +11,7 @@ const COPY = {state: {title: "Build the state",description:- "Choose which observation fields the decision model sees, as one JSON object. Questions refer to these fields by name; the live preview shows the exact object sent.",+ "Choose which observation fields the decision model sees, as one JSON object. The live preview shows the exact object sent.",},} as const;diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsxindex cbdc7d15f5..928ae38565 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx@@ -1,12 +1,15 @@import { memo, useMemo, useRef } from "react";+import { isOpenAIDecisionModel } from "@langfuse/shared";import { useStore } from "zustand";import { useShallow } from "zustand/react/shallow";import { DecisionModelStateEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor";+import { OpenAIDecisionModelStateEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor";import { VariableMapping } from "@/src/features/evals/v2/components/VariableMapping/VariableMapping";import { buildDecisionModelStateFields } from "@/src/features/evals/v2/fns/variableMapping/buildDecisionModelStateFields";import { buildEvaluatorVariableMappings } from "@/src/features/evals/v2/fns/variableMapping/buildEvaluatorVariableMappings";import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";const StableVariableMapping = memo(VariableMapping);@@ -28,7 +31,9 @@ export function VariableMappingEditorContainer({type: state.type,promptMessages: state.promptMessages,stateKeys: state.stateKeys,+ questions: state.questions,variableFields: state.variableFields,+ selectedModel: state.selectedModel,activeMapping: state.activeMapping,selectedObservationId: state.selectedObservation?.id ?? null,actions: state.actions,@@ -47,6 +52,11 @@ export function VariableMappingEditorContainer({}),[state.type, state.stateKeys, state.promptMessages, state.variableFields],);+ const fallbackDecisionModel = useFallbackDecisionModel(+ projectId,+ state.type === "DECISION_MODEL",+ );+ const decisionModel = state.selectedModel ?? fallbackDecisionModel;const lastCompletedSample = useRef(resolvedSample);if (!state.selectedObservationId) lastCompletedSample.current = null;@@ -54,6 +64,34 @@ export function VariableMappingEditorContainer({const displayedSample = resolvedSample ?? lastCompletedSample.current;+ if (+ state.type === "DECISION_MODEL" &&+ decisionModel != null &&+ isOpenAIDecisionModel(decisionModel.model)+ ) {+ const inputField = mappings.find((mapping) => mapping.variable === "input");+ return (+ <OpenAIDecisionModelStateEditor+ field={{+ key: "input",+ fieldState: inputField?.fieldState ?? {+ selectedColumnId: "input",+ jsonSelector: null,+ },+ }}+ activeMapping={state.activeMapping}+ onActiveMappingChange={state.actions.setActiveMapping}+ onChangeField={(fieldState) =>+ state.actions.setVariableField("input", fieldState)+ }+ questions={state.questions}+ sourceObject={displayedSample}+ hasMatchingObservations={Boolean(displayedSample)}+ sourceUnavailableMessage={SOURCE_UNAVAILABLE_MESSAGE}+ />+ );+ }+if (state.type === "DECISION_MODEL") {return (<DecisionModelStateEditordiff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsxindex f7b5b2de28..e54fc082b1 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx@@ -24,6 +24,7 @@ describe("EvaluatorSetupFooter", () => {isSaving={false}nameAIAssistanceAvailable={false}codeValidation={{ isValid: false, isPending: false }}+ fallbackDecisionModel={null}onClose={vi.fn()}onSave={vi.fn()}/>@@ -53,6 +54,7 @@ describe("EvaluatorSetupFooter", () => {isSaving={false}nameAIAssistanceAvailable={false}codeValidation={null}+ fallbackDecisionModel={null}onClose={vi.fn()}onSave={vi.fn()}/>@@ -96,6 +98,7 @@ describe("EvaluatorSetupFooter", () => {isSaving={false}nameAIAssistanceAvailable={false}codeValidation={null}+ fallbackDecisionModel={null}onClose={vi.fn()}onSave={vi.fn()}/>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsxindex 2d53d7e67d..6935299129 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx@@ -1,6 +1,8 @@import { useStore } from "zustand";import { useShallow } from "zustand/react/shallow";+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";+import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";import {@@ -17,6 +19,7 @@ export function EvaluatorSetupFooter({isSaving,nameAIAssistanceAvailable,codeValidation,+ fallbackDecisionModel,onClose,onSave,}: {@@ -26,6 +29,7 @@ export function EvaluatorSetupFooter({isSaving: boolean;nameAIAssistanceAvailable: boolean;codeValidation: { isValid: boolean; isPending: boolean } | null;+ fallbackDecisionModel: JudgeModel | null;onClose: () => void;onSave: () => void;}) {@@ -39,7 +43,8 @@ export function EvaluatorSetupFooter({} = useStore(store,useShallow((state) => {- const { definition, mappings } = prepareEvaluatorDraft(state);+ const draft = applyFallbackDecisionModel(state, fallbackDecisionModel);+ const { definition, mappings } = prepareEvaluatorDraft(draft);const hasCompleteMappings =state.type !== "LLM_AS_JUDGE" ||mappings.every(({ fieldState }) =>@@ -50,7 +55,7 @@ export function EvaluatorSetupFooter({currentSnapshot: JSON.stringify({name: state.name.trim(),description: state.description.trim() || null,- definition,+ definition: prepareEvaluatorDraft(state).definition,}),canSubmit: Boolean(definition) && hasCompleteMappings,promptMessagesReason:@@ -62,7 +67,10 @@ export function EvaluatorSetupFooter({? getScoreOutputValidation(state.scoreOutput).reason: null,nameMissing: !state.name.trim(),- hasValidModel: selectHasValidModel(state),+ hasValidModel:+ state.type === "DECISION_MODEL"+ ? Boolean(draft.selectedModel)+ : selectHasValidModel(state),};}),);diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsxindex c83e226ab9..6882170f65 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx@@ -56,7 +56,7 @@ describe("EvaluatorVersionHistorySheet", () => {);expect(screen.getByText("Version 1").parentElement).toHaveClass(- "items-baseline",+ "items-center",);expect(screen.queryByRole("button", { name: "Restore version 2" }),diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsxindex 9d57196003..4efd35cccc 100644--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx@@ -11,7 +11,7 @@ import {type EvaluatorJudgeModelPickerProps = Exclude<JudgeModelPickerProps,- { purpose: "projectDefault" }+ { purpose: "projectDefault" } | { purpose: "decision" }>;function EvaluatorJudgeModelPicker(props: EvaluatorJudgeModelPickerProps) {diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsxindex a9c68b9d10..4f957d0776 100644--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx@@ -35,8 +35,6 @@ type JudgeModelPickerCommonProps = {onOpenChange: (open: boolean) => void;providerGroups: Array<[string, string[]]>;onConfigureProviders: () => void;- onConfigureModel: () => void;- hasModelConfiguration?: boolean;};type EvaluatorJudgeModelPickerProps = JudgeModelPickerCommonProps & {@@ -46,19 +44,30 @@ type EvaluatorJudgeModelPickerProps = JudgeModelPickerCommonProps & {selectedModel: JudgeModel | null;onModeChange: (mode: JudgeModelMode) => void;onSelectCustom: (model: JudgeModel) => void;+ onConfigureModel: () => void;+ hasModelConfiguration?: boolean;canSetProjectDefault: boolean;onSetProjectDefault: () => void;};+type DecisionJudgeModelPickerProps = JudgeModelPickerCommonProps & {+ purpose: "decision";+ selectedModel: JudgeModel | null;+ onSelect: (model: JudgeModel) => void;+};+type ProjectDefaultJudgeModelPickerProps = JudgeModelPickerCommonProps & {purpose: "projectDefault";defaultModel: JudgeModel | null;onSelectProjectDefault: (model: JudgeModel) => void;+ onConfigureModel: () => void;+ hasModelConfiguration?: boolean;};export type JudgeModelPickerProps =| EvaluatorJudgeModelPickerProps- | ProjectDefaultJudgeModelPickerProps;+ | ProjectDefaultJudgeModelPickerProps+ | DecisionJudgeModelPickerProps;type JudgeModelPickerTriggerProps = Omit<ButtonProps,@@ -180,22 +189,28 @@ export const JudgeModelPickerTrigger = forwardRef<);JudgeModelPickerTrigger.displayName = "JudgeModelPickerTrigger";+function selectionShowsCheck(props: JudgeModelPickerProps) {+ if (props.purpose === "decision" || props.purpose === "projectDefault") {+ return true;+ }+ return props.mode === "custom";+}+/** A controlled model selection menu for evaluators and the project default. */export function JudgeModelPicker(props: JudgeModelPickerProps) {- const {- children,- open,- onOpenChange,- defaultModel,- providerGroups,- onConfigureProviders,- onConfigureModel,- hasModelConfiguration = false,- } = props;+ const { children, open, onOpenChange, providerGroups, onConfigureProviders } =+ props;const selectsProjectDefault = props.purpose === "projectDefault";- const selectedModel = selectsProjectDefault- ? (defaultModel ?? null)- : props.selectedModel;+ const selectsDecision = props.purpose === "decision";+ const defaultModel = selectsDecision ? null : props.defaultModel;+ const hasModelConfiguration = selectsDecision+ ? false+ : (props.hasModelConfiguration ?? false);+ const selectedModel = selectsDecision+ ? props.selectedModel+ : selectsProjectDefault+ ? (defaultModel ?? null)+ : props.selectedModel;const selectedModelIsDefault =!selectsProjectDefault &&selectedModel !== null &&@@ -220,11 +235,13 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {<CommandInput placeholder="Find a model..." /><CommandList><CommandEmpty>No model found.</CommandEmpty>- {!selectsProjectDefault && defaultModel ? (+ {props.purpose !== "decision" &&+ props.purpose !== "projectDefault" &&+ props.defaultModel ? (<><CommandGroup><CommandItem- value={`project-default ${defaultModel.provider} ${defaultModel.model}`}+ value={`project-default ${props.defaultModel.provider} ${props.defaultModel.model}`}onSelect={() =>selectAndClose(() => props.onModeChange("default"))}@@ -238,9 +255,9 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {<Sparkles className="text-muted-foreground icon-base mr-2 shrink-0" /><spanclassName="truncate"- title={`${defaultModel.provider} / ${defaultModel.model}`}+ title={`${props.defaultModel.provider} / ${props.defaultModel.model}`}>- {defaultModel.provider} / {defaultModel.model}+ {props.defaultModel.provider} / {props.defaultModel.model}</span><Badge variant="secondary" className="ml-auto shrink-0">Project default@@ -256,7 +273,7 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {const isSelected =selectedModel?.provider === provider &&selectedModel.model === model &&- (selectsProjectDefault || props.mode === "custom");+ selectionShowsCheck(props);const isProjectDefault =defaultModel?.provider === provider &&defaultModel.model === model;@@ -269,6 +286,8 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {selectAndClose(() => {if (selectsProjectDefault) {props.onSelectProjectDefault({ provider, model });+ } else if (selectsDecision) {+ props.onSelect({ provider, model });} else {props.onSelectCustom({ provider, model });}@@ -309,33 +328,36 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {Configure AI providers<ExternalLink className="icon-base text-muted-foreground ml-auto" /></Button>- <Button- type="button"- variant="ghost"- className="font-regular justify-start"- disabled={- selectsProjectDefault- ? !defaultModel- : props.mode !== "custom" || !selectedModel- }- onClick={() => selectAndClose(onConfigureModel)}- >- <Settings2 className="icon-base text-muted-foreground mr-2" />- Model configuration- {hasModelConfiguration ? (- <>- <span className="sr-only">Customized</span>- <span- aria-hidden="true"- className="relative ml-auto inline-flex h-2.5 w-2.5"- >- <span className="bg-dark-yellow absolute inline-flex h-full w-full animate-ping rounded-full opacity-75" />- <span className="bg-dark-yellow relative inline-flex h-2.5 w-2.5 rounded-full" />- </span>- </>- ) : null}- </Button>- {!selectsProjectDefault ? (+ {props.purpose === "decision" ? null : (+ <Button+ type="button"+ variant="ghost"+ className="font-regular justify-start"+ disabled={+ props.purpose === "projectDefault"+ ? !props.defaultModel+ : props.mode !== "custom" || !selectedModel+ }+ onClick={() => selectAndClose(props.onConfigureModel)}+ >+ <Settings2 className="icon-base text-muted-foreground mr-2" />+ Model configuration+ {hasModelConfiguration ? (+ <>+ <span className="sr-only">Customized</span>+ <span+ aria-hidden="true"+ className="relative ml-auto inline-flex h-2.5 w-2.5"+ >+ <span className="bg-dark-yellow absolute inline-flex h-full w-full animate-ping rounded-full opacity-75" />+ <span className="bg-dark-yellow relative inline-flex h-2.5 w-2.5 rounded-full" />+ </span>+ </>+ ) : null}+ </Button>+ )}+ {props.purpose !== "projectDefault" &&+ props.purpose !== "decision" ? (<Buttontype="button"variant="ghost"diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsxindex fd107a60e3..40ba59bcb0 100644--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx@@ -46,11 +46,11 @@ function ResultStat({}) {return (<span- className="text-muted-foreground flex -translate-y-px items-center gap-1 font-mono text-xs leading-none tabular-nums"+ className="text-muted-foreground flex items-center gap-1 font-mono text-xs leading-none tabular-nums"title={title}>- <Icon className="icon-sm" />- <span className="translate-y-0.5">{children}</span>+ <Icon className="icon-sm shrink-0" />+ {children}</span>);}diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsxindex 938605737f..f12e314520 100644--- a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx+++ b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx@@ -4,7 +4,7 @@ import { type ReactNode, useState } from "react";import { useStore } from "zustand";import { useShallow } from "zustand/react/shallow";-import { Button } from "@/src/components/ui/button";+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import {Command,CommandEmpty,@@ -100,22 +100,11 @@ export function EvaluatorAssignmentsEditor({</span></button>) : (- <Button- type="button"- variant="ghost"- size="sm"+ <TextActionButton+ text={additionalActionLabel}disabled={disabled}- className={cn(- "text-foreground hover:text-foreground inline-flex h-auto items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline",- pickerWidth === "trigger" && "w-full justify-start",- )}- >- <Plus- className="icon-base text-icon-foreground shrink-0"- aria-hidden="true"- />- {additionalActionLabel}- </Button>+ width={pickerWidth === "trigger" ? "fill" : "content"}+ />)}</PopoverTrigger><PopoverContentdiff --git a/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts b/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.tsindex 3a86f99189..2c393288b0 100644--- a/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts+++ b/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts@@ -1,3 +1,5 @@+import { LLMAdapter } from "@langfuse/shared";+import type { DecisionModelQuestionResult } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView";const isRecord = (value: unknown): value is Record<string, unknown> =>@@ -12,11 +14,17 @@ const toProbabilities = (value: unknown): Record<string, number> =>): {};+function readDecisionDetails(metadata: unknown) {+ if (!isRecord(metadata)) return null;+ const details = metadata[LLMAdapter.OpenAI] ?? metadata[LLMAdapter.TypeSafe];+ return isRecord(details) ? details : null;+}+/*** Reads the scores of a decision-model test run into one row per question.- * Values come from the score itself; distributions and confidence from the- * `typesafe` metadata namespace the executor writes; the question text from- * the request that was sent.+ * Values come from the score itself. Distributions and confidence come from+ * the provider namespace the executor writes (`openai` or `typesafe`). The+ * question text comes from the request that was sent.*/export function toDecisionModelResults(response: Record<string, unknown>,@@ -29,10 +37,10 @@ export function toDecisionModelResults(return scores.flatMap((score, index): DecisionModelQuestionResult[] => {if (!isRecord(score)) return [];- const typesafe = isRecord(score.metadata) ? score.metadata.typesafe : null;- if (!isRecord(typesafe)) return [];+ const details = readDecisionDetails(score.metadata);+ if (!details) return [];- const questionId = String(typesafe.questionId ?? index);+ const questionId = String(details.questionId ?? index);const question = requestQuestions[questionId];const common = {questionId,@@ -43,21 +51,21 @@ export function toDecisionModelResults(: "",};const confidence =- typeof typesafe.confidence === "number" ? typesafe.confidence : null;+ typeof details.confidence === "number" ? details.confidence : null;- switch (typesafe.type) {+ switch (details.type) {case "choice":return [{...common,type: "choice",- choice: String(score.value ?? typesafe.choice ?? ""),- probabilities: toProbabilities(typesafe.probabilities),+ choice: String(score.value ?? details.choice ?? ""),+ probabilities: toProbabilities(details.probabilities),confidence,},];case "score": {- const legend = isRecord(typesafe.legend) ? typesafe.legend : {};+ const legend = isRecord(details.legend) ? details.legend : {};return [{...common,@@ -66,7 +74,7 @@ export function toDecisionModelResults(levels: Object.keys(legend).sort((a, b) => Number(a) - Number(b)).map((level) => String(legend[level] ?? "")),- probabilities: toProbabilities(typesafe.probabilities),+ probabilities: toProbabilities(details.probabilities),confidence,},];diff --git a/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts b/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.tsindex f778be55bc..173dada5c7 100644--- a/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts@@ -85,6 +85,69 @@ describe("toTestResultPanelState", () => {},);+ it("reads OpenAI and TypeSafe decision details into the result panel", () => {+ const response = {+ success: true,+ request: { questions: { topic: { instructions: "How severe?" } } },+ scores: [+ {+ name: "topic",+ value: 0.24,+ metadata: {+ openai: {+ questionId: "topic",+ type: "score",+ model: "gpt-6-luna",+ confidence: 0.52,+ probabilities: { "0": 0.76, "1": 0.24 },+ legend: { "0": "Fine", "1": "Blocked" },+ },+ },+ },+ ],+ };++ expect(+ toTestResultPanelState({+ type: "DECISION_MODEL",+ isPending: false,+ result: response,+ }),+ ).toEqual({+ status: "decision-success",+ results: [+ {+ questionId: "topic",+ scoreName: "topic",+ instructions: "How severe?",+ type: "score",+ score: 0.24,+ levels: ["Fine", "Blocked"],+ probabilities: { "0": 0.76, "1": 0.24 },+ confidence: 0.52,+ },+ ],+ });++ const typesafePanel = toTestResultPanelState({+ type: "DECISION_MODEL",+ isPending: false,+ result: {+ ...response,+ scores: [+ {+ ...response.scores[0],+ metadata: { typesafe: response.scores[0].metadata.openai },+ },+ ],+ },+ });+ expect(typesafePanel).toMatchObject({+ status: "decision-success",+ results: [{ confidence: 0.52, score: 0.24 }],+ });+ });+it("keeps pending and failed runs distinct", () => {expect(toTestResultPanelState({diff --git a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.tsnew file mode 100644index 0000000000..02af90cbfd--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.ts@@ -0,0 +1,104 @@+import { DecisionModelQuestionType } from "@langfuse/shared";+import { describe, expect, it } from "vitest";++import {+ previewOpenAIDecisionQuestions,+ usesPlainDecisionInstructions,+} from "./decisionModelQuestions";+import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";++const draft = (+ overrides: Partial<DecisionModelQuestionDraft> &+ Pick<DecisionModelQuestionDraft, "id" | "type">,+): DecisionModelQuestionDraft => ({+ scoreName: "",+ instructions: "",+ options: [+ { value: "", description: "" },+ { value: "", description: "" },+ ],+ levels: [{ description: "" }, { description: "" }],+ criteria: { true: "", false: "" },+ ...overrides,+});++describe("usesPlainDecisionInstructions", () => {+ it("drops field chips for an OpenAI connection, including before the model id matches", () => {+ expect(+ usesPlainDecisionInstructions({+ model: "gpt-4.1",+ adapter: "openai",+ }),+ ).toBe(true);+ expect(+ usesPlainDecisionInstructions({+ model: "gpt-6-luna",+ adapter: undefined,+ }),+ ).toBe(true);+ expect(+ usesPlainDecisionInstructions({+ model: "jev-latest",+ adapter: "typesafe",+ }),+ ).toBe(false);+ });+});++describe("previewOpenAIDecisionQuestions", () => {+ it("shows choices, labeled levels, and folded yes/no criteria", () => {+ expect(+ previewOpenAIDecisionQuestions([+ draft({+ id: "department",+ type: DecisionModelQuestionType.CHOICE,+ instructions: " Which team? ",+ options: [+ { value: "billing", description: " Payments " },+ { value: "other", description: " " },+ ],+ }),+ draft({+ id: "severity",+ type: DecisionModelQuestionType.SCORE,+ instructions: "How severe?",+ levels: [+ { description: "Appearance only" },+ { description: "Blocked" },+ ],+ }),+ draft({+ id: "refund",+ type: DecisionModelQuestionType.NOUL,+ instructions: "Does it request a refund?",+ criteria: { true: "Asks for money back", false: "" },+ }),+ ]),+ ).toEqual([+ {+ type: "choice",+ name: "department",+ instructions: "Which team?",+ choices: [+ { value: "billing", description: "Payments" },+ { value: "other" },+ ],+ },+ {+ type: "score",+ name: "severity",+ instructions: "How severe?",+ levels: [+ { label: "0", description: "Appearance only" },+ { label: "1", description: "Blocked" },+ ],+ },+ {+ type: "predicate",+ name: "refund",+ instructions:+ "Does it request a refund?\n\nCriteria for true:\nAsks for money back",+ },+ ]);+ });+});diff --git a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.tsindex 958f99ef2e..ed60a26fba 100644--- a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts+++ b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts@@ -2,6 +2,8 @@ import {DECISION_MODEL_LIMITS,DecisionModelQuestionType,DecisionModelQuestionsSchema,+ isOpenAIDecisionModel,+ LLMAdapter,type DecisionModelEntry,type DecisionModelQuestion,type DecisionModelQuestions,@@ -123,6 +125,15 @@ export function getQuestionDraftErrors(return errors;}+/** OpenAI questions are plain text. Field chips are only for TypeSafe. */+export function usesPlainDecisionInstructions(params: {+ model: string | undefined;+ adapter: string | undefined;+}) {+ if (params.adapter === LLMAdapter.OpenAI) return true;+ return params.model != null && isOpenAIDecisionModel(params.model);+}+/** Persistable questions, or null while any draft is incomplete. */export function draftsToQuestions(drafts: DecisionModelQuestionDraft[],@@ -173,3 +184,72 @@ export function draftsToQuestions(const parsed = DecisionModelQuestionsSchema.safeParse(questions);return parsed.success ? parsed.data : null;}++export type OpenAIDecisionQuestionPreview =+ | {+ type: "choice";+ name: string;+ instructions: string;+ choices: Array<{ value: string; description?: string }>;+ }+ | {+ type: "score";+ name: string;+ instructions: string;+ levels: Array<{ label?: string; description?: string }>;+ }+ | { type: "predicate"; name: string; instructions: string };++/** Questions as the Decisions API receives them, including unfinished drafts. */+export function previewOpenAIDecisionQuestions(+ drafts: DecisionModelQuestionDraft[],+): OpenAIDecisionQuestionPreview[] {+ return drafts.map((draft) => {+ const name = draft.id;+ const instructions = openAIInstructions(draft);+ switch (draft.type) {+ case DecisionModelQuestionType.CHOICE:+ return {+ type: "choice",+ name,+ instructions,+ choices: draft.options.flatMap((option) => {+ const value = option.value.trim();+ if (!value) return [];+ const description = option.description.trim();+ return [+ {+ value,+ ...(description ? { description } : {}),+ },+ ];+ }),+ };+ case DecisionModelQuestionType.SCORE:+ return {+ type: "score",+ name,+ instructions,+ levels: draft.levels.flatMap((level, index) => {+ const description = level.description.trim();+ if (!description) return [];+ return [{ label: String(index), description }];+ }),+ };+ case DecisionModelQuestionType.NOUL:+ return { type: "predicate", name, instructions };+ }+ });+}++function openAIInstructions(draft: DecisionModelQuestionDraft) {+ if (draft.type !== DecisionModelQuestionType.NOUL) {+ return draft.instructions.trim();+ }+ const parts = [draft.instructions.trim()];+ const whenTrue = draft.criteria.true.trim();+ const whenFalse = draft.criteria.false.trim();+ if (whenTrue) parts.push(`Criteria for true:\n${whenTrue}`);+ if (whenFalse) parts.push(`Criteria for false:\n${whenFalse}`);+ return parts.filter((part) => part.length > 0).join("\n\n");+}diff --git a/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.ts b/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.tsnew file mode 100644index 0000000000..a1714120a7--- /dev/null+++ b/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.ts@@ -0,0 +1,74 @@+import {+ isAllowedDecisionModel,+ LLMAdapter,+ OPENAI_DECISION_MODEL_IDS,+ supportedModels,+ supportsDecisionModels,+} from "@langfuse/shared";++import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";++type DecisionModelConnection = {+ provider: string;+ adapter: string;+ customModels: readonly string[];+ withDefaultModels: boolean;+};++function defaultModelsFor(adapter: string): readonly string[] {+ if (adapter === LLMAdapter.OpenAI) return supportedModels[LLMAdapter.OpenAI];+ if (adapter === LLMAdapter.TypeSafe) {+ return supportedModels[LLMAdapter.TypeSafe];+ }+ return [];+}++/** Decision models each connection can run, keyed by provider. */+export function decisionModelsByProvider(+ connections: readonly DecisionModelConnection[],+): Map<string, string[]> {+ const modelsByProvider = new Map<string, string[]>();+ for (const connection of connections) {+ if (!supportsDecisionModels(connection.adapter)) continue;+ const models = Array.from(+ new Set([+ ...connection.customModels,+ ...(connection.withDefaultModels+ ? defaultModelsFor(connection.adapter)+ : []),+ ]),+ ).filter((model) => isAllowedDecisionModel(connection.adapter, model));+ if (models.length > 0) modelsByProvider.set(connection.provider, models);+ }+ return modelsByProvider;+}++/**+ * OpenAI is the default decision provider. The first allowlisted model a+ * connection actually offers wins. TypeSafe is never chosen for the user.+ */+export function preferredDecisionModel(+ connections: readonly DecisionModelConnection[],+): JudgeModel | null {+ const modelsByProvider = decisionModelsByProvider(connections);+ for (const model of OPENAI_DECISION_MODEL_IDS) {+ for (const [provider, models] of modelsByProvider) {+ if (models.includes(model)) return { provider, model };+ }+ }+ return null;+}++/** Use the saved choice. Fill an empty decision-model choice from the connection list. */+export function applyFallbackDecisionModel<+ T extends { type: string; selectedModel: JudgeModel | null },+>(state: T, fallback: JudgeModel | null): T {+ if (+ state.type !== "DECISION_MODEL" ||+ state.selectedModel != null ||+ fallback == null+ ) {+ return state;+ }+ return { ...state, selectedModel: fallback };+}diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.tsindex 5985ba86a8..4d5b36990b 100644--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts@@ -201,6 +201,19 @@ describe("prepareEvaluatorDraft", () => {],});+ expect(+ prepareEvaluatorDraft({+ ...base,+ selectedModel: { provider: "openai", model: "gpt-6-luna" },+ }).definition?.variableMapping,+ ).toEqual([+ {+ templateVariable: "input",+ selectedColumnId: "input",+ jsonSelector: null,+ },+ ]);+// An unmapped state key blocks saving; the model would never see it.expect(prepareEvaluatorDraft({diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.tsindex 90eaf5b229..8ecd9340ad 100644--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts@@ -1,4 +1,6 @@/* eslint-disable no-nested-ternary */+import { isOpenAIDecisionModel } from "@langfuse/shared";+import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";import { buildScoreOutputDefinition } from "@/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition";import { buildEvaluatorVariableMappings } from "@/src/features/evals/v2/fns/variableMapping/buildEvaluatorVariableMappings";@@ -36,7 +38,11 @@ export function prepareEvaluatorDraft(params: EvaluatorSetupDraftState) {}if (params.type === "DECISION_MODEL") {return buildDecisionModelStateFields({- stateKeys: params.stateKeys,+ stateKeys:+ params.selectedModel != null &&+ isOpenAIDecisionModel(params.selectedModel.model)+ ? ["input"]+ : params.stateKeys,variableFields: params.variableFields,});}diff --git a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsxindex 26095d4506..e88bc8b09c 100644--- a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx+++ b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx@@ -9,6 +9,16 @@ vi.mock("@/src/features/evals/v2/hooks/useEvaluatorSetupSample", () => ({useEvaluatorSetupSample: () => ({ id: "sample" }),}));+vi.mock("@/src/utils/api", () => ({+ api: {+ llmApiKey: {+ all: {+ useQuery: () => ({ data: undefined }),+ },+ },+ },+}));+describe("useEvaluatorTestAvailability", () => {const store = createEvaluatorSetupStore({initialEvaluator: null,diff --git a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.tsindex ea7cfd09d1..77dcada7c4 100644--- a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts+++ b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts@@ -1,8 +1,10 @@import { useStore } from "zustand";+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";export function useEvaluatorTestAvailability({@@ -19,8 +21,17 @@ export function useEvaluatorTestAvailability({store,(state) => state.selectedObservation,);+ const evaluatorType = useStore(store, (state) => state.type);+ const fallbackDecisionModel = useFallbackDecisionModel(+ projectId,+ evaluatorType === "DECISION_MODEL",+ );const definitionAvailable = useStore(store, (state) =>- Boolean(prepareEvaluatorDraft(state).definition),+ Boolean(+ prepareEvaluatorDraft(+ applyFallbackDecisionModel(state, fallbackDecisionModel),+ ).definition,+ ),);const scoreOutputReason = useStore(store, (state) =>state.type === "LLM_AS_JUDGE"diff --git a/web/src/features/evals/v2/hooks/useFallbackDecisionModel.ts b/web/src/features/evals/v2/hooks/useFallbackDecisionModel.tsnew file mode 100644index 0000000000..d06560d217--- /dev/null+++ b/web/src/features/evals/v2/hooks/useFallbackDecisionModel.ts@@ -0,0 +1,11 @@+import { preferredDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";+import { api } from "@/src/utils/api";++/** The decision model to show and save when the user has not chosen one yet. */+export function useFallbackDecisionModel(projectId: string, enabled: boolean) {+ const connections = api.llmApiKey.all.useQuery(+ { projectId, includeDecisionModels: true },+ { enabled },+ );+ return preferredDecisionModel(connections.data?.data ?? []);+}diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsxindex ac45529048..52545b2fc0 100644--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx@@ -29,7 +29,9 @@ import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluatimport { EvaluatorSetupFooter } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter";import { SampleObservationSelectorContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer";import { EvaluatorTestPanelContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer";+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";import { draftsToQuestions } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";import type { NormalizedEvaluatorDefinition } from "../server/evaluators/evaluatorTypes";import { api } from "@/src/utils/api";@@ -288,6 +290,12 @@ export function EvaluatorSetupPage(sourceCode: codeDraft.sourceCode,sourceCodeLanguage: codeDraft.sourceCodeLanguage,});+ const fallbackDecisionModel = useFallbackDecisionModel(+ projectId,+ modelDraft.type === "DECISION_MODEL",+ );+ // The default decision model arrives with the connection list. It is applied+ // on save, not stored, so it must not count as an edit.const getCurrentSnapshot = (state = evaluatorSetupStore.getState()) =>JSON.stringify({name: state.name.trim(),@@ -581,7 +589,9 @@ export function EvaluatorSetupPage(return;}}- const { definition } = prepareEvaluatorDraft(state);+ const { definition } = prepareEvaluatorDraft(+ applyFallbackDecisionModel(state, fallbackDecisionModel),+ );if (!definition) return;const { name, description } = metadata;@@ -717,7 +727,9 @@ export function EvaluatorSetupPage(const runTest = () => {const state = evaluatorSetupStore.getState();- const { definition } = prepareEvaluatorDraft(state);+ const { definition } = prepareEvaluatorDraft(+ applyFallbackDecisionModel(state, fallbackDecisionModel),+ );const selectedObservation = state.selectedObservation;if (!definition || !selectedObservation?.traceId) return;capture("evaluators:test", {@@ -948,6 +960,7 @@ export function EvaluatorSetupPage(}: null}+ fallbackDecisionModel={fallbackDecisionModel}onClose={requestClose}onSave={save}/>diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.tsindex 36bb4cd2ab..e4060e7821 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts@@ -8,7 +8,10 @@ import {DefaultEvalModelService,getClientInitiatedNonStreamingLlmTimeoutMs,getLLMErrorInfo,+ isAllowedDecisionModel,isDecisionModelAdapter,+ LLMAdapter,+ OPENAI_DECISION_MODEL_IDS,} from "@langfuse/shared/src/server";import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";@@ -212,7 +215,7 @@ export async function getDecisionModelConfigurationError(params: {name: string;definition: Pick<Extract<EvaluatorDefinition, { type: "DECISION_MODEL" }>,- "provider" | "model"+ "provider" | "model" | "questions">;}): Promise<string | null> {const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(@@ -221,10 +224,18 @@ export async function getDecisionModelConfigurationError(params: {params.definition.model,);if (!modelConfig.valid) {- return `No decision-model connection found for evaluator "${params.name}". ${modelConfig.error}. Add a TypeSafe connection under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) first.`;+ return `No decision-model connection found for evaluator "${params.name}". ${modelConfig.error}. Add an OpenAI connection using ${OPENAI_DECISION_MODEL_IDS.join(", ")}, or a TypeSafe connection, under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) first.`;}- if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {- return `Connection "${params.definition.provider}" is not a decision-model connection. Decision-model evaluators need a TypeSafe connection.`;+ if (+ !isAllowedDecisionModel(+ modelConfig.config.apiKey.adapter,+ modelConfig.config.model,+ )+ ) {+ if (modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI) {+ return `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`;+ }+ return `Connection "${params.definition.provider}" is not a decision-model connection. Decision-model evaluators need an OpenAI connection using ${OPENAI_DECISION_MODEL_IDS.join(", ")}, or a TypeSafe connection.`;}return null;}diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 7c5b4846ff..3f8ac76f10 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -6,7 +6,7 @@ import { decrypt } from "@langfuse/shared/encryption";import {buildEvalExecutionData,compileLangfuseMediaMessages,- createTypeSafeDecisionModelClient,+ createDecisionModelClient,createW3CTraceId,decryptAndParseExtraHeaders,DefaultEvalModelService,@@ -16,7 +16,9 @@ import {extractObservationVariables,findModel,generateLLMText,- isDecisionModelAdapter,+ isAllowedDecisionModel,+ LLMAdapter,+ OPENAI_DECISION_MODEL_IDS,LangfuseInternalTraceEnvironment,mapLegacyLLMCompletionParams,matchPricingTier,@@ -134,17 +136,26 @@ async function testDecisionModelEvaluator(params: {if (!modelConfig.valid) {return { success: false as const, error: modelConfig.error };}- if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {+ if (+ !isAllowedDecisionModel(+ modelConfig.config.apiKey.adapter,+ modelConfig.config.model,+ )+ ) {return {success: false as const,- error: `Connection "${params.definition.provider}" is not a decision-model connection.`,+ error:+ modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI+ ? `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`+ : `Connection "${params.definition.provider}" is not a decision-model connection.`,};}const executionTraceId = createW3CTraceId();let request: DecisionModelRequest | undefined;try {- const client = createTypeSafeDecisionModelClient({+ const client = createDecisionModelClient({+ adapter: modelConfig.config.apiKey.adapter,apiKey: decrypt(modelConfig.config.apiKey.secretKey),model: modelConfig.config.model,baseURL: modelConfig.config.apiKey.baseURL,diff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsxindex 1a78bb195a..2c2d3d1038 100644--- a/web/src/features/filters/components/filter-builder.tsx+++ b/web/src/features/filters/components/filter-builder.tsx@@ -1,5 +1,6 @@/* eslint-disable no-nested-ternary *//* eslint-disable @repo/no-style-props */+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";import { Button } from "@/src/components/ui/button";import { Input } from "@/src/components/ui/input";import { Textarea } from "@/src/components/ui/textarea";@@ -1263,26 +1264,26 @@ function FilterBuilderForm({</table>)}{!disabled ? (- <Button- onClick={() => addNewFilter()}- type="button" // required as it will otherwise submit forms where this component is used- className={cn(- subtleAddButton- ? "text-foreground hover:text-foreground mt-2 h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"- : "mt-2",- compact && !subtleAddButton && "mt-4 self-start",- )}- variant={subtleAddButton ? "ghost" : "outline"}- size="sm"- >- <Plus- className={cn(- "icon-base text-icon-foreground shrink-0",- !subtleAddButton && "mr-2",- )}- />- Add filter- </Button>+ subtleAddButton ? (+ <div className="mt-2">+ <TextActionButton+ text="Add filter"+ width="fill"+ onClick={() => addNewFilter()}+ />+ </div>+ ) : (+ <Button+ onClick={() => addNewFilter()}+ type="button" // required as it will otherwise submit forms where this component is used+ className={cn("mt-2", compact && "mt-4 self-start")}+ variant="outline"+ size="sm"+ >+ <Plus className="icon-base text-icon-foreground mr-2 shrink-0" />+ Add filter+ </Button>+ )) : null}</>)}diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.tsindex d88ed1ace4..da83681a08 100644--- a/web/src/features/llm-api-key/server/router.ts+++ b/web/src/features/llm-api-key/server/router.ts@@ -128,7 +128,7 @@ async function testDecisionModelConnection(params: {kind: {type: "choice",instructions: "What kind of message is `message`?",- criteria: { greeting: null, other: null },+ choices: [{ value: "greeting" }, { value: "other" }],},},});diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.tsindex 5d1c1c166a..2cbd682895 100644--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({name: "createEvaluator",description: ["Create a new evaluator with a stable id. Names do not act as identity and may be reused.",- "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.","Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",].join(" "),action: "evaluator:CUD",diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.tsindex e58226b951..8987b31f80 100644--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts@@ -25,7 +25,7 @@ export const [description: ["List the evaluator templates maintained by Langfuse and partners.","LLM-as-a-judge and code definitions can be copied into createEvaluator.",- "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a TypeSafe modelConfig.",+ "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a modelConfig for an OpenAI connection using an allowlisted decision model (currently `gpt-6-luna`) or a TypeSafe connection.",].join(" "),action: "evaluator:read",baseSchema: ListManagedEvaluatorTemplatesInput,diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.tsindex 34200bc103..133776e94c 100644--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({name: "updateEvaluator",description: ["Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",- "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",].join(" "),action: "evaluator:CUD",baseSchema: McpUpdateEvaluatorInputBase,diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.tsindex f26065415b..4c6a2ce432 100644--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts@@ -197,10 +197,10 @@ describe("runDecisionModelEvaluation", () => {readiness: {type: "choice",instructions: "Is `output` ready to send as an answer to `input`?",- criteria: { ready: null, needs_revision: null },+ choices: [{ value: "ready" }, { value: "needs_revision" }],},refund: {- type: "boolean",+ type: "predicate",instructions: "Does `input` request a refund?",},},diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.tsindex 0b6bd02edd..bc27764b1a 100644--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts@@ -18,7 +18,9 @@ import {EvaluatorBlockSource,executeDecisionModelEvaluator,instrumentAsync,- isDecisionModelAdapter,+ isAllowedDecisionModel,+ LLMAdapter,+ OPENAI_DECISION_MODEL_IDS,logger,type ExtractedVariable,} from "@langfuse/shared/src/server";@@ -113,8 +115,16 @@ export async function runDecisionModelEvaluation({let modelConfigError: string | null = null;if (!modelConfig.valid) {modelConfigError = modelConfig.error;- } else if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {- modelConfigError = `Connection "${modelConfig.config.provider}" is not a decision-model connection`;+ } else if (+ !isAllowedDecisionModel(+ modelConfig.config.apiKey.adapter,+ modelConfig.config.model,+ )+ ) {+ modelConfigError =+ modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI+ ? `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`+ : `Connection "${modelConfig.config.provider}" is not a decision-model connection`;}if (!modelConfig.valid || modelConfigError !== null) {const blockReason = getBlockReasonForInvalidModelConfig({diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 0577f7b944..99216fad09 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -8,7 +8,7 @@ import {buildEventBucketPrefix,compileLangfuseMediaMessages,createLLMOutput,- createTypeSafeDecisionModelClient,+ createDecisionModelClient,decryptAndParseExtraHeaders,DefaultEvalModelService,generateLLMText,@@ -338,15 +338,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {callDecisionModel: async (params) => {const { apiKey } = params.modelConfig;- if (apiKey.adapter !== LLMAdapter.TypeSafe) {- throw new Error(- `Decision-model adapter is not supported: ${apiKey.adapter}`,- );- }const secretKey = apiKey.secretKey;if (typeof secretKey !== "string") {throw new UnrecoverableError(- "TypeSafe connection is missing its secret key",+ "Decision-model connection is missing its secret key",);}@@ -359,11 +354,12 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {);} catch {throw new UnrecoverableError(- "TypeSafe connection secrets could not be decrypted",+ "Decision-model connection secrets could not be decrypted",);}- const client = createTypeSafeDecisionModelClient({+ const client = createDecisionModelClient({+ adapter: apiKey.adapter,apiKey: decryptedSecretKey,model: params.modelConfig.model,baseURL: typeof apiKey.baseURL === "string" ? apiKey.baseURL : null,@@ -400,7 +396,7 @@ export function createMockEvalExecutionDeps(Object.entries(request.questions).map(([id, question]) => {switch (question.type) {case "choice": {- const [choice = "mock"] = Object.keys(question.criteria);+ const choice = question.choices[0]?.value ?? "mock";return [id,{@@ -416,7 +412,7 @@ export function createMockEvalExecutionDeps(id,{ type: "score", score: 0, probabilities: {}, confidence: 1 },];- case "boolean":+ case "predicate":return [id, { type: "boolean", probability: 1 }];}}),