Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): support OpenAI decision models (#18389)

Tobias Wochingerc539db55570a
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 1391c43a3d..662d3fc6b2 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -199,7 +199,7 @@
"@ai-sdk/azure": "^4.0.5",
"@ai-sdk/google": "^4.0.6",
"@ai-sdk/google-vertex": "^5.0.8",
- "@ai-sdk/openai": "^4.0.0",
+ "@ai-sdk/openai": "^4.0.86",
"@ai-sdk/openai-compatible": "^3.0.3",
"@ai-sdk/typesafe-ai": "^3.0.4",
"@aws-sdk/client-cloudwatch": "catalog:",
@@ -223,7 +223,7 @@
"@slack/web-api": "^7.15.0",
"@smithy/node-http-handler": "4.5.1",
"@types/bcryptjs": "^2.4.6",
- "ai": "^7.0.0",
+ "ai": "^7.0.130",
"ajv": "^8.18.0",
"ajv-formats": "^3.0.1",
"bcryptjs": "^2.4.3",
diff --git a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts
index 97f5afeee1..2e75f3f1d0 100644
--- a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts
+++ b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.test.ts
@@ -37,6 +37,7 @@ const questions: DecisionModelQuestions = DecisionModelQuestionsSchema.parse([
scoreName: "refund_requested",
type: "noul",
instructions: "Does `question` request a refund?",
+ criteria: { true: "Asks for money back", false: "Does not" },
},
]);
@@ -91,19 +92,27 @@ describe("executeDecisionModelEvaluator", () => {
readiness: {
type: "choice",
instructions: "Is `reply` ready to send as an answer to `question`?",
- criteria: {
- ready: "Answers and states the next step",
- needs_revision: null,
- },
+ choices: [
+ {
+ value: "ready",
+ description: "Answers and states the next step",
+ },
+ { value: "needs_revision" },
+ ],
},
frustration: {
type: "score",
instructions: "How frustrated is the customer in `question`?",
- criteria: ["Calm", "Frustrated but civil", "Very angry"],
+ levels: [
+ { description: "Calm" },
+ { description: "Frustrated but civil" },
+ { description: "Very angry" },
+ ],
},
refund: {
- type: "boolean",
+ type: "predicate",
instructions: "Does `question` request a refund?",
+ criteria: { true: "Asks for money back", false: "Does not" },
},
},
});
@@ -172,6 +181,25 @@ describe("executeDecisionModelEvaluator", () => {
]);
});
+ it("writes OpenAI score details under metadata.openai", async () => {
+ const { client } = createClient({ ...evaluation, model: "gpt-6-luna" });
+
+ const { scores } = await executeDecisionModelEvaluator({
+ variables: [{ var: "question", value: "refund?" }],
+ questions,
+ client,
+ });
+
+ expect(scores.map((score) => Object.keys(score.metadata ?? {}))).toEqual([
+ ["openai"],
+ ["openai"],
+ ["openai"],
+ ]);
+ expect(scores[1]?.metadata).toMatchObject({
+ openai: { model: "gpt-6-luna", questionId: "frustration", type: "score" },
+ });
+ });
+
it("fails permanently when the state is empty", async () => {
const { client, evaluate } = createClient(evaluation);
diff --git a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts
index 7bf4dcdf37..3eee28abe6 100644
--- a/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts
+++ b/packages/shared/src/server/evals/decisionModelEvaluatorExecution.ts
@@ -12,26 +12,33 @@ import {
type InternalTraceEventInput,
} from "../llm/internalTraceEvents";
import {
+ isOpenAIDecisionModel,
LangfuseInternalTraceEnvironment,
+ LLMAdapter,
type InternalTraceWriteInput,
} from "../llm/types";
import type { CodeEvalScoreWithName } from "./codeEvalDispatcherTypes";
import type { ExtractedVariable } from "./extractObservationVariables";
+/** OpenAI Decisions question shape. TypeSafe inference projects this down. */
export type DecisionModelRequestQuestion =
| {
type: "choice";
instructions: DecisionModelEntry;
- criteria: Record<string, DecisionModelEntry | null>;
+ choices: Array<{
+ value: string;
+ description?: DecisionModelEntry | null;
+ }>;
}
| {
type: "score";
instructions: DecisionModelEntry;
- criteria: DecisionModelEntry[];
+ levels: Array<{ description: DecisionModelEntry }>;
}
| {
- type: "boolean";
+ type: "predicate";
instructions: DecisionModelEntry;
+ /** Yes/no notes from a saved TypeSafe question. Each SDK attaches them. */
criteria?: {
true?: DecisionModelEntry | null;
false?: DecisionModelEntry | null;
@@ -97,28 +104,32 @@ export function toDecisionModelRequestQuestion(
return {
type: "choice",
instructions: question.instructions,
- criteria: Object.fromEntries(
- question.options.map((option) => [
- option.value,
- option.description ?? null,
- ]),
- ),
+ choices: question.options.map((option) => ({
+ value: option.value,
+ ...(option.description != null
+ ? { description: option.description }
+ : {}),
+ })),
};
case DecisionModelQuestionType.SCORE:
return {
type: "score",
instructions: question.instructions,
- criteria: question.levels.map((level) => level.description),
+ levels: question.levels,
};
case DecisionModelQuestionType.NOUL:
return {
- type: "boolean",
+ type: "predicate",
instructions: question.instructions,
- ...(question.criteria
+ ...(question.criteria?.true != null || question.criteria?.false != null
? {
criteria: {
- true: question.criteria.true ?? null,
- false: question.criteria.false ?? null,
+ ...(question.criteria.true != null
+ ? { true: question.criteria.true }
+ : {}),
+ ...(question.criteria.false != null
+ ? { false: question.criteria.false }
+ : {}),
},
}
: {}),
@@ -184,13 +195,15 @@ export function formatDecisionModelComment(params: {
? question.levels.length - 1
: 0,
);
- const levelDescription =
+ const level =
question.type === DecisionModelQuestionType.SCORE
- ? question.levels[nearestLevel]?.description
+ ? question.levels[nearestLevel]
: undefined;
+ const levelName =
+ typeof level?.description === "string" ? level.description : undefined;
parts.push(
- typeof levelDescription === "string"
- ? `${formatNumber(answer.score)} ≈ level ${nearestLevel} "${levelDescription}"`
+ levelName != null
+ ? `${formatNumber(answer.score)} ≈ level ${nearestLevel} "${levelName}"`
: `${formatNumber(answer.score)} ≈ level ${nearestLevel}`,
);
if (answer.confidence !== null) {
@@ -202,6 +215,11 @@ export function formatDecisionModelComment(params: {
return parts.join("; ");
}
+/** Score details are namespaced by the API that produced them. */
+function decisionScoreMetadataKey(model: string) {
+ return isOpenAIDecisionModel(model) ? LLMAdapter.OpenAI : LLMAdapter.TypeSafe;
+}
+
function toScoreMetadata(params: {
question: DecisionModelQuestion;
answer: DecisionModelAnswer;
@@ -209,10 +227,11 @@ function toScoreMetadata(params: {
}): Record<string, unknown> {
const { question, answer, model } = params;
const base = { questionId: question.id, type: question.type, model };
+ const key = decisionScoreMetadataKey(model);
switch (answer.type) {
case "choice":
return {
- typesafe: {
+ [key]: {
...base,
choice: answer.choice,
confidence: answer.confidence,
@@ -221,7 +240,7 @@ function toScoreMetadata(params: {
};
case "score":
return {
- typesafe: {
+ [key]: {
...base,
confidence: answer.confidence,
probabilities: answer.probabilities,
@@ -230,14 +249,16 @@ function toScoreMetadata(params: {
? Object.fromEntries(
question.levels.map((level, index) => [
String(index),
- level.description,
+ typeof level.description === "string"
+ ? level.description
+ : JSON.stringify(level.description),
]),
)
: undefined,
},
};
case "boolean":
- return { typesafe: base };
+ return { [key]: base };
}
}
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 097234cbc8..9ed589de76 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -39,6 +39,7 @@ export * from "./evals/extractObservationVariables";
export * from "./evals/classifyEvaluatorLlmError";
export * from "./evals/llmEvaluatorExecution";
export * from "./evals/decisionModelEvaluatorExecution";
+export * from "./llm/createDecisionModelClient";
export * from "./llm/typesafe/typeSafeDecisionModelClient";
export * from "./utils/traceId";
export * from "./auth/apiKeyCache";
diff --git a/packages/shared/src/server/llm/createDecisionModelClient.ts b/packages/shared/src/server/llm/createDecisionModelClient.ts
new file mode 100644
index 0000000000..4c1a0b2820
--- /dev/null
+++ b/packages/shared/src/server/llm/createDecisionModelClient.ts
@@ -0,0 +1,32 @@
+import {
+ DecisionModelEvaluatorError,
+ type DecisionModelClient,
+} from "../evals/decisionModelEvaluatorExecution";
+import {
+ isAllowedDecisionModel,
+ LLMAdapter,
+ OPENAI_DECISION_MODEL_IDS,
+} from "./types";
+import { createOpenAIDecisionModelClient } from "./openai/openAIDecisionModelClient";
+import { createTypeSafeDecisionModelClient } from "./typesafe/typeSafeDecisionModelClient";
+
+export function createDecisionModelClient(params: {
+ adapter: string;
+ apiKey: string;
+ model: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ fetchImpl?: typeof fetch;
+}): DecisionModelClient {
+ if (!isAllowedDecisionModel(params.adapter, params.model)) {
+ throw new DecisionModelEvaluatorError(
+ params.adapter === LLMAdapter.OpenAI
+ ? `Model "${params.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`
+ : `Decision-model adapter is not supported: ${params.adapter}`,
+ );
+ }
+ if (params.adapter === LLMAdapter.OpenAI) {
+ return createOpenAIDecisionModelClient(params);
+ }
+ return createTypeSafeDecisionModelClient(params);
+}
diff --git a/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.ts b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.ts
new file mode 100644
index 0000000000..631febea5e
--- /dev/null
+++ b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.test.ts
@@ -0,0 +1,220 @@
+import { describe, expect, it, vi } from "vitest";
+import type { DecisionModelRequest } from "../../evals/decisionModelEvaluatorExecution";
+import { createOpenAIDecisionModelClient } from "./openAIDecisionModelClient";
+
+function jsonResponse(body: unknown, status = 200) {
+ return new Response(JSON.stringify(body), {
+ status,
+ headers: { "Content-Type": "application/json" },
+ });
+}
+
+const request: DecisionModelRequest = {
+ state: { input: "I was charged twice.", output: "Refund issued." },
+ questions: {
+ department: {
+ type: "choice",
+ instructions: "Which team should handle `input`?",
+ choices: [
+ { value: "billing", description: "Payments and refunds" },
+ { value: "other" },
+ ],
+ },
+ severity: {
+ type: "score",
+ instructions: "How severe is the issue?",
+ levels: [{ description: "Appearance only" }, { description: "Blocked" }],
+ },
+ refund: {
+ type: "predicate",
+ instructions: "Does `input` request a refund?",
+ criteria: { true: "Asks for money back", false: { note: "money" } },
+ },
+ },
+};
+
+describe("createOpenAIDecisionModelClient", () => {
+ it("posts text input and maps predicate, choice, and score answers", async () => {
+ const fetchImpl = vi.fn().mockResolvedValue(
+ jsonResponse({
+ model: "gpt-6-luna",
+ answers: [
+ {
+ type: "choice",
+ name: "department",
+ choice: "billing",
+ confidence: 0.8,
+ probabilities: [
+ { value: "billing", probability: 0.9 },
+ { value: "other", probability: 0.1 },
+ ],
+ },
+ {
+ type: "score",
+ name: "severity",
+ score: 0.4,
+ confidence: 0.7,
+ probabilities: [
+ { value: 0, probability: 0.6 },
+ { value: 1, probability: 0.4 },
+ ],
+ },
+ { type: "predicate", name: "refund", probability: 0.95 },
+ ],
+ usage: { input_tokens: 120, output_tokens: 3 },
+ }),
+ );
+
+ const client = createOpenAIDecisionModelClient({
+ apiKey: "sk-test",
+ model: "gpt-6-luna",
+ fetchImpl,
+ });
+ const evaluation = await client.evaluate(request);
+
+ const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit];
+ expect(url).toBe("https://api.openai.com/v1/decisions");
+ expect(new Headers(init.headers).get("authorization")).toBe(
+ "Bearer sk-test",
+ );
+ expect(JSON.parse(init.body as string)).toEqual({
+ model: "gpt-6-luna",
+ input: JSON.stringify(request.state),
+ questions: [
+ {
+ type: "choice",
+ name: "department",
+ instructions: "Which team should handle `input`?",
+ choices: [
+ { value: "billing", description: "Payments and refunds" },
+ { value: "other" },
+ ],
+ },
+ {
+ type: "score",
+ name: "severity",
+ instructions: "How severe is the issue?",
+ levels: [
+ { label: "0", description: "Appearance only" },
+ { label: "1", description: "Blocked" },
+ ],
+ },
+ {
+ type: "predicate",
+ name: "refund",
+ instructions:
+ 'Does `input` request a refund?\n\nCriteria for true:\nAsks for money back\n\nCriteria for false:\n{"note":"money"}',
+ },
+ ],
+ });
+ expect(evaluation).toEqual({
+ model: "gpt-6-luna",
+ answers: {
+ department: {
+ type: "choice",
+ choice: "billing",
+ probabilities: { billing: 0.9, other: 0.1 },
+ confidence: 0.8,
+ },
+ severity: {
+ type: "score",
+ score: 0.4,
+ probabilities: { "0": 0.6, "1": 0.4 },
+ confidence: 0.7,
+ },
+ refund: { type: "boolean", probability: 0.95 },
+ },
+ usage: { inputTokens: 120, outputTokens: 3 },
+ });
+ });
+
+ it.each([
+ [401, false],
+ [500, true],
+ ])(
+ "surfaces HTTP %s as an AI SDK call error (retryable: %s)",
+ async (status, isRetryable) => {
+ const fetchImpl = vi
+ .fn()
+ .mockImplementation(() =>
+ jsonResponse(
+ { error: { message: "nope", type: "invalid_request_error" } },
+ status,
+ ),
+ );
+ const client = createOpenAIDecisionModelClient({
+ apiKey: "sk-bad",
+ model: "gpt-6-luna",
+ fetchImpl,
+ });
+
+ await expect(client.evaluate(request)).rejects.toMatchObject({
+ name: "AI_APICallError",
+ statusCode: status,
+ isRetryable,
+ });
+ },
+ );
+
+ it("fails the call when any question is refused", async () => {
+ const fetchImpl = vi.fn().mockImplementation(() =>
+ jsonResponse({
+ model: "gpt-6-luna",
+ answers: [
+ {
+ type: "choice",
+ name: "department",
+ choice: "billing",
+ probabilities: [
+ { value: "billing", probability: 1 },
+ { value: "other", probability: 0 },
+ ],
+ },
+ {
+ type: "score",
+ name: "severity",
+ score: 0,
+ probabilities: [
+ { value: 0, probability: 1 },
+ { value: 1, probability: 0 },
+ ],
+ },
+ { type: "refusal", name: "refund" },
+ ],
+ }),
+ );
+ const client = createOpenAIDecisionModelClient({
+ apiKey: "sk-test",
+ model: "gpt-6-luna",
+ fetchImpl,
+ });
+
+ await expect(client.evaluate(request)).rejects.toMatchObject({
+ name: "DecisionModelEvaluatorError",
+ message: "OpenAI declined to answer: refund.",
+ });
+ });
+
+ it("appends /decisions to a custom base URL", async () => {
+ const fetchImpl = vi.fn().mockResolvedValue(
+ jsonResponse({
+ model: "gpt-6-luna",
+ answers: [{ type: "predicate", name: "refund", probability: 0.5 }],
+ }),
+ );
+ const client = createOpenAIDecisionModelClient({
+ apiKey: "sk-test",
+ model: "gpt-6-luna",
+ baseURL: "https://llm-proxy.example.com/v1/",
+ fetchImpl,
+ });
+
+ await client.evaluate({
+ state: request.state,
+ questions: { refund: request.questions.refund },
+ });
+
+ const [url] = fetchImpl.mock.calls[0] as [string, RequestInit];
+ expect(url).toBe("https://llm-proxy.example.com/v1/decisions");
+ });
+});
diff --git a/packages/shared/src/server/llm/openai/openAIDecisionModelClient.ts b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.ts
new file mode 100644
index 0000000000..5177d19877
--- /dev/null
+++ b/packages/shared/src/server/llm/openai/openAIDecisionModelClient.ts
@@ -0,0 +1,167 @@
+import { createOpenAI } from "@ai-sdk/openai";
+import {
+ experimental_decide as decide,
+ Experimental_DecisionRefusalError,
+ type Experimental_DecisionQuestion,
+} from "ai";
+import {
+ DecisionModelEvaluatorError,
+ type DecisionModelAnswer,
+ type DecisionModelClient,
+ type DecisionModelEvaluation,
+ type DecisionModelRequest,
+ type DecisionModelRequestQuestion,
+} from "../../evals/decisionModelEvaluatorExecution";
+import type { DecisionModelEntry } from "../../../features/evals/decisionModel";
+import { createSecureLlmFetch } from "../secureLlmFetch";
+
+/** Same text conversion the OpenAI SDK applies before POST /v1/decisions. */
+function toDecisionText(entry: DecisionModelEntry): string {
+ return typeof entry === "string" ? entry : JSON.stringify(entry);
+}
+
+/** The SDK's score question has descriptions only. It sends index labels. */
+function toSdkQuestion(
+ question: DecisionModelRequestQuestion,
+): Experimental_DecisionQuestion {
+ switch (question.type) {
+ case "choice":
+ return {
+ type: "choice",
+ instructions: toDecisionText(question.instructions),
+ criteria: Object.fromEntries(
+ question.choices.map((choice) => [
+ choice.value,
+ choice.description == null
+ ? null
+ : toDecisionText(choice.description),
+ ]),
+ ),
+ };
+ case "score":
+ return {
+ type: "score",
+ instructions: toDecisionText(question.instructions),
+ criteria: question.levels.map((level) =>
+ toDecisionText(level.description),
+ ),
+ };
+ case "predicate": {
+ const criteria = {
+ ...(question.criteria?.true != null
+ ? { true: toDecisionText(question.criteria.true) }
+ : {}),
+ ...(question.criteria?.false != null
+ ? { false: toDecisionText(question.criteria.false) }
+ : {}),
+ };
+ return {
+ type: "boolean",
+ instructions: toDecisionText(question.instructions),
+ ...(Object.keys(criteria).length > 0 ? { criteria } : {}),
+ };
+ }
+ }
+}
+
+function readConfidence(providerMetadata: unknown): Record<string, number> {
+ if (typeof providerMetadata !== "object" || providerMetadata === null) {
+ return {};
+ }
+ const openai = (providerMetadata as Record<string, unknown>).openai;
+ if (typeof openai !== "object" || openai === null) return {};
+ const confidence = (openai as Record<string, unknown>).confidence;
+ if (typeof confidence !== "object" || confidence === null) return {};
+ return Object.fromEntries(
+ Object.entries(confidence as Record<string, unknown>).flatMap(
+ ([id, value]) =>
+ typeof value === "number" && value >= 0 && value <= 1
+ ? [[id, value]]
+ : [],
+ ),
+ );
+}
+
+export function createOpenAIDecisionModelClient(params: {
+ apiKey: string;
+ model: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ fetchImpl?: typeof fetch;
+}): DecisionModelClient {
+ const provider = createOpenAI({
+ apiKey: params.apiKey,
+ baseURL: params.baseURL ?? undefined,
+ headers: params.extraHeaders,
+ fetch:
+ params.fetchImpl ??
+ createSecureLlmFetch({
+ logContext: "OpenAI decision model",
+ additionalSensitiveHeaders: Object.keys(params.extraHeaders ?? {}),
+ }),
+ });
+
+ return {
+ evaluate: async (request: DecisionModelRequest) => {
+ const questions: Record<string, Experimental_DecisionQuestion> =
+ Object.fromEntries(
+ Object.entries(request.questions).map(([id, question]) => [
+ id,
+ toSdkQuestion(question),
+ ]),
+ );
+ let result;
+ try {
+ result = await decide({
+ model: provider.decisionModel(params.model),
+ state: JSON.stringify(request.state),
+ questions,
+ maxRetries: 0,
+ });
+ } catch (error) {
+ if (Experimental_DecisionRefusalError.isInstance(error)) {
+ throw new DecisionModelEvaluatorError(
+ `OpenAI declined to answer: ${error.questionIds.join(", ")}.`,
+ );
+ }
+ throw error;
+ }
+
+ const confidenceById = readConfidence(result.providerMetadata);
+ const answers: Record<string, DecisionModelAnswer> = {};
+ for (const [id, answer] of Object.entries(result.answers)) {
+ switch (answer.type) {
+ case "choice":
+ answers[id] = {
+ type: "choice",
+ choice: answer.choice,
+ probabilities: answer.probabilities ?? { [answer.choice]: 1 },
+ confidence: confidenceById[id] ?? null,
+ };
+ break;
+ case "score":
+ answers[id] = {
+ type: "score",
+ score: answer.score,
+ probabilities: answer.probabilities ?? {},
+ confidence: confidenceById[id] ?? null,
+ };
+ break;
+ case "boolean":
+ answers[id] = { type: "boolean", probability: answer.probability };
+ break;
+ }
+ }
+
+ const evaluation: DecisionModelEvaluation = {
+ model: result.response.modelId,
+ answers,
+ usage: {
+ inputTokens: result.usage.inputTokens ?? null,
+ outputTokens: result.usage.outputTokens ?? null,
+ },
+ };
+ return evaluation;
+ },
+ };
+}
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index a38684c0fe..de479c968a 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -258,10 +258,31 @@ export const DECISION_MODEL_ADAPTERS: readonly LLMAdapter[] = [
LLMAdapter.TypeSafe,
];
+/** OpenAI models the Decisions API accepts. */
+export const OPENAI_DECISION_MODEL_IDS: readonly string[] = ["gpt-6-luna"];
+
+export function isOpenAIDecisionModel(model: string): boolean {
+ return OPENAI_DECISION_MODEL_IDS.includes(model);
+}
+
export function isDecisionModelAdapter(adapter: string): boolean {
return DECISION_MODEL_ADAPTERS.includes(adapter as LLMAdapter);
}
+/** Adapters that can answer a decision-model evaluator. OpenAI stays a text adapter too. */
+export function supportsDecisionModels(adapter: string): boolean {
+ return adapter === LLMAdapter.TypeSafe || adapter === LLMAdapter.OpenAI;
+}
+
+export function isAllowedDecisionModel(
+ adapter: string,
+ model: string,
+): boolean {
+ if (adapter === LLMAdapter.TypeSafe) return model.length > 0;
+ if (adapter === LLMAdapter.OpenAI) return isOpenAIDecisionModel(model);
+ return false;
+}
+
// Some providers require at least one user message. The persisted-message
// conversion boundary turns a lone message into a user message for them.
export const PROVIDERS_WITH_REQUIRED_USER_MESSAGE: readonly LLMAdapter[] = [
diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
index a68682db9f..21b01a44e2 100644
--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.test.ts
@@ -25,15 +25,19 @@ const request: DecisionModelRequest = {
readiness: {
type: "choice",
instructions: "Is `reply` ready to send?",
- criteria: { ready: null, needs_revision: null },
+ choices: [{ value: "ready" }, { value: "needs_revision" }],
},
frustration: {
type: "score",
instructions: "How frustrated is the customer?",
- criteria: ["Calm", "Frustrated", "Angry"],
+ levels: [
+ { description: "Calm" },
+ { description: "Frustrated" },
+ { description: "Angry" },
+ ],
},
refund: {
- type: "boolean",
+ type: "predicate",
instructions: "Does `question` request a refund?",
},
},
@@ -77,14 +81,25 @@ describe("createTypeSafeDecisionModelClient", () => {
expect(new Headers(init.headers).get("authorization")).toBe(
"Bearer sk-test",
);
- // The SDK sends TypeSafe's native `noul` type for boolean questions.
+ // Score levels become a description list. Predicates go out as TypeSafe's noul type.
expect(JSON.parse(init.body as string)).toEqual({
model: "jev-latest",
state: request.state,
questions: {
- readiness: request.questions.readiness,
- frustration: request.questions.frustration,
- refund: { ...request.questions.refund, type: "noul" },
+ readiness: {
+ type: "choice",
+ instructions: "Is `reply` ready to send?",
+ criteria: { ready: null, needs_revision: null },
+ },
+ frustration: {
+ type: "score",
+ instructions: "How frustrated is the customer?",
+ criteria: ["Calm", "Frustrated", "Angry"],
+ },
+ refund: {
+ type: "noul",
+ instructions: "Does `question` request a refund?",
+ },
},
});
diff --git a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
index da80d48d01..5a17a25923 100644
--- a/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
+++ b/packages/shared/src/server/llm/typesafe/typeSafeDecisionModelClient.ts
@@ -1,16 +1,53 @@
import { createTypeSafeAi } from "@ai-sdk/typesafe-ai";
import {
- experimental_evaluate as evaluate,
- type Experimental_EvaluationQuestion as EvaluationQuestion,
+ experimental_decide as decide,
+ type Experimental_DecisionQuestion,
type JSONValue,
} from "ai";
import type {
DecisionModelAnswer,
DecisionModelClient,
DecisionModelEvaluation,
+ DecisionModelRequestQuestion,
} from "../../evals/decisionModelEvaluatorExecution";
import { createSecureLlmFetch } from "../secureLlmFetch";
+/** TypeSafe reads a description per level, in order. */
+function toTypeSafeQuestion(question: DecisionModelRequestQuestion) {
+ switch (question.type) {
+ case "choice":
+ return {
+ type: "choice" as const,
+ instructions: question.instructions,
+ criteria: Object.fromEntries(
+ question.choices.map((choice) => [
+ choice.value,
+ choice.description ?? null,
+ ]),
+ ),
+ };
+ case "score":
+ return {
+ type: "score" as const,
+ instructions: question.instructions,
+ criteria: question.levels.map((level) => level.description),
+ };
+ case "predicate":
+ return {
+ type: "boolean" as const,
+ instructions: question.instructions,
+ ...(question.criteria
+ ? {
+ criteria: {
+ true: question.criteria.true ?? null,
+ false: question.criteria.false ?? null,
+ },
+ }
+ : {}),
+ };
+ }
+}
+
export function createTypeSafeDecisionModelClient(params: {
apiKey: string;
model: string;
@@ -33,10 +70,15 @@ export function createTypeSafeDecisionModelClient(params: {
return {
evaluate: async (request) => {
- const result = await evaluate({
+ const result = await decide({
model,
state: request.state as Record<string, JSONValue>,
- questions: request.questions as Record<string, EvaluationQuestion>,
+ questions: Object.fromEntries(
+ Object.entries(request.questions).map(([id, question]) => [
+ id,
+ toTypeSafeQuestion(question),
+ ]),
+ ) as Record<string, Experimental_DecisionQuestion>,
maxRetries: 1,
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 29891f81a0..6e845181a4 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -297,8 +297,8 @@ importers:
specifier: ^5.0.8
'@ai-sdk/openai':
- specifier: ^4.0.0
- version: 4.0.78([email protected])
+ specifier: ^4.0.86
+ version: 4.0.86([email protected])
'@ai-sdk/openai-compatible':
specifier: ^3.0.3
version: 3.0.3([email protected])
@@ -369,8 +369,8 @@ importers:
specifier: ^2.4.6
version: 2.4.6
ai:
- specifier: ^7.0.0
- version: 7.0.107([email protected])
+ specifier: ^7.0.130
+ version: 7.0.130([email protected])
ajv:
specifier: ^8.18.0
version: 8.20.0
@@ -1379,8 +1379,8 @@ packages:
peerDependencies:
zod: 4.3.6
- '@ai-sdk/[email protected]':
- resolution: {integrity: sha512-6wJmX/D5OO7tvDJbtFdXdBuRt8AkCnW+FLK+W35UHe/tHgolAa4hclXtJcMSYnlTDy/t5DNRgdesVaoMXtwzow==}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-NprSdFzyVnbEp2SDWtIgsCFIUHFCehOD9S6FQkF/dLrCmnke8w4SOK3KcmUUiEbwtVzlUF8prkaELp+ohcQaQw==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
@@ -1439,6 +1439,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-sbUHayMvjAYV6ELydYXFTr5UBX5qyYmRELqZR3tV0dieUhgUYd8dED9bwEgqZJNJU6V8GGLOlaEpL1ltbNXA/g==}
+ engines: {node: '>=22'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==}
engines: {node: '>=18'}
@@ -1481,6 +1487,12 @@ packages:
peerDependencies:
zod: 4.3.6
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-F7/U1YpVXEfSxRHbxScm0PVdr9lN6GaUWHnt8ws2UjuGeuxWHqCkV25dGpHAkpDHkuIXoC+aak/OqSylKSJmHw==}
+ engines: {node: '>=22'}
+ peerDependencies:
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==}
engines: {node: '>=18'}
@@ -1521,6 +1533,10 @@ packages:
resolution: {integrity: sha512-+GZJIgz1jk86pwEbb3f1BD2bdoSKyWE4Jg4YUc7NMnMozbWemSKYZcw2F4nMaO5qwsL5A8RmioAKW81YktRpIQ==}
engines: {node: '>=22'}
+ '@ai-sdk/[email protected]':
+ resolution: {integrity: sha512-iztFqJTdxNN0ZzqZAAnVPMhRMaL1bfQFa3GIPAffWQ0+dP2FXIkyMD8kHeEI7EpB/C6TR45cqOkrS7YZZ1sATA==}
+ engines: {node: '>=22'}
+
'@ai-sdk/[email protected]':
resolution: {integrity: sha512-LQvqtf6cVla5iyH/LYs9rvl63NDWRRqggz9ilGNmjixhi5TuONAWo+SC6+xT46izn/bpyx9hmXNdMLg3ZSSW0A==}
engines: {node: '>=22'}
@@ -7319,8 +7335,8 @@ packages:
peerDependencies:
zod: 4.3.6
- resolution: {integrity: sha512-PVYQ3W9kR8oYiGQxbg4aFIlhQzKskZm8UV+4E0zB2uKOzFr8ju3bXQncOOZtLsOgJY8vdpgyu3cXtomsfcdQYg==}
+ resolution: {integrity: sha512-QkBkCogl46jXUg0SrSsdqQlOI2ZVMCjfnbBQS8t3gE21U/SO17oNMVFyjmcC9z20wOoYUA+2O2vsso0maoS7dw==}
engines: {node: '>=22'}
peerDependencies:
zod: 4.3.6
@@ -12803,10 +12819,10 @@ snapshots:
'@vercel/oidc': 3.2.0
zod: 4.3.6
dependencies:
- '@ai-sdk/provider': 4.0.17
- '@ai-sdk/provider-utils': 5.0.45([email protected])
+ '@ai-sdk/provider': 4.0.24
+ '@ai-sdk/provider-utils': 5.0.56([email protected])
'@vercel/oidc': 3.2.0
zod: 4.3.6
@@ -12876,6 +12892,12 @@ snapshots:
'@ai-sdk/provider-utils': 5.0.49([email protected])
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 4.0.24
+ '@ai-sdk/provider-utils': 5.0.56([email protected])
+ zod: 4.3.6
+
dependencies:
'@ai-sdk/provider': 1.1.3
@@ -12931,6 +12953,15 @@ snapshots:
undici: 7.29.1
zod: 4.3.6
+ dependencies:
+ '@ai-sdk/provider': 4.0.24
+ '@standard-schema/spec': 1.1.0
+ '@workflow/serde': 4.1.0
+ eventsource-parser: 3.1.0
+ undici: 7.29.1
+ zod: 4.3.6
+
'@ai-sdk/[email protected]':
dependencies:
json-schema: 0.4.0
@@ -12971,6 +13002,10 @@ snapshots:
dependencies:
json-schema: 0.4.0
+ '@ai-sdk/[email protected]':
+ dependencies:
+ json-schema: 0.4.0
+
dependencies:
'@ai-sdk/provider': 4.0.17
@@ -19139,11 +19174,11 @@ snapshots:
'@opentelemetry/api': 1.9.1
zod: 4.3.6
dependencies:
- '@ai-sdk/gateway': 4.0.87([email protected])
- '@ai-sdk/provider': 4.0.17
- '@ai-sdk/provider-utils': 5.0.45([email protected])
+ '@ai-sdk/gateway': 4.0.106([email protected])
+ '@ai-sdk/provider': 4.0.24
+ '@ai-sdk/provider-utils': 5.0.56([email protected])
zod: 4.3.6
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index f1aec88a81..c5394f1f38 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -201,6 +201,12 @@ minimumReleaseAgeExclude:
+ # AI SDK Decisions API (`experimental_decide`); eligible after 2026-10-12 01:47 UTC.
+ - "@ai-sdk/[email protected]"
+ - "@ai-sdk/[email protected]"
+ - "@ai-sdk/[email protected]"
+ - "@ai-sdk/[email protected]"
# Turbo and lint companions; eligible after 2026-10-06 04:01 UTC.
diff --git a/web/src/__tests__/server/llm-api-key.servertest.ts b/web/src/__tests__/server/llm-api-key.servertest.ts
index 080dad7e96..a501e956b3 100644
--- a/web/src/__tests__/server/llm-api-key.servertest.ts
+++ b/web/src/__tests__/server/llm-api-key.servertest.ts
@@ -375,6 +375,87 @@ describe("llmApiKey.all RPC", () => {
});
});
+ it("should treat a decision-model connection as working when the probe is answered", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(
+ JSON.stringify({
+ model: "jev-1.13.0",
+ answers: {
+ kind: {
+ type: "choice",
+ choice: "greeting",
+ confidence: 0.9,
+ probabilities: { greeting: 0.9, other: 0.1 },
+ },
+ },
+ }),
+ { status: 200, headers: { "Content-Type": "application/json" } },
+ ),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ try {
+ const result = await caller.llmApiKey.test({
+ projectId,
+ secretKey: "sk-typesafe",
+ provider: "jev",
+ adapter: LLMAdapter.TypeSafe,
+ baseURL: "https://example.com/typesafe/v1",
+ extraHeaders: { "x-team": "evals" },
+ });
+
+ expect(result).toEqual({ success: true });
+ expect(mockGenerateLLMText).not.toHaveBeenCalled();
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
+ expect(url).toBe("https://example.com/typesafe/v1/systemone");
+ expect(new Headers(init.headers).get("authorization")).toBe(
+ "Bearer sk-typesafe",
+ );
+ expect(new Headers(init.headers).get("x-team")).toBe("evals");
+ expect(JSON.parse(init.body as string)).toEqual({
+ model: "jev-latest",
+ state: { message: "Hello, is anyone there?" },
+ questions: {
+ kind: {
+ type: "choice",
+ instructions: "What kind of message is `message`?",
+ criteria: { greeting: null, other: null },
+ },
+ },
+ });
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
+ it("should report the upstream error when a decision-model connection test fails", async () => {
+ const fetchMock = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ error: { message: "invalid api key" } }), {
+ status: 401,
+ headers: { "Content-Type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ try {
+ const result = await caller.llmApiKey.test({
+ projectId,
+ secretKey: "sk-typesafe",
+ provider: "jev",
+ adapter: LLMAdapter.TypeSafe,
+ baseURL: "https://example.com/typesafe/v1",
+ });
+
+ expect(result.success).toBe(false);
+ expect(result.error).toEqual(expect.any(String));
+ expect(result.error!.length).toBeGreaterThan(0);
+ expect(mockGenerateLLMText).not.toHaveBeenCalled();
+ } finally {
+ vi.unstubAllGlobals();
+ }
+ });
+
it("should block decision-model connections with an internal custom base URL", async () => {
await expect(
caller.llmApiKey.create({
diff --git a/web/src/components/TextActionButton/TextActionButton.stories.tsx b/web/src/components/TextActionButton/TextActionButton.stories.tsx
new file mode 100644
index 0000000000..22667490d5
--- /dev/null
+++ b/web/src/components/TextActionButton/TextActionButton.stories.tsx
@@ -0,0 +1,24 @@
+import { fn } from "storybook/test";
+
+import preview from "../../../.storybook/preview";
+import { TextActionButton } from "./TextActionButton";
+
+const meta = preview.meta({
+ component: TextActionButton,
+ args: { text: "Add message", onClick: fn() },
+});
+
+export const Default = meta.story({});
+
+export const Disabled = meta.story({
+ args: { text: "Add option", disabled: true },
+});
+
+export const Fill = meta.story({
+ parameters: { controls: { disable: true } },
+ render: () => (
+ <div className="w-64 border border-dashed">
+ <TextActionButton text="Add message" width="fill" onClick={fn()} />
+ </div>
+ ),
+});
diff --git a/web/src/components/TextActionButton/TextActionButton.tsx b/web/src/components/TextActionButton/TextActionButton.tsx
new file mode 100644
index 0000000000..9d7b04a5cc
--- /dev/null
+++ b/web/src/components/TextActionButton/TextActionButton.tsx
@@ -0,0 +1,49 @@
+import { Plus, type LucideIcon } from "lucide-react";
+import {
+ type ComponentPropsWithoutRef,
+ type MouseEventHandler,
+ type Ref,
+} from "react";
+
+import { cn } from "@/src/utils/tailwind";
+
+/** Underlined text action with a leading icon. */
+export function TextActionButton({
+ text,
+ icon: Icon = Plus,
+ width = "content",
+ type = "button",
+ disabled,
+ onClick,
+ ref,
+ ...props
+}: {
+ text: string;
+ icon?: LucideIcon;
+ width?: "content" | "fill";
+ ref?: Ref<HTMLButtonElement>;
+} & Omit<
+ ComponentPropsWithoutRef<"button">,
+ "className" | "children" | "type" | "disabled" | "onClick"
+> & {
+ type?: "button" | "submit";
+ disabled?: boolean;
+ onClick?: MouseEventHandler<HTMLButtonElement>;
+ }) {
+ return (
+ <button
+ ref={ref}
+ type={type}
+ disabled={disabled}
+ onClick={onClick}
+ {...props}
+ className={cn(
+ "text-foreground hover:text-foreground inline-flex h-6 items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline disabled:cursor-not-allowed disabled:opacity-50",
+ width === "fill" && "w-full justify-start",
+ )}
+ >
+ <Icon className="icon-base text-icon-foreground shrink-0" aria-hidden />
+ {text}
+ </button>
+ );
+}
diff --git a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
index 2ff554ccc7..064fc31065 100644
--- a/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
+++ b/web/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard.tsx
@@ -32,11 +32,11 @@ export function CollapsibleCard({
<>
<ChevronDown
className={cn(
- "text-foreground-tertiary icon-base shrink-0 translate-y-px transition-transform",
+ "text-foreground-tertiary icon-base shrink-0 transition-transform",
open ? "rotate-0" : "-rotate-90",
)}
/>
- <span className="flex min-w-0 flex-1 items-baseline gap-2">{header}</span>
+ <span className="flex min-w-0 flex-1 items-center gap-2">{header}</span>
</>
);
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
index 4549ffad27..2861aace19 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
@@ -178,7 +178,7 @@ export function EvaluatorGalleryView({
type="button"
variant="secondary"
className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"
- title="Ask TypeSafe Jev typed questions and get calibrated answers in one call. Experimental."
+ title="Ask Jev or the OpenAI Decisions API typed questions in one call. Experimental."
onClick={() =>
onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)
}
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
index 030aaf75a6..870fa68e42 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
@@ -19,9 +19,8 @@ export function EvaluatorGalleryDecisionModelBanner({
<div className="min-w-0 flex-1">
<span className="font-bold">New: decision-model evaluators.</span>{" "}
<span className="text-muted-foreground">
- Ask TypeSafe Jev typed questions about each observation and get
- calibrated answers in one call, at a fraction of an LLM judge&apos;s
- cost. Experimental.
+ Ask Jev or the OpenAI Decisions API typed questions about each
+ observation in one call. Experimental.
</span>
</div>
</div>
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
index 2c30ecbf16..4f7bd2a0b3 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor.tsx
@@ -1,6 +1,7 @@
import { DECISION_MODEL_LIMITS } from "@langfuse/shared";
-import { Plus, Trash2 } from "lucide-react";
+import { Trash2 } from "lucide-react";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Input } from "@/src/components/ui/input";
@@ -36,16 +37,20 @@ export function ChoiceOptionsEditor({
return (
<div className="flex flex-col gap-2">
- <Label className="flex items-center gap-1.5">
+ <Label className="flex items-baseline gap-1.5">
Options
- <InfoTooltip label="About options">
- One label per option. A short description sharpens the boundary
- between options; add an “other” option when the list may not cover
- every input.
- </InfoTooltip>
- <span className="text-muted-foreground text-xs font-normal">
- {options.length} of {DECISION_MODEL_LIMITS.maxChoiceOptions}
+ <span className="inline-flex self-center">
+ <InfoTooltip label="About options">
+ One label per option. A short description sharpens the boundary
+ between options; add an “other” option when the list may not cover
+ every input.
+ </InfoTooltip>
</span>
+ {options.length >= 240 ? (
+ <span className="text-muted-foreground text-xs leading-none font-normal">
+ {options.length} of {DECISION_MODEL_LIMITS.maxChoiceOptions}
+ </span>
+ ) : null}
</Label>
<SortableList
items={options}
@@ -87,18 +92,11 @@ export function ChoiceOptionsEditor({
</div>
)}
/>
- <div>
- <Button
- type="button"
- variant="outline"
- size="sm"
- disabled={options.length >= DECISION_MODEL_LIMITS.maxChoiceOptions}
- onClick={() => onChange([...options, { value: "", description: "" }])}
- >
- <Plus className="icon-base text-icon-foreground mr-1" />
- Add option
- </Button>
- </div>
+ <TextActionButton
+ text="Add option"
+ disabled={options.length >= DECISION_MODEL_LIMITS.maxChoiceOptions}
+ onClick={() => onChange([...options, { value: "", description: "" }])}
+ />
{error ? <p className="text-destructive text-xs">{error}</p> : null}
</div>
);
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx
index 97476870ff..aaee96eba7 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionCard/DecisionModelQuestionCard.tsx
@@ -10,6 +10,7 @@ import { Label } from "@/src/components/ui/label";
import { CollapsibleCard } from "@/src/features/evals/v2/components/CollapsibleCard/CollapsibleCard";
import { ChoiceOptionsEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/ChoiceOptionsEditor/ChoiceOptionsEditor";
import { NoulCriteriaEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/NoulCriteriaEditor/NoulCriteriaEditor";
+import { OpenAIQuestionInstructions } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions";
import { QuestionInstructionsField } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionInstructionsField/QuestionInstructionsField";
import {
QUESTION_TYPE_COPY,
@@ -32,6 +33,8 @@ export type DecisionModelQuestionCardProps = {
onChange: (question: DecisionModelQuestionDraft) => void;
onRemove: (() => void) | null;
errors?: DecisionModelQuestionDraftErrors;
+ /** OpenAI questions are plain text. Jev questions name state fields in backticks. */
+ plainInstructions?: boolean;
};
function writesLine(question: DecisionModelQuestionDraft) {
@@ -95,6 +98,7 @@ export function DecisionModelQuestionCard({
onChange,
onRemove,
errors = {},
+ plainInstructions = false,
}: DecisionModelQuestionCardProps) {
const id = useId();
const copy = QUESTION_TYPE_COPY[question.type];
@@ -109,14 +113,14 @@ export function DecisionModelQuestionCard({
triggerTitle={expanded ? "Collapse question" : "Expand question"}
header={
<span className="flex min-w-0 items-center gap-2">
- <span className="text-muted-foreground shrink-0 font-mono text-xs">
+ <span className="text-muted-foreground shrink-0 font-mono text-xs leading-none">
{index + 1}
</span>
<copy.icon className="icon-base shrink-0" aria-label={copy.label} />
<Badge
variant="secondary"
className={cn(
- "shrink-0 font-mono",
+ "shrink-0 pb-0 font-mono",
!question.scoreName.trim() && "text-dark-yellow",
)}
>
@@ -124,7 +128,7 @@ export function DecisionModelQuestionCard({
</Badge>
<span
className={cn(
- "truncate",
+ "truncate leading-none",
!question.instructions.trim() && "text-muted-foreground italic",
)}
title={summary}
@@ -172,23 +176,42 @@ export function DecisionModelQuestionCard({
/>
</div>
- <QuestionInstructionsField
- value={question.instructions}
- onChange={(instructions) =>
- onChange({
- ...question,
- instructions,
- scoreName:
- question.scoreName === "" ||
- question.scoreName === suggestScoreName(question.instructions)
- ? suggestScoreName(instructions)
- : question.scoreName,
- })
- }
- stateKeys={stateKeys}
- placeholder={copy.example}
- error={errors.instructions}
- />
+ {plainInstructions ? (
+ <OpenAIQuestionInstructions
+ value={question.instructions}
+ onChange={(instructions) =>
+ onChange({
+ ...question,
+ instructions,
+ scoreName:
+ question.scoreName === "" ||
+ question.scoreName === suggestScoreName(question.instructions)
+ ? suggestScoreName(instructions)
+ : question.scoreName,
+ })
+ }
+ placeholder={copy.example}
+ error={errors.instructions}
+ />
+ ) : (
+ <QuestionInstructionsField
+ value={question.instructions}
+ onChange={(instructions) =>
+ onChange({
+ ...question,
+ instructions,
+ scoreName:
+ question.scoreName === "" ||
+ question.scoreName === suggestScoreName(question.instructions)
+ ? suggestScoreName(instructions)
+ : question.scoreName,
+ })
+ }
+ stateKeys={stateKeys}
+ placeholder={copy.example}
+ error={errors.instructions}
+ />
+ )}
<CriteriaEditor
question={question}
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx
index 6762edcc05..2503b792fe 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.stories.tsx
@@ -8,6 +8,7 @@ import { moveItem } from "@/src/features/evals/v2/fns/moveItem";
import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";
import {
DecisionModelQuestionList,
+ OPENAI_QUESTION_EXAMPLES,
QUESTION_EXAMPLES,
} from "./DecisionModelQuestionList";
@@ -61,7 +62,10 @@ function InteractiveQuestionList(args: StoryArgs) {
args.onAdd();
}}
onAddExample={(type) => {
- const question = { id: nextId(), ...QUESTION_EXAMPLES[type] };
+ const question = {
+ id: nextId(),
+ ...(args.examples ?? QUESTION_EXAMPLES)[type],
+ };
setQuestions((current) => [...current, question]);
setExpandedId(question.id);
args.onAddExample(type);
@@ -97,6 +101,19 @@ export const Empty = meta.story({
questions: [],
expandedId: null,
stateKeys: STATE_KEYS,
+ examples: QUESTION_EXAMPLES,
+ },
+ render: (args) => <InteractiveQuestionList {...args} />,
+});
+
+export const EmptyOpenAI = meta.story({
+ args: {
+ ...callbacks,
+ questions: [],
+ expandedId: null,
+ stateKeys: ["input"],
+ plainInstructions: true,
+ examples: OPENAI_QUESTION_EXAMPLES,
},
render: (args) => <InteractiveQuestionList {...args} />,
});
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
index cb2d3fd69b..711de7626e 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList.tsx
@@ -2,8 +2,9 @@ import {
DECISION_MODEL_LIMITS,
DecisionModelQuestionType,
} from "@langfuse/shared";
-import { Plus, Sparkles } from "lucide-react";
+import { Sparkles } from "lucide-react";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Label } from "@/src/components/ui/label";
@@ -48,7 +49,9 @@ export const QUESTION_EXAMPLES: Record<
levels: [
{ description: "Calm, just stating facts" },
{ description: "Frustrated but civil" },
- { description: "Very angry, strong language or threatening to leave" },
+ {
+ description: "Very angry, strong language or threatening to leave",
+ },
],
criteria: { true: "", false: "" },
},
@@ -62,6 +65,25 @@ export const QUESTION_EXAMPLES: Record<
},
};
+/** OpenAI questions cannot name state fields, so these examples stay plain. */
+export const OPENAI_QUESTION_EXAMPLES: Record<
+ DecisionModelQuestionType,
+ Omit<DecisionModelQuestionDraft, "id">
+> = {
+ [DecisionModelQuestionType.CHOICE]: {
+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.CHOICE],
+ instructions: "Is the reply ready to send to the customer?",
+ },
+ [DecisionModelQuestionType.SCORE]: {
+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.SCORE],
+ instructions: "How frustrated is the customer?",
+ },
+ [DecisionModelQuestionType.NOUL]: {
+ ...QUESTION_EXAMPLES[DecisionModelQuestionType.NOUL],
+ instructions: "Does the customer request a refund?",
+ },
+};
+
/**
* The ordered questions of a decision-model evaluator. Every question is
* answered in the same call, so adding one costs only its own tokens; the
@@ -78,6 +100,8 @@ export function DecisionModelQuestionList({
onAddExample,
onRemove,
onReorder,
+ plainInstructions = false,
+ examples = QUESTION_EXAMPLES,
}: {
questions: DecisionModelQuestionDraft[];
expandedId: string | null;
@@ -89,19 +113,28 @@ export function DecisionModelQuestionList({
onAddExample: (type: DecisionModelQuestionType) => void;
onRemove: (id: string) => void;
onReorder: (fromIndex: number, toIndex: number) => void;
+ plainInstructions?: boolean;
+ examples?: Record<
+ DecisionModelQuestionType,
+ Omit<DecisionModelQuestionDraft, "id">
+ >;
}) {
return (
<div className="flex flex-col gap-3">
- <Label className="flex items-center gap-1.5">
+ <Label className="flex items-baseline gap-1.5">
Questions
- <InfoTooltip label="About questions">
- Each question is one snap judgment about the state and writes one
- score. All questions are answered in a single model call, so a second
- or tenth question costs only its own tokens.
- </InfoTooltip>
- <span className="text-muted-foreground text-xs font-normal">
- {questions.length} of {DECISION_MODEL_LIMITS.maxQuestions}
+ <span className="inline-flex self-center">
+ <InfoTooltip label="About questions">
+ Each question is one snap judgment about the state and writes one
+ score. All questions are answered in a single model call, so a
+ second or tenth question costs only its own tokens.
+ </InfoTooltip>
</span>
+ {questions.length > 40 ? (
+ <span className="text-muted-foreground text-xs leading-none font-normal">
+ {questions.length} of {DECISION_MODEL_LIMITS.maxQuestions}
+ </span>
+ ) : null}
</Label>
{questions.length === 0 ? (
@@ -122,8 +155,7 @@ export function DecisionModelQuestionList({
onClick={() => onAddExample(type)}
>
<Sparkles className="icon-base text-icon-foreground mr-1" />
- {copy.label}: “
- {QUESTION_EXAMPLES[type].instructions.replace(/`/g, "")}”
+ {copy.label}: “{examples[type].instructions}”
</Button>
);
})}
@@ -152,20 +184,18 @@ export function DecisionModelQuestionList({
questions.length > 1 ? () => onRemove(question.id) : null
}
errors={errorsById[question.id]}
+ plainInstructions={plainInstructions}
/>
)}
/>
)}
- <div className="flex flex-wrap items-center gap-3">
- <Button type="button" variant="outline" size="sm" onClick={onAdd}>
- <Plus className="icon-base text-icon-foreground mr-1" />
- Add question
- </Button>
- <span className="text-muted-foreground text-xs">
+ <div className="flex items-center gap-1.5">
+ <TextActionButton text="Add question" onClick={onAdd} />
+ <InfoTooltip label="About extra questions">
Answered together in one call; extra questions add only their own
tokens.
- </span>
+ </InfoTooltip>
</div>
</div>
);
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx
index f835b1f4c7..790759f00f 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionSummary/DecisionModelQuestionSummary.tsx
@@ -37,7 +37,9 @@ function Criteria({ question }: { question: DecisionModelQuestion }) {
<span className="text-muted-foreground shrink-0 font-mono">
{index}
</span>
- <span>{entryText(level.description)}</span>
+ {level.description ? (
+ <span>{entryText(level.description)}</span>
+ ) : null}
</li>
))}
</ol>
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
index 30a6126d55..d85ec06f11 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.stories.tsx
@@ -65,7 +65,7 @@ export const LowConfidence = meta.story({
},
});
-/** Expected values at both ends of the scale keep their labels inside the track. */
+/** Scores at the ends of the scale highlight that level's bar. */
export const ScoreAtEdges = meta.story({
args: {
results: [
@@ -97,7 +97,7 @@ export const ScoreAtEdges = meta.story({
},
});
-/** Maximum-length scales keep descriptions available without rendering them inline. */
+/** A long scale lists every level as its own bar. */
export const TenLevels = meta.story({
name: "(Test) Ten Levels",
args: {
@@ -138,15 +138,9 @@ export const TenLevels = meta.story({
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
- const page = within(canvasElement.ownerDocument.body);
- const levelDescription = canvas.getByRole("button", {
- name: "Level 0 description",
- });
-
- levelDescription.focus();
-
- await expect(await page.findByRole("tooltip")).toHaveTextContent(
- "No technical detail",
- );
+ await expect(canvas.getByText("No technical detail")).toBeInTheDocument();
+ await expect(
+ canvas.getByText("Explains system behavior"),
+ ).toBeInTheDocument();
},
});
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
index 65801a86a1..45814186d6 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView.tsx
@@ -1,12 +1,4 @@
-import { DECISION_MODEL_LIMITS } from "@langfuse/shared";
-import { InfoIcon } from "lucide-react";
-
import { Badge } from "@/src/components/ui/badge";
-import {
- Tooltip,
- TooltipContent,
- TooltipTrigger,
-} from "@/src/components/ui/tooltip";
import { QUESTION_TYPE_COPY } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/QuestionTypeSelector/QuestionTypeSelector";
import { cn } from "@/src/utils/tailwind";
@@ -50,27 +42,6 @@ function ConfidenceBadge({ confidence }: { confidence: number }) {
);
}
-function LevelDescriptionTooltip({
- index,
- description,
-}: {
- index: number;
- description: string;
-}) {
- return (
- <Tooltip>
- <TooltipTrigger
- type="button"
- className="text-muted-foreground focus-visible:ring-ring cursor-help rounded-sm focus-visible:ring-2 focus-visible:outline-none"
- aria-label={`Level ${index} description`}
- >
- <InfoIcon className="icon-base" aria-hidden="true" />
- </TooltipTrigger>
- <TooltipContent className="max-w-xs">{description}</TooltipContent>
- </Tooltip>
- );
-}
-
function ProbabilityBars({
entries,
highlight,
@@ -120,74 +91,6 @@ function ProbabilityBars({
);
}
-function ScoreDistribution({
- score,
- levels,
- probabilities,
-}: {
- score: number;
- levels: string[];
- probabilities: Record<string, number>;
-}) {
- const top = Math.max(levels.length - 1, 1);
- const nearest = Math.min(Math.max(Math.round(score), 0), levels.length - 1);
- const position = Math.min(Math.max(score / top, 0), 1) * 100;
- const showDescriptions = levels.length < DECISION_MODEL_LIMITS.maxScoreLevels;
- return (
- <div className="flex flex-col gap-2">
- <div className="relative pt-4">
- <div className="bg-muted h-2 rounded-full" />
- <span
- className="bg-primary-accent absolute top-3 h-4 w-0.5 -translate-x-1/2 rounded"
- style={{ left: `${position}%` }}
- aria-hidden="true"
- />
- <span
- className="text-primary-accent absolute top-0 font-mono text-xs font-bold"
- style={{
- left: `${position}%`,
- transform: `translateX(-${position}%)`,
- }}
- >
- {score.toFixed(2)}
- </span>
- </div>
- <ol
- className="grid gap-1"
- style={{
- gridTemplateColumns: `repeat(${levels.length}, minmax(0, 1fr))`,
- }}
- >
- {levels.map((description, index) => (
- <li
- key={index}
- className={cn(
- "flex flex-col gap-0.5 text-xs",
- index === nearest ? "" : "text-muted-foreground",
- )}
- >
- <span className="flex items-center gap-0.5 font-mono">
- {index}
- {!showDescriptions ? (
- <LevelDescriptionTooltip
- index={index}
- description={description}
- />
- ) : null}
- · {percent(probabilities[String(index)] ?? 0)}
- </span>
- {showDescriptions ? (
- <span className="line-clamp-2" title={description}>
- {description}
- </span>
- ) : null}
- </li>
- ))}
- </ol>
- </div>
- );
-}
-
function NoulGauge({ probability }: { probability: number }) {
const leaning = probability >= 0.5 ? "yes" : "no";
return (
@@ -213,12 +116,12 @@ function NoulGauge({ probability }: { probability: number }) {
);
}
-function resultValue(result: DecisionModelQuestionResult) {
+function resultValue(
+ result: Exclude<DecisionModelQuestionResult, { type: "score" }>,
+) {
switch (result.type) {
case "choice":
return result.choice;
- case "score":
- return result.score.toFixed(2);
case "noul":
return result.probability.toFixed(2);
}
@@ -235,14 +138,23 @@ function ResultVisual({ result }: { result: DecisionModelQuestionResult }) {
highlight={result.choice}
/>
);
- case "score":
+ case "score": {
+ const nearest = Math.min(
+ Math.max(Math.round(result.score), 0),
+ Math.max(result.levels.length - 1, 0),
+ );
return (
- <ScoreDistribution
- score={result.score}
- levels={result.levels}
- probabilities={result.probabilities}
+ <ProbabilityBars
+ entries={result.levels
+ .map((description, index) => ({
+ label: description || String(index),
+ value: result.probabilities[String(index)] ?? 0,
+ }))
+ .sort((left, right) => right.value - left.value)}
+ highlight={result.levels[nearest] || String(nearest)}
/>
);
+ }
case "noul":
return <NoulGauge probability={result.probability} />;
}
@@ -269,7 +181,9 @@ function ResultRow({ result }: { result: DecisionModelQuestionResult }) {
{result.type !== "noul" && result.confidence !== null ? (
<ConfidenceBadge confidence={result.confidence} />
) : null}
- <Badge className="font-mono">{resultValue(result)}</Badge>
+ {result.type === "score" ? null : (
+ <Badge className="font-mono">{resultValue(result)}</Badge>
+ )}
</span>
</div>
<ResultVisual result={result} />
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx
index 03f350fff4..c6ecc10eac 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor.tsx
@@ -1,52 +1,17 @@
-import { useMemo, useState } from "react";
-import {
- DecisionModelStateKeySchema,
- deepParseJsonIterative,
-} from "@langfuse/shared";
-import { ChevronDown, Plus, TriangleAlert } from "lucide-react";
-
-import { Button } from "@/src/components/ui/button";
-import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";
+import { DecisionModelStateKeySchema } from "@langfuse/shared";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
+import { DecisionModelStatePreview } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview";
import { EditableVariableMapping } from "@/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping";
-import { extractVariableMappingValue } from "@/src/features/evals/v2/fns/variableMapping/extractVariableMappingValue";
import type {
ActiveVariableMapping,
VariableFieldState,
} from "@/src/features/evals/v2/types/variableMapping";
-import { cn } from "@/src/utils/tailwind";
export type DecisionModelStateField = {
key: string;
fieldState: VariableFieldState;
};
-/** TypeSafe accepts ~32k for state plus the longest question; warn well below. */
-const STATE_SIZE_WARNING_CHARS = 24_000;
-
-/**
- * Builds the JSON object the model receives from the mapped fields and the
- * sample observation, mirroring the worker's state builder.
- */
-function buildStatePreview(
- fields: DecisionModelStateField[],
- sourceObject: Record<string, unknown> | null,
-): Record<string, unknown> | null {
- if (!sourceObject) return null;
- const state: Record<string, unknown> = {};
- for (const field of fields) {
- const columnId = field.fieldState.selectedColumnId;
- if (!columnId) continue;
- const { value } = extractVariableMappingValue(
- sourceObject,
- columnId,
- field.fieldState.jsonSelector ?? undefined,
- );
- if (value === null || value === undefined || value === "") continue;
- state[field.key] = deepParseJsonIterative(value) ?? value;
- }
- return state;
-}
-
/** Mirrors the shared state-key rule plus uniqueness within this evaluator. */
function validateStateKey(
next: string,
@@ -92,14 +57,6 @@ export function DecisionModelStateEditor({
hasMatchingObservations: boolean;
sourceUnavailableMessage?: string;
}) {
- const [previewOpen, setPreviewOpen] = useState(false);
- const statePreview = useMemo(
- () => buildStatePreview(fields, sourceObject),
- [fields, sourceObject],
- );
- const stateSize = statePreview ? JSON.stringify(statePreview).length : 0;
- const tooLarge = stateSize > STATE_SIZE_WARNING_CHARS;
-
return (
<div className="flex flex-col gap-3">
<EditableVariableMapping
@@ -121,63 +78,9 @@ export function DecisionModelStateEditor({
sourceUnavailableMessage={sourceUnavailableMessage}
/>
- <div>
- <Button type="button" variant="outline" size="sm" onClick={onAddField}>
- <Plus className="icon-base text-icon-foreground mr-1" />
- Add field
- </Button>
- </div>
+ <TextActionButton text="Add field" onClick={onAddField} />
- <div className="rounded-md border">
- <button
- type="button"
- className="hover:bg-muted/50 flex w-full items-center justify-between gap-2 px-3 py-2 text-left text-sm"
- aria-expanded={previewOpen}
- onClick={() => setPreviewOpen((open) => !open)}
- >
- <span className="flex items-center gap-2">
- <span className="font-bold">State sent to the model</span>
- {statePreview ? (
- <span
- className={cn(
- "text-muted-foreground font-mono text-xs",
- tooLarge && "text-dark-yellow flex items-center gap-1",
- )}
- >
- {tooLarge ? <TriangleAlert className="icon-base" /> : null}
- {Object.keys(statePreview).length} field
- {Object.keys(statePreview).length === 1 ? "" : "s"} ·{" "}
- {stateSize.toLocaleString()} chars
- {tooLarge ? " · trim fields, ~32k limit" : ""}
- </span>
- ) : (
- <span className="text-muted-foreground text-xs">
- select a sample observation to preview
- </span>
- )}
- </span>
- <ChevronDown
- className={cn(
- "text-foreground-tertiary icon-base shrink-0 transition-transform",
- previewOpen ? "rotate-180" : "rotate-0",
- )}
- />
- </button>
- {previewOpen && statePreview ? (
- <div className="border-t">
- <PrettyJsonView
- json={statePreview}
- currentView="pretty"
- isLoading={false}
- showNullValues={true}
- stickyTopLevelKey={false}
- showObservationTypeBadge={false}
- scrollable={true}
- className="max-h-80 [&_.border]:border-0 [&_.rounded-sm]:rounded-none"
- />
- </div>
- ) : null}
- </div>
+ <DecisionModelStatePreview fields={fields} sourceObject={sourceObject} />
</div>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsx
new file mode 100644
index 0000000000..fea8fcdf8a
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview.tsx
@@ -0,0 +1,112 @@
+import { useMemo, useState } from "react";
+import { deepParseJsonIterative } from "@langfuse/shared";
+import { ChevronDown, TriangleAlert } from "lucide-react";
+
+import { PrettyJsonView } from "@/src/components/ui/PrettyJsonView";
+import { extractVariableMappingValue } from "@/src/features/evals/v2/fns/variableMapping/extractVariableMappingValue";
+import type { VariableFieldState } from "@/src/features/evals/v2/types/variableMapping";
+import { cn } from "@/src/utils/tailwind";
+
+type PreviewField = {
+ key: string;
+ fieldState: VariableFieldState;
+};
+
+/** TypeSafe accepts ~32k for state plus the longest question; warn well below. */
+const STATE_SIZE_WARNING_CHARS = 24_000;
+
+function buildStatePreview(
+ fields: PreviewField[],
+ sourceObject: Record<string, unknown> | null,
+): Record<string, unknown> | null {
+ if (!sourceObject) return null;
+ const state: Record<string, unknown> = {};
+ for (const field of fields) {
+ const columnId = field.fieldState.selectedColumnId;
+ if (!columnId) continue;
+ const { value } = extractVariableMappingValue(
+ sourceObject,
+ columnId,
+ field.fieldState.jsonSelector ?? undefined,
+ );
+ if (value === null || value === undefined || value === "") continue;
+ state[field.key] = deepParseJsonIterative(value) ?? value;
+ }
+ return state;
+}
+
+/** The JSON object that will be sent, assembled from the mapped fields. */
+export function DecisionModelStatePreview({
+ fields,
+ sourceObject,
+ questions,
+ title = "State sent to the model",
+}: {
+ fields: PreviewField[];
+ sourceObject: Record<string, unknown> | null;
+ /** Included beside the state. OpenAI sends these on the same request. */
+ questions?: unknown[];
+ title?: string;
+}) {
+ const [open, setOpen] = useState(false);
+ const statePreview = useMemo(() => {
+ const state = buildStatePreview(fields, sourceObject);
+ if (!state) return null;
+ return questions ? { ...state, questions } : state;
+ }, [fields, questions, sourceObject]);
+ const stateSize = statePreview ? JSON.stringify(statePreview).length : 0;
+ const tooLarge = stateSize > STATE_SIZE_WARNING_CHARS;
+
+ return (
+ <div className="rounded-md border">
+ <button
+ type="button"
+ className="hover:bg-muted/50 flex w-full items-center justify-between gap-2 px-3 py-2 text-left text-sm"
+ aria-expanded={open}
+ onClick={() => setOpen((current) => !current)}
+ >
+ <span className="flex items-center gap-2">
+ <span className="font-bold">{title}</span>
+ {statePreview ? (
+ <span
+ className={cn(
+ "text-muted-foreground font-mono text-xs",
+ tooLarge && "text-dark-yellow flex items-center gap-1",
+ )}
+ >
+ {tooLarge ? <TriangleAlert className="icon-base" /> : null}
+ {Object.keys(statePreview).length} field
+ {Object.keys(statePreview).length === 1 ? "" : "s"} ·{" "}
+ {stateSize.toLocaleString()} chars
+ {tooLarge ? " · trim fields, ~32k limit" : ""}
+ </span>
+ ) : (
+ <span className="text-muted-foreground text-xs">
+ select a sample observation to preview
+ </span>
+ )}
+ </span>
+ <ChevronDown
+ className={cn(
+ "text-foreground-tertiary icon-base shrink-0 transition-transform",
+ open ? "rotate-180" : "rotate-0",
+ )}
+ />
+ </button>
+ {open && statePreview ? (
+ <div className="border-t">
+ <PrettyJsonView
+ json={statePreview}
+ currentView="pretty"
+ isLoading={false}
+ showNullValues={true}
+ stickyTopLevelKey={false}
+ showObservationTypeBadge={false}
+ scrollable={true}
+ className="max-h-80 [&_.border]:border-0 [&_.rounded-sm]:rounded-none"
+ />
+ </div>
+ ) : null}
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsx
new file mode 100644
index 0000000000..a5cc8ada05
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.stories.tsx
@@ -0,0 +1,86 @@
+import { useState } from "react";
+import { DecisionModelQuestionType } from "@langfuse/shared";
+import { fn } from "storybook/test";
+
+import preview from "../../../../../../../../.storybook/preview";
+import type {
+ ActiveVariableMapping,
+ VariableFieldState,
+} from "@/src/features/evals/v2/types/variableMapping";
+import { OpenAIDecisionModelStateEditor } from "./OpenAIDecisionModelStateEditor";
+
+const QUESTIONS = [
+ {
+ id: "severity",
+ type: DecisionModelQuestionType.SCORE,
+ scoreName: "severity",
+ instructions: "How severe is the issue in the input?",
+ options: [
+ { value: "", description: "" },
+ { value: "", description: "" },
+ ],
+ levels: [
+ { description: "Appearance only" },
+ { description: "Cannot continue" },
+ ],
+ criteria: { true: "", false: "" },
+ },
+];
+
+const meta = preview.meta({ component: OpenAIDecisionModelStateEditor });
+
+const SAMPLE = {
+ input: "I was charged twice. Please refund the extra charge.",
+ output: "I've issued the refund.",
+};
+
+export const MappedInput = meta.story({
+ args: {
+ field: {
+ key: "input",
+ fieldState: { selectedColumnId: "input", jsonSelector: null },
+ },
+ activeMapping: null,
+ onActiveMappingChange: fn(),
+ onChangeField: fn(),
+ questions: QUESTIONS,
+ sourceObject: SAMPLE,
+ hasMatchingObservations: true,
+ },
+ render: (args) => {
+ const [fieldState, setFieldState] = useState<VariableFieldState>(
+ args.field.fieldState,
+ );
+ const [activeMapping, setActiveMapping] =
+ useState<ActiveVariableMapping>(null);
+ return (
+ <OpenAIDecisionModelStateEditor
+ {...args}
+ field={{ key: "input", fieldState }}
+ activeMapping={activeMapping}
+ onActiveMappingChange={setActiveMapping}
+ onChangeField={(next) => {
+ setFieldState(next);
+ args.onChangeField(next);
+ }}
+ />
+ );
+ },
+});
+
+export const NoSample = meta.story({
+ args: {
+ field: {
+ key: "input",
+ fieldState: { selectedColumnId: "input", jsonSelector: null },
+ },
+ activeMapping: null,
+ onActiveMappingChange: fn(),
+ onChangeField: fn(),
+ questions: QUESTIONS,
+ sourceObject: null,
+ hasMatchingObservations: false,
+ sourceUnavailableMessage:
+ "Select a sample observation in the test panel to preview mapped values.",
+ },
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsx
new file mode 100644
index 0000000000..27718510bb
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor.tsx
@@ -0,0 +1,55 @@
+import { DecisionModelStatePreview } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStatePreview/DecisionModelStatePreview";
+import type { DecisionModelStateField } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor";
+import { previewOpenAIDecisionQuestions } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";
+import { EditableVariableMapping } from "@/src/features/evals/v2/components/VariableMapping/components/EditableVariableMapping/EditableVariableMapping";
+import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";
+import type {
+ ActiveVariableMapping,
+ VariableFieldState,
+} from "@/src/features/evals/v2/types/variableMapping";
+
+/**
+ * OpenAI decisions take one `input` plus the questions. The question text does
+ * not name fields, so this mapping cannot be renamed or extended. The preview
+ * is that input beside the questions posted with it.
+ */
+export function OpenAIDecisionModelStateEditor({
+ field,
+ questions,
+ activeMapping,
+ onActiveMappingChange,
+ onChangeField,
+ sourceObject,
+ hasMatchingObservations,
+ sourceUnavailableMessage,
+}: {
+ field: DecisionModelStateField;
+ questions: DecisionModelQuestionDraft[];
+ activeMapping: ActiveVariableMapping;
+ onActiveMappingChange: (activeMapping: ActiveVariableMapping) => void;
+ onChangeField: (fieldState: VariableFieldState) => void;
+ sourceObject: Record<string, unknown> | null;
+ hasMatchingObservations: boolean;
+ sourceUnavailableMessage?: string;
+}) {
+ return (
+ <div className="flex flex-col gap-3">
+ <EditableVariableMapping
+ mappings={[{ variable: "input", fieldState: field.fieldState }]}
+ variableDisplay="stateKey"
+ activeMapping={activeMapping}
+ onActiveMappingChange={onActiveMappingChange}
+ onChangeField={(_key, fieldState) => onChangeField(fieldState)}
+ sourceObject={sourceObject}
+ hasMatchingObservations={hasMatchingObservations}
+ sourceUnavailableMessage={sourceUnavailableMessage}
+ />
+ <DecisionModelStatePreview
+ fields={[field]}
+ sourceObject={sourceObject}
+ questions={previewOpenAIDecisionQuestions(questions)}
+ title="Sent to the model"
+ />
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsx
new file mode 100644
index 0000000000..3a8350305e
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.stories.tsx
@@ -0,0 +1,43 @@
+import { useState } from "react";
+import { fn } from "storybook/test";
+
+import preview from "../../../../../../../../.storybook/preview";
+import { OpenAIQuestionInstructions } from "./OpenAIQuestionInstructions";
+
+const meta = preview.meta({ component: OpenAIQuestionInstructions });
+
+export const Default = meta.story({
+ args: {
+ value: "Which team should handle this ticket?",
+ onChange: fn(),
+ },
+ render: (args) => {
+ const [value, setValue] = useState(args.value);
+ return (
+ <OpenAIQuestionInstructions
+ {...args}
+ value={value}
+ onChange={(next) => {
+ setValue(next);
+ args.onChange(next);
+ }}
+ />
+ );
+ },
+});
+
+export const Empty = meta.story({
+ args: {
+ value: "",
+ placeholder: "Which team should handle this ticket?",
+ onChange: fn(),
+ },
+});
+
+export const Error = meta.story({
+ args: {
+ value: "",
+ error: "Every question needs instructions.",
+ onChange: fn(),
+ },
+});
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsx
new file mode 100644
index 0000000000..a57f5bfc5e
--- /dev/null
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIQuestionInstructions/OpenAIQuestionInstructions.tsx
@@ -0,0 +1,46 @@
+import { useId } from "react";
+
+import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
+import { Label } from "@/src/components/ui/label";
+import { Textarea } from "@/src/components/ui/textarea";
+import { cn } from "@/src/utils/tailwind";
+
+/**
+ * Plain question text for OpenAI decisions. The API has no variable syntax;
+ * the mapped `input` object is the only evidence the model receives.
+ */
+export function OpenAIQuestionInstructions({
+ value,
+ onChange,
+ placeholder,
+ error,
+}: {
+ value: string;
+ onChange: (value: string) => void;
+ placeholder?: string;
+ error?: string;
+}) {
+ const id = useId();
+
+ return (
+ <div className="flex flex-col gap-1.5">
+ <Label htmlFor={id} className="flex items-center gap-1.5">
+ Question
+ <InfoTooltip label="About questions">
+ Ask for one snap judgment. This text is sent as written. The model
+ reads the input object, so the question does not insert fields.
+ </InfoTooltip>
+ </Label>
+ <Textarea
+ id={id}
+ value={value}
+ onChange={(event) => onChange(event.target.value)}
+ placeholder={placeholder}
+ rows={3}
+ aria-invalid={Boolean(error)}
+ className={cn(error && "border-destructive")}
+ />
+ {error ? <p className="text-destructive text-xs">{error}</p> : null}
+ </div>
+ );
+}
diff --git a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
index 58dd22164e..207f0dccda 100644
--- a/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/DecisionModel/ScoreLevelsEditor/ScoreLevelsEditor.tsx
@@ -1,6 +1,7 @@
import { DECISION_MODEL_LIMITS } from "@langfuse/shared";
-import { Plus, Trash2 } from "lucide-react";
+import { Trash2 } from "lucide-react";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import { Button } from "@/src/components/ui/button";
import { InfoTooltip } from "@/src/components/ui/InfoTooltip/InfoTooltip";
import { Input } from "@/src/components/ui/input";
@@ -19,9 +20,8 @@ function levelPlaceholder(index: number, count: number) {
}
/**
- * Ordered levels of a Score question, low to high. The level number is its
- * position; the model only sees the descriptions, so each one has to describe
- * a recognisable situation rather than a degree.
+ * Ordered score levels, low to high. The position is the level number. The
+ * description is what the model reads. The stored score is that position.
*/
export function ScoreLevelsEditor({
levels,
@@ -32,23 +32,27 @@ export function ScoreLevelsEditor({
onChange: (levels: ScoreLevelDraft[]) => void;
error?: string;
}) {
- const update = (index: number, description: string) =>
- onChange(levels.map((level, i) => (i === index ? { description } : level)));
+ const update = (index: number, patch: Partial<ScoreLevelDraft>) =>
+ onChange(
+ levels.map((level, i) => (i === index ? { ...level, ...patch } : level)),
+ );
const remove = (index: number) =>
onChange(levels.filter((_, i) => i !== index));
const canRemove = levels.length > DECISION_MODEL_LIMITS.minScoreLevels;
return (
<div className="flex flex-col gap-2">
- <Label className="flex items-center gap-1.5">
+ <Label className="flex items-baseline gap-1.5">
Levels, low to high
- <InfoTooltip label="About levels">
- Each level is one point on the scale. The model judges every level on
- its own against the state and returns a position between them, so
- describe situations (“broken, but a workaround exists”), not degrees
- (“moderately severe”). Two to ten levels; drag to reorder.
- </InfoTooltip>
- <span className="text-muted-foreground text-xs font-normal">
+ <span className="inline-flex self-center">
+ <InfoTooltip label="About levels">
+ Each level is one point on the scale. The model judges every level
+ on its own against the state and returns a position between them, so
+ describe situations (“broken, but a workaround exists”), not degrees
+ (“moderately severe”). Two to ten levels; drag to reorder.
+ </InfoTooltip>
+ </span>
+ <span className="text-muted-foreground text-xs leading-none font-normal">
{levels.length} of {DECISION_MODEL_LIMITS.maxScoreLevels}
</span>
</Label>
@@ -67,7 +71,9 @@ export function ScoreLevelsEditor({
</span>
<Input
value={level.description}
- onChange={(event) => update(index, event.target.value)}
+ onChange={(event) =>
+ update(index, { description: event.target.value })
+ }
placeholder={levelPlaceholder(index, levels.length)}
aria-label={`Level ${index} description`}
/>
@@ -89,21 +95,16 @@ export function ScoreLevelsEditor({
</div>
)}
/>
- <div className="flex items-center gap-3">
- <Button
- type="button"
- variant="outline"
- size="sm"
+ <div className="flex items-center gap-1.5">
+ <TextActionButton
+ text="Add level"
disabled={levels.length >= DECISION_MODEL_LIMITS.maxScoreLevels}
onClick={() => onChange([...levels, { description: "" }])}
- >
- <Plus className="icon-base text-icon-foreground mr-1" />
- Add level
- </Button>
- <span className="text-muted-foreground text-xs">
+ />
+ <InfoTooltip label="About the score">
The score is the probability-weighted average of the level numbers, so
it can be a decimal from 0 to {Math.max(levels.length - 1, 0)}.
- </span>
+ </InfoTooltip>
</div>
{error ? <p className="text-destructive text-xs">{error}</p> : null}
</div>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx
index ba942b224a..044b794806 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelQuestionsEditor/DecisionModelQuestionsEditor.tsx
@@ -4,18 +4,24 @@ import { useShallow } from "zustand/react/shallow";
import {
DecisionModelQuestionList,
+ OPENAI_QUESTION_EXAMPLES,
QUESTION_EXAMPLES,
} from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelQuestionList/DecisionModelQuestionList";
import {
createEmptyQuestion,
getQuestionDraftErrors,
+ usesPlainDecisionInstructions,
} from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";
+import { preferredDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
+import { api } from "@/src/utils/api";
import { safeRandomUUID } from "@/src/utils/safe-random-uuid";
export function DecisionModelQuestionsEditor({
+ projectId,
store,
}: {
+ projectId: string;
store: EvaluatorSetupStore;
}) {
const state = useStore(
@@ -24,9 +30,23 @@ export function DecisionModelQuestionsEditor({
questions: state.questions,
expandedQuestionId: state.expandedQuestionId,
stateKeys: state.stateKeys,
+ selectedModel: state.selectedModel,
actions: state.actions,
})),
);
+ const connections = api.llmApiKey.all.useQuery({
+ projectId,
+ includeDecisionModels: true,
+ });
+ const model =
+ state.selectedModel ?? preferredDecisionModel(connections.data?.data ?? []);
+ const adapter = connections.data?.data.find(
+ (connection) => connection.provider === model?.provider,
+ )?.adapter;
+ const openaiDecision = usesPlainDecisionInstructions({
+ model: model?.model,
+ adapter,
+ });
const errorsById = useMemo(() => {
const errors = getQuestionDraftErrors(state.questions);
for (const question of state.questions) {
@@ -37,6 +57,10 @@ export function DecisionModelQuestionsEditor({
return errors;
}, [state.questions]);
+ const examples = openaiDecision
+ ? OPENAI_QUESTION_EXAMPLES
+ : QUESTION_EXAMPLES;
+
return (
<DecisionModelQuestionList
questions={state.questions}
@@ -46,14 +70,16 @@ export function DecisionModelQuestionsEditor({
onExpandedChange={state.actions.setExpandedQuestionId}
onChange={state.actions.setQuestion}
onAdd={() => state.actions.addQuestion(createEmptyQuestion())}
+ examples={examples}
onAddExample={(type) =>
state.actions.addQuestion({
id: safeRandomUUID(),
- ...QUESTION_EXAMPLES[type],
+ ...examples[type],
})
}
onRemove={state.actions.removeQuestion}
onReorder={state.actions.reorderQuestion}
+ plainInstructions={openaiDecision}
/>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx
index 2ec3947d37..98e94d91bf 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.clienttest.tsx
@@ -34,6 +34,8 @@ describe("DecisionModelSelector", () => {
});
beforeEach(() => {
+ connection.provider = "jev";
+ connection.adapter = "typesafe";
connection.withDefaultModels = true;
connection.customModels = [];
});
@@ -55,9 +57,31 @@ describe("DecisionModelSelector", () => {
{ wrapper: LayerProvider },
);
- return screen.getByRole("combobox", { name: "Decision model" });
+ return screen.getByRole("button");
};
+ it("selects gpt-6-luna when nothing is selected and the connection offers it", () => {
+ connection.provider = "openai";
+ connection.adapter = "openai";
+ const store = createEvaluatorSetupStore({
+ initialEvaluator: null,
+ initialType: "DECISION_MODEL",
+ mode: "create",
+ });
+
+ render(
+ <DecisionModelSelector
+ projectId="project"
+ store={store}
+ onConfigureProviders={vi.fn()}
+ />,
+ { wrapper: LayerProvider },
+ );
+
+ expect(screen.getByRole("button")).toHaveTextContent("openai / gpt-6-luna");
+ expect(store.getState().selectedModel).toBeNull();
+ });
+
it.each([
["lists only default models", true],
["lists no models", false],
@@ -67,7 +91,7 @@ describe("DecisionModelSelector", () => {
connection.withDefaultModels = withDefaultModels;
expect(renderWithSelectedModel("jev-1.13.0")).toHaveTextContent(
- "jev: jev-1.13.0",
+ "jev / jev-1.13.0",
);
},
);
@@ -84,7 +108,7 @@ describe("DecisionModelSelector", () => {
connection.customModels = customModels;
expect(renderWithSelectedModel("jev-latest").textContent).toBe(
- "jev: jev-latest",
+ "jev / jev-latest",
);
},
);
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
index 1b33c28cab..f202bb734c 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DecisionModelSelector/DecisionModelSelector.tsx
@@ -1,22 +1,27 @@
-import { isDecisionModelAdapter, supportedModels } from "@langfuse/shared";
+import { useState } from "react";
+import {
+ isAllowedDecisionModel,
+ isOpenAIDecisionModel,
+ OPENAI_DECISION_MODEL_IDS,
+} from "@langfuse/shared";
import { useStore } from "zustand";
import { Button } from "@/src/components/ui/button";
-import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";
+import { PopoverTrigger } from "@/src/components/ui/popover";
+import {
+ JudgeModelPicker,
+ JudgeModelPickerTrigger,
+} from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";
+import {
+ decisionModelsByProvider,
+ preferredDecisionModel,
+} from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
import { api } from "@/src/utils/api";
-const SEPARATOR = "::";
-
-const toOption = (provider: string, model: string) => ({
- value: `${provider}${SEPARATOR}${model}`,
- label: `${provider}: ${model}`,
-});
-
/**
- * Picks the decision-model connection and model version. Decision models have
- * no project default and no sampling parameters, so this replaces the judge
- * model picker with a flat list of `provider :: model` options.
+ * Same content-width model menu as an LLM-as-a-judge, limited to decision
+ * models. No project default and no sampling parameters.
*/
export function DecisionModelSelector({
projectId,
@@ -27,6 +32,7 @@ export function DecisionModelSelector({
store: EvaluatorSetupStore;
onConfigureProviders: () => void;
}) {
+ const [open, setOpen] = useState(false);
const selectedModel = useStore(store, (state) => state.selectedModel);
const selectModel = store.getState().actions.selectModel;
const connections = api.llmApiKey.all.useQuery({
@@ -34,54 +40,74 @@ export function DecisionModelSelector({
includeDecisionModels: true,
});
- const connectionOptions = (connections.data?.data ?? [])
- .filter((connection) => isDecisionModelAdapter(connection.adapter))
- .flatMap((connection) => {
- const models = Array.from(
- new Set([
- ...connection.customModels,
- ...(connection.withDefaultModels
- ? supportedModels[connection.adapter]
- : []),
- ]),
- );
- return models.map((model) => toOption(connection.provider, model));
- });
- const selectedOption = selectedModel
- ? toOption(selectedModel.provider, selectedModel.model)
- : null;
- // Radix renders a blank trigger for a value without a matching option, e.g.
- // a model the connection no longer lists.
- const options =
- selectedOption &&
- !connectionOptions.some((option) => option.value === selectedOption.value)
- ? [...connectionOptions, selectedOption]
- : connectionOptions;
+ const rows = connections.data?.data ?? [];
+ const modelsByProvider = decisionModelsByProvider(rows);
+
+ const selectedConnection = rows.find(
+ (connection) => connection.provider === selectedModel?.provider,
+ );
+ // A saved TypeSafe version can outlive the connection's current list. An
+ // OpenAI model outside the decision-model list must not be offered again.
+ const selectedIsAllowed =
+ selectedModel != null &&
+ (selectedConnection == null ||
+ isAllowedDecisionModel(selectedConnection.adapter, selectedModel.model));
+ if (selectedModel && selectedIsAllowed) {
+ const listed = modelsByProvider.get(selectedModel.provider) ?? [];
+ if (!listed.includes(selectedModel.model)) {
+ modelsByProvider.set(selectedModel.provider, [
+ ...listed,
+ selectedModel.model,
+ ]);
+ }
+ }
+
+ const providerGroups = Array.from(modelsByProvider.entries()).sort(
+ (left, right) =>
+ Number(right[1].some(isOpenAIDecisionModel)) -
+ Number(left[1].some(isOpenAIDecisionModel)),
+ );
+ const unsupported = selectedModel != null && !selectedIsAllowed;
+ // Shown until the user picks a model. The store stays untouched.
+ const displayedModel = selectedModel ?? preferredDecisionModel(rows);
- if (connections.isSuccess && options.length === 0) {
+ if (
+ connections.isSuccess &&
+ providerGroups.length === 0 &&
+ selectedModel == null
+ ) {
return (
<Button type="button" variant="outline" onClick={onConfigureProviders}>
- Add a TypeSafe connection
+ Add an OpenAI or TypeSafe connection
</Button>
);
}
return (
- <div className="max-w-xs min-w-56">
- <SelectInput
- aria-label="Decision model"
- placeholder="Select a decision model"
- value={selectedOption?.value ?? ""}
- options={options}
- onValueChange={(value) => {
- const separatorIndex = value.indexOf(SEPARATOR);
- if (separatorIndex === -1) return;
- selectModel({
- provider: value.slice(0, separatorIndex),
- model: value.slice(separatorIndex + SEPARATOR.length),
- });
- }}
- />
+ <div className="flex w-fit max-w-full flex-col items-start gap-1.5">
+ <JudgeModelPicker
+ purpose="decision"
+ open={open}
+ onOpenChange={setOpen}
+ providerGroups={providerGroups}
+ selectedModel={displayedModel}
+ onSelect={selectModel}
+ onConfigureProviders={onConfigureProviders}
+ >
+ <PopoverTrigger asChild>
+ <JudgeModelPickerTrigger
+ mode="custom"
+ selectedModel={displayedModel}
+ disabled={false}
+ />
+ </PopoverTrigger>
+ </JudgeModelPicker>
+ {unsupported ? (
+ <p className="text-destructive text-xs">
+ {selectedModel.model} is not supported for decision models. Choose{" "}
+ {OPENAI_DECISION_MODEL_IDS.join(", ")}.
+ </p>
+ ) : null}
</div>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
index 39e2771ee8..dc3125f13b 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/DefinitionStepContainer/DefinitionStepContainer.tsx
@@ -120,7 +120,9 @@ export function DefinitionStepContainer({
onConfigureProviders={onConfigureProviders}
/>
}
- questionsEditor={<DecisionModelQuestionsEditor store={store} />}
+ questionsEditor={
+ <DecisionModelQuestionsEditor projectId={projectId} store={store} />
+ }
/>
);
}
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
index 3e963e4ac8..18203f1f2d 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/DefinitionStep/components/PromptEditor/PromptEditor.tsx
@@ -6,7 +6,6 @@ import {
Copy,
GripVertical,
MoreVertical,
- Plus,
Trash2,
TriangleAlert,
} from "lucide-react";
@@ -33,6 +32,7 @@ import { useStore } from "zustand";
import { useShallow } from "zustand/react/shallow";
import { Badge } from "@/src/components/ui/badge";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import { Button } from "@/src/components/ui/button";
import {
Tooltip,
@@ -189,11 +189,9 @@ export function PromptEditorContent({
/>
))}
</SortableContext>
- <Button
- type="button"
- variant="ghost"
- size="sm"
- className="text-foreground hover:text-foreground h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"
+ <TextActionButton
+ text="Add message"
+ width="fill"
onClick={() => {
state.actions.setPromptPreviewEnabled(false);
state.actions.addPromptMessage();
@@ -201,10 +199,7 @@ export function PromptEditorContent({
modification: "message_added",
});
}}
- >
- <Plus className="icon-base text-icon-foreground shrink-0" />
- Add message
- </Button>
+ />
</div>
<DragOverlay dropAnimation={null}>
{activeMessage ? (
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx
index 7573ad1650..f913b9526b 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/VariableMappingStep.tsx
@@ -11,7 +11,7 @@ const COPY = {
state: {
title: "Build the state",
description:
- "Choose which observation fields the decision model sees, as one JSON object. Questions refer to these fields by name; the live preview shows the exact object sent.",
+ "Choose which observation fields the decision model sees, as one JSON object. The live preview shows the exact object sent.",
},
} as const;
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx
index cbdc7d15f5..928ae38565 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupEditor/components/VariableMappingStep/components/VariableMappingEditorContainer/VariableMappingEditorContainer.tsx
@@ -1,12 +1,15 @@
import { memo, useMemo, useRef } from "react";
+import { isOpenAIDecisionModel } from "@langfuse/shared";
import { useStore } from "zustand";
import { useShallow } from "zustand/react/shallow";
import { DecisionModelStateEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelStateEditor/DecisionModelStateEditor";
+import { OpenAIDecisionModelStateEditor } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/OpenAIDecisionModelStateEditor/OpenAIDecisionModelStateEditor";
import { VariableMapping } from "@/src/features/evals/v2/components/VariableMapping/VariableMapping";
import { buildDecisionModelStateFields } from "@/src/features/evals/v2/fns/variableMapping/buildDecisionModelStateFields";
import { buildEvaluatorVariableMappings } from "@/src/features/evals/v2/fns/variableMapping/buildEvaluatorVariableMappings";
import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";
+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
const StableVariableMapping = memo(VariableMapping);
@@ -28,7 +31,9 @@ export function VariableMappingEditorContainer({
type: state.type,
promptMessages: state.promptMessages,
stateKeys: state.stateKeys,
+ questions: state.questions,
variableFields: state.variableFields,
+ selectedModel: state.selectedModel,
activeMapping: state.activeMapping,
selectedObservationId: state.selectedObservation?.id ?? null,
actions: state.actions,
@@ -47,6 +52,11 @@ export function VariableMappingEditorContainer({
}),
[state.type, state.stateKeys, state.promptMessages, state.variableFields],
);
+ const fallbackDecisionModel = useFallbackDecisionModel(
+ projectId,
+ state.type === "DECISION_MODEL",
+ );
+ const decisionModel = state.selectedModel ?? fallbackDecisionModel;
const lastCompletedSample = useRef(resolvedSample);
if (!state.selectedObservationId) lastCompletedSample.current = null;
@@ -54,6 +64,34 @@ export function VariableMappingEditorContainer({
const displayedSample = resolvedSample ?? lastCompletedSample.current;
+ if (
+ state.type === "DECISION_MODEL" &&
+ decisionModel != null &&
+ isOpenAIDecisionModel(decisionModel.model)
+ ) {
+ const inputField = mappings.find((mapping) => mapping.variable === "input");
+ return (
+ <OpenAIDecisionModelStateEditor
+ field={{
+ key: "input",
+ fieldState: inputField?.fieldState ?? {
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ }}
+ activeMapping={state.activeMapping}
+ onActiveMappingChange={state.actions.setActiveMapping}
+ onChangeField={(fieldState) =>
+ state.actions.setVariableField("input", fieldState)
+ }
+ questions={state.questions}
+ sourceObject={displayedSample}
+ hasMatchingObservations={Boolean(displayedSample)}
+ sourceUnavailableMessage={SOURCE_UNAVAILABLE_MESSAGE}
+ />
+ );
+ }
+
if (state.type === "DECISION_MODEL") {
return (
<DecisionModelStateEditor
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx
index f7b5b2de28..e54fc082b1 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.clienttest.tsx
@@ -24,6 +24,7 @@ describe("EvaluatorSetupFooter", () => {
isSaving={false}
nameAIAssistanceAvailable={false}
codeValidation={{ isValid: false, isPending: false }}
+ fallbackDecisionModel={null}
onClose={vi.fn()}
onSave={vi.fn()}
/>
@@ -53,6 +54,7 @@ describe("EvaluatorSetupFooter", () => {
isSaving={false}
nameAIAssistanceAvailable={false}
codeValidation={null}
+ fallbackDecisionModel={null}
onClose={vi.fn()}
onSave={vi.fn()}
/>
@@ -96,6 +98,7 @@ describe("EvaluatorSetupFooter", () => {
isSaving={false}
nameAIAssistanceAvailable={false}
codeValidation={null}
+ fallbackDecisionModel={null}
onClose={vi.fn()}
onSave={vi.fn()}
/>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
index 2d53d7e67d..6935299129 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter.tsx
@@ -1,6 +1,8 @@
import { useStore } from "zustand";
import { useShallow } from "zustand/react/shallow";
+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";
+import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";
import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";
import {
@@ -17,6 +19,7 @@ export function EvaluatorSetupFooter({
isSaving,
nameAIAssistanceAvailable,
codeValidation,
+ fallbackDecisionModel,
onClose,
onSave,
}: {
@@ -26,6 +29,7 @@ export function EvaluatorSetupFooter({
isSaving: boolean;
nameAIAssistanceAvailable: boolean;
codeValidation: { isValid: boolean; isPending: boolean } | null;
+ fallbackDecisionModel: JudgeModel | null;
onClose: () => void;
onSave: () => void;
}) {
@@ -39,7 +43,8 @@ export function EvaluatorSetupFooter({
} = useStore(
store,
useShallow((state) => {
- const { definition, mappings } = prepareEvaluatorDraft(state);
+ const draft = applyFallbackDecisionModel(state, fallbackDecisionModel);
+ const { definition, mappings } = prepareEvaluatorDraft(draft);
const hasCompleteMappings =
state.type !== "LLM_AS_JUDGE" ||
mappings.every(({ fieldState }) =>
@@ -50,7 +55,7 @@ export function EvaluatorSetupFooter({
currentSnapshot: JSON.stringify({
name: state.name.trim(),
description: state.description.trim() || null,
- definition,
+ definition: prepareEvaluatorDraft(state).definition,
}),
canSubmit: Boolean(definition) && hasCompleteMappings,
promptMessagesReason:
@@ -62,7 +67,10 @@ export function EvaluatorSetupFooter({
? getScoreOutputValidation(state.scoreOutput).reason
: null,
nameMissing: !state.name.trim(),
- hasValidModel: selectHasValidModel(state),
+ hasValidModel:
+ state.type === "DECISION_MODEL"
+ ? Boolean(draft.selectedModel)
+ : selectHasValidModel(state),
};
}),
);
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx
index c83e226ab9..6882170f65 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluatorVersionHistorySheet/EvaluatorVersionHistorySheet.clienttest.tsx
@@ -56,7 +56,7 @@ describe("EvaluatorVersionHistorySheet", () => {
);
expect(screen.getByText("Version 1").parentElement).toHaveClass(
- "items-baseline",
+ "items-center",
);
expect(
screen.queryByRole("button", { name: "Restore version 2" }),
diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx
index 9d57196003..4efd35cccc 100644
--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.stories.tsx
@@ -11,7 +11,7 @@ import {
type EvaluatorJudgeModelPickerProps = Exclude<
JudgeModelPickerProps,
- { purpose: "projectDefault" }
+ { purpose: "projectDefault" } | { purpose: "decision" }
>;
function EvaluatorJudgeModelPicker(props: EvaluatorJudgeModelPickerProps) {
diff --git a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
index a9c68b9d10..4f957d0776 100644
--- a/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker.tsx
@@ -35,8 +35,6 @@ type JudgeModelPickerCommonProps = {
onOpenChange: (open: boolean) => void;
providerGroups: Array<[string, string[]]>;
onConfigureProviders: () => void;
- onConfigureModel: () => void;
- hasModelConfiguration?: boolean;
};
type EvaluatorJudgeModelPickerProps = JudgeModelPickerCommonProps & {
@@ -46,19 +44,30 @@ type EvaluatorJudgeModelPickerProps = JudgeModelPickerCommonProps & {
selectedModel: JudgeModel | null;
onModeChange: (mode: JudgeModelMode) => void;
onSelectCustom: (model: JudgeModel) => void;
+ onConfigureModel: () => void;
+ hasModelConfiguration?: boolean;
canSetProjectDefault: boolean;
onSetProjectDefault: () => void;
};
+type DecisionJudgeModelPickerProps = JudgeModelPickerCommonProps & {
+ purpose: "decision";
+ selectedModel: JudgeModel | null;
+ onSelect: (model: JudgeModel) => void;
+};
+
type ProjectDefaultJudgeModelPickerProps = JudgeModelPickerCommonProps & {
purpose: "projectDefault";
defaultModel: JudgeModel | null;
onSelectProjectDefault: (model: JudgeModel) => void;
+ onConfigureModel: () => void;
+ hasModelConfiguration?: boolean;
};
export type JudgeModelPickerProps =
| EvaluatorJudgeModelPickerProps
- | ProjectDefaultJudgeModelPickerProps;
+ | ProjectDefaultJudgeModelPickerProps
+ | DecisionJudgeModelPickerProps;
type JudgeModelPickerTriggerProps = Omit<
ButtonProps,
@@ -180,22 +189,28 @@ export const JudgeModelPickerTrigger = forwardRef<
);
JudgeModelPickerTrigger.displayName = "JudgeModelPickerTrigger";
+function selectionShowsCheck(props: JudgeModelPickerProps) {
+ if (props.purpose === "decision" || props.purpose === "projectDefault") {
+ return true;
+ }
+ return props.mode === "custom";
+}
+
/** A controlled model selection menu for evaluators and the project default. */
export function JudgeModelPicker(props: JudgeModelPickerProps) {
- const {
- children,
- open,
- onOpenChange,
- defaultModel,
- providerGroups,
- onConfigureProviders,
- onConfigureModel,
- hasModelConfiguration = false,
- } = props;
+ const { children, open, onOpenChange, providerGroups, onConfigureProviders } =
+ props;
const selectsProjectDefault = props.purpose === "projectDefault";
- const selectedModel = selectsProjectDefault
- ? (defaultModel ?? null)
- : props.selectedModel;
+ const selectsDecision = props.purpose === "decision";
+ const defaultModel = selectsDecision ? null : props.defaultModel;
+ const hasModelConfiguration = selectsDecision
+ ? false
+ : (props.hasModelConfiguration ?? false);
+ const selectedModel = selectsDecision
+ ? props.selectedModel
+ : selectsProjectDefault
+ ? (defaultModel ?? null)
+ : props.selectedModel;
const selectedModelIsDefault =
!selectsProjectDefault &&
selectedModel !== null &&
@@ -220,11 +235,13 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {
<CommandInput placeholder="Find a model..." />
<CommandList>
<CommandEmpty>No model found.</CommandEmpty>
- {!selectsProjectDefault && defaultModel ? (
+ {props.purpose !== "decision" &&
+ props.purpose !== "projectDefault" &&
+ props.defaultModel ? (
<>
<CommandGroup>
<CommandItem
- value={`project-default ${defaultModel.provider} ${defaultModel.model}`}
+ value={`project-default ${props.defaultModel.provider} ${props.defaultModel.model}`}
onSelect={() =>
selectAndClose(() => props.onModeChange("default"))
}
@@ -238,9 +255,9 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {
<Sparkles className="text-muted-foreground icon-base mr-2 shrink-0" />
<span
className="truncate"
- title={`${defaultModel.provider} / ${defaultModel.model}`}
+ title={`${props.defaultModel.provider} / ${props.defaultModel.model}`}
>
- {defaultModel.provider} / {defaultModel.model}
+ {props.defaultModel.provider} / {props.defaultModel.model}
</span>
<Badge variant="secondary" className="ml-auto shrink-0">
Project default
@@ -256,7 +273,7 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {
const isSelected =
selectedModel?.provider === provider &&
selectedModel.model === model &&
- (selectsProjectDefault || props.mode === "custom");
+ selectionShowsCheck(props);
const isProjectDefault =
defaultModel?.provider === provider &&
defaultModel.model === model;
@@ -269,6 +286,8 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {
selectAndClose(() => {
if (selectsProjectDefault) {
props.onSelectProjectDefault({ provider, model });
+ } else if (selectsDecision) {
+ props.onSelect({ provider, model });
} else {
props.onSelectCustom({ provider, model });
}
@@ -309,33 +328,36 @@ export function JudgeModelPicker(props: JudgeModelPickerProps) {
Configure AI providers
<ExternalLink className="icon-base text-muted-foreground ml-auto" />
</Button>
- <Button
- type="button"
- variant="ghost"
- className="font-regular justify-start"
- disabled={
- selectsProjectDefault
- ? !defaultModel
- : props.mode !== "custom" || !selectedModel
- }
- onClick={() => selectAndClose(onConfigureModel)}
- >
- <Settings2 className="icon-base text-muted-foreground mr-2" />
- Model configuration
- {hasModelConfiguration ? (
- <>
- <span className="sr-only">Customized</span>
- <span
- aria-hidden="true"
- className="relative ml-auto inline-flex h-2.5 w-2.5"
- >
- <span className="bg-dark-yellow absolute inline-flex h-full w-full animate-ping rounded-full opacity-75" />
- <span className="bg-dark-yellow relative inline-flex h-2.5 w-2.5 rounded-full" />
- </span>
- </>
- ) : null}
- </Button>
- {!selectsProjectDefault ? (
+ {props.purpose === "decision" ? null : (
+ <Button
+ type="button"
+ variant="ghost"
+ className="font-regular justify-start"
+ disabled={
+ props.purpose === "projectDefault"
+ ? !props.defaultModel
+ : props.mode !== "custom" || !selectedModel
+ }
+ onClick={() => selectAndClose(props.onConfigureModel)}
+ >
+ <Settings2 className="icon-base text-muted-foreground mr-2" />
+ Model configuration
+ {hasModelConfiguration ? (
+ <>
+ <span className="sr-only">Customized</span>
+ <span
+ aria-hidden="true"
+ className="relative ml-auto inline-flex h-2.5 w-2.5"
+ >
+ <span className="bg-dark-yellow absolute inline-flex h-full w-full animate-ping rounded-full opacity-75" />
+ <span className="bg-dark-yellow relative inline-flex h-2.5 w-2.5 rounded-full" />
+ </span>
+ </>
+ ) : null}
+ </Button>
+ )}
+ {props.purpose !== "projectDefault" &&
+ props.purpose !== "decision" ? (
<Button
type="button"
variant="ghost"
diff --git a/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx b/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx
index fd107a60e3..40ba59bcb0 100644
--- a/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/Testing/components/TestResultPanelView/TestResultPanelView.tsx
@@ -46,11 +46,11 @@ function ResultStat({
}) {
return (
<span
- className="text-muted-foreground flex -translate-y-px items-center gap-1 font-mono text-xs leading-none tabular-nums"
+ className="text-muted-foreground flex items-center gap-1 font-mono text-xs leading-none tabular-nums"
title={title}
>
- <Icon className="icon-sm" />
- <span className="translate-y-0.5">{children}</span>
+ <Icon className="icon-sm shrink-0" />
+ {children}
</span>
);
}
diff --git a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
index 938605737f..f12e314520 100644
--- a/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
+++ b/web/src/features/evals/v2/components/Rules/EvaluatorAssignmentsEditor/EvaluatorAssignmentsEditor.tsx
@@ -4,7 +4,7 @@ import { type ReactNode, useState } from "react";
import { useStore } from "zustand";
import { useShallow } from "zustand/react/shallow";
-import { Button } from "@/src/components/ui/button";
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import {
Command,
CommandEmpty,
@@ -100,22 +100,11 @@ export function EvaluatorAssignmentsEditor({
</span>
</button>
) : (
- <Button
- type="button"
- variant="ghost"
- size="sm"
+ <TextActionButton
+ text={additionalActionLabel}
disabled={disabled}
- className={cn(
- "text-foreground hover:text-foreground inline-flex h-auto items-center gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline",
- pickerWidth === "trigger" && "w-full justify-start",
- )}
- >
- <Plus
- className="icon-base text-icon-foreground shrink-0"
- aria-hidden="true"
- />
- {additionalActionLabel}
- </Button>
+ width={pickerWidth === "trigger" ? "fill" : "content"}
+ />
)}
</PopoverTrigger>
<PopoverContent
diff --git a/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts b/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts
index 3a86f99189..2c393288b0 100644
--- a/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts
+++ b/web/src/features/evals/v2/fns/evaluatorTesting/toDecisionModelResults.ts
@@ -1,3 +1,5 @@
+import { LLMAdapter } from "@langfuse/shared";
+
import type { DecisionModelQuestionResult } from "@/src/features/evals/v2/components/Evaluators/DecisionModel/DecisionModelResultView/DecisionModelResultView";
const isRecord = (value: unknown): value is Record<string, unknown> =>
@@ -12,11 +14,17 @@ const toProbabilities = (value: unknown): Record<string, number> =>
)
: {};
+function readDecisionDetails(metadata: unknown) {
+ if (!isRecord(metadata)) return null;
+ const details = metadata[LLMAdapter.OpenAI] ?? metadata[LLMAdapter.TypeSafe];
+ return isRecord(details) ? details : null;
+}
+
/**
* Reads the scores of a decision-model test run into one row per question.
- * Values come from the score itself; distributions and confidence from the
- * `typesafe` metadata namespace the executor writes; the question text from
- * the request that was sent.
+ * Values come from the score itself. Distributions and confidence come from
+ * the provider namespace the executor writes (`openai` or `typesafe`). The
+ * question text comes from the request that was sent.
*/
export function toDecisionModelResults(
response: Record<string, unknown>,
@@ -29,10 +37,10 @@ export function toDecisionModelResults(
return scores.flatMap((score, index): DecisionModelQuestionResult[] => {
if (!isRecord(score)) return [];
- const typesafe = isRecord(score.metadata) ? score.metadata.typesafe : null;
- if (!isRecord(typesafe)) return [];
+ const details = readDecisionDetails(score.metadata);
+ if (!details) return [];
- const questionId = String(typesafe.questionId ?? index);
+ const questionId = String(details.questionId ?? index);
const question = requestQuestions[questionId];
const common = {
questionId,
@@ -43,21 +51,21 @@ export function toDecisionModelResults(
: "",
};
const confidence =
- typeof typesafe.confidence === "number" ? typesafe.confidence : null;
+ typeof details.confidence === "number" ? details.confidence : null;
- switch (typesafe.type) {
+ switch (details.type) {
case "choice":
return [
{
...common,
type: "choice",
- choice: String(score.value ?? typesafe.choice ?? ""),
- probabilities: toProbabilities(typesafe.probabilities),
+ choice: String(score.value ?? details.choice ?? ""),
+ probabilities: toProbabilities(details.probabilities),
confidence,
},
];
case "score": {
- const legend = isRecord(typesafe.legend) ? typesafe.legend : {};
+ const legend = isRecord(details.legend) ? details.legend : {};
return [
{
...common,
@@ -66,7 +74,7 @@ export function toDecisionModelResults(
levels: Object.keys(legend)
.sort((a, b) => Number(a) - Number(b))
.map((level) => String(legend[level] ?? "")),
- probabilities: toProbabilities(typesafe.probabilities),
+ probabilities: toProbabilities(details.probabilities),
confidence,
},
];
diff --git a/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts b/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts
index f778be55bc..173dada5c7 100644
--- a/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluatorTesting/toTestResultPanelState.clienttest.ts
@@ -85,6 +85,69 @@ describe("toTestResultPanelState", () => {
},
);
+ it("reads OpenAI and TypeSafe decision details into the result panel", () => {
+ const response = {
+ success: true,
+ request: { questions: { topic: { instructions: "How severe?" } } },
+ scores: [
+ {
+ name: "topic",
+ value: 0.24,
+ metadata: {
+ openai: {
+ questionId: "topic",
+ type: "score",
+ model: "gpt-6-luna",
+ confidence: 0.52,
+ probabilities: { "0": 0.76, "1": 0.24 },
+ legend: { "0": "Fine", "1": "Blocked" },
+ },
+ },
+ },
+ ],
+ };
+
+ expect(
+ toTestResultPanelState({
+ type: "DECISION_MODEL",
+ isPending: false,
+ result: response,
+ }),
+ ).toEqual({
+ status: "decision-success",
+ results: [
+ {
+ questionId: "topic",
+ scoreName: "topic",
+ instructions: "How severe?",
+ type: "score",
+ score: 0.24,
+ levels: ["Fine", "Blocked"],
+ probabilities: { "0": 0.76, "1": 0.24 },
+ confidence: 0.52,
+ },
+ ],
+ });
+
+ const typesafePanel = toTestResultPanelState({
+ type: "DECISION_MODEL",
+ isPending: false,
+ result: {
+ ...response,
+ scores: [
+ {
+ ...response.scores[0],
+ metadata: { typesafe: response.scores[0].metadata.openai },
+ },
+ ],
+ },
+ });
+ expect(typesafePanel).toMatchObject({
+ status: "decision-success",
+ results: [{ confidence: 0.52, score: 0.24 }],
+ });
+ });
+
it("keeps pending and failed runs distinct", () => {
expect(
toTestResultPanelState({
diff --git a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.ts
new file mode 100644
index 0000000000..02af90cbfd
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.clienttest.ts
@@ -0,0 +1,104 @@
+import { DecisionModelQuestionType } from "@langfuse/shared";
+import { describe, expect, it } from "vitest";
+
+import {
+ previewOpenAIDecisionQuestions,
+ usesPlainDecisionInstructions,
+} from "./decisionModelQuestions";
+import type { DecisionModelQuestionDraft } from "@/src/features/evals/v2/types/decisionModel";
+
+const draft = (
+ overrides: Partial<DecisionModelQuestionDraft> &
+ Pick<DecisionModelQuestionDraft, "id" | "type">,
+): DecisionModelQuestionDraft => ({
+ scoreName: "",
+ instructions: "",
+ options: [
+ { value: "", description: "" },
+ { value: "", description: "" },
+ ],
+ levels: [{ description: "" }, { description: "" }],
+ criteria: { true: "", false: "" },
+ ...overrides,
+});
+
+describe("usesPlainDecisionInstructions", () => {
+ it("drops field chips for an OpenAI connection, including before the model id matches", () => {
+ expect(
+ usesPlainDecisionInstructions({
+ model: "gpt-4.1",
+ adapter: "openai",
+ }),
+ ).toBe(true);
+ expect(
+ usesPlainDecisionInstructions({
+ model: "gpt-6-luna",
+ adapter: undefined,
+ }),
+ ).toBe(true);
+ expect(
+ usesPlainDecisionInstructions({
+ model: "jev-latest",
+ adapter: "typesafe",
+ }),
+ ).toBe(false);
+ });
+});
+
+describe("previewOpenAIDecisionQuestions", () => {
+ it("shows choices, labeled levels, and folded yes/no criteria", () => {
+ expect(
+ previewOpenAIDecisionQuestions([
+ draft({
+ id: "department",
+ type: DecisionModelQuestionType.CHOICE,
+ instructions: " Which team? ",
+ options: [
+ { value: "billing", description: " Payments " },
+ { value: "other", description: " " },
+ ],
+ }),
+ draft({
+ id: "severity",
+ type: DecisionModelQuestionType.SCORE,
+ instructions: "How severe?",
+ levels: [
+ { description: "Appearance only" },
+ { description: "Blocked" },
+ ],
+ }),
+ draft({
+ id: "refund",
+ type: DecisionModelQuestionType.NOUL,
+ instructions: "Does it request a refund?",
+ criteria: { true: "Asks for money back", false: "" },
+ }),
+ ]),
+ ).toEqual([
+ {
+ type: "choice",
+ name: "department",
+ instructions: "Which team?",
+ choices: [
+ { value: "billing", description: "Payments" },
+ { value: "other" },
+ ],
+ },
+ {
+ type: "score",
+ name: "severity",
+ instructions: "How severe?",
+ levels: [
+ { label: "0", description: "Appearance only" },
+ { label: "1", description: "Blocked" },
+ ],
+ },
+ {
+ type: "predicate",
+ name: "refund",
+ instructions:
+ "Does it request a refund?\n\nCriteria for true:\nAsks for money back",
+ },
+ ]);
+ });
+});
diff --git a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts
index 958f99ef2e..ed60a26fba 100644
--- a/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts
+++ b/web/src/features/evals/v2/fns/evaluators/decisionModelQuestions.ts
@@ -2,6 +2,8 @@ import {
DECISION_MODEL_LIMITS,
DecisionModelQuestionType,
DecisionModelQuestionsSchema,
+ isOpenAIDecisionModel,
+ LLMAdapter,
type DecisionModelEntry,
type DecisionModelQuestion,
type DecisionModelQuestions,
@@ -123,6 +125,15 @@ export function getQuestionDraftErrors(
return errors;
}
+/** OpenAI questions are plain text. Field chips are only for TypeSafe. */
+export function usesPlainDecisionInstructions(params: {
+ model: string | undefined;
+ adapter: string | undefined;
+}) {
+ if (params.adapter === LLMAdapter.OpenAI) return true;
+ return params.model != null && isOpenAIDecisionModel(params.model);
+}
+
/** Persistable questions, or null while any draft is incomplete. */
export function draftsToQuestions(
drafts: DecisionModelQuestionDraft[],
@@ -173,3 +184,72 @@ export function draftsToQuestions(
const parsed = DecisionModelQuestionsSchema.safeParse(questions);
return parsed.success ? parsed.data : null;
}
+
+export type OpenAIDecisionQuestionPreview =
+ | {
+ type: "choice";
+ name: string;
+ instructions: string;
+ choices: Array<{ value: string; description?: string }>;
+ }
+ | {
+ type: "score";
+ name: string;
+ instructions: string;
+ levels: Array<{ label?: string; description?: string }>;
+ }
+ | { type: "predicate"; name: string; instructions: string };
+
+/** Questions as the Decisions API receives them, including unfinished drafts. */
+export function previewOpenAIDecisionQuestions(
+ drafts: DecisionModelQuestionDraft[],
+): OpenAIDecisionQuestionPreview[] {
+ return drafts.map((draft) => {
+ const name = draft.id;
+ const instructions = openAIInstructions(draft);
+ switch (draft.type) {
+ case DecisionModelQuestionType.CHOICE:
+ return {
+ type: "choice",
+ name,
+ instructions,
+ choices: draft.options.flatMap((option) => {
+ const value = option.value.trim();
+ if (!value) return [];
+ const description = option.description.trim();
+ return [
+ {
+ value,
+ ...(description ? { description } : {}),
+ },
+ ];
+ }),
+ };
+ case DecisionModelQuestionType.SCORE:
+ return {
+ type: "score",
+ name,
+ instructions,
+ levels: draft.levels.flatMap((level, index) => {
+ const description = level.description.trim();
+ if (!description) return [];
+ return [{ label: String(index), description }];
+ }),
+ };
+ case DecisionModelQuestionType.NOUL:
+ return { type: "predicate", name, instructions };
+ }
+ });
+}
+
+function openAIInstructions(draft: DecisionModelQuestionDraft) {
+ if (draft.type !== DecisionModelQuestionType.NOUL) {
+ return draft.instructions.trim();
+ }
+ const parts = [draft.instructions.trim()];
+ const whenTrue = draft.criteria.true.trim();
+ const whenFalse = draft.criteria.false.trim();
+ if (whenTrue) parts.push(`Criteria for true:\n${whenTrue}`);
+ if (whenFalse) parts.push(`Criteria for false:\n${whenFalse}`);
+ return parts.filter((part) => part.length > 0).join("\n\n");
+}
diff --git a/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.ts b/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.ts
new file mode 100644
index 0000000000..a1714120a7
--- /dev/null
+++ b/web/src/features/evals/v2/fns/evaluators/preferredDecisionModel.ts
@@ -0,0 +1,74 @@
+import {
+ isAllowedDecisionModel,
+ LLMAdapter,
+ OPENAI_DECISION_MODEL_IDS,
+ supportedModels,
+ supportsDecisionModels,
+} from "@langfuse/shared";
+
+import type { JudgeModel } from "@/src/features/evals/v2/judgeModel";
+
+type DecisionModelConnection = {
+ provider: string;
+ adapter: string;
+ customModels: readonly string[];
+ withDefaultModels: boolean;
+};
+
+function defaultModelsFor(adapter: string): readonly string[] {
+ if (adapter === LLMAdapter.OpenAI) return supportedModels[LLMAdapter.OpenAI];
+ if (adapter === LLMAdapter.TypeSafe) {
+ return supportedModels[LLMAdapter.TypeSafe];
+ }
+ return [];
+}
+
+/** Decision models each connection can run, keyed by provider. */
+export function decisionModelsByProvider(
+ connections: readonly DecisionModelConnection[],
+): Map<string, string[]> {
+ const modelsByProvider = new Map<string, string[]>();
+ for (const connection of connections) {
+ if (!supportsDecisionModels(connection.adapter)) continue;
+ const models = Array.from(
+ new Set([
+ ...connection.customModels,
+ ...(connection.withDefaultModels
+ ? defaultModelsFor(connection.adapter)
+ : []),
+ ]),
+ ).filter((model) => isAllowedDecisionModel(connection.adapter, model));
+ if (models.length > 0) modelsByProvider.set(connection.provider, models);
+ }
+ return modelsByProvider;
+}
+
+/**
+ * OpenAI is the default decision provider. The first allowlisted model a
+ * connection actually offers wins. TypeSafe is never chosen for the user.
+ */
+export function preferredDecisionModel(
+ connections: readonly DecisionModelConnection[],
+): JudgeModel | null {
+ const modelsByProvider = decisionModelsByProvider(connections);
+ for (const model of OPENAI_DECISION_MODEL_IDS) {
+ for (const [provider, models] of modelsByProvider) {
+ if (models.includes(model)) return { provider, model };
+ }
+ }
+ return null;
+}
+
+/** Use the saved choice. Fill an empty decision-model choice from the connection list. */
+export function applyFallbackDecisionModel<
+ T extends { type: string; selectedModel: JudgeModel | null },
+>(state: T, fallback: JudgeModel | null): T {
+ if (
+ state.type !== "DECISION_MODEL" ||
+ state.selectedModel != null ||
+ fallback == null
+ ) {
+ return state;
+ }
+ return { ...state, selectedModel: fallback };
+}
diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
index 5985ba86a8..4d5b36990b 100644
--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.clienttest.ts
@@ -201,6 +201,19 @@ describe("prepareEvaluatorDraft", () => {
],
});
+ expect(
+ prepareEvaluatorDraft({
+ ...base,
+ selectedModel: { provider: "openai", model: "gpt-6-luna" },
+ }).definition?.variableMapping,
+ ).toEqual([
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ ]);
+
// An unmapped state key blocks saving; the model would never see it.
expect(
prepareEvaluatorDraft({
diff --git a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
index 90eaf5b229..8ecd9340ad 100644
--- a/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
+++ b/web/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft.ts
@@ -1,4 +1,6 @@
/* eslint-disable no-nested-ternary */
+import { isOpenAIDecisionModel } from "@langfuse/shared";
+
import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";
import { buildScoreOutputDefinition } from "@/src/features/evals/v2/fns/scoreOutput/buildScoreOutputDefinition";
import { buildEvaluatorVariableMappings } from "@/src/features/evals/v2/fns/variableMapping/buildEvaluatorVariableMappings";
@@ -36,7 +38,11 @@ export function prepareEvaluatorDraft(params: EvaluatorSetupDraftState) {
}
if (params.type === "DECISION_MODEL") {
return buildDecisionModelStateFields({
- stateKeys: params.stateKeys,
+ stateKeys:
+ params.selectedModel != null &&
+ isOpenAIDecisionModel(params.selectedModel.model)
+ ? ["input"]
+ : params.stateKeys,
variableFields: params.variableFields,
});
}
diff --git a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx
index 26095d4506..e88bc8b09c 100644
--- a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx
+++ b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.clienttest.tsx
@@ -9,6 +9,16 @@ vi.mock("@/src/features/evals/v2/hooks/useEvaluatorSetupSample", () => ({
useEvaluatorSetupSample: () => ({ id: "sample" }),
}));
+vi.mock("@/src/utils/api", () => ({
+ api: {
+ llmApiKey: {
+ all: {
+ useQuery: () => ({ data: undefined }),
+ },
+ },
+ },
+}));
+
describe("useEvaluatorTestAvailability", () => {
const store = createEvaluatorSetupStore({
initialEvaluator: null,
diff --git a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts
index ea7cfd09d1..77dcada7c4 100644
--- a/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts
+++ b/web/src/features/evals/v2/hooks/useEvaluatorTestAvailability.ts
@@ -1,8 +1,10 @@
import { useStore } from "zustand";
+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";
import { getScoreOutputValidation } from "@/src/features/evals/v2/fns/scoreOutput/getScoreOutputValidation";
import { useEvaluatorSetupSample } from "@/src/features/evals/v2/hooks/useEvaluatorSetupSample";
+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";
import type { EvaluatorSetupStore } from "@/src/features/evals/v2/store/evaluatorSetupStore/evaluatorSetupStore";
export function useEvaluatorTestAvailability({
@@ -19,8 +21,17 @@ export function useEvaluatorTestAvailability({
store,
(state) => state.selectedObservation,
);
+ const evaluatorType = useStore(store, (state) => state.type);
+ const fallbackDecisionModel = useFallbackDecisionModel(
+ projectId,
+ evaluatorType === "DECISION_MODEL",
+ );
const definitionAvailable = useStore(store, (state) =>
- Boolean(prepareEvaluatorDraft(state).definition),
+ Boolean(
+ prepareEvaluatorDraft(
+ applyFallbackDecisionModel(state, fallbackDecisionModel),
+ ).definition,
+ ),
);
const scoreOutputReason = useStore(store, (state) =>
state.type === "LLM_AS_JUDGE"
diff --git a/web/src/features/evals/v2/hooks/useFallbackDecisionModel.ts b/web/src/features/evals/v2/hooks/useFallbackDecisionModel.ts
new file mode 100644
index 0000000000..d06560d217
--- /dev/null
+++ b/web/src/features/evals/v2/hooks/useFallbackDecisionModel.ts
@@ -0,0 +1,11 @@
+import { preferredDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
+import { api } from "@/src/utils/api";
+
+/** The decision model to show and save when the user has not chosen one yet. */
+export function useFallbackDecisionModel(projectId: string, enabled: boolean) {
+ const connections = api.llmApiKey.all.useQuery(
+ { projectId, includeDecisionModels: true },
+ { enabled },
+ );
+ return preferredDecisionModel(connections.data?.data ?? []);
+}
diff --git a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
index ac45529048..52545b2fc0 100644
--- a/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
+++ b/web/src/features/evals/v2/pages/EvaluatorSetupPage.tsx
@@ -29,7 +29,9 @@ import { getEvaluatorNameStep } from "@/src/features/evals/v2/components/Evaluat
import { EvaluatorSetupFooter } from "@/src/features/evals/v2/components/Evaluators/EvaluatorSetupFooter/EvaluatorSetupFooter";
import { SampleObservationSelectorContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/SampleObservationSelectorContainer/SampleObservationSelectorContainer";
import { EvaluatorTestPanelContainer } from "@/src/features/evals/v2/components/EvaluatorTestPanel/components/EvaluatorTestPanelContainer/EvaluatorTestPanelContainer";
+import { applyFallbackDecisionModel } from "@/src/features/evals/v2/fns/evaluators/preferredDecisionModel";
import { prepareEvaluatorDraft } from "@/src/features/evals/v2/fns/evaluators/prepareEvaluatorDraft";
+import { useFallbackDecisionModel } from "@/src/features/evals/v2/hooks/useFallbackDecisionModel";
import { draftsToQuestions } from "@/src/features/evals/v2/fns/evaluators/decisionModelQuestions";
import type { NormalizedEvaluatorDefinition } from "../server/evaluators/evaluatorTypes";
import { api } from "@/src/utils/api";
@@ -288,6 +290,12 @@ export function EvaluatorSetupPage(
sourceCode: codeDraft.sourceCode,
sourceCodeLanguage: codeDraft.sourceCodeLanguage,
});
+ const fallbackDecisionModel = useFallbackDecisionModel(
+ projectId,
+ modelDraft.type === "DECISION_MODEL",
+ );
+ // The default decision model arrives with the connection list. It is applied
+ // on save, not stored, so it must not count as an edit.
const getCurrentSnapshot = (state = evaluatorSetupStore.getState()) =>
JSON.stringify({
name: state.name.trim(),
@@ -581,7 +589,9 @@ export function EvaluatorSetupPage(
return;
}
}
- const { definition } = prepareEvaluatorDraft(state);
+ const { definition } = prepareEvaluatorDraft(
+ applyFallbackDecisionModel(state, fallbackDecisionModel),
+ );
if (!definition) return;
const { name, description } = metadata;
@@ -717,7 +727,9 @@ export function EvaluatorSetupPage(
const runTest = () => {
const state = evaluatorSetupStore.getState();
- const { definition } = prepareEvaluatorDraft(state);
+ const { definition } = prepareEvaluatorDraft(
+ applyFallbackDecisionModel(state, fallbackDecisionModel),
+ );
const selectedObservation = state.selectedObservation;
if (!definition || !selectedObservation?.traceId) return;
capture("evaluators:test", {
@@ -948,6 +960,7 @@ export function EvaluatorSetupPage(
}
: null
}
+ fallbackDecisionModel={fallbackDecisionModel}
onClose={requestClose}
onSave={save}
/>
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
index 36bb4cd2ab..e4060e7821 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorValidation.ts
@@ -8,7 +8,10 @@ import {
DefaultEvalModelService,
getClientInitiatedNonStreamingLlmTimeoutMs,
getLLMErrorInfo,
+ isAllowedDecisionModel,
isDecisionModelAdapter,
+ LLMAdapter,
+ OPENAI_DECISION_MODEL_IDS,
} from "@langfuse/shared/src/server";
import { getEvaluatorDefinitionPreflightError } from "@/src/features/evals/server/evaluator-preflight";
import { getPromptMessagesValidationError } from "@/src/features/evals/v2/fns/promptMessages/hasInvalidSystemPromptMessage";
@@ -212,7 +215,7 @@ export async function getDecisionModelConfigurationError(params: {
name: string;
definition: Pick<
Extract<EvaluatorDefinition, { type: "DECISION_MODEL" }>,
- "provider" | "model"
+ "provider" | "model" | "questions"
>;
}): Promise<string | null> {
const modelConfig = await DefaultEvalModelService.fetchValidModelConfig(
@@ -221,10 +224,18 @@ export async function getDecisionModelConfigurationError(params: {
params.definition.model,
);
if (!modelConfig.valid) {
- return `No decision-model connection found for evaluator "${params.name}". ${modelConfig.error}. Add a TypeSafe connection under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) first.`;
+ return `No decision-model connection found for evaluator "${params.name}". ${modelConfig.error}. Add an OpenAI connection using ${OPENAI_DECISION_MODEL_IDS.join(", ")}, or a TypeSafe connection, under Settings → LLM Connections (/project/${params.projectId}/settings/llm-connections) first.`;
}
- if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {
- return `Connection "${params.definition.provider}" is not a decision-model connection. Decision-model evaluators need a TypeSafe connection.`;
+ if (
+ !isAllowedDecisionModel(
+ modelConfig.config.apiKey.adapter,
+ modelConfig.config.model,
+ )
+ ) {
+ if (modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI) {
+ return `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`;
+ }
+ return `Connection "${params.definition.provider}" is not a decision-model connection. Decision-model evaluators need an OpenAI connection using ${OPENAI_DECISION_MODEL_IDS.join(", ")}, or a TypeSafe connection.`;
}
return null;
}
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 7c5b4846ff..3f8ac76f10 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -6,7 +6,7 @@ import { decrypt } from "@langfuse/shared/encryption";
import {
buildEvalExecutionData,
compileLangfuseMediaMessages,
- createTypeSafeDecisionModelClient,
+ createDecisionModelClient,
createW3CTraceId,
decryptAndParseExtraHeaders,
DefaultEvalModelService,
@@ -16,7 +16,9 @@ import {
extractObservationVariables,
findModel,
generateLLMText,
- isDecisionModelAdapter,
+ isAllowedDecisionModel,
+ LLMAdapter,
+ OPENAI_DECISION_MODEL_IDS,
LangfuseInternalTraceEnvironment,
mapLegacyLLMCompletionParams,
matchPricingTier,
@@ -134,17 +136,26 @@ async function testDecisionModelEvaluator(params: {
if (!modelConfig.valid) {
return { success: false as const, error: modelConfig.error };
}
- if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {
+ if (
+ !isAllowedDecisionModel(
+ modelConfig.config.apiKey.adapter,
+ modelConfig.config.model,
+ )
+ ) {
return {
success: false as const,
- error: `Connection "${params.definition.provider}" is not a decision-model connection.`,
+ error:
+ modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI
+ ? `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`
+ : `Connection "${params.definition.provider}" is not a decision-model connection.`,
};
}
const executionTraceId = createW3CTraceId();
let request: DecisionModelRequest | undefined;
try {
- const client = createTypeSafeDecisionModelClient({
+ const client = createDecisionModelClient({
+ adapter: modelConfig.config.apiKey.adapter,
apiKey: decrypt(modelConfig.config.apiKey.secretKey),
model: modelConfig.config.model,
baseURL: modelConfig.config.apiKey.baseURL,
diff --git a/web/src/features/filters/components/filter-builder.tsx b/web/src/features/filters/components/filter-builder.tsx
index 1a78bb195a..2c2d3d1038 100644
--- a/web/src/features/filters/components/filter-builder.tsx
+++ b/web/src/features/filters/components/filter-builder.tsx
@@ -1,5 +1,6 @@
/* eslint-disable no-nested-ternary */
/* eslint-disable @repo/no-style-props */
+import { TextActionButton } from "@/src/components/TextActionButton/TextActionButton";
import { Button } from "@/src/components/ui/button";
import { Input } from "@/src/components/ui/input";
import { Textarea } from "@/src/components/ui/textarea";
@@ -1263,26 +1264,26 @@ function FilterBuilderForm({
</table>
)}
{!disabled ? (
- <Button
- onClick={() => addNewFilter()}
- type="button" // required as it will otherwise submit forms where this component is used
- className={cn(
- subtleAddButton
- ? "text-foreground hover:text-foreground mt-2 h-6 w-full justify-start gap-1.5 px-0 py-0 text-xs leading-none underline-offset-4 hover:bg-transparent hover:underline"
- : "mt-2",
- compact && !subtleAddButton && "mt-4 self-start",
- )}
- variant={subtleAddButton ? "ghost" : "outline"}
- size="sm"
- >
- <Plus
- className={cn(
- "icon-base text-icon-foreground shrink-0",
- !subtleAddButton && "mr-2",
- )}
- />
- Add filter
- </Button>
+ subtleAddButton ? (
+ <div className="mt-2">
+ <TextActionButton
+ text="Add filter"
+ width="fill"
+ onClick={() => addNewFilter()}
+ />
+ </div>
+ ) : (
+ <Button
+ onClick={() => addNewFilter()}
+ type="button" // required as it will otherwise submit forms where this component is used
+ className={cn("mt-2", compact && "mt-4 self-start")}
+ variant="outline"
+ size="sm"
+ >
+ <Plus className="icon-base text-icon-foreground mr-2 shrink-0" />
+ Add filter
+ </Button>
+ )
) : null}
</>
)}
diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.ts
index d88ed1ace4..da83681a08 100644
--- a/web/src/features/llm-api-key/server/router.ts
+++ b/web/src/features/llm-api-key/server/router.ts
@@ -128,7 +128,7 @@ async function testDecisionModelConnection(params: {
kind: {
type: "choice",
instructions: "What kind of message is `message`?",
- criteria: { greeting: null, other: null },
+ choices: [{ value: "greeting" }, { value: "other" }],
},
},
});
diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
index 5d1c1c166a..2cbd682895 100644
--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({
name: "createEvaluator",
description: [
"Create a new evaluator with a stable id. Names do not act as identity and may be reused.",
- "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",
+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",
"Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",
].join(" "),
action: "evaluator:CUD",
diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
index e58226b951..8987b31f80 100644
--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
@@ -25,7 +25,7 @@ export const [
description: [
"List the evaluator templates maintained by Langfuse and partners.",
"LLM-as-a-judge and code definitions can be copied into createEvaluator.",
- "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a TypeSafe modelConfig.",
+ "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a modelConfig for an OpenAI connection using an allowlisted decision model (currently `gpt-6-luna`) or a TypeSafe connection.",
].join(" "),
action: "evaluator:read",
baseSchema: ListManagedEvaluatorTemplatesInput,
diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
index 34200bc103..133776e94c 100644
--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({
name: "updateEvaluator",
description: [
"Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",
- "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",
+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",
].join(" "),
action: "evaluator:CUD",
baseSchema: McpUpdateEvaluatorInputBase,
diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts
index f26065415b..4c6a2ce432 100644
--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts
+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.test.ts
@@ -197,10 +197,10 @@ describe("runDecisionModelEvaluation", () => {
readiness: {
type: "choice",
instructions: "Is `output` ready to send as an answer to `input`?",
- criteria: { ready: null, needs_revision: null },
+ choices: [{ value: "ready" }, { value: "needs_revision" }],
},
refund: {
- type: "boolean",
+ type: "predicate",
instructions: "Does `input` request a refund?",
},
},
diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
index 0b6bd02edd..bc27764b1a 100644
--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
@@ -18,7 +18,9 @@ import {
EvaluatorBlockSource,
executeDecisionModelEvaluator,
instrumentAsync,
- isDecisionModelAdapter,
+ isAllowedDecisionModel,
+ LLMAdapter,
+ OPENAI_DECISION_MODEL_IDS,
logger,
type ExtractedVariable,
} from "@langfuse/shared/src/server";
@@ -113,8 +115,16 @@ export async function runDecisionModelEvaluation({
let modelConfigError: string | null = null;
if (!modelConfig.valid) {
modelConfigError = modelConfig.error;
- } else if (!isDecisionModelAdapter(modelConfig.config.apiKey.adapter)) {
- modelConfigError = `Connection "${modelConfig.config.provider}" is not a decision-model connection`;
+ } else if (
+ !isAllowedDecisionModel(
+ modelConfig.config.apiKey.adapter,
+ modelConfig.config.model,
+ )
+ ) {
+ modelConfigError =
+ modelConfig.config.apiKey.adapter === LLMAdapter.OpenAI
+ ? `Model "${modelConfig.config.model}" is not supported for decision models. Use ${OPENAI_DECISION_MODEL_IDS.join(", ")}.`
+ : `Connection "${modelConfig.config.provider}" is not a decision-model connection`;
}
if (!modelConfig.valid || modelConfigError !== null) {
const blockReason = getBlockReasonForInvalidModelConfig({
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 0577f7b944..99216fad09 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -8,7 +8,7 @@ import {
buildEventBucketPrefix,
compileLangfuseMediaMessages,
createLLMOutput,
- createTypeSafeDecisionModelClient,
+ createDecisionModelClient,
decryptAndParseExtraHeaders,
DefaultEvalModelService,
generateLLMText,
@@ -338,15 +338,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
callDecisionModel: async (params) => {
const { apiKey } = params.modelConfig;
- if (apiKey.adapter !== LLMAdapter.TypeSafe) {
- throw new Error(
- `Decision-model adapter is not supported: ${apiKey.adapter}`,
- );
- }
const secretKey = apiKey.secretKey;
if (typeof secretKey !== "string") {
throw new UnrecoverableError(
- "TypeSafe connection is missing its secret key",
+ "Decision-model connection is missing its secret key",
);
}
@@ -359,11 +354,12 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
);
} catch {
throw new UnrecoverableError(
- "TypeSafe connection secrets could not be decrypted",
+ "Decision-model connection secrets could not be decrypted",
);
}
- const client = createTypeSafeDecisionModelClient({
+ const client = createDecisionModelClient({
+ adapter: apiKey.adapter,
apiKey: decryptedSecretKey,
model: params.modelConfig.model,
baseURL: typeof apiKey.baseURL === "string" ? apiKey.baseURL : null,
@@ -400,7 +396,7 @@ export function createMockEvalExecutionDeps(
Object.entries(request.questions).map(([id, question]) => {
switch (question.type) {
case "choice": {
- const [choice = "mock"] = Object.keys(question.criteria);
+ const choice = question.choices[0]?.value ?? "mock";
return [
id,
{
@@ -416,7 +412,7 @@ export function createMockEvalExecutionDeps(
id,
{ type: "score", score: 0, probabilities: {}, confidence: 1 },
];
- case "boolean":
+ case "predicate":
return [id, { type: "boolean", probability: 1 }];
}
}),