🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): graduate decision model evaluators (#18541)
Tobias Wochinger615a0f2256e9
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsxindex 2861aace19..93674274ac 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx@@ -178,7 +178,7 @@ export function EvaluatorGalleryView({type="button"variant="secondary"className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"- title="Ask Jev or the OpenAI Decisions API typed questions in one call. Experimental."+ title="Ask Jev or the OpenAI Decisions API typed questions in one call."onClick={() =>onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)}diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsxindex 870fa68e42..69281634e0 100644--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx@@ -20,7 +20,7 @@ export function EvaluatorGalleryDecisionModelBanner({<span className="font-bold">New: decision-model evaluators.</span>{" "}<span className="text-muted-foreground">Ask Jev or the OpenAI Decisions API typed questions about each- observation in one call. Experimental.+ observation in one call.</span></div></div>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsxindex 55086bc148..a4f23e37c1 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx@@ -27,8 +27,8 @@ export function EvaluationTypeConfiguration({Use custom code for deterministic checks like exact matches, regex,or schema validation. Use an LLM when the check needs judgment, suchas rating helpfulness, tone, or answer quality. Use a decision model- (experimental) for a fast, cheap label with calibrated probabilities- when the possible answers are known upfront.+ for a fast, cheap label with calibrated probabilities when the+ possible answers are known upfront.</InfoTooltip></span></Label>diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsxindex d3f86d7d08..30c31e1c4c 100644--- a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx+++ b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx@@ -23,7 +23,7 @@ const evaluationTypes = [},{value: EvalTemplateTypeEnum.DECISION_MODEL,- label: "Decision model (experimental)",+ label: "Decision model",icon: Scale,},] as const;diff --git a/web/src/features/evals/v2/constants/tableFilterColumns.ts b/web/src/features/evals/v2/constants/tableFilterColumns.tsindex 1d9231a344..7a39bb9f1e 100644--- a/web/src/features/evals/v2/constants/tableFilterColumns.ts+++ b/web/src/features/evals/v2/constants/tableFilterColumns.ts@@ -15,7 +15,7 @@ const evaluatorTypeOptions = [{ value: EvalTemplateType.CODE, displayValue: "Code" },{value: EvalTemplateType.DECISION_MODEL,- displayValue: "Decision model (experimental)",+ displayValue: "Decision model",},];diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.tsindex 7a3ac04bb0..c400c38333 100644--- a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts@@ -6,7 +6,7 @@ export function evaluatorTypeLabel(type: EvalTemplateType): string {case EvalTemplateTypeEnum.CODE:return "Code";case EvalTemplateTypeEnum.DECISION_MODEL:- return "Decision model (experimental)";+ return "Decision model";case EvalTemplateTypeEnum.LLM_AS_JUDGE:return "LLM as a judge";case EvalTemplateTypeEnum.FACET:diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.tsindex 404d9ce900..db0e8dd237 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts@@ -73,10 +73,10 @@ export const CodeEvaluatorDefinitionSchema = z.object({});/**- * Decision-model evaluators (experimental): the state is the JSON object- * built from the variable mapping (key → extractor), `questions` are the- * typed questions asked about it, and each question writes its own score.- * The model connection is always explicit; there is no project default.+ * Decision-model evaluators use the JSON object built from the variable+ * mapping (key → extractor) as state. `questions` are the typed questions+ * asked about it, and each question writes its own score. The model connection+ * is always explicit; there is no project default.*/const DecisionModelEvaluatorDefinitionSchema =EvaluatorVersionBaseSchema.extend({diff --git a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsxindex 3ef956f338..71fb131c8c 100644--- a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx+++ b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx@@ -12,7 +12,6 @@ import {BEDROCK_USE_DEFAULT_CREDENTIALS,TYPESAFE_UPSTREAMS,VERTEXAI_USE_DEFAULT_CREDENTIALS,- isDecisionModelAdapter,resolveTypeSafeUpstream,type TypeSafeUpstream,} from "@langfuse/shared";@@ -760,9 +759,7 @@ export function CreateLLMApiKeyForm({<SelectContent>{adapterOptions.map((provider) => (<SelectItem value={provider} key={provider}>- {isDecisionModelAdapter(provider)- ? `${provider} (experimental)`- : provider}+ {provider}</SelectItem>))}{mode === "create" && (diff --git a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts b/web/src/features/public-api/server/evaluation/evaluationAdapters.tsindex 72adf362d5..b76007e83d 100644--- a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts+++ b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts@@ -84,7 +84,7 @@ export function toPublicEvaluatorType(type: EvalTemplateType) {return PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE;case EvalTemplateType.DECISION_MODEL:throw new InvalidRequestError(- "Decision-model evaluators are experimental and not available through the public API",+ "Decision-model evaluators are not available through the public API",);}}@@ -286,7 +286,7 @@ export function toPublicEvaluatorVersion(if (evaluatorType === EvalTemplateType.DECISION_MODEL) {throw new InvalidRequestError(- "Decision-model evaluators are experimental and not available through the public API",+ "Decision-model evaluators are not available through the public API",);}diff --git a/web/src/features/public-api/types/llm-connections.ts b/web/src/features/public-api/types/llm-connections.tsindex 3224531799..29b49b8e58 100644--- a/web/src/features/public-api/types/llm-connections.ts+++ b/web/src/features/public-api/types/llm-connections.ts@@ -49,7 +49,7 @@ export const GetLlmConnectionsV1Response = z}).strict();-// Decision-model adapters are experimental and not part of the public contract.+// Decision-model adapters are not part of the public contract.const PUBLIC_LLM_ADAPTERS = Object.values(LLMAdapter).filter((adapter) => !isDecisionModelAdapter(adapter),) as [LLMAdapter, ...LLMAdapter[]];diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 99216fad09..b0dc745a45 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -157,7 +157,7 @@ export interface EvalExecutionDeps {params: FetchModelConfigParams,) => Promise<ModelConfigResult>;- // Decision-model operations (experimental)+ // Decision-model operationscallDecisionModel: (params: DecisionModelCallParams,) => Promise<DecisionModelEvaluation>;