Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): graduate decision model evaluators (#18541)

Tobias Wochinger615a0f2256e9
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
index 2861aace19..93674274ac 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/EvaluatorGalleryView.tsx
@@ -178,7 +178,7 @@ export function EvaluatorGalleryView({
type="button"
variant="secondary"
className="flex-1 shrink-0 gap-1.5 @2xl:flex-none"
- title="Ask Jev or the OpenAI Decisions API typed questions in one call. Experimental."
+ title="Ask Jev or the OpenAI Decisions API typed questions in one call."
onClick={() =>
onCreateFromScratch(EvalTemplateTypeEnum.DECISION_MODEL)
}
diff --git a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
index 870fa68e42..69281634e0 100644
--- a/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
+++ b/web/src/features/evals/v2/components/EvaluatorGalleryView/components/EvaluatorGalleryDecisionModelBanner/EvaluatorGalleryDecisionModelBanner.tsx
@@ -20,7 +20,7 @@ export function EvaluatorGalleryDecisionModelBanner({
<span className="font-bold">New: decision-model evaluators.</span>{" "}
<span className="text-muted-foreground">
Ask Jev or the OpenAI Decisions API typed questions about each
- observation in one call. Experimental.
+ observation in one call.
</span>
</div>
</div>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx
index 55086bc148..a4f23e37c1 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/EvaluationTypeConfiguration.tsx
@@ -27,8 +27,8 @@ export function EvaluationTypeConfiguration({
Use custom code for deterministic checks like exact matches, regex,
or schema validation. Use an LLM when the check needs judgment, such
as rating helpfulness, tone, or answer quality. Use a decision model
- (experimental) for a fast, cheap label with calibrated probabilities
- when the possible answers are known upfront.
+ for a fast, cheap label with calibrated probabilities when the
+ possible answers are known upfront.
</InfoTooltip>
</span>
</Label>
diff --git a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx
index d3f86d7d08..30c31e1c4c 100644
--- a/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx
+++ b/web/src/features/evals/v2/components/Evaluators/EvaluationTypeConfiguration/components/EvaluationTypeToggle/EvaluationTypeToggle.tsx
@@ -23,7 +23,7 @@ const evaluationTypes = [
},
{
value: EvalTemplateTypeEnum.DECISION_MODEL,
- label: "Decision model (experimental)",
+ label: "Decision model",
icon: Scale,
},
] as const;
diff --git a/web/src/features/evals/v2/constants/tableFilterColumns.ts b/web/src/features/evals/v2/constants/tableFilterColumns.ts
index 1d9231a344..7a39bb9f1e 100644
--- a/web/src/features/evals/v2/constants/tableFilterColumns.ts
+++ b/web/src/features/evals/v2/constants/tableFilterColumns.ts
@@ -15,7 +15,7 @@ const evaluatorTypeOptions = [
{ value: EvalTemplateType.CODE, displayValue: "Code" },
{
value: EvalTemplateType.DECISION_MODEL,
- displayValue: "Decision model (experimental)",
+ displayValue: "Decision model",
},
];
diff --git a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
index 7a3ac04bb0..c400c38333 100644
--- a/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
+++ b/web/src/features/evals/v2/fns/evaluators/evaluatorTypeLabel.ts
@@ -6,7 +6,7 @@ export function evaluatorTypeLabel(type: EvalTemplateType): string {
case EvalTemplateTypeEnum.CODE:
return "Code";
case EvalTemplateTypeEnum.DECISION_MODEL:
- return "Decision model (experimental)";
+ return "Decision model";
case EvalTemplateTypeEnum.LLM_AS_JUDGE:
return "LLM as a judge";
case EvalTemplateTypeEnum.FACET:
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
index 404d9ce900..db0e8dd237 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorTypes.ts
@@ -73,10 +73,10 @@ export const CodeEvaluatorDefinitionSchema = z.object({
});
/**
- * Decision-model evaluators (experimental): the state is the JSON object
- * built from the variable mapping (key → extractor), `questions` are the
- * typed questions asked about it, and each question writes its own score.
- * The model connection is always explicit; there is no project default.
+ * Decision-model evaluators use the JSON object built from the variable
+ * mapping (key → extractor) as state. `questions` are the typed questions
+ * asked about it, and each question writes its own score. The model connection
+ * is always explicit; there is no project default.
*/
const DecisionModelEvaluatorDefinitionSchema =
EvaluatorVersionBaseSchema.extend({
diff --git a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
index 3ef956f338..71fb131c8c 100644
--- a/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
+++ b/web/src/features/public-api/components/CreateLLMApiKeyForm.tsx
@@ -12,7 +12,6 @@ import {
BEDROCK_USE_DEFAULT_CREDENTIALS,
TYPESAFE_UPSTREAMS,
VERTEXAI_USE_DEFAULT_CREDENTIALS,
- isDecisionModelAdapter,
resolveTypeSafeUpstream,
type TypeSafeUpstream,
} from "@langfuse/shared";
@@ -760,9 +759,7 @@ export function CreateLLMApiKeyForm({
<SelectContent>
{adapterOptions.map((provider) => (
<SelectItem value={provider} key={provider}>
- {isDecisionModelAdapter(provider)
- ? `${provider} (experimental)`
- : provider}
+ {provider}
</SelectItem>
))}
{mode === "create" && (
diff --git a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts
index 72adf362d5..b76007e83d 100644
--- a/web/src/features/public-api/server/evaluation/evaluationAdapters.ts
+++ b/web/src/features/public-api/server/evaluation/evaluationAdapters.ts
@@ -84,7 +84,7 @@ export function toPublicEvaluatorType(type: EvalTemplateType) {
return PUBLIC_EVALUATOR_TYPE_LLM_AS_JUDGE;
case EvalTemplateType.DECISION_MODEL:
throw new InvalidRequestError(
- "Decision-model evaluators are experimental and not available through the public API",
+ "Decision-model evaluators are not available through the public API",
);
}
}
@@ -286,7 +286,7 @@ export function toPublicEvaluatorVersion(
if (evaluatorType === EvalTemplateType.DECISION_MODEL) {
throw new InvalidRequestError(
- "Decision-model evaluators are experimental and not available through the public API",
+ "Decision-model evaluators are not available through the public API",
);
}
diff --git a/web/src/features/public-api/types/llm-connections.ts b/web/src/features/public-api/types/llm-connections.ts
index 3224531799..29b49b8e58 100644
--- a/web/src/features/public-api/types/llm-connections.ts
+++ b/web/src/features/public-api/types/llm-connections.ts
@@ -49,7 +49,7 @@ export const GetLlmConnectionsV1Response = z
})
.strict();
-// Decision-model adapters are experimental and not part of the public contract.
+// Decision-model adapters are not part of the public contract.
const PUBLIC_LLM_ADAPTERS = Object.values(LLMAdapter).filter(
(adapter) => !isDecisionModelAdapter(adapter),
) as [LLMAdapter, ...LLMAdapter[]];
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 99216fad09..b0dc745a45 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -157,7 +157,7 @@ export interface EvalExecutionDeps {
params: FetchModelConfigParams,
) => Promise<ModelConfigResult>;
- // Decision-model operations (experimental)
+ // Decision-model operations
callDecisionModel: (
params: DecisionModelCallParams,
) => Promise<DecisionModelEvaluation>;