Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(mcp): support decision-model evaluators (#18166)

Tobias Wochingere59efe007e74
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.ts
index c16a6f000b..ad3a6acc11 100644
--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts
@@ -591,6 +591,31 @@ describe("MCP Read Tools", () => {
),
).toBe(true);
});
+
+ it("lists decision-model templates", async () => {
+ const result = (await handleListManagedEvaluatorTemplates(
+ { type: "DECISION_MODEL" },
+ mockServerContext(),
+ )) as {
+ templates: Array<{ key: string; evaluator: { type: string } }>;
+ };
+
+ expect(result.templates).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ key: "topic-decision-model",
+ evaluator: expect.objectContaining({
+ type: "DECISION_MODEL",
+ }),
+ }),
+ ]),
+ );
+ expect(
+ result.templates.every(
+ (template) => template.evaluator.type === "DECISION_MODEL",
+ ),
+ ).toBe(true);
+ });
});
describe("getEvaluator tool", () => {
diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.ts
index e6d980b299..c27849beae 100644
--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts
@@ -244,6 +244,7 @@ describe("MCP Write Tools", () => {
evaluatorId: expect.any(Object),
type: expect.any(Object),
prompt: expect.any(Object),
+ questions: expect.any(Object),
sourceCode: expect.any(Object),
observationId: expect.any(Object),
traceId: expect.any(Object),
@@ -302,6 +303,65 @@ describe("MCP Write Tools", () => {
startTime: new Date(startTime),
});
+ await expect(
+ handleTestEvaluator(
+ {
+ type: "DECISION_MODEL",
+ questions: [
+ {
+ id: "relevant",
+ type: "noul",
+ scoreName: "relevant",
+ instructions: "Determine whether the input is relevant.",
+ criteria: null,
+ },
+ ],
+ modelConfig: {
+ provider: "typesafe",
+ model: "jev-latest",
+ },
+ variableMapping: [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ },
+ ],
+ observationId,
+ traceId,
+ startTime,
+ } as never,
+ setup.context,
+ ),
+ ).resolves.toEqual(unifiedResult);
+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(3, {
+ orgId: setup.orgId,
+ projectId: setup.projectId,
+ definition: {
+ type: "DECISION_MODEL",
+ questions: [
+ {
+ id: "relevant",
+ type: "noul",
+ scoreName: "relevant",
+ instructions: "Determine whether the input is relevant.",
+ criteria: null,
+ },
+ ],
+ provider: "typesafe",
+ model: "jev-latest",
+ vars: ["input"],
+ variableMapping: [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ },
+ ],
+ },
+ observationId,
+ traceId,
+ startTime: new Date(startTime),
+ });
+
await expect(
handleTestEvaluator(
{ observationId, traceId, startTime } as never,
diff --git a/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.ts b/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.ts
new file mode 100644
index 0000000000..18f3e4c2b7
--- /dev/null
+++ b/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.ts
@@ -0,0 +1,199 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+ McpEvaluatorInput,
+ toEvaluatorServiceInput,
+} from "@/src/features/mcp/server/evals/tools/evaluatorInput";
+import {
+ assertRuleAssignmentsReplaceableViaMcp,
+ toMcpEvaluationRule,
+} from "@/src/features/mcp/server/evals/rule-service";
+
+const decisionModelInput = {
+ name: "Support classifier",
+ type: "DECISION_MODEL" as const,
+ questions: [
+ {
+ id: "topic",
+ type: "choice" as const,
+ scoreName: "topic",
+ instructions: "Classify the support request.",
+ options: [
+ { value: "billing", description: "Billing questions." },
+ { value: "technical", description: "Technical questions." },
+ ],
+ },
+ ],
+ modelConfig: {
+ provider: "typesafe",
+ model: "jev-latest",
+ },
+ variableMapping: [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input" as const,
+ },
+ ],
+};
+
+describe("MCP evaluator input", () => {
+ it("converts a decision-model definition to evaluator service input", () => {
+ expect(toEvaluatorServiceInput(decisionModelInput)).toEqual({
+ name: decisionModelInput.name,
+ description: null,
+ definition: {
+ type: "DECISION_MODEL",
+ questions: decisionModelInput.questions,
+ provider: "typesafe",
+ model: "jev-latest",
+ vars: ["input"],
+ variableMapping: [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ },
+ ],
+ },
+ });
+ });
+
+ it.each([
+ ["questions", { questions: undefined }],
+ ["modelConfig", { modelConfig: undefined }],
+ ["variableMapping", { variableMapping: undefined }],
+ ])("requires decision-model %s", (field, override) => {
+ const result = McpEvaluatorInput.safeParse({
+ ...decisionModelInput,
+ ...override,
+ });
+
+ expect(result.success).toBe(false);
+ if (!result.success) {
+ expect(result.error.issues).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: expect.arrayContaining([field]),
+ }),
+ ]),
+ );
+ }
+ });
+
+ it("rejects invalid decision-model state keys", () => {
+ const result = McpEvaluatorInput.safeParse({
+ ...decisionModelInput,
+ variableMapping: [
+ {
+ templateVariable: "support topic",
+ selectedColumnId: "input",
+ },
+ ],
+ });
+
+ expect(result.success).toBe(false);
+ if (!result.success) {
+ expect(result.error.issues).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: ["variableMapping", 0, "templateVariable"],
+ }),
+ ]),
+ );
+ }
+ });
+
+ it("rejects model parameters for decision-model evaluators", () => {
+ const result = McpEvaluatorInput.safeParse({
+ ...decisionModelInput,
+ modelConfig: {
+ ...decisionModelInput.modelConfig,
+ modelParams: { temperature: 0 },
+ },
+ });
+
+ expect(result.success).toBe(false);
+ if (!result.success) {
+ expect(result.error.issues).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ path: ["modelConfig", "modelParams"],
+ }),
+ ]),
+ );
+ }
+ });
+
+ it("exposes decision-model assignments in MCP evaluation rules", () => {
+ const createdAt = new Date("2026-10-02T00:00:00.000Z");
+ const updatedAt = new Date("2026-10-02T01:00:00.000Z");
+
+ expect(
+ toMcpEvaluationRule({
+ id: "rule-1",
+ name: "Support routing",
+ enabled: true,
+ sampling: 1,
+ filter: [],
+ assignments: [
+ {
+ evaluator: {
+ id: "evaluator-1",
+ name: "Support classifier",
+ type: "DECISION_MODEL",
+ },
+ variableMapping: [
+ {
+ templateVariable: "input",
+ selectedColumnId: "input",
+ jsonSelector: null,
+ },
+ ],
+ },
+ ],
+ createdAt,
+ updatedAt,
+ } as never),
+ ).toEqual({
+ id: "rule-1",
+ name: "Support routing",
+ enabled: true,
+ sampling: 1,
+ filter: [],
+ evaluators: [
+ {
+ evaluatorId: "evaluator-1",
+ evaluatorName: "Support classifier",
+ evaluatorType: "decision_model",
+ variableMapping: [
+ {
+ variable: "input",
+ source: "input",
+ },
+ ],
+ },
+ ],
+ createdAt,
+ updatedAt,
+ });
+ });
+
+ it("allows MCP to replace rules with decision-model assignments", async () => {
+ await expect(
+ assertRuleAssignmentsReplaceableViaMcp(
+ {
+ get: vi.fn().mockResolvedValue({
+ assignments: [
+ {
+ evaluator: {
+ type: "DECISION_MODEL",
+ },
+ },
+ ],
+ }),
+ } as never,
+ "project-1",
+ "rule-1",
+ ),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/web/src/features/mcp/server/evals/rule-schema.ts b/web/src/features/mcp/server/evals/rule-schema.ts
index 4b4ef80005..4b9273cbad 100644
--- a/web/src/features/mcp/server/evals/rule-schema.ts
+++ b/web/src/features/mcp/server/evals/rule-schema.ts
@@ -15,6 +15,11 @@ import {
} from "@/src/features/public-api/server";
import { McpAdvancedFilterBaseSchema } from "../../core/filter-schema";
+export const McpEvaluatorType = z.enum([
+ ...PublicEvaluatorType.options,
+ "decision_model",
+]);
+
const RuleFilterBaseSchema = McpAdvancedFilterBaseSchema.describe(
'Observation filter condition, e.g. {"column":"version","operator":"=","value":"1.0.0","type":"string"}. Use `key` for object columns such as `metadata`.',
);
@@ -33,7 +38,7 @@ const EvaluationRuleEvaluatorInputBase = {
const VariableMappingSchema = z
.array(ObservationPromptVariableMappingInput)
.describe(
- "Optional rule-specific variable mapping for LLM evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",
+ "Optional rule-specific variable mapping for LLM-as-a-judge and decision-model evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",
);
export const AttachEvaluatorToEvaluationRuleBaseSchema = z.object({
@@ -158,7 +163,7 @@ export const EvaluationRuleResponseSchema = z
z.object({
evaluatorId: z.string(),
evaluatorName: z.string(),
- evaluatorType: PublicEvaluatorType,
+ evaluatorType: McpEvaluatorType,
variableMapping: z.array(PromptVariableMappingRead).nullable(),
}),
),
diff --git a/web/src/features/mcp/server/evals/rule-service.ts b/web/src/features/mcp/server/evals/rule-service.ts
index 843cd12f9f..9a27d7c6ca 100644
--- a/web/src/features/mcp/server/evals/rule-service.ts
+++ b/web/src/features/mcp/server/evals/rule-service.ts
@@ -7,15 +7,30 @@ import {
toStoredMappingList,
} from "@/src/features/public-api/server";
import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";
-import { InvalidRequestError } from "@langfuse/shared";
+import { EvalTemplateType, InvalidRequestError } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import type { z } from "zod";
import type { ServerContext } from "../../types";
import {
+ McpEvaluatorType,
EvaluationRuleResponseSchema,
type EvaluationRuleAssignmentInput,
} from "./rule-schema";
+function isMcpEvaluatorType(type: EvalTemplateType) {
+ return (
+ type === EvalTemplateType.DECISION_MODEL || isPublicApiEvaluatorType(type)
+ );
+}
+
+function toMcpEvaluatorType(type: EvalTemplateType) {
+ return McpEvaluatorType.parse(
+ type === EvalTemplateType.DECISION_MODEL
+ ? "decision_model"
+ : toPublicEvaluatorType(type),
+ );
+}
+
export function createMcpRuleService(context: ServerContext) {
return new RuleService(prisma, ({ action, ruleId }) =>
auditLog({
@@ -51,11 +66,11 @@ export async function assertRuleAssignmentsReplaceableViaMcp(
) {
const rule = await service.get(projectId, ruleId);
const hidden = rule.assignments.filter(
- (assignment) => !isPublicApiEvaluatorType(assignment.evaluator.type),
+ (assignment) => !isMcpEvaluatorType(assignment.evaluator.type),
);
if (hidden.length > 0) {
throw new InvalidRequestError(
- "This rule uses experimental evaluators that are not visible through MCP. Update its evaluator assignments in the Langfuse UI.",
+ "This rule uses internal evaluators that are not visible through MCP. Update its evaluator assignments in the Langfuse UI.",
);
}
}
@@ -70,13 +85,11 @@ export function toMcpEvaluationRule(
sampling: rule.sampling,
filter: rule.filter,
evaluators: rule.assignments
- .filter((assignment) =>
- isPublicApiEvaluatorType(assignment.evaluator.type),
- )
+ .filter((assignment) => isMcpEvaluatorType(assignment.evaluator.type))
.map((assignment) => ({
evaluatorId: assignment.evaluator.id,
evaluatorName: assignment.evaluator.name,
- evaluatorType: toPublicEvaluatorType(assignment.evaluator.type),
+ evaluatorType: toMcpEvaluatorType(assignment.evaluator.type),
variableMapping:
assignment.variableMapping === null
? null
diff --git a/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts b/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts
index 09ffa24aa2..dc8b204dfd 100644
--- a/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts
+++ b/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts
@@ -16,7 +16,7 @@ export const [createEvaluationRuleTool, handleCreateEvaluationRule] =
name: "createEvaluationRule",
description: [
"Create an observation evaluation rule with one or more evaluator assignments.",
- "Each assignment references a project evaluator by stable ID. LLM evaluator assignments may override their variable mapping; code evaluator assignments must omit it.",
+ "Each assignment references a project evaluator by stable ID. LLM-as-a-judge and decision-model assignments may override their variable mapping; code evaluator assignments must omit it.",
].join(" "),
action: "evaluationRule:CUD",
baseSchema: CreateEvaluationRuleBaseSchema,
diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
index 6857ac500a..5d1c1c166a 100644
--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({
name: "createEvaluator",
description: [
"Create a new evaluator with a stable id. Names do not act as identity and may be reused.",
- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",
+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",
"Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",
].join(" "),
action: "evaluator:CUD",
diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
index 40a108858e..4f08af9b3f 100644
--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
@@ -1,4 +1,10 @@
import {
+ DECISION_MODEL_LIMITS,
+ DecisionModelChoiceOptionSchema,
+ DecisionModelChoiceQuestionSchema,
+ DecisionModelQuestionType,
+ DecisionModelQuestionsSchema,
+ DecisionModelStateKeySchema,
EvalOutputDataTypeSchema,
EvalTemplateType,
observationVariableMapping,
@@ -31,6 +37,52 @@ const McpObservationVariableMappingSchema = observationVariableMapping.extend({
jsonSelector: z.string().optional(),
});
+const McpDecisionModelQuestionBaseSchema = z.object({
+ id: DecisionModelChoiceQuestionSchema.shape.id.describe(
+ "Stable question identifier.",
+ ),
+ type: z.enum(DecisionModelQuestionType).describe("Decision question type."),
+ scoreName: DecisionModelChoiceQuestionSchema.shape.scoreName.describe(
+ "Name of the score produced by the question.",
+ ),
+ instructions: z
+ .unknown()
+ .describe("Question instructions as text or structured JSON."),
+ options: z
+ .array(
+ z.object({
+ value: DecisionModelChoiceOptionSchema.shape.value,
+ description: z
+ .unknown()
+ .optional()
+ .describe("Option description as text or structured JSON."),
+ }),
+ )
+ .min(DECISION_MODEL_LIMITS.minChoiceOptions)
+ .max(DECISION_MODEL_LIMITS.maxChoiceOptions)
+ .optional()
+ .describe("Choice options. Required when type is `choice`."),
+ levels: z
+ .array(
+ z.object({
+ description: z
+ .unknown()
+ .describe("Level description as text or structured JSON."),
+ }),
+ )
+ .min(DECISION_MODEL_LIMITS.minScoreLevels)
+ .max(DECISION_MODEL_LIMITS.maxScoreLevels)
+ .optional()
+ .describe("Ordered score levels. Required when type is `score`."),
+ criteria: z
+ .object({
+ true: z.unknown().optional(),
+ false: z.unknown().optional(),
+ })
+ .optional()
+ .describe("Optional true/false criteria when type is `noul`."),
+});
+
const McpEvalOutputDefinitionSchema = z.object({
dataType: EvalOutputDataTypeSchema.describe(
"The score type returned by the evaluator.",
@@ -78,21 +130,35 @@ const McpEvalOutputDefinitionSchema = z.object({
export const McpEvaluatorInputBase = z.object({
name: CreateEvaluatorSchema.shape.name,
description: CreateEvaluatorSchema.shape.description.unwrap().optional(),
- type: z.enum([EvalTemplateType.LLM_AS_JUDGE, EvalTemplateType.CODE]),
+ type: z.enum([
+ EvalTemplateType.LLM_AS_JUDGE,
+ EvalTemplateType.CODE,
+ EvalTemplateType.DECISION_MODEL,
+ ]),
prompt: z.string().min(1).optional(),
modelConfig: McpEvaluatorModelConfigSchema.optional().describe(
- "Optional custom model configuration. Omit to use the project default model.",
+ "Model configuration. Required for decision-model evaluators. Optional for LLM-as-a-judge evaluators, which use the project default when omitted.",
),
outputDefinition: McpEvalOutputDefinitionSchema.optional().describe(
"Required for LLM-as-a-judge evaluators. Defines the reasoning and score returned by the evaluator.",
),
+ questions: z
+ .array(McpDecisionModelQuestionBaseSchema)
+ .min(1)
+ .max(DECISION_MODEL_LIMITS.maxQuestions)
+ .optional()
+ .describe(
+ "Required for decision-model evaluators. Each question produces its own score.",
+ ),
sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),
sourceCodeLanguage:
CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),
variableMapping: z
.array(McpObservationVariableMappingSchema)
.optional()
- .describe("Variable mappings for LLM-as-a-judge evaluators only."),
+ .describe(
+ "Variable mappings for LLM-as-a-judge and decision-model evaluators. Decision-model mappings define the state and require at least one entry.",
+ ),
});
export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({
@@ -100,7 +166,13 @@ export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({
description: true,
});
-function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {
+const McpEvaluatorRuntimeInputBase = McpEvaluatorInputBase.extend({
+ questions: DecisionModelQuestionsSchema.optional().describe(
+ "Required for decision-model evaluators. Each question produces its own score.",
+ ),
+});
+
+function toEvaluatorInput(input: z.infer<typeof McpEvaluatorRuntimeInputBase>) {
if (input.type === EvalTemplateType.LLM_AS_JUDGE) {
return {
name: input.name,
@@ -117,6 +189,19 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {
};
}
+ if (input.type === EvalTemplateType.DECISION_MODEL) {
+ return {
+ name: input.name,
+ description: input.description ?? null,
+ definition: {
+ type: input.type,
+ questions: input.questions,
+ modelConfig: input.modelConfig,
+ variableMapping: input.variableMapping,
+ },
+ };
+ }
+
return {
name: input.name,
description: input.description ?? null,
@@ -129,7 +214,7 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {
}
function validateEvaluatorInput(
- input: z.infer<typeof McpEvaluatorInputBase>,
+ input: z.infer<typeof McpEvaluatorRuntimeInputBase>,
ctx: z.RefinementCtx,
) {
if (input.type === EvalTemplateType.LLM_AS_JUDGE && !input.prompt?.trim()) {
@@ -140,6 +225,43 @@ function validateEvaluatorInput(
});
}
+ if (
+ input.type === EvalTemplateType.DECISION_MODEL &&
+ input.variableMapping?.length === 0
+ ) {
+ ctx.addIssue({
+ code: "custom",
+ path: ["variableMapping"],
+ message:
+ "Decision-model evaluators require at least one state variable mapping.",
+ });
+ }
+
+ if (input.type === EvalTemplateType.DECISION_MODEL) {
+ input.variableMapping?.forEach(({ templateVariable }, index) => {
+ const parsedStateKey =
+ DecisionModelStateKeySchema.safeParse(templateVariable);
+ if (parsedStateKey.success) return;
+
+ for (const issue of parsedStateKey.error.issues) {
+ ctx.addIssue({
+ code: "custom",
+ path: ["variableMapping", index, "templateVariable"],
+ message: issue.message,
+ });
+ }
+ });
+
+ if (input.modelConfig?.modelParams !== undefined) {
+ ctx.addIssue({
+ code: "custom",
+ path: ["modelConfig", "modelParams"],
+ message:
+ "Decision-model evaluators do not support model parameters. Provide only provider and model.",
+ });
+ }
+ }
+
if (
input.type === EvalTemplateType.CODE &&
input.variableMapping !== undefined
@@ -166,7 +288,7 @@ function validateEvaluatorInput(
}
}
-export const McpEvaluatorInput = McpEvaluatorInputBase.superRefine(
+export const McpEvaluatorInput = McpEvaluatorRuntimeInputBase.superRefine(
validateEvaluatorInput,
);
@@ -174,13 +296,18 @@ export const McpUpdateEvaluatorInputBase = McpEvaluatorInputBase.extend({
evaluatorId: z.string(),
});
-export const McpUpdateEvaluatorInput = McpUpdateEvaluatorInputBase.superRefine(
- ({ evaluatorId: _evaluatorId, ...input }, ctx) =>
- validateEvaluatorInput(input, ctx),
-);
+const McpUpdateEvaluatorRuntimeInputBase = McpEvaluatorRuntimeInputBase.extend({
+ evaluatorId: z.string(),
+});
+
+export const McpUpdateEvaluatorInput =
+ McpUpdateEvaluatorRuntimeInputBase.superRefine(
+ ({ evaluatorId: _evaluatorId, ...input }, ctx) =>
+ validateEvaluatorInput(input, ctx),
+ );
export function toEvaluatorServiceInput(
- input: z.infer<typeof McpEvaluatorInputBase>,
+ input: z.infer<typeof McpEvaluatorRuntimeInputBase>,
) {
return CreateEvaluatorWithoutProjectSchema.parse(toEvaluatorInput(input));
}
diff --git a/web/src/features/mcp/server/evals/tools/getEvaluator.ts b/web/src/features/mcp/server/evals/tools/getEvaluator.ts
index c245333e19..3636d95869 100644
--- a/web/src/features/mcp/server/evals/tools/getEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/getEvaluator.ts
@@ -9,7 +9,7 @@ const GetEvaluatorInput = z.object({ evaluatorId: z.string() });
export const [getEvaluatorTool, handleGetEvaluator] = defineTool({
name: "getEvaluator",
description:
- "Fetch a single evaluator by id, including its prompt or source code, output definition, and how many evaluation rules reference it.",
+ "Fetch a single evaluator by id, including its LLM prompt, code, or decision-model questions and state mapping, plus how many evaluation rules reference it.",
action: "evaluator:read",
baseSchema: GetEvaluatorInput,
inputSchema: GetEvaluatorInput,
diff --git a/web/src/features/mcp/server/evals/tools/listEvaluators.ts b/web/src/features/mcp/server/evals/tools/listEvaluators.ts
index 813beda195..652313118d 100644
--- a/web/src/features/mcp/server/evals/tools/listEvaluators.ts
+++ b/web/src/features/mcp/server/evals/tools/listEvaluators.ts
@@ -12,7 +12,7 @@ const ListEvaluatorsInput = z.object({
export const [listEvaluatorsTool, handleListEvaluators] = defineTool({
name: "listEvaluators",
description:
- "List evaluators (llm_as_judge and code) defined in the current Langfuse project. Results are paginated.",
+ "List LLM-as-a-judge, code, and decision-model evaluators defined in the current Langfuse project. Results are paginated.",
action: "evaluator:read",
baseSchema: ListEvaluatorsInput,
inputSchema: ListEvaluatorsInput,
diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
index 172d4e412c..e58226b951 100644
--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
@@ -1,4 +1,5 @@
import { z } from "zod";
+import { EvalTemplateType } from "@langfuse/shared";
import { managedEvaluatorTemplateService } from "@/src/features/evals";
import { defineTool } from "@/src/features/mcp/core/define-tool";
@@ -7,7 +8,13 @@ import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";
const ListManagedEvaluatorTemplatesInput = z.object({
search: z.string().trim().max(200).optional(),
category: z.string().trim().min(1).optional(),
- type: z.enum(["LLM_AS_JUDGE", "CODE"]).optional(),
+ type: z
+ .enum([
+ EvalTemplateType.LLM_AS_JUDGE,
+ EvalTemplateType.CODE,
+ EvalTemplateType.DECISION_MODEL,
+ ])
+ .optional(),
});
export const [
@@ -15,8 +22,11 @@ export const [
handleListManagedEvaluatorTemplates,
] = defineTool({
name: "listManagedEvaluatorTemplates",
- description:
- "List the evaluator templates maintained by Langfuse and partners. Copy a returned definition into createEvaluator to create a project evaluator.",
+ description: [
+ "List the evaluator templates maintained by Langfuse and partners.",
+ "LLM-as-a-judge and code definitions can be copied into createEvaluator.",
+ "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a TypeSafe modelConfig.",
+ ].join(" "),
action: "evaluator:read",
baseSchema: ListManagedEvaluatorTemplatesInput,
inputSchema: ListManagedEvaluatorTemplatesInput,
diff --git a/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts b/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts
index b366823d5c..299d4399b2 100644
--- a/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts
+++ b/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts
@@ -14,7 +14,7 @@ export const [
name: "attachEvaluatorToEvaluationRule",
description: [
"Attach a project evaluator to an observation evaluation rule using their stable IDs.",
- "For LLM evaluators, omit variableMapping to inherit the evaluator version's mapping or provide an override. Code evaluators must omit variableMapping.",
+ "For LLM-as-a-judge and decision-model evaluators, omit variableMapping to inherit the evaluator version's mapping or provide an override. Code evaluators must omit variableMapping.",
].join(" "),
action: "evaluationRule:CUD",
baseSchema: AttachEvaluatorToEvaluationRuleBaseSchema,
diff --git a/web/src/features/mcp/server/evals/tools/testEvaluator.ts b/web/src/features/mcp/server/evals/tools/testEvaluator.ts
index 43d1e42020..0cabbbf069 100644
--- a/web/src/features/mcp/server/evals/tools/testEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/testEvaluator.ts
@@ -1,3 +1,4 @@
+import { DecisionModelQuestionsSchema } from "@langfuse/shared";
import { z } from "zod";
import { defineTool } from "../../../core/define-tool";
import { runMcpTool } from "../../../core/run-mcp-tool";
@@ -29,36 +30,42 @@ const TestEvaluatorInputBase = z
})
.strict();
-const TestEvaluatorInput = TestEvaluatorInputBase.superRefine((input, ctx) => {
- const hasDraftDefinition = DRAFT_FIELD_NAMES.some(
- (field) => input[field] !== undefined,
- );
+const TestEvaluatorRuntimeInputBase = TestEvaluatorInputBase.extend({
+ questions: DecisionModelQuestionsSchema.optional(),
+});
- if (Boolean(input.evaluatorId) === hasDraftDefinition) {
- ctx.addIssue({
- code: "custom",
- message:
- "Provide either evaluatorId or a draft evaluator definition, but not both.",
- });
- return;
- }
+const TestEvaluatorInput = TestEvaluatorRuntimeInputBase.superRefine(
+ (input, ctx) => {
+ const hasDraftDefinition = DRAFT_FIELD_NAMES.some(
+ (field) => input[field] !== undefined,
+ );
- if (!hasDraftDefinition) return;
+ if (Boolean(input.evaluatorId) === hasDraftDefinition) {
+ ctx.addIssue({
+ code: "custom",
+ message:
+ "Provide either evaluatorId or a draft evaluator definition, but not both.",
+ });
+ return;
+ }
- const parsedDraft = McpEvaluatorInput.safeParse({
- ...input,
- name: "Evaluator test",
- });
- if (parsedDraft.success) return;
+ if (!hasDraftDefinition) return;
- for (const issue of parsedDraft.error.issues) {
- ctx.addIssue({
- code: "custom",
- path: issue.path,
- message: issue.message,
+ const parsedDraft = McpEvaluatorInput.safeParse({
+ ...input,
+ name: "Evaluator test",
});
- }
-}).transform((input) => ({
+ if (parsedDraft.success) return;
+
+ for (const issue of parsedDraft.error.issues) {
+ ctx.addIssue({
+ code: "custom",
+ path: issue.path,
+ message: issue.message,
+ });
+ }
+ },
+).transform((input) => ({
...input,
startTime: new Date(input.startTime),
}));
@@ -67,7 +74,7 @@ export const [testEvaluatorTool, handleTestEvaluator] = defineTool({
name: "testEvaluator",
description: [
"Test an evaluator draft or the latest saved version of an evaluator against one observation in the current Langfuse project.",
- "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge or code evaluator fields.",
+ "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge, code, or decision-model evaluator fields.",
"Pass the observationId, traceId, and startTime returned by the observation tools.",
"This executes the evaluator, emits an internal trace, and may incur model or code execution cost.",
].join(" "),
diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
index 1dad016a56..34200bc103 100644
--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({
name: "updateEvaluator",
description: [
"Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",
- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",
+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",
].join(" "),
action: "evaluator:CUD",
baseSchema: McpUpdateEvaluatorInputBase,