🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): support decision-model evaluators (#18166)
Tobias Wochingere59efe007e74
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.tsindex c16a6f000b..ad3a6acc11 100644--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts@@ -591,6 +591,31 @@ describe("MCP Read Tools", () => {),).toBe(true);});++ it("lists decision-model templates", async () => {+ const result = (await handleListManagedEvaluatorTemplates(+ { type: "DECISION_MODEL" },+ mockServerContext(),+ )) as {+ templates: Array<{ key: string; evaluator: { type: string } }>;+ };++ expect(result.templates).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ key: "topic-decision-model",+ evaluator: expect.objectContaining({+ type: "DECISION_MODEL",+ }),+ }),+ ]),+ );+ expect(+ result.templates.every(+ (template) => template.evaluator.type === "DECISION_MODEL",+ ),+ ).toBe(true);+ });});describe("getEvaluator tool", () => {diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.tsindex e6d980b299..c27849beae 100644--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts@@ -244,6 +244,7 @@ describe("MCP Write Tools", () => {evaluatorId: expect.any(Object),type: expect.any(Object),prompt: expect.any(Object),+ questions: expect.any(Object),sourceCode: expect.any(Object),observationId: expect.any(Object),traceId: expect.any(Object),@@ -302,6 +303,65 @@ describe("MCP Write Tools", () => {startTime: new Date(startTime),});+ await expect(+ handleTestEvaluator(+ {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "relevant",+ type: "noul",+ scoreName: "relevant",+ instructions: "Determine whether the input is relevant.",+ criteria: null,+ },+ ],+ modelConfig: {+ provider: "typesafe",+ model: "jev-latest",+ },+ variableMapping: [+ {+ templateVariable: "input",+ selectedColumnId: "input",+ },+ ],+ observationId,+ traceId,+ startTime,+ } as never,+ setup.context,+ ),+ ).resolves.toEqual(unifiedResult);+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(3, {+ orgId: setup.orgId,+ projectId: setup.projectId,+ definition: {+ type: "DECISION_MODEL",+ questions: [+ {+ id: "relevant",+ type: "noul",+ scoreName: "relevant",+ instructions: "Determine whether the input is relevant.",+ criteria: null,+ },+ ],+ provider: "typesafe",+ model: "jev-latest",+ vars: ["input"],+ variableMapping: [+ {+ templateVariable: "input",+ selectedColumnId: "input",+ },+ ],+ },+ observationId,+ traceId,+ startTime: new Date(startTime),+ });+await expect(handleTestEvaluator({ observationId, traceId, startTime } as never,diff --git a/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.ts b/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.tsnew file mode 100644index 0000000000..18f3e4c2b7--- /dev/null+++ b/web/src/__tests__/server/unit/mcp-evaluator-input.servertest.ts@@ -0,0 +1,199 @@+import { describe, expect, it, vi } from "vitest";++import {+ McpEvaluatorInput,+ toEvaluatorServiceInput,+} from "@/src/features/mcp/server/evals/tools/evaluatorInput";+import {+ assertRuleAssignmentsReplaceableViaMcp,+ toMcpEvaluationRule,+} from "@/src/features/mcp/server/evals/rule-service";++const decisionModelInput = {+ name: "Support classifier",+ type: "DECISION_MODEL" as const,+ questions: [+ {+ id: "topic",+ type: "choice" as const,+ scoreName: "topic",+ instructions: "Classify the support request.",+ options: [+ { value: "billing", description: "Billing questions." },+ { value: "technical", description: "Technical questions." },+ ],+ },+ ],+ modelConfig: {+ provider: "typesafe",+ model: "jev-latest",+ },+ variableMapping: [+ {+ templateVariable: "input",+ selectedColumnId: "input" as const,+ },+ ],+};++describe("MCP evaluator input", () => {+ it("converts a decision-model definition to evaluator service input", () => {+ expect(toEvaluatorServiceInput(decisionModelInput)).toEqual({+ name: decisionModelInput.name,+ description: null,+ definition: {+ type: "DECISION_MODEL",+ questions: decisionModelInput.questions,+ provider: "typesafe",+ model: "jev-latest",+ vars: ["input"],+ variableMapping: [+ {+ templateVariable: "input",+ selectedColumnId: "input",+ },+ ],+ },+ });+ });++ it.each([+ ["questions", { questions: undefined }],+ ["modelConfig", { modelConfig: undefined }],+ ["variableMapping", { variableMapping: undefined }],+ ])("requires decision-model %s", (field, override) => {+ const result = McpEvaluatorInput.safeParse({+ ...decisionModelInput,+ ...override,+ });++ expect(result.success).toBe(false);+ if (!result.success) {+ expect(result.error.issues).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: expect.arrayContaining([field]),+ }),+ ]),+ );+ }+ });++ it("rejects invalid decision-model state keys", () => {+ const result = McpEvaluatorInput.safeParse({+ ...decisionModelInput,+ variableMapping: [+ {+ templateVariable: "support topic",+ selectedColumnId: "input",+ },+ ],+ });++ expect(result.success).toBe(false);+ if (!result.success) {+ expect(result.error.issues).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: ["variableMapping", 0, "templateVariable"],+ }),+ ]),+ );+ }+ });++ it("rejects model parameters for decision-model evaluators", () => {+ const result = McpEvaluatorInput.safeParse({+ ...decisionModelInput,+ modelConfig: {+ ...decisionModelInput.modelConfig,+ modelParams: { temperature: 0 },+ },+ });++ expect(result.success).toBe(false);+ if (!result.success) {+ expect(result.error.issues).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ path: ["modelConfig", "modelParams"],+ }),+ ]),+ );+ }+ });++ it("exposes decision-model assignments in MCP evaluation rules", () => {+ const createdAt = new Date("2026-10-02T00:00:00.000Z");+ const updatedAt = new Date("2026-10-02T01:00:00.000Z");++ expect(+ toMcpEvaluationRule({+ id: "rule-1",+ name: "Support routing",+ enabled: true,+ sampling: 1,+ filter: [],+ assignments: [+ {+ evaluator: {+ id: "evaluator-1",+ name: "Support classifier",+ type: "DECISION_MODEL",+ },+ variableMapping: [+ {+ templateVariable: "input",+ selectedColumnId: "input",+ jsonSelector: null,+ },+ ],+ },+ ],+ createdAt,+ updatedAt,+ } as never),+ ).toEqual({+ id: "rule-1",+ name: "Support routing",+ enabled: true,+ sampling: 1,+ filter: [],+ evaluators: [+ {+ evaluatorId: "evaluator-1",+ evaluatorName: "Support classifier",+ evaluatorType: "decision_model",+ variableMapping: [+ {+ variable: "input",+ source: "input",+ },+ ],+ },+ ],+ createdAt,+ updatedAt,+ });+ });++ it("allows MCP to replace rules with decision-model assignments", async () => {+ await expect(+ assertRuleAssignmentsReplaceableViaMcp(+ {+ get: vi.fn().mockResolvedValue({+ assignments: [+ {+ evaluator: {+ type: "DECISION_MODEL",+ },+ },+ ],+ }),+ } as never,+ "project-1",+ "rule-1",+ ),+ ).resolves.toBeUndefined();+ });+});diff --git a/web/src/features/mcp/server/evals/rule-schema.ts b/web/src/features/mcp/server/evals/rule-schema.tsindex 4b4ef80005..4b9273cbad 100644--- a/web/src/features/mcp/server/evals/rule-schema.ts+++ b/web/src/features/mcp/server/evals/rule-schema.ts@@ -15,6 +15,11 @@ import {} from "@/src/features/public-api/server";import { McpAdvancedFilterBaseSchema } from "../../core/filter-schema";+export const McpEvaluatorType = z.enum([+ ...PublicEvaluatorType.options,+ "decision_model",+]);+const RuleFilterBaseSchema = McpAdvancedFilterBaseSchema.describe('Observation filter condition, e.g. {"column":"version","operator":"=","value":"1.0.0","type":"string"}. Use `key` for object columns such as `metadata`.',);@@ -33,7 +38,7 @@ const EvaluationRuleEvaluatorInputBase = {const VariableMappingSchema = z.array(ObservationPromptVariableMappingInput).describe(- "Optional rule-specific variable mapping for LLM evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",+ "Optional rule-specific variable mapping for LLM-as-a-judge and decision-model evaluators. Omit for code evaluators, whose mapping is managed by Langfuse.",);export const AttachEvaluatorToEvaluationRuleBaseSchema = z.object({@@ -158,7 +163,7 @@ export const EvaluationRuleResponseSchema = zz.object({evaluatorId: z.string(),evaluatorName: z.string(),- evaluatorType: PublicEvaluatorType,+ evaluatorType: McpEvaluatorType,variableMapping: z.array(PromptVariableMappingRead).nullable(),}),),diff --git a/web/src/features/mcp/server/evals/rule-service.ts b/web/src/features/mcp/server/evals/rule-service.tsindex 843cd12f9f..9a27d7c6ca 100644--- a/web/src/features/mcp/server/evals/rule-service.ts+++ b/web/src/features/mcp/server/evals/rule-service.ts@@ -7,15 +7,30 @@ import {toStoredMappingList,} from "@/src/features/public-api/server";import { RuleService } from "@/src/features/evals/v2/server/rules/ruleService";-import { InvalidRequestError } from "@langfuse/shared";+import { EvalTemplateType, InvalidRequestError } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import type { z } from "zod";import type { ServerContext } from "../../types";import {+ McpEvaluatorType,EvaluationRuleResponseSchema,type EvaluationRuleAssignmentInput,} from "./rule-schema";+function isMcpEvaluatorType(type: EvalTemplateType) {+ return (+ type === EvalTemplateType.DECISION_MODEL || isPublicApiEvaluatorType(type)+ );+}++function toMcpEvaluatorType(type: EvalTemplateType) {+ return McpEvaluatorType.parse(+ type === EvalTemplateType.DECISION_MODEL+ ? "decision_model"+ : toPublicEvaluatorType(type),+ );+}+export function createMcpRuleService(context: ServerContext) {return new RuleService(prisma, ({ action, ruleId }) =>auditLog({@@ -51,11 +66,11 @@ export async function assertRuleAssignmentsReplaceableViaMcp() {const rule = await service.get(projectId, ruleId);const hidden = rule.assignments.filter(- (assignment) => !isPublicApiEvaluatorType(assignment.evaluator.type),+ (assignment) => !isMcpEvaluatorType(assignment.evaluator.type),);if (hidden.length > 0) {throw new InvalidRequestError(- "This rule uses experimental evaluators that are not visible through MCP. Update its evaluator assignments in the Langfuse UI.",+ "This rule uses internal evaluators that are not visible through MCP. Update its evaluator assignments in the Langfuse UI.",);}}@@ -70,13 +85,11 @@ export function toMcpEvaluationRule(sampling: rule.sampling,filter: rule.filter,evaluators: rule.assignments- .filter((assignment) =>- isPublicApiEvaluatorType(assignment.evaluator.type),- )+ .filter((assignment) => isMcpEvaluatorType(assignment.evaluator.type)).map((assignment) => ({evaluatorId: assignment.evaluator.id,evaluatorName: assignment.evaluator.name,- evaluatorType: toPublicEvaluatorType(assignment.evaluator.type),+ evaluatorType: toMcpEvaluatorType(assignment.evaluator.type),variableMapping:assignment.variableMapping === null? nulldiff --git a/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts b/web/src/features/mcp/server/evals/tools/createEvaluationRule.tsindex 09ffa24aa2..dc8b204dfd 100644--- a/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts+++ b/web/src/features/mcp/server/evals/tools/createEvaluationRule.ts@@ -16,7 +16,7 @@ export const [createEvaluationRuleTool, handleCreateEvaluationRule] =name: "createEvaluationRule",description: ["Create an observation evaluation rule with one or more evaluator assignments.",- "Each assignment references a project evaluator by stable ID. LLM evaluator assignments may override their variable mapping; code evaluator assignments must omit it.",+ "Each assignment references a project evaluator by stable ID. LLM-as-a-judge and decision-model assignments may override their variable mapping; code evaluator assignments must omit it.",].join(" "),action: "evaluationRule:CUD",baseSchema: CreateEvaluationRuleBaseSchema,diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.tsindex 6857ac500a..5d1c1c166a 100644--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts@@ -12,7 +12,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({name: "createEvaluator",description: ["Create a new evaluator with a stable id. Names do not act as identity and may be reused.",- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.","Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",].join(" "),action: "evaluator:CUD",diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.tsindex 40a108858e..4f08af9b3f 100644--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts@@ -1,4 +1,10 @@import {+ DECISION_MODEL_LIMITS,+ DecisionModelChoiceOptionSchema,+ DecisionModelChoiceQuestionSchema,+ DecisionModelQuestionType,+ DecisionModelQuestionsSchema,+ DecisionModelStateKeySchema,EvalOutputDataTypeSchema,EvalTemplateType,observationVariableMapping,@@ -31,6 +37,52 @@ const McpObservationVariableMappingSchema = observationVariableMapping.extend({jsonSelector: z.string().optional(),});+const McpDecisionModelQuestionBaseSchema = z.object({+ id: DecisionModelChoiceQuestionSchema.shape.id.describe(+ "Stable question identifier.",+ ),+ type: z.enum(DecisionModelQuestionType).describe("Decision question type."),+ scoreName: DecisionModelChoiceQuestionSchema.shape.scoreName.describe(+ "Name of the score produced by the question.",+ ),+ instructions: z+ .unknown()+ .describe("Question instructions as text or structured JSON."),+ options: z+ .array(+ z.object({+ value: DecisionModelChoiceOptionSchema.shape.value,+ description: z+ .unknown()+ .optional()+ .describe("Option description as text or structured JSON."),+ }),+ )+ .min(DECISION_MODEL_LIMITS.minChoiceOptions)+ .max(DECISION_MODEL_LIMITS.maxChoiceOptions)+ .optional()+ .describe("Choice options. Required when type is `choice`."),+ levels: z+ .array(+ z.object({+ description: z+ .unknown()+ .describe("Level description as text or structured JSON."),+ }),+ )+ .min(DECISION_MODEL_LIMITS.minScoreLevels)+ .max(DECISION_MODEL_LIMITS.maxScoreLevels)+ .optional()+ .describe("Ordered score levels. Required when type is `score`."),+ criteria: z+ .object({+ true: z.unknown().optional(),+ false: z.unknown().optional(),+ })+ .optional()+ .describe("Optional true/false criteria when type is `noul`."),+});+const McpEvalOutputDefinitionSchema = z.object({dataType: EvalOutputDataTypeSchema.describe("The score type returned by the evaluator.",@@ -78,21 +130,35 @@ const McpEvalOutputDefinitionSchema = z.object({export const McpEvaluatorInputBase = z.object({name: CreateEvaluatorSchema.shape.name,description: CreateEvaluatorSchema.shape.description.unwrap().optional(),- type: z.enum([EvalTemplateType.LLM_AS_JUDGE, EvalTemplateType.CODE]),+ type: z.enum([+ EvalTemplateType.LLM_AS_JUDGE,+ EvalTemplateType.CODE,+ EvalTemplateType.DECISION_MODEL,+ ]),prompt: z.string().min(1).optional(),modelConfig: McpEvaluatorModelConfigSchema.optional().describe(- "Optional custom model configuration. Omit to use the project default model.",+ "Model configuration. Required for decision-model evaluators. Optional for LLM-as-a-judge evaluators, which use the project default when omitted.",),outputDefinition: McpEvalOutputDefinitionSchema.optional().describe("Required for LLM-as-a-judge evaluators. Defines the reasoning and score returned by the evaluator.",),+ questions: z+ .array(McpDecisionModelQuestionBaseSchema)+ .min(1)+ .max(DECISION_MODEL_LIMITS.maxQuestions)+ .optional()+ .describe(+ "Required for decision-model evaluators. Each question produces its own score.",+ ),sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),sourceCodeLanguage:CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),variableMapping: z.array(McpObservationVariableMappingSchema).optional()- .describe("Variable mappings for LLM-as-a-judge evaluators only."),+ .describe(+ "Variable mappings for LLM-as-a-judge and decision-model evaluators. Decision-model mappings define the state and require at least one entry.",+ ),});export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({@@ -100,7 +166,13 @@ export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({description: true,});-function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {+const McpEvaluatorRuntimeInputBase = McpEvaluatorInputBase.extend({+ questions: DecisionModelQuestionsSchema.optional().describe(+ "Required for decision-model evaluators. Each question produces its own score.",+ ),+});++function toEvaluatorInput(input: z.infer<typeof McpEvaluatorRuntimeInputBase>) {if (input.type === EvalTemplateType.LLM_AS_JUDGE) {return {name: input.name,@@ -117,6 +189,19 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {};}+ if (input.type === EvalTemplateType.DECISION_MODEL) {+ return {+ name: input.name,+ description: input.description ?? null,+ definition: {+ type: input.type,+ questions: input.questions,+ modelConfig: input.modelConfig,+ variableMapping: input.variableMapping,+ },+ };+ }+return {name: input.name,description: input.description ?? null,@@ -129,7 +214,7 @@ function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {}function validateEvaluatorInput(- input: z.infer<typeof McpEvaluatorInputBase>,+ input: z.infer<typeof McpEvaluatorRuntimeInputBase>,ctx: z.RefinementCtx,) {if (input.type === EvalTemplateType.LLM_AS_JUDGE && !input.prompt?.trim()) {@@ -140,6 +225,43 @@ function validateEvaluatorInput(});}+ if (+ input.type === EvalTemplateType.DECISION_MODEL &&+ input.variableMapping?.length === 0+ ) {+ ctx.addIssue({+ code: "custom",+ path: ["variableMapping"],+ message:+ "Decision-model evaluators require at least one state variable mapping.",+ });+ }++ if (input.type === EvalTemplateType.DECISION_MODEL) {+ input.variableMapping?.forEach(({ templateVariable }, index) => {+ const parsedStateKey =+ DecisionModelStateKeySchema.safeParse(templateVariable);+ if (parsedStateKey.success) return;++ for (const issue of parsedStateKey.error.issues) {+ ctx.addIssue({+ code: "custom",+ path: ["variableMapping", index, "templateVariable"],+ message: issue.message,+ });+ }+ });++ if (input.modelConfig?.modelParams !== undefined) {+ ctx.addIssue({+ code: "custom",+ path: ["modelConfig", "modelParams"],+ message:+ "Decision-model evaluators do not support model parameters. Provide only provider and model.",+ });+ }+ }+if (input.type === EvalTemplateType.CODE &&input.variableMapping !== undefined@@ -166,7 +288,7 @@ function validateEvaluatorInput(}}-export const McpEvaluatorInput = McpEvaluatorInputBase.superRefine(+export const McpEvaluatorInput = McpEvaluatorRuntimeInputBase.superRefine(validateEvaluatorInput,);@@ -174,13 +296,18 @@ export const McpUpdateEvaluatorInputBase = McpEvaluatorInputBase.extend({evaluatorId: z.string(),});-export const McpUpdateEvaluatorInput = McpUpdateEvaluatorInputBase.superRefine(- ({ evaluatorId: _evaluatorId, ...input }, ctx) =>- validateEvaluatorInput(input, ctx),-);+const McpUpdateEvaluatorRuntimeInputBase = McpEvaluatorRuntimeInputBase.extend({+ evaluatorId: z.string(),+});++export const McpUpdateEvaluatorInput =+ McpUpdateEvaluatorRuntimeInputBase.superRefine(+ ({ evaluatorId: _evaluatorId, ...input }, ctx) =>+ validateEvaluatorInput(input, ctx),+ );export function toEvaluatorServiceInput(- input: z.infer<typeof McpEvaluatorInputBase>,+ input: z.infer<typeof McpEvaluatorRuntimeInputBase>,) {return CreateEvaluatorWithoutProjectSchema.parse(toEvaluatorInput(input));}diff --git a/web/src/features/mcp/server/evals/tools/getEvaluator.ts b/web/src/features/mcp/server/evals/tools/getEvaluator.tsindex c245333e19..3636d95869 100644--- a/web/src/features/mcp/server/evals/tools/getEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/getEvaluator.ts@@ -9,7 +9,7 @@ const GetEvaluatorInput = z.object({ evaluatorId: z.string() });export const [getEvaluatorTool, handleGetEvaluator] = defineTool({name: "getEvaluator",description:- "Fetch a single evaluator by id, including its prompt or source code, output definition, and how many evaluation rules reference it.",+ "Fetch a single evaluator by id, including its LLM prompt, code, or decision-model questions and state mapping, plus how many evaluation rules reference it.",action: "evaluator:read",baseSchema: GetEvaluatorInput,inputSchema: GetEvaluatorInput,diff --git a/web/src/features/mcp/server/evals/tools/listEvaluators.ts b/web/src/features/mcp/server/evals/tools/listEvaluators.tsindex 813beda195..652313118d 100644--- a/web/src/features/mcp/server/evals/tools/listEvaluators.ts+++ b/web/src/features/mcp/server/evals/tools/listEvaluators.ts@@ -12,7 +12,7 @@ const ListEvaluatorsInput = z.object({export const [listEvaluatorsTool, handleListEvaluators] = defineTool({name: "listEvaluators",description:- "List evaluators (llm_as_judge and code) defined in the current Langfuse project. Results are paginated.",+ "List LLM-as-a-judge, code, and decision-model evaluators defined in the current Langfuse project. Results are paginated.",action: "evaluator:read",baseSchema: ListEvaluatorsInput,inputSchema: ListEvaluatorsInput,diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.tsindex 172d4e412c..e58226b951 100644--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts@@ -1,4 +1,5 @@import { z } from "zod";+import { EvalTemplateType } from "@langfuse/shared";import { managedEvaluatorTemplateService } from "@/src/features/evals";import { defineTool } from "@/src/features/mcp/core/define-tool";@@ -7,7 +8,13 @@ import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";const ListManagedEvaluatorTemplatesInput = z.object({search: z.string().trim().max(200).optional(),category: z.string().trim().min(1).optional(),- type: z.enum(["LLM_AS_JUDGE", "CODE"]).optional(),+ type: z+ .enum([+ EvalTemplateType.LLM_AS_JUDGE,+ EvalTemplateType.CODE,+ EvalTemplateType.DECISION_MODEL,+ ])+ .optional(),});export const [@@ -15,8 +22,11 @@ export const [handleListManagedEvaluatorTemplates,] = defineTool({name: "listManagedEvaluatorTemplates",- description:- "List the evaluator templates maintained by Langfuse and partners. Copy a returned definition into createEvaluator to create a project evaluator.",+ description: [+ "List the evaluator templates maintained by Langfuse and partners.",+ "LLM-as-a-judge and code definitions can be copied into createEvaluator.",+ "For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a TypeSafe modelConfig.",+ ].join(" "),action: "evaluator:read",baseSchema: ListManagedEvaluatorTemplatesInput,inputSchema: ListManagedEvaluatorTemplatesInput,diff --git a/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts b/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.tsindex b366823d5c..299d4399b2 100644--- a/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts+++ b/web/src/features/mcp/server/evals/tools/manageEvaluationRuleEvaluators.ts@@ -14,7 +14,7 @@ export const [name: "attachEvaluatorToEvaluationRule",description: ["Attach a project evaluator to an observation evaluation rule using their stable IDs.",- "For LLM evaluators, omit variableMapping to inherit the evaluator version's mapping or provide an override. Code evaluators must omit variableMapping.",+ "For LLM-as-a-judge and decision-model evaluators, omit variableMapping to inherit the evaluator version's mapping or provide an override. Code evaluators must omit variableMapping.",].join(" "),action: "evaluationRule:CUD",baseSchema: AttachEvaluatorToEvaluationRuleBaseSchema,diff --git a/web/src/features/mcp/server/evals/tools/testEvaluator.ts b/web/src/features/mcp/server/evals/tools/testEvaluator.tsindex 43d1e42020..0cabbbf069 100644--- a/web/src/features/mcp/server/evals/tools/testEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/testEvaluator.ts@@ -1,3 +1,4 @@+import { DecisionModelQuestionsSchema } from "@langfuse/shared";import { z } from "zod";import { defineTool } from "../../../core/define-tool";import { runMcpTool } from "../../../core/run-mcp-tool";@@ -29,36 +30,42 @@ const TestEvaluatorInputBase = z}).strict();-const TestEvaluatorInput = TestEvaluatorInputBase.superRefine((input, ctx) => {- const hasDraftDefinition = DRAFT_FIELD_NAMES.some(- (field) => input[field] !== undefined,- );+const TestEvaluatorRuntimeInputBase = TestEvaluatorInputBase.extend({+ questions: DecisionModelQuestionsSchema.optional(),+});- if (Boolean(input.evaluatorId) === hasDraftDefinition) {- ctx.addIssue({- code: "custom",- message:- "Provide either evaluatorId or a draft evaluator definition, but not both.",- });- return;- }+const TestEvaluatorInput = TestEvaluatorRuntimeInputBase.superRefine(+ (input, ctx) => {+ const hasDraftDefinition = DRAFT_FIELD_NAMES.some(+ (field) => input[field] !== undefined,+ );- if (!hasDraftDefinition) return;+ if (Boolean(input.evaluatorId) === hasDraftDefinition) {+ ctx.addIssue({+ code: "custom",+ message:+ "Provide either evaluatorId or a draft evaluator definition, but not both.",+ });+ return;+ }- const parsedDraft = McpEvaluatorInput.safeParse({- ...input,- name: "Evaluator test",- });- if (parsedDraft.success) return;+ if (!hasDraftDefinition) return;- for (const issue of parsedDraft.error.issues) {- ctx.addIssue({- code: "custom",- path: issue.path,- message: issue.message,+ const parsedDraft = McpEvaluatorInput.safeParse({+ ...input,+ name: "Evaluator test",});- }-}).transform((input) => ({+ if (parsedDraft.success) return;++ for (const issue of parsedDraft.error.issues) {+ ctx.addIssue({+ code: "custom",+ path: issue.path,+ message: issue.message,+ });+ }+ },+).transform((input) => ({...input,startTime: new Date(input.startTime),}));@@ -67,7 +74,7 @@ export const [testEvaluatorTool, handleTestEvaluator] = defineTool({name: "testEvaluator",description: ["Test an evaluator draft or the latest saved version of an evaluator against one observation in the current Langfuse project.",- "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge or code evaluator fields.",+ "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge, code, or decision-model evaluator fields.","Pass the observationId, traceId, and startTime returned by the observation tools.","This executes the evaluator, emits an internal trace, and may incur model or code execution cost.",].join(" "),diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.tsindex 1dad016a56..34200bc103 100644--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts@@ -12,7 +12,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({name: "updateEvaluator",description: ["Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",- "Set type to `LLM_AS_JUDGE` and provide prompt + outputDefinition. Omit modelConfig to use the project default, or provide modelConfig with provider, model, and optional modelParams. For `CODE`, provide sourceCode + sourceCodeLanguage.",+ "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, an explicit TypeSafe modelConfig, and variableMapping entries defining its state.",].join(" "),action: "evaluator:CUD",baseSchema: McpUpdateEvaluatorInputBase,