Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(mcp): describe evaluator output definitions (#16816)

Tobias Wochinger57a5b6cf0417
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.ts
index 1137a4305f..6194525ba2 100644
--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts
@@ -214,7 +214,7 @@ const createLlmEvaluatorForMcpReadTest = async (
name,
type: "LLM_AS_JUDGE",
prompt: "Judge {{input}} against {{output}}",
- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },
+ outputDefinition: mcpEvalOutputDefinition,
},
setup.context,
)) as { id: string; name: string };
diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.ts
index 89d0f71a8b..b95a7f2aff 100644
--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts
@@ -131,7 +131,7 @@ const createStableLlmEvaluatorForMcpWriteTest = async (
name,
type: "LLM_AS_JUDGE",
prompt: "Judge {{input}} against {{output}}",
- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },
+ outputDefinition: mcpEvalOutputDefinition,
variableMapping: [
{ templateVariable: "input", selectedColumnId: "input" },
{ templateVariable: "output", selectedColumnId: "output" },
@@ -203,7 +203,7 @@ describe("MCP Write Tools", () => {
name: evaluator.name,
type: "LLM_AS_JUDGE",
prompt: "Judge {{input}} very strictly",
- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },
+ outputDefinition: mcpEvalOutputDefinition,
},
setup.context,
)) as { id: string; versions: Array<{ version: number }> };
diff --git a/web/src/__tests__/server/unit/evaluator-input.servertest.ts b/web/src/__tests__/server/unit/evaluator-input.servertest.ts
index f95c04d096..08b371575d 100644
--- a/web/src/__tests__/server/unit/evaluator-input.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-input.servertest.ts
@@ -44,6 +44,35 @@ describe("MCP evaluator input", () => {
type: "object",
required: ["provider", "model"],
});
+ expect(schema.properties?.outputDefinition).toMatchObject({
+ type: "object",
+ required: ["dataType", "reasoning", "score"],
+ properties: {
+ dataType: {
+ type: "string",
+ enum: ["NUMERIC", "CATEGORICAL", "BOOLEAN"],
+ },
+ reasoning: {
+ type: "object",
+ properties: {
+ description: { type: "string" },
+ },
+ },
+ score: {
+ type: "object",
+ properties: {
+ description: { type: "string" },
+ minValue: { type: "number" },
+ maxValue: { type: "number" },
+ categories: {
+ type: "array",
+ items: { type: "string" },
+ },
+ shouldAllowMultipleMatches: { type: "boolean" },
+ },
+ },
+ },
+ });
expect(hasJsonSchemaComposition(schema)).toBe(false);
});
diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
index af1472953a..3dcf4b81c5 100644
--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
@@ -1,4 +1,8 @@
-import { EvalTemplateType, observationVariableMapping } from "@langfuse/shared";
+import {
+ EvalOutputDataTypeSchema,
+ EvalTemplateType,
+ observationVariableMapping,
+} from "@langfuse/shared";
import { z } from "zod";
import {
CodeEvaluatorDefinitionSchema,
@@ -27,6 +31,50 @@ const McpObservationVariableMappingSchema = observationVariableMapping.extend({
jsonSelector: z.string().optional(),
});
+const McpEvalOutputDefinitionSchema = z.object({
+ dataType: EvalOutputDataTypeSchema.describe(
+ "The score type returned by the evaluator.",
+ ),
+ reasoning: z
+ .object({
+ description: z
+ .string()
+ .optional()
+ .describe("Instructions for the evaluator's reasoning output."),
+ })
+ .describe("Definition of the evaluator's textual reasoning output."),
+ score: z
+ .object({
+ description: z
+ .string()
+ .optional()
+ .describe("Instructions for the evaluator's score output."),
+ minValue: z
+ .number()
+ .optional()
+ .describe("Optional minimum score for NUMERIC evaluators."),
+ maxValue: z
+ .number()
+ .optional()
+ .describe("Optional maximum score for NUMERIC evaluators."),
+ categories: z
+ .array(z.string())
+ .optional()
+ .describe(
+ "Allowed score labels for CATEGORICAL evaluators. Provide at least two unique values.",
+ ),
+ shouldAllowMultipleMatches: z
+ .boolean()
+ .optional()
+ .describe(
+ "Whether CATEGORICAL evaluators may return multiple score labels.",
+ ),
+ })
+ .describe(
+ "Definition of the evaluator's typed score output. Type-specific fields are validated against dataType.",
+ ),
+});
+
export const McpEvaluatorInputBase = z.object({
name: CreateEvaluatorSchema.shape.name,
description: CreateEvaluatorSchema.shape.description.unwrap().optional(),
@@ -35,7 +83,9 @@ export const McpEvaluatorInputBase = z.object({
modelConfig: McpEvaluatorModelConfigSchema.optional().describe(
"Optional custom model configuration. Omit to use the project default model.",
),
- outputDefinition: z.record(z.string(), z.unknown()).optional(),
+ outputDefinition: McpEvalOutputDefinitionSchema.optional().describe(
+ "Required for LLM-as-a-judge evaluators. Defines the reasoning and score returned by the evaluator.",
+ ),
sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),
sourceCodeLanguage:
CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),