🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): describe evaluator output definitions (#16816)
Tobias Wochinger57a5b6cf0417
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.tsindex 1137a4305f..6194525ba2 100644--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts@@ -214,7 +214,7 @@ const createLlmEvaluatorForMcpReadTest = async (name,type: "LLM_AS_JUDGE",prompt: "Judge {{input}} against {{output}}",- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },+ outputDefinition: mcpEvalOutputDefinition,},setup.context,)) as { id: string; name: string };diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.tsindex 89d0f71a8b..b95a7f2aff 100644--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts@@ -131,7 +131,7 @@ const createStableLlmEvaluatorForMcpWriteTest = async (name,type: "LLM_AS_JUDGE",prompt: "Judge {{input}} against {{output}}",- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },+ outputDefinition: mcpEvalOutputDefinition,variableMapping: [{ templateVariable: "input", selectedColumnId: "input" },{ templateVariable: "output", selectedColumnId: "output" },@@ -203,7 +203,7 @@ describe("MCP Write Tools", () => {name: evaluator.name,type: "LLM_AS_JUDGE",prompt: "Judge {{input}} very strictly",- outputDefinition: { version: 2, ...mcpEvalOutputDefinition },+ outputDefinition: mcpEvalOutputDefinition,},setup.context,)) as { id: string; versions: Array<{ version: number }> };diff --git a/web/src/__tests__/server/unit/evaluator-input.servertest.ts b/web/src/__tests__/server/unit/evaluator-input.servertest.tsindex f95c04d096..08b371575d 100644--- a/web/src/__tests__/server/unit/evaluator-input.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-input.servertest.ts@@ -44,6 +44,35 @@ describe("MCP evaluator input", () => {type: "object",required: ["provider", "model"],});+ expect(schema.properties?.outputDefinition).toMatchObject({+ type: "object",+ required: ["dataType", "reasoning", "score"],+ properties: {+ dataType: {+ type: "string",+ enum: ["NUMERIC", "CATEGORICAL", "BOOLEAN"],+ },+ reasoning: {+ type: "object",+ properties: {+ description: { type: "string" },+ },+ },+ score: {+ type: "object",+ properties: {+ description: { type: "string" },+ minValue: { type: "number" },+ maxValue: { type: "number" },+ categories: {+ type: "array",+ items: { type: "string" },+ },+ shouldAllowMultipleMatches: { type: "boolean" },+ },+ },+ },+ });expect(hasJsonSchemaComposition(schema)).toBe(false);});diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.tsindex af1472953a..3dcf4b81c5 100644--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts@@ -1,4 +1,8 @@-import { EvalTemplateType, observationVariableMapping } from "@langfuse/shared";+import {+ EvalOutputDataTypeSchema,+ EvalTemplateType,+ observationVariableMapping,+} from "@langfuse/shared";import { z } from "zod";import {CodeEvaluatorDefinitionSchema,@@ -27,6 +31,50 @@ const McpObservationVariableMappingSchema = observationVariableMapping.extend({jsonSelector: z.string().optional(),});+const McpEvalOutputDefinitionSchema = z.object({+ dataType: EvalOutputDataTypeSchema.describe(+ "The score type returned by the evaluator.",+ ),+ reasoning: z+ .object({+ description: z+ .string()+ .optional()+ .describe("Instructions for the evaluator's reasoning output."),+ })+ .describe("Definition of the evaluator's textual reasoning output."),+ score: z+ .object({+ description: z+ .string()+ .optional()+ .describe("Instructions for the evaluator's score output."),+ minValue: z+ .number()+ .optional()+ .describe("Optional minimum score for NUMERIC evaluators."),+ maxValue: z+ .number()+ .optional()+ .describe("Optional maximum score for NUMERIC evaluators."),+ categories: z+ .array(z.string())+ .optional()+ .describe(+ "Allowed score labels for CATEGORICAL evaluators. Provide at least two unique values.",+ ),+ shouldAllowMultipleMatches: z+ .boolean()+ .optional()+ .describe(+ "Whether CATEGORICAL evaluators may return multiple score labels.",+ ),+ })+ .describe(+ "Definition of the evaluator's typed score output. Type-specific fields are validated against dataType.",+ ),+});+export const McpEvaluatorInputBase = z.object({name: CreateEvaluatorSchema.shape.name,description: CreateEvaluatorSchema.shape.description.unwrap().optional(),@@ -35,7 +83,9 @@ export const McpEvaluatorInputBase = z.object({modelConfig: McpEvaluatorModelConfigSchema.optional().describe("Optional custom model configuration. Omit to use the project default model.",),- outputDefinition: z.record(z.string(), z.unknown()).optional(),+ outputDefinition: McpEvalOutputDefinitionSchema.optional().describe(+ "Required for LLM-as-a-judge evaluators. Defines the reasoning and score returned by the evaluator.",+ ),sourceCode: CodeEvaluatorDefinitionSchema.shape.sourceCode.optional(),sourceCodeLanguage:CodeEvaluatorDefinitionSchema.shape.sourceCodeLanguage.optional(),