🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): add evaluator testing tool (#16870)
Tobias Wochinger2b70ed280ea7
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex 9df9515268..2014dc3d51 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -144,6 +144,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {approval: "auto",availability: { scope: "evaluator:read" },},+ testEvaluator: {+ approval: "approval",+ availability: { scope: "evaluator:CUD" },+ },createEvaluator: {approval: "approval",availability: { scope: "evaluator:CUD" },diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.tsindex 3f14fd47db..f31df9b949 100644--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts@@ -812,7 +812,16 @@ describe("EvaluatorService", () => {}),).resolves.toEqual({ score: 1 });- expect(mocks.testEvaluator).toHaveBeenCalledTimes(2);+ await expect(+ service.testEvaluator({+ orgId,+ projectId,+ definition: llmInput("Test evaluator").definition,+ ...observation,+ }),+ ).resolves.toEqual({ score: 1 });++ expect(mocks.testEvaluator).toHaveBeenCalledTimes(3);expect(mocks.testEvaluator).toHaveBeenCalledWith(expect.objectContaining({orgId,@@ -829,6 +838,15 @@ describe("EvaluatorService", () => {...observation,}),);+ expect(mocks.testEvaluator).toHaveBeenNthCalledWith(+ 3,+ expect.objectContaining({+ orgId,+ evaluatorId: expect.any(String),+ includeEvaluatorLink: false,+ ...observation,+ }),+ );await expect(prisma.evaluatorVersion.count({ where: { evaluator: { projectId } } }),).resolves.toBe(before);@@ -844,6 +862,49 @@ describe("EvaluatorService", () => {).rejects.toThrow("Evaluator not found");});+ it("tests the latest version of a saved evaluator", async () => {+ mocks.testEvaluator.mockResolvedValue({ score: 1 });+ const service = createService();+ const evaluator = await service.create(llmInput("Saved evaluator"), null);+ const latestDefinition = {+ ...llmInput("Saved evaluator").definition,+ prompt: "Judge the latest {{output}}",+ };+ await service.update(+ {+ ...llmInput("Saved evaluator"),+ evaluatorId: evaluator.id,+ definition: latestDefinition,+ },+ null,+ );++ const observation = {+ observationId: "test-observation-id",+ traceId: "test-trace-id",+ startTime: new Date("2026-08-11T00:00:00.000Z"),+ shouldReadFromObservationsTable: false,+ };++ await expect(+ service.testEvaluator({+ orgId,+ projectId,+ evaluatorId: evaluator.id,+ ...observation,+ }),+ ).resolves.toEqual({ score: 1 });++ expect(mocks.testEvaluator).toHaveBeenCalledWith({+ orgId,+ projectId,+ evaluatorId: evaluator.id,+ definition: latestDefinition,+ includeEvaluatorLink: true,+ ...observation,+ });+ });+it("suggests a safe name and silently returns null when unavailable", async () => {mocks.generateLangfuseAIText.mockResolvedValueOnce(' "Concise quality judge" ')diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.tsindex b95a7f2aff..e6d980b299 100644--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts@@ -88,7 +88,13 @@ import {deleteEvaluatorTool,handleDeleteEvaluator,} from "@/src/features/mcp/server/evals/tools/deleteEvaluator";+import {+ handleTestEvaluator,+ testEvaluatorTool,+} from "@/src/features/mcp/server/evals/tools/testEvaluator";+import { evalsFeature } from "@/src/features/mcp/server/evals";import { handleGetEvaluationRule } from "@/src/features/mcp/server/evals/tools/getEvaluationRule";+import { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";import {attachEvaluatorToEvaluationRuleTool,detachEvaluatorFromEvaluationRuleTool,@@ -191,6 +197,151 @@ describe("MCP Write Tools", () => {});describe("stable evaluator tools", () => {+ it("tests the latest saved evaluator against a project observation", async () => {+ const setup = await createMcpTestSetup();+ const evaluator = await createStableLlmEvaluatorForMcpWriteTest(setup);+ await handleUpdateEvaluator(+ {+ evaluatorId: evaluator.id,+ name: evaluator.name,+ type: "LLM_AS_JUDGE",+ prompt: "Latest evaluator prompt for {{input}}",+ outputDefinition: mcpEvalOutputDefinition,+ variableMapping: [+ { templateVariable: "input", selectedColumnId: "input" },+ ],+ },+ setup.context,+ );++ const observationId = nanoid();+ const traceId = nanoid();+ const startTime = "2026-08-11T12:00:00.000Z";+ const unifiedResult = {+ success: true as const,+ result: {+ dataType: "NUMERIC" as const,+ score: 1,+ reasoning: "passes",+ },+ interpolatedPrompt: "Latest evaluator prompt for sample input",+ model: "test-model",+ provider: "test-provider",+ executionTraceId: "execution-trace-id",+ estimatedCostUsd: null,+ durationMs: 25,+ };+ const testEvaluatorSpy = vi+ .spyOn(EvaluatorService.prototype, "testEvaluator")+ .mockResolvedValue(unifiedResult);++ try {+ verifyToolAnnotations(testEvaluatorTool, { expensiveHint: true });+ expect(testEvaluatorTool.annotations?.readOnlyHint).not.toBe(true);+ expect(testEvaluatorTool.annotations?.destructiveHint).not.toBe(true);+ expect(testEvaluatorTool.inputSchema.properties).toEqual(+ expect.objectContaining({+ evaluatorId: expect.any(Object),+ type: expect.any(Object),+ prompt: expect.any(Object),+ sourceCode: expect.any(Object),+ observationId: expect.any(Object),+ traceId: expect.any(Object),+ startTime: expect.any(Object),+ }),+ );+ expect(+ evalsFeature.tools.some(+ ({ definition }) => definition.name === "testEvaluator",+ ),+ ).toBe(true);++ await expect(+ handleTestEvaluator(+ {+ evaluatorId: evaluator.id,+ observationId,+ traceId,+ startTime,+ } as never,+ setup.context,+ ),+ ).resolves.toEqual(unifiedResult);+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(1, {+ orgId: setup.orgId,+ projectId: setup.projectId,+ evaluatorId: evaluator.id,+ observationId,+ traceId,+ startTime: new Date(startTime),+ });++ await expect(+ handleTestEvaluator(+ {+ type: "CODE",+ sourceCode: "return { score: 1 };",+ sourceCodeLanguage: "TYPESCRIPT",+ observationId,+ traceId,+ startTime,+ } as never,+ setup.context,+ ),+ ).resolves.toEqual(unifiedResult);+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(2, {+ orgId: setup.orgId,+ projectId: setup.projectId,+ definition: {+ type: "CODE",+ sourceCode: "return { score: 1 };",+ sourceCodeLanguage: "TYPESCRIPT",+ },+ observationId,+ traceId,+ startTime: new Date(startTime),+ });++ await expect(+ handleTestEvaluator(+ { observationId, traceId, startTime } as never,+ setup.context,+ ),+ ).rejects.toThrow(+ "Provide either evaluatorId or a draft evaluator definition, but not both.",+ );+ await expect(+ handleTestEvaluator(+ {+ evaluatorId: evaluator.id,+ type: "CODE",+ sourceCode: "return { score: 1 };",+ sourceCodeLanguage: "TYPESCRIPT",+ observationId,+ traceId,+ startTime,+ } as never,+ setup.context,+ ),+ ).rejects.toThrow(+ "Provide either evaluatorId or a draft evaluator definition, but not both.",+ );+ await expect(+ handleTestEvaluator(+ {+ evaluatorId: evaluator.id,+ observationId,+ traceId,+ startTime: null,+ } as never,+ setup.context,+ ),+ ).rejects.toThrow();+ } finally {+ testEvaluatorSpy.mockRestore();+ }+ });+it("creates and versions an evaluator by stable id", async () => {const setup = await createMcpTestSetup();verifyToolAnnotations(createEvaluatorTool, { destructiveHint: true });diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.tsindex 71d58061da..a7fbf57766 100644--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts@@ -144,6 +144,58 @@ describe("testEvaluator", () => {);});+ it("treats a missing mapping as empty when the prompt has no variables", async () => {+ await expect(+ testEvaluator({+ orgId: "org-1",+ projectId: "project-1",+ evaluatorId: "evaluator-1",+ observationId: "observation-1",+ traceId: "trace-1",+ startTime: new Date("2026-08-11T12:00:00.000Z"),+ definition: {+ type: "LLM_AS_JUDGE",+ prompt: "Judge this response",+ provider: null,+ model: null,+ modelParams: null,+ vars: [],+ variableMapping: null,+ outputDefinition: createNumericEvalOutputDefinition({+ scoreDescription: "Correctness",+ reasoningDescription: "Why the answer is correct",+ }),+ },+ }),+ ).resolves.toMatchObject({ success: true });+ });++ it("rejects a missing mapping when the prompt has variables", async () => {+ await expect(+ testEvaluator({+ orgId: "org-1",+ projectId: "project-1",+ evaluatorId: "evaluator-1",+ observationId: "observation-1",+ traceId: "trace-1",+ startTime: new Date("2026-08-11T12:00:00.000Z"),+ definition: {+ type: "LLM_AS_JUDGE",+ prompt: "Judge {{answer}}",+ provider: null,+ model: null,+ modelParams: null,+ vars: ["answer"],+ variableMapping: null,+ outputDefinition: createNumericEvalOutputDefinition({+ scoreDescription: "Correctness",+ reasoningDescription: "Why the answer is correct",+ }),+ },+ }),+ ).rejects.toThrow("Missing mappings for evaluator variables: answer");+ });+it("loads the selected observation and dispatches a code evaluator with canonical variables", async () => {const startTime = new Date("2026-08-11T12:00:00.000Z");vi.mocked(getObservationForEvalById).mockResolvedValue({diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.tsindex 85ca3c6b8c..b968d79a2e 100644--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts@@ -1,3 +1,4 @@+import { randomUUID } from "node:crypto";import { isDeepStrictEqual } from "node:util";import { z } from "zod";import {@@ -8,6 +9,7 @@ import {getCodeEvalVariableMapping,getEvaluatorBlockMetadata,isEvaluatorBlockReasonRecoverableByDefinitionUpdate,+ InvalidRequestError,LangfuseConflictError,LangfuseNotFoundError,} from "@langfuse/shared";@@ -547,21 +549,62 @@ export class EvaluatorService {return evaluatorIds;}- async testEvaluator(params: Parameters<typeof executeEvaluatorTest>[0]) {- const evaluator = await this.prisma.evaluator.findFirst({- where: { id: params.evaluatorId, projectId: params.projectId },- select: { id: true },- });- // The setup editor pre-generates a UUID so a test run can be attributed to- // the evaluator before it is first saved. Every other id must resolve- // inside the project — never look it up unscoped, which would turn the- // response into a cross-project existence oracle.- if (!evaluator && !isPregeneratedEvaluatorId(params.evaluatorId)) {- throw new LangfuseNotFoundError("Evaluator not found");+ async testEvaluator(+ params: Omit<+ Parameters<typeof executeEvaluatorTest>[0],+ "evaluatorId" | "definition" | "includeEvaluatorLink"+ > & {+ evaluatorId?: string;+ definition?: EvaluatorDefinition;+ },+ ) {+ const {+ evaluatorId: requestedEvaluatorId,+ definition: requestedDefinition,+ ...executionParams+ } = params;++ let evaluatorId = requestedEvaluatorId;+ let definition = requestedDefinition;+ let includeEvaluatorLink = false;++ if (definition) {+ if (evaluatorId) {+ const evaluator = await this.prisma.evaluator.findFirst({+ where: { id: evaluatorId, projectId: params.projectId },+ select: { id: true },+ });+ // The setup editor pre-generates a UUID so a test run can be attributed+ // to the evaluator before it is first saved. Every other id must resolve+ // inside the project — never look it up unscoped, which would turn the+ // response into a cross-project existence oracle.+ if (!evaluator && !isPregeneratedEvaluatorId(evaluatorId)) {+ throw new LangfuseNotFoundError("Evaluator not found");+ }+ includeEvaluatorLink = Boolean(evaluator);+ } else {+ evaluatorId = randomUUID();+ }+ } else {+ if (!evaluatorId) {+ throw new InvalidRequestError(+ "Either evaluatorId or definition is required",+ );+ }+ const evaluator = await this.get(params.projectId, evaluatorId);+ const latestVersion = evaluator.versions[0];+ if (!latestVersion) {+ throw new LangfuseNotFoundError("Evaluator version not found");+ }+ definition = toEvaluatorDefinition(evaluator.type, latestVersion);+ includeEvaluatorLink = true;}+return executeEvaluatorTest({- ...params,- includeEvaluatorLink: Boolean(evaluator),+ ...executionParams,+ evaluatorId,+ definition,+ includeEvaluatorLink,});}diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 760d3be631..863cef7afe 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -20,6 +20,7 @@ import {} from "@langfuse/shared/src/server";import { getObservationForEvalById } from "@/src/features/evals/server/getObservationForEvalById";import type { EvaluatorDefinition } from "./evaluatorTypes";+import { assertCompleteEvaluatorVariableMapping } from "./evaluatorValidation";export async function testEvaluator(params: {orgId: string;@@ -40,10 +41,17 @@ export async function testEvaluator(params: {startTime: params.startTime,shouldReadFromObservationsTable: params.shouldReadFromObservationsTable,});- const variableMapping =- params.definition.type === "CODE"- ? getCodeEvalVariableMapping()- : observationVariableMappingList.parse(params.definition.variableMapping);+ let variableMapping;+ if (params.definition.type === "CODE") {+ variableMapping = getCodeEvalVariableMapping();+ } else {+ const llmVariableMapping = params.definition.variableMapping ?? [];+ assertCompleteEvaluatorVariableMapping({+ prompt: params.definition.prompt,+ variableMapping: llmVariableMapping,+ });+ variableMapping = observationVariableMappingList.parse(llmVariableMapping);+ }const variables = extractObservationVariables({observation,variableMapping,diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.tsindex cc4e1fa5f7..93df4c27d0 100644--- a/web/src/features/in-app-agent/components/utils/side-effects.ts+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts@@ -70,6 +70,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {langfuse_listEvaluators: [],langfuse_listManagedEvaluatorTemplates: [],langfuse_getEvaluator: [],+ langfuse_testEvaluator: ["evals"],langfuse_createEvaluator: ["evals", "models"],langfuse_updateEvaluator: ["evals", "models"],langfuse_deleteEvaluator: ["evals", "models"],diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.tsindex dab84c6757..8199863a31 100644--- a/web/src/features/in-app-agent/components/utils/utils.ts+++ b/web/src/features/in-app-agent/components/utils/utils.ts@@ -74,6 +74,7 @@ const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {queryMetrics: "Checking metrics",read: "Reading file",submitFeedback: "Submitting user feedback",+ testEvaluator: "Testing evaluator",updateDashboardPlacement: "Moving widget",updateDashboardWidget: "Updating widget",updatePromptLabels: "Updating prompt labels",diff --git a/web/src/features/mcp/server/evals/index.ts b/web/src/features/mcp/server/evals/index.tsindex a61531fbec..dda0457ae4 100644--- a/web/src/features/mcp/server/evals/index.ts+++ b/web/src/features/mcp/server/evals/index.ts@@ -46,6 +46,7 @@ import {handleAttachEvaluatorToEvaluationRule,handleDetachEvaluatorFromEvaluationRule,} from "./tools/manageEvaluationRuleEvaluators";+import { handleTestEvaluator, testEvaluatorTool } from "./tools/testEvaluator";export const evalsFeature = {name: "evals",@@ -64,6 +65,10 @@ export const evalsFeature = {definition: getEvaluatorTool,handler: handleGetEvaluator,},+ {+ definition: testEvaluatorTool,+ handler: handleTestEvaluator,+ },{ definition: createEvaluatorTool, handler: handleCreateEvaluator },{ definition: updateEvaluatorTool, handler: handleUpdateEvaluator },{diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.tsindex 3dcf4b81c5..df4b34e5b7 100644--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts@@ -95,6 +95,11 @@ export const McpEvaluatorInputBase = z.object({.describe("Variable mappings for LLM-as-a-judge evaluators only."),});+export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({+ name: true,+ description: true,+});+function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {if (input.type === EvalTemplateType.LLM_AS_JUDGE) {return {diff --git a/web/src/features/mcp/server/evals/tools/testEvaluator.ts b/web/src/features/mcp/server/evals/tools/testEvaluator.tsnew file mode 100644index 0000000000..f42406c1e2--- /dev/null+++ b/web/src/features/mcp/server/evals/tools/testEvaluator.ts@@ -0,0 +1,106 @@+import { z } from "zod";+import { defineTool } from "../../../core/define-tool";+import { runMcpTool } from "../../../core/run-mcp-tool";+import { createMcpEvaluatorService } from "../evaluator-service";+import {+ McpEvaluatorDefinitionInputBase,+ McpEvaluatorInput,+ toEvaluatorServiceInput,+} from "./evaluatorInput";++const DraftEvaluatorFields = McpEvaluatorDefinitionInputBase.partial();+const DRAFT_FIELD_NAMES = Object.keys(DraftEvaluatorFields.shape) as Array<+ keyof z.infer<typeof DraftEvaluatorFields>+>;++const TestEvaluatorInputBase = z+ .object({+ evaluatorId: z+ .string()+ .min(1)+ .optional()+ .describe(+ "Saved evaluator ID. Provide this or a draft evaluator definition, but not both.",+ ),+ ...DraftEvaluatorFields.shape,+ observationId: z.string().min(1),+ traceId: z.string().min(1),+ startTime: z.iso.datetime({ offset: true }),+ })+ .strict();++const TestEvaluatorInput = TestEvaluatorInputBase.superRefine((input, ctx) => {+ const hasDraftDefinition = DRAFT_FIELD_NAMES.some(+ (field) => input[field] !== undefined,+ );++ if (Boolean(input.evaluatorId) === hasDraftDefinition) {+ ctx.addIssue({+ code: "custom",+ message:+ "Provide either evaluatorId or a draft evaluator definition, but not both.",+ });+ return;+ }++ if (!hasDraftDefinition) return;++ const parsedDraft = McpEvaluatorInput.safeParse({+ ...input,+ name: "Evaluator test",+ });+ if (parsedDraft.success) return;++ for (const issue of parsedDraft.error.issues) {+ ctx.addIssue({+ code: "custom",+ path: issue.path,+ message: issue.message,+ });+ }+}).transform((input) => ({+ ...input,+ startTime: new Date(input.startTime),+}));++export const [testEvaluatorTool, handleTestEvaluator] = defineTool({+ name: "testEvaluator",+ description: [+ "Test an evaluator draft or the latest saved version of an evaluator against one observation in the current Langfuse project.",+ "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge or code evaluator fields.",+ "Pass the observationId, traceId, and startTime returned by the observation tools.",+ "This executes the evaluator, emits an internal trace, and may incur model or code execution cost.",+ ].join(" "),+ baseSchema: TestEvaluatorInputBase,+ inputSchema: TestEvaluatorInput,+ handler: (input, context) =>+ runMcpTool({+ spanName: "mcp.evaluators.test",+ context,+ attributes: {+ ...(input.evaluatorId ? { "mcp.evaluator_id": input.evaluatorId } : {}),+ "mcp.observation_id": input.observationId,+ },+ fn: () => {+ const definition = input.evaluatorId+ ? undefined+ : toEvaluatorServiceInput(+ McpEvaluatorInput.parse({+ ...input,+ name: "Evaluator test",+ }),+ ).definition;++ return createMcpEvaluatorService(context).testEvaluator({+ orgId: context.orgId,+ projectId: context.projectId,+ evaluatorId: input.evaluatorId,+ definition,+ observationId: input.observationId,+ traceId: input.traceId,+ startTime: input.startTime,+ });+ },+ }),+ expensiveHint: true,+});