Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(mcp): add evaluator testing tool (#16870)

Tobias Wochinger2b70ed280ea7
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index 9df9515268..2014dc3d51 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -144,6 +144,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {
approval: "auto",
availability: { scope: "evaluator:read" },
},
+ testEvaluator: {
+ approval: "approval",
+ availability: { scope: "evaluator:CUD" },
+ },
createEvaluator: {
approval: "approval",
availability: { scope: "evaluator:CUD" },
diff --git a/web/src/__tests__/server/evaluator-v2-service.servertest.ts b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
index 3f14fd47db..f31df9b949 100644
--- a/web/src/__tests__/server/evaluator-v2-service.servertest.ts
+++ b/web/src/__tests__/server/evaluator-v2-service.servertest.ts
@@ -812,7 +812,16 @@ describe("EvaluatorService", () => {
}),
).resolves.toEqual({ score: 1 });
- expect(mocks.testEvaluator).toHaveBeenCalledTimes(2);
+ await expect(
+ service.testEvaluator({
+ orgId,
+ projectId,
+ definition: llmInput("Test evaluator").definition,
+ ...observation,
+ }),
+ ).resolves.toEqual({ score: 1 });
+
+ expect(mocks.testEvaluator).toHaveBeenCalledTimes(3);
expect(mocks.testEvaluator).toHaveBeenCalledWith(
expect.objectContaining({
orgId,
@@ -829,6 +838,15 @@ describe("EvaluatorService", () => {
...observation,
}),
);
+ expect(mocks.testEvaluator).toHaveBeenNthCalledWith(
+ 3,
+ expect.objectContaining({
+ orgId,
+ evaluatorId: expect.any(String),
+ includeEvaluatorLink: false,
+ ...observation,
+ }),
+ );
await expect(
prisma.evaluatorVersion.count({ where: { evaluator: { projectId } } }),
).resolves.toBe(before);
@@ -844,6 +862,49 @@ describe("EvaluatorService", () => {
).rejects.toThrow("Evaluator not found");
});
+ it("tests the latest version of a saved evaluator", async () => {
+ mocks.testEvaluator.mockResolvedValue({ score: 1 });
+ const service = createService();
+ const evaluator = await service.create(llmInput("Saved evaluator"), null);
+ const latestDefinition = {
+ ...llmInput("Saved evaluator").definition,
+ prompt: "Judge the latest {{output}}",
+ };
+ await service.update(
+ {
+ ...llmInput("Saved evaluator"),
+ evaluatorId: evaluator.id,
+ definition: latestDefinition,
+ },
+ null,
+ );
+
+ const observation = {
+ observationId: "test-observation-id",
+ traceId: "test-trace-id",
+ startTime: new Date("2026-08-11T00:00:00.000Z"),
+ shouldReadFromObservationsTable: false,
+ };
+
+ await expect(
+ service.testEvaluator({
+ orgId,
+ projectId,
+ evaluatorId: evaluator.id,
+ ...observation,
+ }),
+ ).resolves.toEqual({ score: 1 });
+
+ expect(mocks.testEvaluator).toHaveBeenCalledWith({
+ orgId,
+ projectId,
+ evaluatorId: evaluator.id,
+ definition: latestDefinition,
+ includeEvaluatorLink: true,
+ ...observation,
+ });
+ });
+
it("suggests a safe name and silently returns null when unavailable", async () => {
mocks.generateLangfuseAIText
.mockResolvedValueOnce(' "Concise quality judge" ')
diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.ts
index b95a7f2aff..e6d980b299 100644
--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts
@@ -88,7 +88,13 @@ import {
deleteEvaluatorTool,
handleDeleteEvaluator,
} from "@/src/features/mcp/server/evals/tools/deleteEvaluator";
+import {
+ handleTestEvaluator,
+ testEvaluatorTool,
+} from "@/src/features/mcp/server/evals/tools/testEvaluator";
+import { evalsFeature } from "@/src/features/mcp/server/evals";
import { handleGetEvaluationRule } from "@/src/features/mcp/server/evals/tools/getEvaluationRule";
+import { EvaluatorService } from "@/src/features/evals/v2/server/evaluators/evaluatorService";
import {
attachEvaluatorToEvaluationRuleTool,
detachEvaluatorFromEvaluationRuleTool,
@@ -191,6 +197,151 @@ describe("MCP Write Tools", () => {
});
describe("stable evaluator tools", () => {
+ it("tests the latest saved evaluator against a project observation", async () => {
+ const setup = await createMcpTestSetup();
+ const evaluator = await createStableLlmEvaluatorForMcpWriteTest(setup);
+ await handleUpdateEvaluator(
+ {
+ evaluatorId: evaluator.id,
+ name: evaluator.name,
+ type: "LLM_AS_JUDGE",
+ prompt: "Latest evaluator prompt for {{input}}",
+ outputDefinition: mcpEvalOutputDefinition,
+ variableMapping: [
+ { templateVariable: "input", selectedColumnId: "input" },
+ ],
+ },
+ setup.context,
+ );
+
+ const observationId = nanoid();
+ const traceId = nanoid();
+ const startTime = "2026-08-11T12:00:00.000Z";
+ const unifiedResult = {
+ success: true as const,
+ result: {
+ dataType: "NUMERIC" as const,
+ score: 1,
+ reasoning: "passes",
+ },
+ interpolatedPrompt: "Latest evaluator prompt for sample input",
+ model: "test-model",
+ provider: "test-provider",
+ executionTraceId: "execution-trace-id",
+ estimatedCostUsd: null,
+ durationMs: 25,
+ };
+ const testEvaluatorSpy = vi
+ .spyOn(EvaluatorService.prototype, "testEvaluator")
+ .mockResolvedValue(unifiedResult);
+
+ try {
+ verifyToolAnnotations(testEvaluatorTool, { expensiveHint: true });
+ expect(testEvaluatorTool.annotations?.readOnlyHint).not.toBe(true);
+ expect(testEvaluatorTool.annotations?.destructiveHint).not.toBe(true);
+ expect(testEvaluatorTool.inputSchema.properties).toEqual(
+ expect.objectContaining({
+ evaluatorId: expect.any(Object),
+ type: expect.any(Object),
+ prompt: expect.any(Object),
+ sourceCode: expect.any(Object),
+ observationId: expect.any(Object),
+ traceId: expect.any(Object),
+ startTime: expect.any(Object),
+ }),
+ );
+ expect(
+ evalsFeature.tools.some(
+ ({ definition }) => definition.name === "testEvaluator",
+ ),
+ ).toBe(true);
+
+ await expect(
+ handleTestEvaluator(
+ {
+ evaluatorId: evaluator.id,
+ observationId,
+ traceId,
+ startTime,
+ } as never,
+ setup.context,
+ ),
+ ).resolves.toEqual(unifiedResult);
+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(1, {
+ orgId: setup.orgId,
+ projectId: setup.projectId,
+ evaluatorId: evaluator.id,
+ observationId,
+ traceId,
+ startTime: new Date(startTime),
+ });
+
+ await expect(
+ handleTestEvaluator(
+ {
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ sourceCodeLanguage: "TYPESCRIPT",
+ observationId,
+ traceId,
+ startTime,
+ } as never,
+ setup.context,
+ ),
+ ).resolves.toEqual(unifiedResult);
+ expect(testEvaluatorSpy).toHaveBeenNthCalledWith(2, {
+ orgId: setup.orgId,
+ projectId: setup.projectId,
+ definition: {
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ sourceCodeLanguage: "TYPESCRIPT",
+ },
+ observationId,
+ traceId,
+ startTime: new Date(startTime),
+ });
+
+ await expect(
+ handleTestEvaluator(
+ { observationId, traceId, startTime } as never,
+ setup.context,
+ ),
+ ).rejects.toThrow(
+ "Provide either evaluatorId or a draft evaluator definition, but not both.",
+ );
+ await expect(
+ handleTestEvaluator(
+ {
+ evaluatorId: evaluator.id,
+ type: "CODE",
+ sourceCode: "return { score: 1 };",
+ sourceCodeLanguage: "TYPESCRIPT",
+ observationId,
+ traceId,
+ startTime,
+ } as never,
+ setup.context,
+ ),
+ ).rejects.toThrow(
+ "Provide either evaluatorId or a draft evaluator definition, but not both.",
+ );
+ await expect(
+ handleTestEvaluator(
+ {
+ evaluatorId: evaluator.id,
+ observationId,
+ traceId,
+ startTime: null,
+ } as never,
+ setup.context,
+ ),
+ ).rejects.toThrow();
+ } finally {
+ testEvaluatorSpy.mockRestore();
+ }
+ });
+
it("creates and versions an evaluator by stable id", async () => {
const setup = await createMcpTestSetup();
verifyToolAnnotations(createEvaluatorTool, { destructiveHint: true });
diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
index 71d58061da..a7fbf57766 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
@@ -144,6 +144,58 @@ describe("testEvaluator", () => {
);
});
+ it("treats a missing mapping as empty when the prompt has no variables", async () => {
+ await expect(
+ testEvaluator({
+ orgId: "org-1",
+ projectId: "project-1",
+ evaluatorId: "evaluator-1",
+ observationId: "observation-1",
+ traceId: "trace-1",
+ startTime: new Date("2026-08-11T12:00:00.000Z"),
+ definition: {
+ type: "LLM_AS_JUDGE",
+ prompt: "Judge this response",
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: [],
+ variableMapping: null,
+ outputDefinition: createNumericEvalOutputDefinition({
+ scoreDescription: "Correctness",
+ reasoningDescription: "Why the answer is correct",
+ }),
+ },
+ }),
+ ).resolves.toMatchObject({ success: true });
+ });
+
+ it("rejects a missing mapping when the prompt has variables", async () => {
+ await expect(
+ testEvaluator({
+ orgId: "org-1",
+ projectId: "project-1",
+ evaluatorId: "evaluator-1",
+ observationId: "observation-1",
+ traceId: "trace-1",
+ startTime: new Date("2026-08-11T12:00:00.000Z"),
+ definition: {
+ type: "LLM_AS_JUDGE",
+ prompt: "Judge {{answer}}",
+ provider: null,
+ model: null,
+ modelParams: null,
+ vars: ["answer"],
+ variableMapping: null,
+ outputDefinition: createNumericEvalOutputDefinition({
+ scoreDescription: "Correctness",
+ reasoningDescription: "Why the answer is correct",
+ }),
+ },
+ }),
+ ).rejects.toThrow("Missing mappings for evaluator variables: answer");
+ });
+
it("loads the selected observation and dispatches a code evaluator with canonical variables", async () => {
const startTime = new Date("2026-08-11T12:00:00.000Z");
vi.mocked(getObservationForEvalById).mockResolvedValue({
diff --git a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
index 85ca3c6b8c..b968d79a2e 100644
--- a/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
+++ b/web/src/features/evals/v2/server/evaluators/evaluatorService.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
import { isDeepStrictEqual } from "node:util";
import { z } from "zod";
import {
@@ -8,6 +9,7 @@ import {
getCodeEvalVariableMapping,
getEvaluatorBlockMetadata,
isEvaluatorBlockReasonRecoverableByDefinitionUpdate,
+ InvalidRequestError,
LangfuseConflictError,
LangfuseNotFoundError,
} from "@langfuse/shared";
@@ -547,21 +549,62 @@ export class EvaluatorService {
return evaluatorIds;
}
- async testEvaluator(params: Parameters<typeof executeEvaluatorTest>[0]) {
- const evaluator = await this.prisma.evaluator.findFirst({
- where: { id: params.evaluatorId, projectId: params.projectId },
- select: { id: true },
- });
- // The setup editor pre-generates a UUID so a test run can be attributed to
- // the evaluator before it is first saved. Every other id must resolve
- // inside the project — never look it up unscoped, which would turn the
- // response into a cross-project existence oracle.
- if (!evaluator && !isPregeneratedEvaluatorId(params.evaluatorId)) {
- throw new LangfuseNotFoundError("Evaluator not found");
+ async testEvaluator(
+ params: Omit<
+ Parameters<typeof executeEvaluatorTest>[0],
+ "evaluatorId" | "definition" | "includeEvaluatorLink"
+ > & {
+ evaluatorId?: string;
+ definition?: EvaluatorDefinition;
+ },
+ ) {
+ const {
+ evaluatorId: requestedEvaluatorId,
+ definition: requestedDefinition,
+ ...executionParams
+ } = params;
+
+ let evaluatorId = requestedEvaluatorId;
+ let definition = requestedDefinition;
+ let includeEvaluatorLink = false;
+
+ if (definition) {
+ if (evaluatorId) {
+ const evaluator = await this.prisma.evaluator.findFirst({
+ where: { id: evaluatorId, projectId: params.projectId },
+ select: { id: true },
+ });
+ // The setup editor pre-generates a UUID so a test run can be attributed
+ // to the evaluator before it is first saved. Every other id must resolve
+ // inside the project — never look it up unscoped, which would turn the
+ // response into a cross-project existence oracle.
+ if (!evaluator && !isPregeneratedEvaluatorId(evaluatorId)) {
+ throw new LangfuseNotFoundError("Evaluator not found");
+ }
+ includeEvaluatorLink = Boolean(evaluator);
+ } else {
+ evaluatorId = randomUUID();
+ }
+ } else {
+ if (!evaluatorId) {
+ throw new InvalidRequestError(
+ "Either evaluatorId or definition is required",
+ );
+ }
+ const evaluator = await this.get(params.projectId, evaluatorId);
+ const latestVersion = evaluator.versions[0];
+ if (!latestVersion) {
+ throw new LangfuseNotFoundError("Evaluator version not found");
+ }
+ definition = toEvaluatorDefinition(evaluator.type, latestVersion);
+ includeEvaluatorLink = true;
}
+
return executeEvaluatorTest({
- ...params,
- includeEvaluatorLink: Boolean(evaluator),
+ ...executionParams,
+ evaluatorId,
+ definition,
+ includeEvaluatorLink,
});
}
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 760d3be631..863cef7afe 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -20,6 +20,7 @@ import {
} from "@langfuse/shared/src/server";
import { getObservationForEvalById } from "@/src/features/evals/server/getObservationForEvalById";
import type { EvaluatorDefinition } from "./evaluatorTypes";
+import { assertCompleteEvaluatorVariableMapping } from "./evaluatorValidation";
export async function testEvaluator(params: {
orgId: string;
@@ -40,10 +41,17 @@ export async function testEvaluator(params: {
startTime: params.startTime,
shouldReadFromObservationsTable: params.shouldReadFromObservationsTable,
});
- const variableMapping =
- params.definition.type === "CODE"
- ? getCodeEvalVariableMapping()
- : observationVariableMappingList.parse(params.definition.variableMapping);
+ let variableMapping;
+ if (params.definition.type === "CODE") {
+ variableMapping = getCodeEvalVariableMapping();
+ } else {
+ const llmVariableMapping = params.definition.variableMapping ?? [];
+ assertCompleteEvaluatorVariableMapping({
+ prompt: params.definition.prompt,
+ variableMapping: llmVariableMapping,
+ });
+ variableMapping = observationVariableMappingList.parse(llmVariableMapping);
+ }
const variables = extractObservationVariables({
observation,
variableMapping,
diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.ts
index cc4e1fa5f7..93df4c27d0 100644
--- a/web/src/features/in-app-agent/components/utils/side-effects.ts
+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts
@@ -70,6 +70,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {
langfuse_listEvaluators: [],
langfuse_listManagedEvaluatorTemplates: [],
langfuse_getEvaluator: [],
+ langfuse_testEvaluator: ["evals"],
langfuse_createEvaluator: ["evals", "models"],
langfuse_updateEvaluator: ["evals", "models"],
langfuse_deleteEvaluator: ["evals", "models"],
diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.ts
index dab84c6757..8199863a31 100644
--- a/web/src/features/in-app-agent/components/utils/utils.ts
+++ b/web/src/features/in-app-agent/components/utils/utils.ts
@@ -74,6 +74,7 @@ const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {
queryMetrics: "Checking metrics",
read: "Reading file",
submitFeedback: "Submitting user feedback",
+ testEvaluator: "Testing evaluator",
updateDashboardPlacement: "Moving widget",
updateDashboardWidget: "Updating widget",
updatePromptLabels: "Updating prompt labels",
diff --git a/web/src/features/mcp/server/evals/index.ts b/web/src/features/mcp/server/evals/index.ts
index a61531fbec..dda0457ae4 100644
--- a/web/src/features/mcp/server/evals/index.ts
+++ b/web/src/features/mcp/server/evals/index.ts
@@ -46,6 +46,7 @@ import {
handleAttachEvaluatorToEvaluationRule,
handleDetachEvaluatorFromEvaluationRule,
} from "./tools/manageEvaluationRuleEvaluators";
+import { handleTestEvaluator, testEvaluatorTool } from "./tools/testEvaluator";
export const evalsFeature = {
name: "evals",
@@ -64,6 +65,10 @@ export const evalsFeature = {
definition: getEvaluatorTool,
handler: handleGetEvaluator,
},
+ {
+ definition: testEvaluatorTool,
+ handler: handleTestEvaluator,
+ },
{ definition: createEvaluatorTool, handler: handleCreateEvaluator },
{ definition: updateEvaluatorTool, handler: handleUpdateEvaluator },
{
diff --git a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
index 3dcf4b81c5..df4b34e5b7 100644
--- a/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
+++ b/web/src/features/mcp/server/evals/tools/evaluatorInput.ts
@@ -95,6 +95,11 @@ export const McpEvaluatorInputBase = z.object({
.describe("Variable mappings for LLM-as-a-judge evaluators only."),
});
+export const McpEvaluatorDefinitionInputBase = McpEvaluatorInputBase.omit({
+ name: true,
+ description: true,
+});
+
function toEvaluatorInput(input: z.infer<typeof McpEvaluatorInputBase>) {
if (input.type === EvalTemplateType.LLM_AS_JUDGE) {
return {
diff --git a/web/src/features/mcp/server/evals/tools/testEvaluator.ts b/web/src/features/mcp/server/evals/tools/testEvaluator.ts
new file mode 100644
index 0000000000..f42406c1e2
--- /dev/null
+++ b/web/src/features/mcp/server/evals/tools/testEvaluator.ts
@@ -0,0 +1,106 @@
+import { z } from "zod";
+import { defineTool } from "../../../core/define-tool";
+import { runMcpTool } from "../../../core/run-mcp-tool";
+import { createMcpEvaluatorService } from "../evaluator-service";
+import {
+ McpEvaluatorDefinitionInputBase,
+ McpEvaluatorInput,
+ toEvaluatorServiceInput,
+} from "./evaluatorInput";
+
+const DraftEvaluatorFields = McpEvaluatorDefinitionInputBase.partial();
+const DRAFT_FIELD_NAMES = Object.keys(DraftEvaluatorFields.shape) as Array<
+ keyof z.infer<typeof DraftEvaluatorFields>
+>;
+
+const TestEvaluatorInputBase = z
+ .object({
+ evaluatorId: z
+ .string()
+ .min(1)
+ .optional()
+ .describe(
+ "Saved evaluator ID. Provide this or a draft evaluator definition, but not both.",
+ ),
+ ...DraftEvaluatorFields.shape,
+ observationId: z.string().min(1),
+ traceId: z.string().min(1),
+ startTime: z.iso.datetime({ offset: true }),
+ })
+ .strict();
+
+const TestEvaluatorInput = TestEvaluatorInputBase.superRefine((input, ctx) => {
+ const hasDraftDefinition = DRAFT_FIELD_NAMES.some(
+ (field) => input[field] !== undefined,
+ );
+
+ if (Boolean(input.evaluatorId) === hasDraftDefinition) {
+ ctx.addIssue({
+ code: "custom",
+ message:
+ "Provide either evaluatorId or a draft evaluator definition, but not both.",
+ });
+ return;
+ }
+
+ if (!hasDraftDefinition) return;
+
+ const parsedDraft = McpEvaluatorInput.safeParse({
+ ...input,
+ name: "Evaluator test",
+ });
+ if (parsedDraft.success) return;
+
+ for (const issue of parsedDraft.error.issues) {
+ ctx.addIssue({
+ code: "custom",
+ path: issue.path,
+ message: issue.message,
+ });
+ }
+}).transform((input) => ({
+ ...input,
+ startTime: new Date(input.startTime),
+}));
+
+export const [testEvaluatorTool, handleTestEvaluator] = defineTool({
+ name: "testEvaluator",
+ description: [
+ "Test an evaluator draft or the latest saved version of an evaluator against one observation in the current Langfuse project.",
+ "For a saved evaluator, provide evaluatorId. For an unsaved draft, omit evaluatorId and provide type plus the matching LLM-as-a-judge or code evaluator fields.",
+ "Pass the observationId, traceId, and startTime returned by the observation tools.",
+ "This executes the evaluator, emits an internal trace, and may incur model or code execution cost.",
+ ].join(" "),
+ baseSchema: TestEvaluatorInputBase,
+ inputSchema: TestEvaluatorInput,
+ handler: (input, context) =>
+ runMcpTool({
+ spanName: "mcp.evaluators.test",
+ context,
+ attributes: {
+ ...(input.evaluatorId ? { "mcp.evaluator_id": input.evaluatorId } : {}),
+ "mcp.observation_id": input.observationId,
+ },
+ fn: () => {
+ const definition = input.evaluatorId
+ ? undefined
+ : toEvaluatorServiceInput(
+ McpEvaluatorInput.parse({
+ ...input,
+ name: "Evaluator test",
+ }),
+ ).definition;
+
+ return createMcpEvaluatorService(context).testEvaluator({
+ orgId: context.orgId,
+ projectId: context.projectId,
+ evaluatorId: input.evaluatorId,
+ definition,
+ observationId: input.observationId,
+ traceId: input.traceId,
+ startTime: input.startTime,
+ });
+ },
+ }),
+ expensiveHint: true,
+});