Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

2.0 KB
Raw Blame
import { defineTool } from "../../../core/define-tool";
import { runMcpTool } from "../../../core/run-mcp-tool";
import { buildEvaluatorUrl } from "@langfuse/shared/src/server";
import { createMcpEvaluatorService } from "../evaluator-service";
import {
  McpEvaluatorInput,
  McpEvaluatorInputBase,
  toEvaluatorServiceInput,
} from "./evaluatorInput";

export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({
  name: "createEvaluator",
  description: [
    "Create a new evaluator with a stable id. Names do not act as identity and may be reused.",
    "For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",
    "Call listLlmConnections to find valid modelConfig provider names and models.",
    "Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",
  ].join(" "),
  action: "evaluator:CUD",
  baseSchema: McpEvaluatorInputBase,
  inputSchema: McpEvaluatorInput,
  handler: (input, context) =>
    runMcpTool({
      spanName: "mcp.evaluators.create",
      context,
      attributes: {
        "mcp.evaluator_name": input.name,
        "mcp.evaluator_type": input.type,
      },
      fn: async (span) => {
        const service = createMcpEvaluatorService(context);
        const evaluator = await service.create(
          {
            ...toEvaluatorServiceInput(input),
            projectId: context.projectId,
          },
          context.userId ?? null,
        );
        span.setAttribute("mcp.evaluator_id", evaluator.id);
        return {
          ...evaluator,
          url: buildEvaluatorUrl({
            projectId: context.projectId,
            evaluatorId: evaluator.id,
          }),
        };
      },
    }),
  destructiveHint: true,
});