🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { EvalTargetObject } from "@langfuse/shared";
import { defineTool } from "../../../core/define-tool";
import { runMcpTool } from "../../../core/run-mcp-tool";
import {
CreateEvaluationRuleBaseSchema,
CreateEvaluationRuleInputSchema,
} from "../rule-schema";
import {
createMcpRuleService,
toMcpEvaluationRule,
toStoredAssignments,
} from "../rule-service";
export const [createEvaluationRuleTool, handleCreateEvaluationRule] =
defineTool({
name: "createEvaluationRule",
description: [
"Create an observation evaluation rule with one or more evaluator assignments.",
"Each assignment references a project evaluator by stable ID. LLM-as-a-judge and decision-model assignments may override their variable mapping; code evaluator assignments must omit it.",
].join(" "),
action: "evaluationRule:CUD",
baseSchema: CreateEvaluationRuleBaseSchema,
inputSchema: CreateEvaluationRuleInputSchema,
handler: async (input, context) =>
runMcpTool({
spanName: "mcp.evaluation_rules.create",
context,
attributes: {
"mcp.evaluation_rule_name": input.name,
"mcp.evaluation_rule_evaluator_count":
input.evaluatorAssignments.length,
},
fn: async (span) => {
const evaluationRule = await createMcpRuleService(context).create(
{
projectId: context.projectId,
name: input.name,
targetObject: EvalTargetObject.EVENT,
enabled: input.enabled,
sampling: input.sampling,
filter: input.filter ?? [],
evaluatorAssignments: toStoredAssignments(
input.evaluatorAssignments,
),
},
context.userId ?? null,
);
span.setAttribute("mcp.evaluation_rule_id", evaluationRule.id);
return toMcpEvaluationRule(evaluationRule);
},
}),
destructiveHint: true,
});