🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { defineTool } from "../../../core/define-tool";
import { runMcpTool } from "../../../core/run-mcp-tool";
import {
UpdateEvaluationRuleBaseSchema,
UpdateEvaluationRuleInputSchema,
} from "../rule-schema";
import {
assertRuleAssignmentsReplaceableViaMcp,
createMcpRuleService,
toMcpEvaluationRule,
toStoredAssignments,
} from "../rule-service";
export const [updateEvaluationRuleTool, handleUpdateEvaluationRule] =
defineTool({
name: "updateEvaluationRule",
description:
"Update an observation evaluation rule, including replacing all evaluator assignments.",
action: "evaluationRule:CUD",
baseSchema: UpdateEvaluationRuleBaseSchema,
inputSchema: UpdateEvaluationRuleInputSchema,
handler: async (input, context) =>
runMcpTool({
spanName: "mcp.evaluation_rules.update",
context,
attributes: { "mcp.evaluation_rule_id": input.evaluationRuleId },
fn: async () => {
const { evaluationRuleId, evaluatorAssignments, ...patch } = input;
const service = createMcpRuleService(context);
if (evaluatorAssignments !== undefined) {
await assertRuleAssignmentsReplaceableViaMcp(
service,
context.projectId,
evaluationRuleId,
);
}
const rule = await service.update({
projectId: context.projectId,
ruleId: evaluationRuleId,
...patch,
...(evaluatorAssignments === undefined
? {}
: {
evaluatorMappings: toStoredAssignments(evaluatorAssignments),
}),
});
return toMcpEvaluationRule(rule);
},
}),
destructiveHint: true,
});