Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

3.5 KB
Raw Blame
import { env } from "@/src/env.mjs";
import type { McpFeatureModule } from "../registry";
import {
  listEvaluatorsTool,
  handleListEvaluators,
} from "./tools/listEvaluators";
import { getEvaluatorTool, handleGetEvaluator } from "./tools/getEvaluator";
import {
  createEvaluatorTool,
  handleCreateEvaluator,
} from "./tools/createEvaluator";
import {
  updateEvaluatorTool,
  handleUpdateEvaluator,
} from "./tools/updateEvaluator";
import {
  deleteEvaluatorTool,
  handleDeleteEvaluator,
} from "./tools/deleteEvaluator";
import {
  listEvaluationRulesTool,
  handleListEvaluationRules,
} from "./tools/listEvaluationRules";
import {
  getEvaluationRuleTool,
  handleGetEvaluationRule,
} from "./tools/getEvaluationRule";
import {
  createEvaluationRuleTool,
  handleCreateEvaluationRule,
} from "./tools/createEvaluationRule";
import {
  updateEvaluationRuleTool,
  handleUpdateEvaluationRule,
} from "./tools/updateEvaluationRule";
import {
  deleteEvaluationRuleTool,
  handleDeleteEvaluationRule,
} from "./tools/deleteEvaluationRule";
import {
  handleListManagedEvaluatorTemplates,
  listManagedEvaluatorTemplatesTool,
} from "./tools/listManagedEvaluatorTemplates";
import {
  attachEvaluatorToEvaluationRuleTool,
  detachEvaluatorFromEvaluationRuleTool,
  handleAttachEvaluatorToEvaluationRule,
  handleDetachEvaluatorFromEvaluationRule,
} from "./tools/manageEvaluationRuleEvaluators";
import { handleTestEvaluator, testEvaluatorTool } from "./tools/testEvaluator";
import {
  handleRunBatchEvaluation,
  runBatchEvaluationTool,
} from "./tools/runBatchEvaluation";

export const evalsFeature = {
  name: "evals",
  description:
    "Manage evaluators and evaluation rules in the current Langfuse project",
  tools: [
    {
      definition: listManagedEvaluatorTemplatesTool,
      handler: handleListManagedEvaluatorTemplates,
    },
    {
      definition: listEvaluatorsTool,
      handler: handleListEvaluators,
    },
    {
      definition: getEvaluatorTool,
      handler: handleGetEvaluator,
    },
    {
      definition: testEvaluatorTool,
      handler: handleTestEvaluator,
    },
    { definition: createEvaluatorTool, handler: handleCreateEvaluator },
    { definition: updateEvaluatorTool, handler: handleUpdateEvaluator },
    {
      definition: deleteEvaluatorTool,
      handler: handleDeleteEvaluator,
    },
    {
      definition: listEvaluationRulesTool,
      handler: handleListEvaluationRules,
    },
    {
      definition: getEvaluationRuleTool,
      handler: handleGetEvaluationRule,
    },
    {
      definition: createEvaluationRuleTool,
      handler: handleCreateEvaluationRule,
    },
    {
      definition: updateEvaluationRuleTool,
      handler: handleUpdateEvaluationRule,
    },
    {
      definition: attachEvaluatorToEvaluationRuleTool,
      handler: handleAttachEvaluatorToEvaluationRule,
    },
    {
      definition: detachEvaluatorFromEvaluationRuleTool,
      handler: handleDetachEvaluatorFromEvaluationRule,
    },
    {
      definition: deleteEvaluationRuleTool,
      handler: handleDeleteEvaluationRule,
    },
  ],
} as const satisfies McpFeatureModule;

export const batchEvaluationsFeature = {
  name: "batchEvaluations",
  description:
    "Run evaluators over existing observations in the current Langfuse project",
  tools: [
    {
      definition: runBatchEvaluationTool,
      handler: handleRunBatchEvaluation,
    },
  ],
  isEnabled: async () =>
    env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true",
} as const satisfies McpFeatureModule;