🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): expose v4 migration data (#16671)
Max Deichmann105154b20a57
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex f22d385162..944d94da55 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -364,6 +364,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {approval: "approval",availability: { scope: "dashboards:CUD" },},+ getV4MigrationData: {+ approval: "auto",+ availability: { scope: "project:read" },+ },} satisfies Record<string, InAppAgentMcpToolPolicy>;export type InAppAgentLangfuseMcpToolName =diff --git a/web/src/__tests__/server/unit/mcp-v4-migration.servertest.ts b/web/src/__tests__/server/unit/mcp-v4-migration.servertest.tsnew file mode 100644index 0000000000..db5d437819--- /dev/null+++ b/web/src/__tests__/server/unit/mcp-v4-migration.servertest.ts@@ -0,0 +1,92 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const { getProjectV4MigrationData } = vi.hoisted(() => ({+ getProjectV4MigrationData: vi.fn(),+}));++vi.mock("@/src/features/v4/server/v4TransitionService", () => ({+ getProjectV4MigrationData,+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {},+}));++import {+ getV4MigrationDataTool,+ handleGetV4MigrationData,+} from "@/src/features/mcp/features/v4Migration/tools/getV4MigrationData";+import { mockServerContext } from "@/src/__tests__/server/mcp-helpers";++describe("getV4MigrationData MCP tool", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("returns the project-scoped data exposed by the v4 migration API", async () => {+ const migrationData = {+ projectId: "project-1",+ forceV3Experience: false,+ sdkUsage: {+ projectId: "project-1",+ experimentInstrumentationMigration: {+ status: "not_required",+ upgradePath: null,+ },+ sdkUsageSeries: [+ {+ sdkName: "langfuse-python",+ sdkVersion: "3.10.0",+ actionLevel: "required",+ },+ ],+ },+ legacyIntegrations: {+ projectId: "project-1",+ legacyIntegrationCount: 1,+ legacyIntegrations: {+ posthog: true,+ mixpanel: false,+ blobStorage: false,+ },+ },+ legacyApiUsage: [+ {+ projectId: "project-1",+ entrypoint: "publicapi: GET /api/public/traces",+ count: 4,+ lastSeen: "2026-08-26T12:00:00.000Z",+ },+ ],+ traceLevelEvals: {+ projectId: "project-1",+ traceLevelEvalCount: 2,+ },+ };+ getProjectV4MigrationData.mockResolvedValue(migrationData);+ const context = mockServerContext({ projectId: "project-1" });++ await expect(handleGetV4MigrationData({}, context)).resolves.toEqual(+ migrationData,+ );+ expect(getProjectV4MigrationData).toHaveBeenCalledWith({+ prisma: expect.anything(),+ projectId: "project-1",+ });+ });++ it("is discoverable as a read-only, expensive tool without project input", () => {+ expect(getV4MigrationDataTool).toMatchObject({+ name: "getV4MigrationData",+ annotations: {+ readOnlyHint: true,+ expensiveHint: true,+ },+ inputSchema: {+ type: "object",+ properties: {},+ additionalProperties: false,+ },+ });+ });+});diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.tsindex 37ff02b97f..6a355ba9e5 100644--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts@@ -604,6 +604,85 @@ describe("v4TransitionRouter", () => {await expect(readRedisJson(legacyApiUsageCacheKey)).resolves.toBeNull();});+ it("returns the complete project migration data for agent consumers", async () => {+ await seedRedisCache({+ [legacyApiUsageCacheKey]: legacyApiBlob([+ {+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: 2,+ lastSeen: "2026-06-24T15:00:00.000000Z",+ },+ ]),+ });+ mockedQueryClickhouse.mockResolvedValueOnce([+ mockSdkUsageRow({+ projectId,+ sdkName: "python",+ sdkVersion: "3.9.0",+ actionLevel: "required",+ }),+ ]);+ const caller = createCaller({+ posthogIntegration: {+ findMany: vi.fn().mockResolvedValue([+ {+ projectId,+ enabled: true,+ exportSource: "TRACES_OBSERVATIONS",+ },+ ]),+ },+ mixpanelIntegration: {+ findMany: vi.fn().mockResolvedValue([]),+ },+ blobStorageIntegration: {+ findMany: vi.fn().mockResolvedValue([]),+ },+ evaluationRule: {+ groupBy: vi+ .fn()+ .mockResolvedValue([{ projectId, _count: { _all: 2 } }]),+ },+ });++ const result = await caller.migrationData({ projectId });++ expect(result).toMatchObject({+ projectId,+ forceV3Experience: false,+ sdkUsage: {+ projectId,+ sdkUsageSeries: expect.arrayContaining([+ expect.objectContaining({+ sdkName: "python",+ sdkVersion: "3.9.0",+ actionLevel: "required",+ }),+ ]),+ },+ legacyIntegrations: {+ projectId,+ legacyIntegrationCount: 1,+ legacyIntegrations: {+ posthog: true,+ mixpanel: false,+ blobStorage: false,+ },+ },+ legacyApiUsage: [+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: 2,+ },+ ],+ traceLevelEvals: {+ projectId,+ traceLevelEvalCount: 2,+ },+ });+ });+it("queries SDK usage for only the authorized project", async () => {mockedQueryClickhouse.mockResolvedValueOnce([mockSdkUsageRow({@@ -657,6 +736,11 @@ describe("v4TransitionRouter", () => {...range,}),).rejects.toThrow("User is not a member of this project");+ await expect(+ caller.migrationData({+ projectId: outsideProjectId,+ }),+ ).rejects.toThrow("User is not a member of this project");expect(mockedQueryClickhouse).not.toHaveBeenCalled();});diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.tsindex 15763255d9..cc4e1fa5f7 100644--- a/web/src/features/in-app-agent/components/utils/side-effects.ts+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts@@ -84,6 +84,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {langfuse_listExperimentItems: [],langfuse_submitFeedback: ["scores", "scoreAnalytics"],langfuse_getHealth: [],+ langfuse_getV4MigrationData: [],langfuse_getMedia: [],langfuse_queryMetrics: [],langfuse_getMetricsSchema: [],diff --git a/web/src/features/in-app-agent/components/utils/utils.clienttest.ts b/web/src/features/in-app-agent/components/utils/utils.clienttest.tsindex 06e62ff258..0270ee7c3a 100644--- a/web/src/features/in-app-agent/components/utils/utils.clienttest.ts+++ b/web/src/features/in-app-agent/components/utils/utils.clienttest.ts@@ -75,6 +75,7 @@ const ACCEPTED_AUTO_IN_APP_AGENT_PROGRESS_LABELS: Record<string, string> = {langfuse_getPrompt: "Inspecting prompt",langfuse_getScore: "Inspecting score",langfuse_getScoreConfig: "Inspecting score config",+ langfuse_getV4MigrationData: "Inspecting v4 migration data",langfuse_listAlerts: "Browsing alerts",langfuse_listAnnotationQueueItems: "Browsing annotation queue items",langfuse_listAnnotationQueues: "Browsing annotation queues",diff --git a/web/src/features/mcp/features/v4Migration/index.ts b/web/src/features/mcp/features/v4Migration/index.tsnew file mode 100644index 0000000000..18ecc1bb97--- /dev/null+++ b/web/src/features/mcp/features/v4Migration/index.ts@@ -0,0 +1,16 @@+import type { McpFeatureModule } from "../../server/registry";+import {+ getV4MigrationDataTool,+ handleGetV4MigrationData,+} from "./tools/getV4MigrationData";++export const v4MigrationFeature = {+ name: "v4Migration",+ description: "Inspect project-specific Langfuse v4 migration evidence",+ tools: [+ {+ definition: getV4MigrationDataTool,+ handler: handleGetV4MigrationData,+ },+ ],+} as const satisfies McpFeatureModule;diff --git a/web/src/features/mcp/features/v4Migration/tools/getV4MigrationData.ts b/web/src/features/mcp/features/v4Migration/tools/getV4MigrationData.tsnew file mode 100644index 0000000000..994432e33e--- /dev/null+++ b/web/src/features/mcp/features/v4Migration/tools/getV4MigrationData.ts@@ -0,0 +1,28 @@+import { prisma } from "@langfuse/shared/src/db";+import { z } from "zod";++import { getProjectV4MigrationData } from "@/src/features/v4/server/v4TransitionService";+import { defineTool } from "@/src/features/mcp/core/define-tool";+import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";++const GetV4MigrationDataInput = z.object({}).strict();++export const [getV4MigrationDataTool, handleGetV4MigrationData] = defineTool({+ name: "getV4MigrationData",+ description:+ "Get project-specific evidence for upgrading to Langfuse v4, including SDK versions and compatibility, experiment instrumentation, legacy integrations, deprecated API usage, and trace-level evaluators. Use this before giving v4 migration guidance.",+ baseSchema: GetV4MigrationDataInput,+ inputSchema: GetV4MigrationDataInput,+ handler: async (_input, context) =>+ runMcpTool({+ spanName: "mcp.v4_migration.get",+ context,+ fn: () =>+ getProjectV4MigrationData({+ prisma,+ projectId: context.projectId,+ }),+ }),+ readOnlyHint: true,+ expensiveHint: true,+});diff --git a/web/src/features/mcp/server/bootstrap.ts b/web/src/features/mcp/server/bootstrap.tsindex ee823fd69f..bdd6d2cb1f 100644--- a/web/src/features/mcp/server/bootstrap.ts+++ b/web/src/features/mcp/server/bootstrap.ts@@ -26,6 +26,7 @@ import { dashboardWidgetsFeature } from "../features/dashboardWidgets";import { feedbackFeature } from "../features/feedback";import { experimentsFeature } from "../features/experiments";import { monitorsFeature } from "../features/monitors";+import { v4MigrationFeature } from "../features/v4Migration";const MCP_FEATURES = [promptsFeature,@@ -43,6 +44,7 @@ const MCP_FEATURES = [feedbackFeature,experimentsFeature,monitorsFeature,+ v4MigrationFeature,] as const satisfies readonly McpFeatureModule[];export type McpFeature = (typeof MCP_FEATURES)[number];diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.tsindex 0c18a6f803..17907fb36f 100644--- a/web/src/features/v4/server/v4TransitionRouter.ts+++ b/web/src/features/v4/server/v4TransitionRouter.ts@@ -7,6 +7,7 @@ import {import { isForceV3ExperienceProject } from "@langfuse/shared/src/server";import {getAccessibleOrganizationProjects,+ getProjectV4MigrationData,getMigrationActions,getLegacyApiUsageSummaries,getLegacyIntegrationSummaries,@@ -19,6 +20,15 @@ export const v4TransitionRouter = createTRPCRouter({.input(z.object({ projectId: z.string() })).query(({ input }) => isForceV3ExperienceProject(input.projectId)),+ migrationData: protectedProjectProcedure+ .input(z.object({ projectId: z.string() }))+ .query(({ input, ctx }) =>+ getProjectV4MigrationData({+ prisma: ctx.prisma,+ projectId: input.projectId,+ }),+ ),+summary: protectedProjectProcedure.input(z.object({ projectId: z.string() })).query(async ({ input, ctx }) => {diff --git a/web/src/features/v4/server/v4TransitionService.ts b/web/src/features/v4/server/v4TransitionService.tsindex 2b4d90a9e4..283d2fc49f 100644--- a/web/src/features/v4/server/v4TransitionService.ts+++ b/web/src/features/v4/server/v4TransitionService.ts@@ -14,14 +14,17 @@ import {readExperimentPostUsageCache,readLegacyApiUsageCache,} from "@/src/features/v4/server/v4TransitionCache";-import { trimLegacyApiUsageRows } from "@/src/features/v4/server/v4TransitionQueryLogUsage";+import {+ getLegacyApiUsageSummaries,+ trimLegacyApiUsageRows,+} from "@/src/features/v4/server/v4TransitionQueryLogUsage";import {deriveExperimentInstrumentationMigration,+ getSdkUsageSummaries,getSdkUsageSeriesByProject,} from "@/src/features/v4/server/v4TransitionSdkUsage";-export { getSdkUsageSummaries } from "@/src/features/v4/server/v4TransitionSdkUsage";-export { getLegacyApiUsageSummaries } from "@/src/features/v4/server/v4TransitionQueryLogUsage";+export { getSdkUsageSummaries, getLegacyApiUsageSummaries };const legacyIntegrationExportSources =new Set<AnalyticsIntegrationExportSource>([@@ -212,6 +215,36 @@ export const getTraceLevelEvalSummaries = async ({}));};+/**+ * Project-scoped migration evidence exposed by the v4 migration API.+ * Keep MCP and UI consumers on the same data sources and detection windows.+ */+export const getProjectV4MigrationData = async ({+ prisma,+ projectId,+}: {+ prisma: V4TransitionPrisma;+ projectId: string;+}) => {+ const projectIds = [projectId];+ const [sdkUsage, legacyIntegrations, legacyApiUsage, traceLevelEvals] =+ await Promise.all([+ getSdkUsageSummaries({ projectIds }),+ getLegacyIntegrationSummaries({ prisma, projectIds }),+ getLegacyApiUsageSummaries({ projectIds }),+ getTraceLevelEvalSummaries({ prisma, projectIds }),+ ]);++ return {+ projectId,+ forceV3Experience: isForceV3ExperienceProject(projectId),+ sdkUsage: sdkUsage[0]!,+ legacyIntegrations: legacyIntegrations[0]!,+ legacyApiUsage,+ traceLevelEvals: traceLevelEvals[0]!,+ };+};+/*** Migration signals for the always-mounted sidebar "Action required" pill.* SDK uses the same Redis + live gap-fill path as the panel summaries so the