🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): add listLlmConnections tool (#19139)
Tobias Wochingere879e1683633
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex 562925a3d4..c81353e76c 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -252,6 +252,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {approval: "approval",availability: { scope: "models:CUD" },},+ listLlmConnections: {+ approval: "auto",+ availability: { scope: "llmApiKeys:read" },+ },listObservations: {approval: "auto",availability: { scope: "project:read" },diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.tsindex 4762e830c0..f29cff8e89 100644--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts@@ -56,7 +56,12 @@ import {buildSessionUrl,buildTraceUrl,} from "@langfuse/shared/src/server";-import { ScoreConfigDataType } from "@langfuse/shared";+import {+ LLMAdapter,+ ScoreConfigDataType,+ supportedModels,+} from "@langfuse/shared";+import { encrypt } from "@langfuse/shared/encryption";import { viewDeclarations } from "@langfuse/shared/query";import { MonitorService } from "@langfuse/shared/monitors/server";import {@@ -197,6 +202,10 @@ import {GetDatasetRunsMcpInput,GetDatasetsMcpInput,} from "@/src/features/mcp/server/datasets/schema";+import {+ handleListLlmConnections,+ listLlmConnectionsTool,+} from "@/src/features/mcp/server/llmConnections/tools/listLlmConnections";import { SkillService } from "@/src/features/skills/server";import { handleListSkills } from "@/src/features/mcp/server/skills/tools/listSkills";import { handleGetSkill } from "@/src/features/mcp/server/skills/tools/getSkill";@@ -726,6 +735,114 @@ describe("MCP Read Tools", () => {});});+ describe("listLlmConnections tool", () => {+ it("should have readOnlyHint annotation", () => {+ verifyToolAnnotations(listLlmConnectionsTool, { readOnlyHint: true });+ });++ it("should be available to in-app agent keys", async () => {+ const context = mockServerContext({+ inAppAgent: { permissions: "read" },+ });++ await expect(+ toolRegistry.getEnabledTool(listLlmConnectionsTool.name, context),+ ).resolves.toBeDefined();+ });++ it("should list effective project and organization connections without credentials", async () => {+ const setup = await createMcpTestSetup();+ const otherSetup = await createMcpTestSetup();+ const secret = `sk-mcp-secret-${nanoid()}`;+ const sharedProvider = `shared-${nanoid()}`;+ const orgOnlyProvider = `org-only-${nanoid()}`;+ const otherOrgProvider = `other-org-${nanoid()}`;+ const connection = (+ owner: { projectId: string } | { organizationId: string },+ data: {+ provider: string;+ adapter: LLMAdapter;+ customModels?: string[];+ withDefaultModels?: boolean;+ },+ ) =>+ prisma.llmApiKeys.create({+ data: {+ ...owner,+ ...data,+ secretKey: encrypt(secret),+ displaySecretKey: "...cret",+ extraHeaders: encrypt(JSON.stringify({ "x-secret": secret })),+ },+ });++ await connection(+ { projectId: setup.projectId },+ {+ provider: sharedProvider,+ adapter: LLMAdapter.OpenAI,+ customModels: ["my-model", "gpt-4o"],+ withDefaultModels: true,+ },+ );+ await connection(+ { organizationId: setup.orgId },+ { provider: sharedProvider, adapter: LLMAdapter.Anthropic },+ );+ await connection(+ { organizationId: setup.orgId },+ {+ provider: orgOnlyProvider,+ adapter: LLMAdapter.Anthropic,+ customModels: ["claude-custom"],+ withDefaultModels: false,+ },+ );+ await connection(+ { organizationId: otherSetup.orgId },+ { provider: otherOrgProvider, adapter: LLMAdapter.OpenAI },+ );++ const result = (await handleListLlmConnections({}, setup.context)) as {+ data: Array<Record<string, unknown>>;+ };++ expect(result.data.map((c) => c.provider).sort()).toEqual(+ [orgOnlyProvider, sharedProvider].sort(),+ );+ expect(result.data).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ provider: sharedProvider,+ adapter: LLMAdapter.OpenAI,+ scope: "project",+ displaySecretKey: "...cret",+ availableModels: Array.from(+ new Set([+ "my-model",+ "gpt-4o",+ ...supportedModels[LLMAdapter.OpenAI],+ ]),+ ),+ }),+ expect.objectContaining({+ provider: orgOnlyProvider,+ adapter: LLMAdapter.Anthropic,+ scope: "organization",+ availableModels: ["claude-custom"],+ }),+ ]),+ );+ for (const item of result.data) {+ expect(item).not.toHaveProperty("secretKey");+ expect(item).not.toHaveProperty("extraHeaders");+ expect(item).not.toHaveProperty("projectId");+ expect(item).not.toHaveProperty("organizationId");+ }+ expect(JSON.stringify(result)).not.toContain(secret);+ });+ });+describe("dataset tool schemas", () => {it("uses dataset IDs for existing dataset read addressing", () => {for (const schema of [diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.tsindex 890b46e5a4..8d80e45d2e 100644--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts@@ -1,10 +1,10 @@import { showSuccessToast } from "@/src/features/notifications";import { useHasProjectAccess } from "@/src/features/rbac";import { useState } from "react";-import { supportedModels } from "@langfuse/shared";import { env } from "@/src/env.mjs";import { getJudgeModelProviderAdapters } from "@/src/features/evals/v2/judgeModel";+import { getConnectionAvailableModels } from "@/src/features/llm-api-key";import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";import { usePostHogClientCapture } from "@/src/features/posthog-analytics";import { api } from "@/src/utils/api";@@ -105,14 +105,7 @@ export function useProjectDefaultModel({connectionsPending: connectionsQuery.isPending,providerGroups: connections.map<[string, string[]]>((connection) => [connection.provider,- Array.from(- new Set([- ...connection.customModels,- ...(connection.withDefaultModels- ? supportedModels[connection.adapter]- : []),- ]),- ),+ getConnectionAvailableModels(connection),]),providerAdapters: getJudgeModelProviderAdapters(connections),canRead,diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.tsindex 92c5bcadc8..824822c860 100644--- a/web/src/features/in-app-agent/components/utils/side-effects.ts+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts@@ -99,6 +99,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {langfuse_createModel: ["models"],langfuse_getModel: [],langfuse_deleteModel: ["models"],+ langfuse_listLlmConnections: [],langfuse_listObservations: [],langfuse_getObservation: [],langfuse_getObservationFieldSchema: [],diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.tsindex 55dc3007e9..964195862f 100644--- a/web/src/features/in-app-agent/components/utils/utils.ts+++ b/web/src/features/in-app-agent/components/utils/utils.ts@@ -77,6 +77,7 @@ const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {getRequestContext: "Checking request context",getSkill: "Inspecting skill",listDashboardWidgets: "Browsing widgets",+ listLlmConnections: "Browsing LLM connections",listSkills: "Browsing skills",loadSkill: "Loading skill",loadSkillResource: "Loading skill resource",diff --git a/web/src/features/llm-api-key/availableModels.ts b/web/src/features/llm-api-key/availableModels.tsnew file mode 100644index 0000000000..fec3352dc4--- /dev/null+++ b/web/src/features/llm-api-key/availableModels.ts@@ -0,0 +1,17 @@+import { type LLMAdapter, supportedModels } from "@langfuse/shared";++/** Returns a connection's custom models plus, if enabled, its adapter's default models. */+export function getConnectionAvailableModels(connection: {+ adapter: LLMAdapter;+ customModels: string[];+ withDefaultModels: boolean;+}): string[] {+ return Array.from(+ new Set([+ ...connection.customModels,+ ...(connection.withDefaultModels+ ? supportedModels[connection.adapter]+ : []),+ ]),+ );+}diff --git a/web/src/features/llm-api-key/index.ts b/web/src/features/llm-api-key/index.tsindex 2961f9872f..0494eba1ed 100644--- a/web/src/features/llm-api-key/index.ts+++ b/web/src/features/llm-api-key/index.ts@@ -3,3 +3,4 @@ export {BedrockAuthMethodSchema,type BedrockAuthMethod,} from "@/src/features/llm-api-key/types";+export { getConnectionAvailableModels } from "@/src/features/llm-api-key/availableModels";diff --git a/web/src/features/llm-api-key/server/index.ts b/web/src/features/llm-api-key/server/index.tsnew file mode 100644index 0000000000..078aa27aeb--- /dev/null+++ b/web/src/features/llm-api-key/server/index.ts@@ -0,0 +1 @@+export { LlmConnectionService } from "./llmConnectionService";diff --git a/web/src/features/llm-api-key/server/llmConnectionService.ts b/web/src/features/llm-api-key/server/llmConnectionService.tsindex 186fe0a7b3..c578415ffa 100644--- a/web/src/features/llm-api-key/server/llmConnectionService.ts+++ b/web/src/features/llm-api-key/server/llmConnectionService.ts@@ -4,6 +4,7 @@ import {AuthMethod,type BedrockAuthMethod,} from "@/src/features/llm-api-key/types";+import { getConnectionAvailableModels } from "@/src/features/llm-api-key/availableModels";import {BEDROCK_USE_DEFAULT_CREDENTIALS,BedrockConfigSchema,@@ -94,6 +95,10 @@ export type SafeLlmConnection = {organizationId?: string;};+type LlmConnectionWithAvailableModels = SafeLlmConnection & {+ availableModels: string[];+};+type TestLlmConnectionParams = Omit<LlmConnectionWriteInput,"withDefaultModels"@@ -494,6 +499,20 @@ export class LlmConnectionService {return { data: connections.map(toSafeConnection) };}+ async listEffectiveWithAvailableModels(params: {+ owner: Extract<LlmConnectionOwner, { type: "project" }>;+ includeDecisionModels?: boolean;+ }): Promise<{ data: LlmConnectionWithAvailableModels[] }> {+ const { data } = await this.listEffective(params);++ return {+ data: data.map((connection) => ({+ ...connection,+ availableModels: getConnectionAvailableModels(connection),+ })),+ };+ }+async exists(params: {owner: LlmConnectionOwner;provider: string;diff --git a/web/src/features/mcp/server/bootstrap.ts b/web/src/features/mcp/server/bootstrap.tsindex 5a284cb2b3..42dd3f91c1 100644--- a/web/src/features/mcp/server/bootstrap.ts+++ b/web/src/features/mcp/server/bootstrap.ts@@ -21,6 +21,7 @@ import { healthFeature } from "./health";import { scoresFeature } from "./scores";import { metricsFeature } from "./metrics";import { modelsFeature } from "./models";+import { llmConnectionsFeature } from "./llmConnections";import { mediaFeature } from "./media";import { evalsFeature } from "./evals";import { dashboardWidgetsFeature } from "./dashboardWidgets";@@ -41,6 +42,7 @@ const MCP_FEATURES = [scoresFeature,metricsFeature,modelsFeature,+ llmConnectionsFeature,mediaFeature,evalsFeature,dashboardWidgetsFeature,diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.tsindex 2cbd682895..acf87f29e8 100644--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts@@ -13,6 +13,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({description: ["Create a new evaluator with a stable id. Names do not act as identity and may be reused.","For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",+ "Call listLlmConnections to find valid modelConfig provider names and models.","Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",].join(" "),action: "evaluator:CUD",diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.tsindex 8987b31f80..66c8688fe5 100644--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts@@ -26,6 +26,7 @@ export const ["List the evaluator templates maintained by Langfuse and partners.","LLM-as-a-judge and code definitions can be copied into createEvaluator.","For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a modelConfig for an OpenAI connection using an allowlisted decision model (currently `gpt-6-luna`) or a TypeSafe connection.",+ "Call listLlmConnections to find valid modelConfig provider names and models.",].join(" "),action: "evaluator:read",baseSchema: ListManagedEvaluatorTemplatesInput,diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.tsindex 133776e94c..5cf3cc9be1 100644--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts@@ -13,6 +13,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({description: ["Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.","For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",+ "Call listLlmConnections to find valid modelConfig provider names and models.",].join(" "),action: "evaluator:CUD",baseSchema: McpUpdateEvaluatorInputBase,diff --git a/web/src/features/mcp/server/llmConnections/index.ts b/web/src/features/mcp/server/llmConnections/index.tsnew file mode 100644index 0000000000..941d63ba25--- /dev/null+++ b/web/src/features/mcp/server/llmConnections/index.ts@@ -0,0 +1,13 @@+import type { McpFeatureModule } from "../registry";+import {+ handleListLlmConnections,+ listLlmConnectionsTool,+} from "./tools/listLlmConnections";++export const llmConnectionsFeature = {+ name: "llmConnections",+ description: "Inspect the project's LLM connections",+ tools: [+ { definition: listLlmConnectionsTool, handler: handleListLlmConnections },+ ],+} as const satisfies McpFeatureModule;diff --git a/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.ts b/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.tsnew file mode 100644index 0000000000..8f066e47f1--- /dev/null+++ b/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.ts@@ -0,0 +1,61 @@+import { z } from "zod";+import { LLMApiKeySchema, LLMConnectionConfigSchema } from "@langfuse/shared";++import { LlmConnectionService } from "@/src/features/llm-api-key/server";+import { BedrockAuthMethodSchema } from "@/src/features/llm-api-key";+import { defineTool } from "@/src/features/mcp/core/define-tool";+import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";++const ListLlmConnectionsInput = z.object({});++/** Non-strict copy of the strict LLMApiKeySchema; parsing strips unpicked fields so only connection metadata is returned. */+const LlmConnectionOutput = z+ .object(LLMApiKeySchema.shape)+ .pick({+ id: true,+ provider: true,+ adapter: true,+ displaySecretKey: true,+ baseURL: true,+ customModels: true,+ withDefaultModels: true,+ createdAt: true,+ updatedAt: true,+ })+ .extend({+ scope: z.enum(["project", "organization"]),+ availableModels: z.array(z.string()),+ config: LLMConnectionConfigSchema.nullable(),+ authMethod: BedrockAuthMethodSchema.optional(),+ });++export const [listLlmConnectionsTool, handleListLlmConnections] = defineTool({+ name: "listLlmConnections",+ description: [+ "List the LLM connections usable in the current project, including connections inherited from the organization. A project connection replaces an organization connection with the same provider name.",+ "`provider` is a user-chosen name, not the vendor; use `adapter` to identify the API (e.g. openai, anthropic, bedrock). Pass `provider` and one of `availableModels` as an evaluator's modelConfig.",+ "Returns metadata only, never credentials.",+ ].join(" "),+ action: "llmApiKeys:read",+ baseSchema: ListLlmConnectionsInput,+ inputSchema: ListLlmConnectionsInput,+ handler: async (_input, context) =>+ runMcpTool({+ spanName: "mcp.llm_connections.list",+ context,+ fn: async () => {+ const { data } =+ await new LlmConnectionService().listEffectiveWithAvailableModels({+ owner: {+ type: "project",+ projectId: context.projectId,+ organizationId: context.orgId,+ },+ includeDecisionModels: true,+ });++ return { data: z.array(LlmConnectionOutput).parse(data) };+ },+ }),+ readOnlyHint: true,+});