Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(mcp): add listLlmConnections tool (#19139)

Tobias Wochingere879e1683633
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index 562925a3d4..c81353e76c 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -252,6 +252,10 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {
approval: "approval",
availability: { scope: "models:CUD" },
},
+ listLlmConnections: {
+ approval: "auto",
+ availability: { scope: "llmApiKeys:read" },
+ },
listObservations: {
approval: "auto",
availability: { scope: "project:read" },
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.ts
index 4762e830c0..f29cff8e89 100644
--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts
@@ -56,7 +56,12 @@ import {
buildSessionUrl,
buildTraceUrl,
} from "@langfuse/shared/src/server";
-import { ScoreConfigDataType } from "@langfuse/shared";
+import {
+ LLMAdapter,
+ ScoreConfigDataType,
+ supportedModels,
+} from "@langfuse/shared";
+import { encrypt } from "@langfuse/shared/encryption";
import { viewDeclarations } from "@langfuse/shared/query";
import { MonitorService } from "@langfuse/shared/monitors/server";
import {
@@ -197,6 +202,10 @@ import {
GetDatasetRunsMcpInput,
GetDatasetsMcpInput,
} from "@/src/features/mcp/server/datasets/schema";
+import {
+ handleListLlmConnections,
+ listLlmConnectionsTool,
+} from "@/src/features/mcp/server/llmConnections/tools/listLlmConnections";
import { SkillService } from "@/src/features/skills/server";
import { handleListSkills } from "@/src/features/mcp/server/skills/tools/listSkills";
import { handleGetSkill } from "@/src/features/mcp/server/skills/tools/getSkill";
@@ -726,6 +735,114 @@ describe("MCP Read Tools", () => {
});
});
+ describe("listLlmConnections tool", () => {
+ it("should have readOnlyHint annotation", () => {
+ verifyToolAnnotations(listLlmConnectionsTool, { readOnlyHint: true });
+ });
+
+ it("should be available to in-app agent keys", async () => {
+ const context = mockServerContext({
+ inAppAgent: { permissions: "read" },
+ });
+
+ await expect(
+ toolRegistry.getEnabledTool(listLlmConnectionsTool.name, context),
+ ).resolves.toBeDefined();
+ });
+
+ it("should list effective project and organization connections without credentials", async () => {
+ const setup = await createMcpTestSetup();
+ const otherSetup = await createMcpTestSetup();
+ const secret = `sk-mcp-secret-${nanoid()}`;
+ const sharedProvider = `shared-${nanoid()}`;
+ const orgOnlyProvider = `org-only-${nanoid()}`;
+ const otherOrgProvider = `other-org-${nanoid()}`;
+ const connection = (
+ owner: { projectId: string } | { organizationId: string },
+ data: {
+ provider: string;
+ adapter: LLMAdapter;
+ customModels?: string[];
+ withDefaultModels?: boolean;
+ },
+ ) =>
+ prisma.llmApiKeys.create({
+ data: {
+ ...owner,
+ ...data,
+ secretKey: encrypt(secret),
+ displaySecretKey: "...cret",
+ extraHeaders: encrypt(JSON.stringify({ "x-secret": secret })),
+ },
+ });
+
+ await connection(
+ { projectId: setup.projectId },
+ {
+ provider: sharedProvider,
+ adapter: LLMAdapter.OpenAI,
+ customModels: ["my-model", "gpt-4o"],
+ withDefaultModels: true,
+ },
+ );
+ await connection(
+ { organizationId: setup.orgId },
+ { provider: sharedProvider, adapter: LLMAdapter.Anthropic },
+ );
+ await connection(
+ { organizationId: setup.orgId },
+ {
+ provider: orgOnlyProvider,
+ adapter: LLMAdapter.Anthropic,
+ customModels: ["claude-custom"],
+ withDefaultModels: false,
+ },
+ );
+ await connection(
+ { organizationId: otherSetup.orgId },
+ { provider: otherOrgProvider, adapter: LLMAdapter.OpenAI },
+ );
+
+ const result = (await handleListLlmConnections({}, setup.context)) as {
+ data: Array<Record<string, unknown>>;
+ };
+
+ expect(result.data.map((c) => c.provider).sort()).toEqual(
+ [orgOnlyProvider, sharedProvider].sort(),
+ );
+ expect(result.data).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ provider: sharedProvider,
+ adapter: LLMAdapter.OpenAI,
+ scope: "project",
+ displaySecretKey: "...cret",
+ availableModels: Array.from(
+ new Set([
+ "my-model",
+ "gpt-4o",
+ ...supportedModels[LLMAdapter.OpenAI],
+ ]),
+ ),
+ }),
+ expect.objectContaining({
+ provider: orgOnlyProvider,
+ adapter: LLMAdapter.Anthropic,
+ scope: "organization",
+ availableModels: ["claude-custom"],
+ }),
+ ]),
+ );
+ for (const item of result.data) {
+ expect(item).not.toHaveProperty("secretKey");
+ expect(item).not.toHaveProperty("extraHeaders");
+ expect(item).not.toHaveProperty("projectId");
+ expect(item).not.toHaveProperty("organizationId");
+ }
+ expect(JSON.stringify(result)).not.toContain(secret);
+ });
+ });
+
describe("dataset tool schemas", () => {
it("uses dataset IDs for existing dataset read addressing", () => {
for (const schema of [
diff --git a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
index 890b46e5a4..8d80e45d2e 100644
--- a/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
+++ b/web/src/features/evals/v2/hooks/useProjectDefaultModel.ts
@@ -1,10 +1,10 @@
import { showSuccessToast } from "@/src/features/notifications";
import { useHasProjectAccess } from "@/src/features/rbac";
import { useState } from "react";
-import { supportedModels } from "@langfuse/shared";
import { env } from "@/src/env.mjs";
import { getJudgeModelProviderAdapters } from "@/src/features/evals/v2/judgeModel";
+import { getConnectionAvailableModels } from "@/src/features/llm-api-key";
import type { ProjectDefaultModelConfig } from "@/src/features/evals/v2/types/ProjectDefaultModelConfig";
import { usePostHogClientCapture } from "@/src/features/posthog-analytics";
import { api } from "@/src/utils/api";
@@ -105,14 +105,7 @@ export function useProjectDefaultModel({
connectionsPending: connectionsQuery.isPending,
providerGroups: connections.map<[string, string[]]>((connection) => [
connection.provider,
- Array.from(
- new Set([
- ...connection.customModels,
- ...(connection.withDefaultModels
- ? supportedModels[connection.adapter]
- : []),
- ]),
- ),
+ getConnectionAvailableModels(connection),
]),
providerAdapters: getJudgeModelProviderAdapters(connections),
canRead,
diff --git a/web/src/features/in-app-agent/components/utils/side-effects.ts b/web/src/features/in-app-agent/components/utils/side-effects.ts
index 92c5bcadc8..824822c860 100644
--- a/web/src/features/in-app-agent/components/utils/side-effects.ts
+++ b/web/src/features/in-app-agent/components/utils/side-effects.ts
@@ -99,6 +99,7 @@ const IN_APP_AGENT_TOOL_TRPC_INVALIDATION_TARGETS = {
langfuse_createModel: ["models"],
langfuse_getModel: [],
langfuse_deleteModel: ["models"],
+ langfuse_listLlmConnections: [],
langfuse_listObservations: [],
langfuse_getObservation: [],
langfuse_getObservationFieldSchema: [],
diff --git a/web/src/features/in-app-agent/components/utils/utils.ts b/web/src/features/in-app-agent/components/utils/utils.ts
index 55dc3007e9..964195862f 100644
--- a/web/src/features/in-app-agent/components/utils/utils.ts
+++ b/web/src/features/in-app-agent/components/utils/utils.ts
@@ -77,6 +77,7 @@ const IN_APP_AGENT_TOOL_PROGRESS_LABEL_OVERRIDES: Record<string, string> = {
getRequestContext: "Checking request context",
getSkill: "Inspecting skill",
listDashboardWidgets: "Browsing widgets",
+ listLlmConnections: "Browsing LLM connections",
listSkills: "Browsing skills",
loadSkill: "Loading skill",
loadSkillResource: "Loading skill resource",
diff --git a/web/src/features/llm-api-key/availableModels.ts b/web/src/features/llm-api-key/availableModels.ts
new file mode 100644
index 0000000000..fec3352dc4
--- /dev/null
+++ b/web/src/features/llm-api-key/availableModels.ts
@@ -0,0 +1,17 @@
+import { type LLMAdapter, supportedModels } from "@langfuse/shared";
+
+/** Returns a connection's custom models plus, if enabled, its adapter's default models. */
+export function getConnectionAvailableModels(connection: {
+ adapter: LLMAdapter;
+ customModels: string[];
+ withDefaultModels: boolean;
+}): string[] {
+ return Array.from(
+ new Set([
+ ...connection.customModels,
+ ...(connection.withDefaultModels
+ ? supportedModels[connection.adapter]
+ : []),
+ ]),
+ );
+}
diff --git a/web/src/features/llm-api-key/index.ts b/web/src/features/llm-api-key/index.ts
index 2961f9872f..0494eba1ed 100644
--- a/web/src/features/llm-api-key/index.ts
+++ b/web/src/features/llm-api-key/index.ts
@@ -3,3 +3,4 @@ export {
BedrockAuthMethodSchema,
type BedrockAuthMethod,
} from "@/src/features/llm-api-key/types";
+export { getConnectionAvailableModels } from "@/src/features/llm-api-key/availableModels";
diff --git a/web/src/features/llm-api-key/server/index.ts b/web/src/features/llm-api-key/server/index.ts
new file mode 100644
index 0000000000..078aa27aeb
--- /dev/null
+++ b/web/src/features/llm-api-key/server/index.ts
@@ -0,0 +1 @@
+export { LlmConnectionService } from "./llmConnectionService";
diff --git a/web/src/features/llm-api-key/server/llmConnectionService.ts b/web/src/features/llm-api-key/server/llmConnectionService.ts
index 186fe0a7b3..c578415ffa 100644
--- a/web/src/features/llm-api-key/server/llmConnectionService.ts
+++ b/web/src/features/llm-api-key/server/llmConnectionService.ts
@@ -4,6 +4,7 @@ import {
AuthMethod,
type BedrockAuthMethod,
} from "@/src/features/llm-api-key/types";
+import { getConnectionAvailableModels } from "@/src/features/llm-api-key/availableModels";
import {
BEDROCK_USE_DEFAULT_CREDENTIALS,
BedrockConfigSchema,
@@ -94,6 +95,10 @@ export type SafeLlmConnection = {
organizationId?: string;
};
+type LlmConnectionWithAvailableModels = SafeLlmConnection & {
+ availableModels: string[];
+};
+
type TestLlmConnectionParams = Omit<
LlmConnectionWriteInput,
"withDefaultModels"
@@ -494,6 +499,20 @@ export class LlmConnectionService {
return { data: connections.map(toSafeConnection) };
}
+ async listEffectiveWithAvailableModels(params: {
+ owner: Extract<LlmConnectionOwner, { type: "project" }>;
+ includeDecisionModels?: boolean;
+ }): Promise<{ data: LlmConnectionWithAvailableModels[] }> {
+ const { data } = await this.listEffective(params);
+
+ return {
+ data: data.map((connection) => ({
+ ...connection,
+ availableModels: getConnectionAvailableModels(connection),
+ })),
+ };
+ }
+
async exists(params: {
owner: LlmConnectionOwner;
provider: string;
diff --git a/web/src/features/mcp/server/bootstrap.ts b/web/src/features/mcp/server/bootstrap.ts
index 5a284cb2b3..42dd3f91c1 100644
--- a/web/src/features/mcp/server/bootstrap.ts
+++ b/web/src/features/mcp/server/bootstrap.ts
@@ -21,6 +21,7 @@ import { healthFeature } from "./health";
import { scoresFeature } from "./scores";
import { metricsFeature } from "./metrics";
import { modelsFeature } from "./models";
+import { llmConnectionsFeature } from "./llmConnections";
import { mediaFeature } from "./media";
import { evalsFeature } from "./evals";
import { dashboardWidgetsFeature } from "./dashboardWidgets";
@@ -41,6 +42,7 @@ const MCP_FEATURES = [
scoresFeature,
metricsFeature,
modelsFeature,
+ llmConnectionsFeature,
mediaFeature,
evalsFeature,
dashboardWidgetsFeature,
diff --git a/web/src/features/mcp/server/evals/tools/createEvaluator.ts b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
index 2cbd682895..acf87f29e8 100644
--- a/web/src/features/mcp/server/evals/tools/createEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/createEvaluator.ts
@@ -13,6 +13,7 @@ export const [createEvaluatorTool, handleCreateEvaluator] = defineTool({
description: [
"Create a new evaluator with a stable id. Names do not act as identity and may be reused.",
"For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",
+ "Call listLlmConnections to find valid modelConfig provider names and models.",
"Use updateEvaluator with the returned evaluatorId to update it or append a new immutable definition version.",
].join(" "),
action: "evaluator:CUD",
diff --git a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
index 8987b31f80..66c8688fe5 100644
--- a/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
+++ b/web/src/features/mcp/server/evals/tools/listManagedEvaluatorTemplates.ts
@@ -26,6 +26,7 @@ export const [
"List the evaluator templates maintained by Langfuse and partners.",
"LLM-as-a-judge and code definitions can be copied into createEvaluator.",
"For decision-model templates, copy questions, convert each state entry into a variableMapping entry, and provide a modelConfig for an OpenAI connection using an allowlisted decision model (currently `gpt-6-luna`) or a TypeSafe connection.",
+ "Call listLlmConnections to find valid modelConfig provider names and models.",
].join(" "),
action: "evaluator:read",
baseSchema: ListManagedEvaluatorTemplatesInput,
diff --git a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
index 133776e94c..5cf3cc9be1 100644
--- a/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
+++ b/web/src/features/mcp/server/evals/tools/updateEvaluator.ts
@@ -13,6 +13,7 @@ export const [updateEvaluatorTool, handleUpdateEvaluator] = defineTool({
description: [
"Update an evaluator by stable id. Definition changes append an immutable version; name and description changes do not.",
"For `LLM_AS_JUDGE`, provide prompt + outputDefinition and optionally modelConfig. For `CODE`, provide sourceCode + sourceCodeLanguage. For `DECISION_MODEL`, provide questions, a modelConfig with provider and model only, and variableMapping entries defining its state. Use an OpenAI connection with an allowlisted decision model (currently `gpt-6-luna`), or a TypeSafe connection.",
+ "Call listLlmConnections to find valid modelConfig provider names and models.",
].join(" "),
action: "evaluator:CUD",
baseSchema: McpUpdateEvaluatorInputBase,
diff --git a/web/src/features/mcp/server/llmConnections/index.ts b/web/src/features/mcp/server/llmConnections/index.ts
new file mode 100644
index 0000000000..941d63ba25
--- /dev/null
+++ b/web/src/features/mcp/server/llmConnections/index.ts
@@ -0,0 +1,13 @@
+import type { McpFeatureModule } from "../registry";
+import {
+ handleListLlmConnections,
+ listLlmConnectionsTool,
+} from "./tools/listLlmConnections";
+
+export const llmConnectionsFeature = {
+ name: "llmConnections",
+ description: "Inspect the project's LLM connections",
+ tools: [
+ { definition: listLlmConnectionsTool, handler: handleListLlmConnections },
+ ],
+} as const satisfies McpFeatureModule;
diff --git a/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.ts b/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.ts
new file mode 100644
index 0000000000..8f066e47f1
--- /dev/null
+++ b/web/src/features/mcp/server/llmConnections/tools/listLlmConnections.ts
@@ -0,0 +1,61 @@
+import { z } from "zod";
+import { LLMApiKeySchema, LLMConnectionConfigSchema } from "@langfuse/shared";
+
+import { LlmConnectionService } from "@/src/features/llm-api-key/server";
+import { BedrockAuthMethodSchema } from "@/src/features/llm-api-key";
+import { defineTool } from "@/src/features/mcp/core/define-tool";
+import { runMcpTool } from "@/src/features/mcp/core/run-mcp-tool";
+
+const ListLlmConnectionsInput = z.object({});
+
+/** Non-strict copy of the strict LLMApiKeySchema; parsing strips unpicked fields so only connection metadata is returned. */
+const LlmConnectionOutput = z
+ .object(LLMApiKeySchema.shape)
+ .pick({
+ id: true,
+ provider: true,
+ adapter: true,
+ displaySecretKey: true,
+ baseURL: true,
+ customModels: true,
+ withDefaultModels: true,
+ createdAt: true,
+ updatedAt: true,
+ })
+ .extend({
+ scope: z.enum(["project", "organization"]),
+ availableModels: z.array(z.string()),
+ config: LLMConnectionConfigSchema.nullable(),
+ authMethod: BedrockAuthMethodSchema.optional(),
+ });
+
+export const [listLlmConnectionsTool, handleListLlmConnections] = defineTool({
+ name: "listLlmConnections",
+ description: [
+ "List the LLM connections usable in the current project, including connections inherited from the organization. A project connection replaces an organization connection with the same provider name.",
+ "`provider` is a user-chosen name, not the vendor; use `adapter` to identify the API (e.g. openai, anthropic, bedrock). Pass `provider` and one of `availableModels` as an evaluator's modelConfig.",
+ "Returns metadata only, never credentials.",
+ ].join(" "),
+ action: "llmApiKeys:read",
+ baseSchema: ListLlmConnectionsInput,
+ inputSchema: ListLlmConnectionsInput,
+ handler: async (_input, context) =>
+ runMcpTool({
+ spanName: "mcp.llm_connections.list",
+ context,
+ fn: async () => {
+ const { data } =
+ await new LlmConnectionService().listEffectiveWithAvailableModels({
+ owner: {
+ type: "project",
+ projectId: context.projectId,
+ organizationId: context.orgId,
+ },
+ includeDecisionModels: true,
+ });
+
+ return { data: z.array(LlmConnectionOutput).parse(data) };
+ },
+ }),
+ readOnlyHint: true,
+});