🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent, mcp): enforce protected prompt labels for in-app agent (#16698)
Lysander Kieselc6391efc9f87
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.tsindex 944d94da55..e82c123699 100644--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts@@ -186,11 +186,11 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {},listExperiments: {approval: "auto",- availability: { scope: "project:read" },+ availability: { scope: "promptExperiments:read" },},listExperimentItems: {approval: "auto",- availability: { scope: "project:read" },+ availability: { scope: "promptExperiments:read" },},submitFeedback: {approval: "approval",diff --git a/packages/shared/src/in-app-agent/server/tools.test.ts b/packages/shared/src/in-app-agent/server/tools.test.tsindex 460b727c6a..d67146bfe4 100644--- a/packages/shared/src/in-app-agent/server/tools.test.ts+++ b/packages/shared/src/in-app-agent/server/tools.test.ts@@ -3,9 +3,28 @@ import { describe, expect, it } from "vitest";import {createInAppAgentToolPolicy,getInAppAgentToolApprovalSource,+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES,} from "./mcpPolicy";describe("createInAppAgentToolPolicy", () => {+ it("lists experiments with promptExperiments:read", () => {+ expect(+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES.listExperiments.availability,+ ).toEqual({ scope: "promptExperiments:read" });+ expect(+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES.listExperimentItems.availability,+ ).toEqual({ scope: "promptExperiments:read" });+ });++ it("keeps listExperiments available to members", () => {+ const asMember = createInAppAgentToolPolicy({+ userAccess: { projectRole: "MEMBER", isAdmin: false },+ });++ expect(asMember.available.has("listExperiments")).toBe(true);+ expect(asMember.available.has("listExperimentItems")).toBe(true);+ });+it("drops a stored grant the user's role no longer covers", () => {const grants = ["langfuse_createModel"];diff --git a/web/src/__tests__/server/mcp-helpers.ts b/web/src/__tests__/server/mcp-helpers.tsindex 0bde322125..aadd1a624a 100644--- a/web/src/__tests__/server/mcp-helpers.ts+++ b/web/src/__tests__/server/mcp-helpers.ts@@ -5,8 +5,10 @@* These helpers allow direct testing of tool handlers without HTTP overhead.*/-import { prisma } from "@langfuse/shared/src/db";+import { randomUUID } from "crypto";+import { prisma, type Role } from "@langfuse/shared/src/db";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";import type { ServerContext } from "@/src/features/mcp/types";/**@@ -55,6 +57,76 @@ export async function createMcpTestSetup(): Promise<{};}+/**+ * Creates a user with an org membership of the given role. Project role is+ * inherited from the org membership unless a project override is added later.+ */+export async function createUserWithOrgRole(params: {+ orgId: string;+ role: Role;+}): Promise<{ userId: string }> {+ const user = await prisma.user.create({+ data: {+ email: `mcp-rbac-${randomUUID()}@langfuse.com`,+ name: `MCP ${params.role} user`,+ },+ });++ await prisma.organizationMembership.create({+ data: {+ orgId: params.orgId,+ userId: user.id,+ role: params.role,+ },+ });++ return { userId: user.id };+}++/**+ * Mints a project API key with isInAppAgentKey + createdByUserId, matching+ * the temporary keys the in-app agent runtime issues for MCP.+ */+export async function createInAppAgentMcpContext(params: {+ projectId: string;+ orgId: string;+ createdByUserId?: string;+}): Promise<{ apiKeyId: string; context: ServerContext }> {+ const apiKey = await createAndAddApiKeysToDb({+ prisma,+ entityId: params.projectId,+ scope: "PROJECT",+ note: "In-app agent MCP session",+ isInAppAgentKey: true,+ createdByUserId: params.createdByUserId,+ });++ return {+ apiKeyId: apiKey.id,+ context: {+ projectId: params.projectId,+ orgId: params.orgId,+ apiKeyId: apiKey.id,+ accessLevel: "project",+ publicKey: apiKey.publicKey,+ plan: "oss",+ rateLimitOverrides: [],+ },+ };+}++export async function protectPromptLabel(params: {+ projectId: string;+ label: string;+}): Promise<void> {+ await prisma.promptProtectedLabels.create({+ data: {+ projectId: params.projectId,+ label: params.label,+ },+ });+}+/*** Creates a mock ServerContext for testing.* Use this when you need a context but don't want to create actual DB records.diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.tsindex 8b70122d8c..89d0f71a8b 100644--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts@@ -41,13 +41,16 @@ vi.mock(}),);-import { prisma } from "@langfuse/shared/src/db";+import { prisma, Role } from "@langfuse/shared/src/db";import { nanoid } from "nanoid";import { z } from "zod";import {+ createInAppAgentMcpContext,createMcpTestSetup,createPromptInDb,+ createUserWithOrgRole,mcpEvalOutputDefinition,+ protectPromptLabel,verifyAuditLog,verifyToolAnnotations,} from "./mcp-helpers";@@ -1758,4 +1761,169 @@ describe("MCP Write Tools", () => {expect(result.name).toBe(promptName);});});++ describe("in-app-agent protected prompt labels", () => {+ it.each([+ {+ name: "rejects adding production with a MEMBER in-app-agent key",+ role: Role.MEMBER,+ inAppAgent: true,+ labels: ["production"],+ expectForbidden: true,+ },+ {+ name: "allows a MEMBER in-app-agent key to add a non-protected label",+ role: Role.MEMBER,+ inAppAgent: true,+ labels: ["staging"],+ expectForbidden: false,+ },+ {+ name: "allows an ADMIN in-app-agent key to add production",+ role: Role.ADMIN,+ inAppAgent: true,+ labels: ["production"],+ expectForbidden: false,+ },+ {+ name: "allows a regular project API key to add production",+ inAppAgent: false,+ labels: ["production"],+ expectForbidden: false,+ },+ ] as const)("$name", async (testCase) => {+ const setup = await createMcpTestSetup();+ await protectPromptLabel({+ projectId: setup.projectId,+ label: "production",+ });++ let context = setup.context;+ if (testCase.inAppAgent) {+ const { userId } = await createUserWithOrgRole({+ orgId: setup.orgId,+ role: testCase.role,+ });+ ({ context } = await createInAppAgentMcpContext({+ projectId: setup.projectId,+ orgId: setup.orgId,+ createdByUserId: userId,+ }));+ }++ const promptName = `protected-labels-${nanoid()}`;+ await createPromptInDb({+ name: promptName,+ prompt: "Test",+ projectId: setup.projectId,+ labels: [],+ version: 1,+ });++ const update = handleUpdatePromptLabels(+ {+ name: promptName,+ version: 1,+ newLabels: [...testCase.labels],+ },+ context,+ );++ if (testCase.expectForbidden) {+ await expect(update).rejects.toMatchObject({+ name: "McpError",+ message: expect.stringContaining("Access forbidden"),+ });++ const prompt = await prisma.prompt.findFirst({+ where: {+ projectId: setup.projectId,+ name: promptName,+ version: 1,+ },+ });+ for (const label of testCase.labels) {+ expect(prompt?.labels).not.toContain(label);+ }+ } else {+ await expect(update).resolves.toMatchObject({+ labels: expect.arrayContaining([...testCase.labels]),+ });+ }+ });++ it("rejects creating a prompt with a custom protected label using a MEMBER in-app-agent key", async () => {+ const setup = await createMcpTestSetup();+ const protectedLabel = "release-gate";+ await protectPromptLabel({+ projectId: setup.projectId,+ label: protectedLabel,+ });+ const { userId } = await createUserWithOrgRole({+ orgId: setup.orgId,+ role: Role.MEMBER,+ });+ const { context } = await createInAppAgentMcpContext({+ projectId: setup.projectId,+ orgId: setup.orgId,+ createdByUserId: userId,+ });+ const promptName = `member-create-protected-${nanoid()}`;++ await expect(+ handleCreateTextPrompt(+ {+ name: promptName,+ prompt: "Protected create",+ labels: [protectedLabel],+ },+ context,+ ),+ ).rejects.toMatchObject({+ name: "McpError",+ message: expect.stringContaining("Access forbidden"),+ });++ await expect(+ prisma.prompt.findFirst({+ where: { projectId: setup.projectId, name: promptName },+ }),+ ).resolves.toBeNull();+ });++ it("rejects adding production when the in-app-agent key has no creator", async () => {+ const setup = await createMcpTestSetup();+ await protectPromptLabel({+ projectId: setup.projectId,+ label: "production",+ });+ const { context } = await createInAppAgentMcpContext({+ projectId: setup.projectId,+ orgId: setup.orgId,+ });+ const promptName = `missing-creator-${nanoid()}`;++ await createPromptInDb({+ name: promptName,+ prompt: "Test",+ projectId: setup.projectId,+ labels: [],+ version: 1,+ });++ await expect(+ handleUpdatePromptLabels(+ {+ name: promptName,+ version: 1,+ newLabels: ["production"],+ },+ context,+ ),+ ).rejects.toMatchObject({+ name: "McpError",+ message: expect.stringContaining("Access forbidden"),+ });+ });+ });});diff --git a/web/src/features/prompts/server/prompt-api-service.ts b/web/src/features/prompts/server/prompt-api-service.tsindex 4b0e69ac27..992512be0b 100644--- a/web/src/features/prompts/server/prompt-api-service.ts+++ b/web/src/features/prompts/server/prompt-api-service.ts@@ -1,12 +1,14 @@-import { prisma } from "@langfuse/shared/src/db";+import { prisma, Role } from "@langfuse/shared/src/db";import {type CreatePromptSchema,type GetPromptByNameSchema,type GetPromptsMetaSchema,type Prompt,+ ForbiddenError,InvalidRequestError,LangfuseConflictError,LangfuseNotFoundError,+ hasProjectAccessByRole,} from "@langfuse/shared";import type { z } from "zod";@@ -15,6 +17,7 @@ import { createPrompt } from "./actions/createPrompt";import { getPromptByName } from "./actions/getPromptByName";import { getPromptsMeta } from "./actions/getPromptsMeta";import { updatePrompt } from "./actions/updatePrompts";+import { checkHasProtectedLabels } from "./utils/checkHasProtectedLabels";type ApiKeyProjectContext = {projectId: string;@@ -38,6 +41,117 @@ export const getPromptForApi = async (input: GetPromptForApiInput) => {return await getPromptByName(input);};+/**+ * Resolve the in-app-agent key creator's project role the same way the worker+ * run executor does: user.admin bypasses membership; otherwise org membership+ * is required and a project membership override wins. Fail closed on missing+ * user, missing org membership, or NONE.+ */+async function resolveApiKeyCreatorProjectAccess(params: {+ userId: string;+ projectId: string;+ orgId: string;+}): Promise<{+ projectRole?: Role;+ isAdmin: boolean;+} | null> {+ const user = await prisma.user.findUnique({+ where: { id: params.userId },+ select: { id: true, admin: true },+ });++ if (!user) {+ return null;+ }++ if (user.admin) {+ return { isAdmin: true };+ }++ const orgMembership = await prisma.organizationMembership.findFirst({+ where: { userId: params.userId, orgId: params.orgId },+ });++ if (!orgMembership) {+ return null;+ }++ const projectMembership = await prisma.projectMembership.findFirst({+ where: {+ userId: params.userId,+ projectId: params.projectId,+ orgMembershipId: orgMembership.id,+ },+ });++ const projectRole = projectMembership?.role ?? orgMembership.role;++ if (projectRole === Role.NONE) {+ return null;+ }++ return {+ projectRole,+ isAdmin: false,+ };+}++/**+ * Ordinary project API keys may still promote protected labels (CI). Temporary+ * in-app-agent keys must honor the creator's promptProtectedLabels:CUD right.+ */+async function assertInAppAgentMayMutateProtectedLabels(params: {+ context: ApiKeyProjectContext;+ labelsToCheck: string[];+ forbiddenErrorMessage: string;+}): Promise<void> {+ const { hasProtectedLabels, protectedLabels } = await checkHasProtectedLabels(+ {+ prisma,+ projectId: params.context.projectId,+ labelsToCheck: params.labelsToCheck,+ },+ );++ if (!hasProtectedLabels) {+ return;+ }++ const apiKey = await prisma.apiKey.findUnique({+ where: { id: params.context.apiKeyId },+ select: {+ isInAppAgentKey: true,+ createdByUserId: true,+ },+ });++ if (!apiKey?.isInAppAgentKey) {+ return;+ }++ const access = apiKey.createdByUserId+ ? await resolveApiKeyCreatorProjectAccess({+ userId: apiKey.createdByUserId,+ projectId: params.context.projectId,+ orgId: params.context.orgId,+ })+ : null;++ const mayMutateProtectedLabels =+ access !== null &&+ hasProjectAccessByRole({+ role: access.projectRole ?? Role.MEMBER,+ admin: access.isAdmin,+ scope: "promptProtectedLabels:CUD",+ });++ if (!mayMutateProtectedLabels) {+ throw new ForbiddenError(+ `${params.forbiddenErrorMessage}\n\n Protected labels are: ${protectedLabels.join(", ")}`,+ );+ }+}+export const createPromptForApi = async ({context,input,@@ -45,6 +159,13 @@ export const createPromptForApi = async ({context: ApiKeyProjectContext;input: z.infer<typeof CreatePromptSchema>;}) => {+ await assertInAppAgentMayMutateProtectedLabels({+ context,+ labelsToCheck: input.labels ?? [],+ forbiddenErrorMessage:+ "You don't have permission to create a prompt with a protected label. Please contact your project admin for assistance.",+ });+const createdPrompt = await createPrompt({...input,config: input.config ?? {},@@ -111,6 +232,19 @@ export const updatePromptLabelsForApi = async ({);}+ // updatePrompt is additive; labels already on this version are not a+ // mutation. Moving a protected label onto this version still counts as add.+ const addedLabels = newLabels.filter(+ (label) => !existingPrompt.labels.includes(label),+ );++ await assertInAppAgentMayMutateProtectedLabels({+ context,+ labelsToCheck: addedLabels,+ forbiddenErrorMessage:+ "You don't have permission to add a protected label to a prompt. Please contact your project admin for assistance.",+ });+const updatedPrompt = await updatePrompt({promptName,projectId: context.projectId,