Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent, mcp): enforce protected prompt labels for in-app agent (#16698)

Lysander Kieselc6391efc9f87
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index 944d94da55..e82c123699 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -186,11 +186,11 @@ export const IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES = {
},
listExperiments: {
approval: "auto",
- availability: { scope: "project:read" },
+ availability: { scope: "promptExperiments:read" },
},
listExperimentItems: {
approval: "auto",
- availability: { scope: "project:read" },
+ availability: { scope: "promptExperiments:read" },
},
submitFeedback: {
approval: "approval",
diff --git a/packages/shared/src/in-app-agent/server/tools.test.ts b/packages/shared/src/in-app-agent/server/tools.test.ts
index 460b727c6a..d67146bfe4 100644
--- a/packages/shared/src/in-app-agent/server/tools.test.ts
+++ b/packages/shared/src/in-app-agent/server/tools.test.ts
@@ -3,9 +3,28 @@ import { describe, expect, it } from "vitest";
import {
createInAppAgentToolPolicy,
getInAppAgentToolApprovalSource,
+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES,
} from "./mcpPolicy";
describe("createInAppAgentToolPolicy", () => {
+ it("lists experiments with promptExperiments:read", () => {
+ expect(
+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES.listExperiments.availability,
+ ).toEqual({ scope: "promptExperiments:read" });
+ expect(
+ IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES.listExperimentItems.availability,
+ ).toEqual({ scope: "promptExperiments:read" });
+ });
+
+ it("keeps listExperiments available to members", () => {
+ const asMember = createInAppAgentToolPolicy({
+ userAccess: { projectRole: "MEMBER", isAdmin: false },
+ });
+
+ expect(asMember.available.has("listExperiments")).toBe(true);
+ expect(asMember.available.has("listExperimentItems")).toBe(true);
+ });
+
it("drops a stored grant the user's role no longer covers", () => {
const grants = ["langfuse_createModel"];
diff --git a/web/src/__tests__/server/mcp-helpers.ts b/web/src/__tests__/server/mcp-helpers.ts
index 0bde322125..aadd1a624a 100644
--- a/web/src/__tests__/server/mcp-helpers.ts
+++ b/web/src/__tests__/server/mcp-helpers.ts
@@ -5,8 +5,10 @@
* These helpers allow direct testing of tool handlers without HTTP overhead.
*/
-import { prisma } from "@langfuse/shared/src/db";
+import { randomUUID } from "crypto";
+import { prisma, type Role } from "@langfuse/shared/src/db";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";
import type { ServerContext } from "@/src/features/mcp/types";
/**
@@ -55,6 +57,76 @@ export async function createMcpTestSetup(): Promise<{
};
}
+/**
+ * Creates a user with an org membership of the given role. Project role is
+ * inherited from the org membership unless a project override is added later.
+ */
+export async function createUserWithOrgRole(params: {
+ orgId: string;
+ role: Role;
+}): Promise<{ userId: string }> {
+ const user = await prisma.user.create({
+ data: {
+ email: `mcp-rbac-${randomUUID()}@langfuse.com`,
+ name: `MCP ${params.role} user`,
+ },
+ });
+
+ await prisma.organizationMembership.create({
+ data: {
+ orgId: params.orgId,
+ userId: user.id,
+ role: params.role,
+ },
+ });
+
+ return { userId: user.id };
+}
+
+/**
+ * Mints a project API key with isInAppAgentKey + createdByUserId, matching
+ * the temporary keys the in-app agent runtime issues for MCP.
+ */
+export async function createInAppAgentMcpContext(params: {
+ projectId: string;
+ orgId: string;
+ createdByUserId?: string;
+}): Promise<{ apiKeyId: string; context: ServerContext }> {
+ const apiKey = await createAndAddApiKeysToDb({
+ prisma,
+ entityId: params.projectId,
+ scope: "PROJECT",
+ note: "In-app agent MCP session",
+ isInAppAgentKey: true,
+ createdByUserId: params.createdByUserId,
+ });
+
+ return {
+ apiKeyId: apiKey.id,
+ context: {
+ projectId: params.projectId,
+ orgId: params.orgId,
+ apiKeyId: apiKey.id,
+ accessLevel: "project",
+ publicKey: apiKey.publicKey,
+ plan: "oss",
+ rateLimitOverrides: [],
+ },
+ };
+}
+
+export async function protectPromptLabel(params: {
+ projectId: string;
+ label: string;
+}): Promise<void> {
+ await prisma.promptProtectedLabels.create({
+ data: {
+ projectId: params.projectId,
+ label: params.label,
+ },
+ });
+}
+
/**
* Creates a mock ServerContext for testing.
* Use this when you need a context but don't want to create actual DB records.
diff --git a/web/src/__tests__/server/mcp-tools-write.servertest.ts b/web/src/__tests__/server/mcp-tools-write.servertest.ts
index 8b70122d8c..89d0f71a8b 100644
--- a/web/src/__tests__/server/mcp-tools-write.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-write.servertest.ts
@@ -41,13 +41,16 @@ vi.mock(
}),
);
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma, Role } from "@langfuse/shared/src/db";
import { nanoid } from "nanoid";
import { z } from "zod";
import {
+ createInAppAgentMcpContext,
createMcpTestSetup,
createPromptInDb,
+ createUserWithOrgRole,
mcpEvalOutputDefinition,
+ protectPromptLabel,
verifyAuditLog,
verifyToolAnnotations,
} from "./mcp-helpers";
@@ -1758,4 +1761,169 @@ describe("MCP Write Tools", () => {
expect(result.name).toBe(promptName);
});
});
+
+ describe("in-app-agent protected prompt labels", () => {
+ it.each([
+ {
+ name: "rejects adding production with a MEMBER in-app-agent key",
+ role: Role.MEMBER,
+ inAppAgent: true,
+ labels: ["production"],
+ expectForbidden: true,
+ },
+ {
+ name: "allows a MEMBER in-app-agent key to add a non-protected label",
+ role: Role.MEMBER,
+ inAppAgent: true,
+ labels: ["staging"],
+ expectForbidden: false,
+ },
+ {
+ name: "allows an ADMIN in-app-agent key to add production",
+ role: Role.ADMIN,
+ inAppAgent: true,
+ labels: ["production"],
+ expectForbidden: false,
+ },
+ {
+ name: "allows a regular project API key to add production",
+ inAppAgent: false,
+ labels: ["production"],
+ expectForbidden: false,
+ },
+ ] as const)("$name", async (testCase) => {
+ const setup = await createMcpTestSetup();
+ await protectPromptLabel({
+ projectId: setup.projectId,
+ label: "production",
+ });
+
+ let context = setup.context;
+ if (testCase.inAppAgent) {
+ const { userId } = await createUserWithOrgRole({
+ orgId: setup.orgId,
+ role: testCase.role,
+ });
+ ({ context } = await createInAppAgentMcpContext({
+ projectId: setup.projectId,
+ orgId: setup.orgId,
+ createdByUserId: userId,
+ }));
+ }
+
+ const promptName = `protected-labels-${nanoid()}`;
+ await createPromptInDb({
+ name: promptName,
+ prompt: "Test",
+ projectId: setup.projectId,
+ labels: [],
+ version: 1,
+ });
+
+ const update = handleUpdatePromptLabels(
+ {
+ name: promptName,
+ version: 1,
+ newLabels: [...testCase.labels],
+ },
+ context,
+ );
+
+ if (testCase.expectForbidden) {
+ await expect(update).rejects.toMatchObject({
+ name: "McpError",
+ message: expect.stringContaining("Access forbidden"),
+ });
+
+ const prompt = await prisma.prompt.findFirst({
+ where: {
+ projectId: setup.projectId,
+ name: promptName,
+ version: 1,
+ },
+ });
+ for (const label of testCase.labels) {
+ expect(prompt?.labels).not.toContain(label);
+ }
+ } else {
+ await expect(update).resolves.toMatchObject({
+ labels: expect.arrayContaining([...testCase.labels]),
+ });
+ }
+ });
+
+ it("rejects creating a prompt with a custom protected label using a MEMBER in-app-agent key", async () => {
+ const setup = await createMcpTestSetup();
+ const protectedLabel = "release-gate";
+ await protectPromptLabel({
+ projectId: setup.projectId,
+ label: protectedLabel,
+ });
+ const { userId } = await createUserWithOrgRole({
+ orgId: setup.orgId,
+ role: Role.MEMBER,
+ });
+ const { context } = await createInAppAgentMcpContext({
+ projectId: setup.projectId,
+ orgId: setup.orgId,
+ createdByUserId: userId,
+ });
+ const promptName = `member-create-protected-${nanoid()}`;
+
+ await expect(
+ handleCreateTextPrompt(
+ {
+ name: promptName,
+ prompt: "Protected create",
+ labels: [protectedLabel],
+ },
+ context,
+ ),
+ ).rejects.toMatchObject({
+ name: "McpError",
+ message: expect.stringContaining("Access forbidden"),
+ });
+
+ await expect(
+ prisma.prompt.findFirst({
+ where: { projectId: setup.projectId, name: promptName },
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it("rejects adding production when the in-app-agent key has no creator", async () => {
+ const setup = await createMcpTestSetup();
+ await protectPromptLabel({
+ projectId: setup.projectId,
+ label: "production",
+ });
+ const { context } = await createInAppAgentMcpContext({
+ projectId: setup.projectId,
+ orgId: setup.orgId,
+ });
+ const promptName = `missing-creator-${nanoid()}`;
+
+ await createPromptInDb({
+ name: promptName,
+ prompt: "Test",
+ projectId: setup.projectId,
+ labels: [],
+ version: 1,
+ });
+
+ await expect(
+ handleUpdatePromptLabels(
+ {
+ name: promptName,
+ version: 1,
+ newLabels: ["production"],
+ },
+ context,
+ ),
+ ).rejects.toMatchObject({
+ name: "McpError",
+ message: expect.stringContaining("Access forbidden"),
+ });
+ });
+ });
});
diff --git a/web/src/features/prompts/server/prompt-api-service.ts b/web/src/features/prompts/server/prompt-api-service.ts
index 4b0e69ac27..992512be0b 100644
--- a/web/src/features/prompts/server/prompt-api-service.ts
+++ b/web/src/features/prompts/server/prompt-api-service.ts
@@ -1,12 +1,14 @@
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma, Role } from "@langfuse/shared/src/db";
import {
type CreatePromptSchema,
type GetPromptByNameSchema,
type GetPromptsMetaSchema,
type Prompt,
+ ForbiddenError,
InvalidRequestError,
LangfuseConflictError,
LangfuseNotFoundError,
+ hasProjectAccessByRole,
} from "@langfuse/shared";
import type { z } from "zod";
@@ -15,6 +17,7 @@ import { createPrompt } from "./actions/createPrompt";
import { getPromptByName } from "./actions/getPromptByName";
import { getPromptsMeta } from "./actions/getPromptsMeta";
import { updatePrompt } from "./actions/updatePrompts";
+import { checkHasProtectedLabels } from "./utils/checkHasProtectedLabels";
type ApiKeyProjectContext = {
projectId: string;
@@ -38,6 +41,117 @@ export const getPromptForApi = async (input: GetPromptForApiInput) => {
return await getPromptByName(input);
};
+/**
+ * Resolve the in-app-agent key creator's project role the same way the worker
+ * run executor does: user.admin bypasses membership; otherwise org membership
+ * is required and a project membership override wins. Fail closed on missing
+ * user, missing org membership, or NONE.
+ */
+async function resolveApiKeyCreatorProjectAccess(params: {
+ userId: string;
+ projectId: string;
+ orgId: string;
+}): Promise<{
+ projectRole?: Role;
+ isAdmin: boolean;
+} | null> {
+ const user = await prisma.user.findUnique({
+ where: { id: params.userId },
+ select: { id: true, admin: true },
+ });
+
+ if (!user) {
+ return null;
+ }
+
+ if (user.admin) {
+ return { isAdmin: true };
+ }
+
+ const orgMembership = await prisma.organizationMembership.findFirst({
+ where: { userId: params.userId, orgId: params.orgId },
+ });
+
+ if (!orgMembership) {
+ return null;
+ }
+
+ const projectMembership = await prisma.projectMembership.findFirst({
+ where: {
+ userId: params.userId,
+ projectId: params.projectId,
+ orgMembershipId: orgMembership.id,
+ },
+ });
+
+ const projectRole = projectMembership?.role ?? orgMembership.role;
+
+ if (projectRole === Role.NONE) {
+ return null;
+ }
+
+ return {
+ projectRole,
+ isAdmin: false,
+ };
+}
+
+/**
+ * Ordinary project API keys may still promote protected labels (CI). Temporary
+ * in-app-agent keys must honor the creator's promptProtectedLabels:CUD right.
+ */
+async function assertInAppAgentMayMutateProtectedLabels(params: {
+ context: ApiKeyProjectContext;
+ labelsToCheck: string[];
+ forbiddenErrorMessage: string;
+}): Promise<void> {
+ const { hasProtectedLabels, protectedLabels } = await checkHasProtectedLabels(
+ {
+ prisma,
+ projectId: params.context.projectId,
+ labelsToCheck: params.labelsToCheck,
+ },
+ );
+
+ if (!hasProtectedLabels) {
+ return;
+ }
+
+ const apiKey = await prisma.apiKey.findUnique({
+ where: { id: params.context.apiKeyId },
+ select: {
+ isInAppAgentKey: true,
+ createdByUserId: true,
+ },
+ });
+
+ if (!apiKey?.isInAppAgentKey) {
+ return;
+ }
+
+ const access = apiKey.createdByUserId
+ ? await resolveApiKeyCreatorProjectAccess({
+ userId: apiKey.createdByUserId,
+ projectId: params.context.projectId,
+ orgId: params.context.orgId,
+ })
+ : null;
+
+ const mayMutateProtectedLabels =
+ access !== null &&
+ hasProjectAccessByRole({
+ role: access.projectRole ?? Role.MEMBER,
+ admin: access.isAdmin,
+ scope: "promptProtectedLabels:CUD",
+ });
+
+ if (!mayMutateProtectedLabels) {
+ throw new ForbiddenError(
+ `${params.forbiddenErrorMessage}\n\n Protected labels are: ${protectedLabels.join(", ")}`,
+ );
+ }
+}
+
export const createPromptForApi = async ({
context,
input,
@@ -45,6 +159,13 @@ export const createPromptForApi = async ({
context: ApiKeyProjectContext;
input: z.infer<typeof CreatePromptSchema>;
}) => {
+ await assertInAppAgentMayMutateProtectedLabels({
+ context,
+ labelsToCheck: input.labels ?? [],
+ forbiddenErrorMessage:
+ "You don't have permission to create a prompt with a protected label. Please contact your project admin for assistance.",
+ });
+
const createdPrompt = await createPrompt({
...input,
config: input.config ?? {},
@@ -111,6 +232,19 @@ export const updatePromptLabelsForApi = async ({
);
}
+ // updatePrompt is additive; labels already on this version are not a
+ // mutation. Moving a protected label onto this version still counts as add.
+ const addedLabels = newLabels.filter(
+ (label) => !existingPrompt.labels.includes(label),
+ );
+
+ await assertInAppAgentMayMutateProtectedLabels({
+ context,
+ labelsToCheck: addedLabels,
+ forbiddenErrorMessage:
+ "You don't have permission to add a protected label to a prompt. Please contact your project admin for assistance.",
+ });
+
const updatedPrompt = await updatePrompt({
promptName,
projectId: context.projectId,