Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): llm completion guard for demo project (#19113)

Nimar040e65aced7f
diff --git a/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts b/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts
index e9326fc663..556f5bdd14 100644
--- a/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts
+++ b/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts
@@ -3,7 +3,7 @@ import type * as SharedServer from "@langfuse/shared/src/server";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
- authorize: vi.fn(),
+ session: vi.fn(),
createOutput: vi.fn(),
createToolSet: vi.fn(),
findConnection: vi.fn(),
@@ -13,11 +13,14 @@ const mocks = vi.hoisted(() => ({
}));
vi.mock("@/src/env.mjs", () => ({
- env: { LANGFUSE_BLOCKED_USERIDS_CHATCOMPLETION: new Map() },
+ env: {
+ LANGFUSE_BLOCKED_USERIDS_CHATCOMPLETION: new Map(),
+ NEXT_PUBLIC_DEMO_PROJECT_ID: "demo-project",
+ },
}));
-vi.mock("@/src/features/playground/server/authorizeRequest", () => ({
- authorizeRequestOrThrow: mocks.authorize,
+vi.mock("@/src/server/auth", () => ({
+ getServerAuthSessionForRequest: mocks.session,
}));
vi.mock("@opentelemetry/api", () => ({
@@ -87,10 +90,20 @@ function createRequest(body: Record<string, unknown>) {
});
}
+function createSession(projectId = baseBody.projectId, admin = false) {
+ return {
+ user: {
+ id: "user-1",
+ admin,
+ organizations: [{ projects: [{ id: projectId, role: "ADMIN" }] }],
+ },
+ };
+}
+
describe("chatCompletionHandler", () => {
beforeEach(() => {
vi.clearAllMocks();
- mocks.authorize.mockResolvedValue({ userId: "user-1" });
+ mocks.session.mockResolvedValue(createSession());
mocks.findConnection.mockResolvedValue({
id: "connection-1",
secretKey: "encrypted-key",
@@ -102,6 +115,34 @@ describe("chatCompletionHandler", () => {
});
});
+ it("blocks demo project admins before resolving credentials or calling a model", async () => {
+ const projectId = "demo-project";
+ mocks.session.mockResolvedValue(createSession(projectId));
+ mocks.generate.mockResolvedValue({ text: "Hello back", finalStep: {} });
+
+ const response = await chatCompletionHandler(
+ createRequest({ ...baseBody, projectId }),
+ );
+
+ expect(response.status).toBe(403);
+ expect(mocks.findConnection).not.toHaveBeenCalled();
+ expect(mocks.generate).not.toHaveBeenCalled();
+ expect(mocks.stream).not.toHaveBeenCalled();
+ });
+
+ it("allows internal admins to run demo completions", async () => {
+ const projectId = "demo-project";
+ mocks.session.mockResolvedValue(createSession(projectId, true));
+ mocks.generate.mockResolvedValue({ text: "Hello back", finalStep: {} });
+
+ const response = await chatCompletionHandler(
+ createRequest({ ...baseBody, projectId }),
+ );
+
+ expect(response.status).toBe(200);
+ expect(mocks.generate).toHaveBeenCalledOnce();
+ });
+
it("maps a native text result to the existing playground JSON response", async () => {
mocks.generate.mockResolvedValue({
text: "Hello back",
diff --git a/web/src/features/rbac/utils/checkAccess.clienttest.ts b/web/src/features/rbac/utils/checkAccess.clienttest.ts
index 22394517e0..537527f742 100644
--- a/web/src/features/rbac/utils/checkAccess.clienttest.ts
+++ b/web/src/features/rbac/utils/checkAccess.clienttest.ts
@@ -2,6 +2,22 @@
import { hasOrganizationAccess } from "./checkOrganizationAccess";
import { hasProjectAccess } from "./checkProjectAccess";
+import { type ProjectScope } from "@langfuse/shared";
+import { type Role } from "@langfuse/shared/src/db";
+import { type Session } from "next-auth";
+
+vi.mock("@/src/env.mjs", () => ({
+ env: { NEXT_PUBLIC_DEMO_PROJECT_ID: "demo-project" },
+}));
+
+function projectSession(projectId: string, role: Role, admin = false) {
+ return {
+ user: {
+ admin,
+ organizations: [{ projects: [{ id: projectId, role }] }],
+ },
+ } as Session;
+}
describe("RBAC access checks", () => {
it("ignores inherited role/admin properties for organization access", () => {
@@ -94,4 +110,51 @@ describe("RBAC access checks", () => {
}),
).toBe(true);
});
+
+ it("reserves demo LLM operations for internal admins even with project OWNER role", () => {
+ const projectId = "demo-project";
+ const session = projectSession(projectId, "OWNER");
+ const paidScopes: ProjectScope[] = [
+ "playground:execute",
+ "topics:CUD",
+ "evaluator:CUD",
+ "evaluationRule:CUD",
+ "promptExperiments:CUD",
+ "evalDefaultModel:CUD",
+ "llmApiKeys:create",
+ "llmApiKeys:update",
+ ];
+ for (const scope of paidScopes) {
+ expect(hasProjectAccess({ session, projectId, scope }), scope).toBe(
+ false,
+ );
+ expect(
+ hasProjectAccess({
+ session: projectSession(projectId, "OWNER", true),
+ projectId,
+ scope,
+ }),
+ scope,
+ ).toBe(true);
+ }
+ });
+
+ it("preserves demo viewer read access", () => {
+ const projectId = "demo-project";
+ const session = projectSession(projectId, "VIEWER");
+ for (const scope of ["topics:read", "llmApiKeys:read"] as const) {
+ expect(hasProjectAccess({ session, projectId, scope }), scope).toBe(true);
+ }
+ });
+
+ it("preserves normal project LLM permissions", () => {
+ const projectId = "project-1";
+ expect(
+ hasProjectAccess({
+ session: projectSession(projectId, "MEMBER"),
+ projectId,
+ scope: "playground:execute",
+ }),
+ ).toBe(true);
+ });
});
diff --git a/web/src/features/rbac/utils/checkProjectAccess.ts b/web/src/features/rbac/utils/checkProjectAccess.ts
index c4968bd0c9..d22c009e43 100644
--- a/web/src/features/rbac/utils/checkProjectAccess.ts
+++ b/web/src/features/rbac/utils/checkProjectAccess.ts
@@ -4,6 +4,19 @@ import { TRPCError } from "@trpc/server";
import { type Session } from "next-auth";
import { useSession } from "next-auth/react";
import { hasOwnRole } from "./hasOwnRole";
+import { env } from "@/src/env.mjs";
+
+/** LLM execution scopes require an internal admin in the demo project. */
+const demoProjectLlmScopes = new Set<ProjectScope>([
+ "playground:execute",
+ "topics:CUD",
+ "evaluator:CUD",
+ "evaluationRule:CUD",
+ "promptExperiments:CUD",
+ "evalDefaultModel:CUD",
+ "llmApiKeys:create",
+ "llmApiKeys:update",
+]);
type HasProjectAccessParams = (
| {
@@ -54,6 +67,8 @@ export function hasProjectAccess(p: HasProjectAccessParams): boolean {
const isAdmin = hasOwnRole(p) ? p.admin : p.session?.user?.admin;
if (isAdmin) return true;
+ if (isDemoProjectLlmScope(p)) return false;
+
const projectRole: Role | undefined = hasOwnRole(p)
? p.role
: p.session?.user?.organizations
@@ -63,3 +78,11 @@ export function hasProjectAccess(p: HasProjectAccessParams): boolean {
return projectRoleAccessRights[projectRole].includes(p.scope);
}
+
+function isDemoProjectLlmScope(p: HasProjectAccessParams): boolean {
+ return (
+ !hasOwnRole(p) &&
+ p.projectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&
+ demoProjectLlmScopes.has(p.scope)
+ );
+}