🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): llm completion guard for demo project (#19113)
Nimar040e65aced7f
diff --git a/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts b/web/src/__tests__/server/unit/chat-completion-handler.servertest.tsindex e9326fc663..556f5bdd14 100644--- a/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts+++ b/web/src/__tests__/server/unit/chat-completion-handler.servertest.ts@@ -3,7 +3,7 @@ import type * as SharedServer from "@langfuse/shared/src/server";import { beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({- authorize: vi.fn(),+ session: vi.fn(),createOutput: vi.fn(),createToolSet: vi.fn(),findConnection: vi.fn(),@@ -13,11 +13,14 @@ const mocks = vi.hoisted(() => ({}));vi.mock("@/src/env.mjs", () => ({- env: { LANGFUSE_BLOCKED_USERIDS_CHATCOMPLETION: new Map() },+ env: {+ LANGFUSE_BLOCKED_USERIDS_CHATCOMPLETION: new Map(),+ NEXT_PUBLIC_DEMO_PROJECT_ID: "demo-project",+ },}));-vi.mock("@/src/features/playground/server/authorizeRequest", () => ({- authorizeRequestOrThrow: mocks.authorize,+vi.mock("@/src/server/auth", () => ({+ getServerAuthSessionForRequest: mocks.session,}));vi.mock("@opentelemetry/api", () => ({@@ -87,10 +90,20 @@ function createRequest(body: Record<string, unknown>) {});}+function createSession(projectId = baseBody.projectId, admin = false) {+ return {+ user: {+ id: "user-1",+ admin,+ organizations: [{ projects: [{ id: projectId, role: "ADMIN" }] }],+ },+ };+}+describe("chatCompletionHandler", () => {beforeEach(() => {vi.clearAllMocks();- mocks.authorize.mockResolvedValue({ userId: "user-1" });+ mocks.session.mockResolvedValue(createSession());mocks.findConnection.mockResolvedValue({id: "connection-1",secretKey: "encrypted-key",@@ -102,6 +115,34 @@ describe("chatCompletionHandler", () => {});});+ it("blocks demo project admins before resolving credentials or calling a model", async () => {+ const projectId = "demo-project";+ mocks.session.mockResolvedValue(createSession(projectId));+ mocks.generate.mockResolvedValue({ text: "Hello back", finalStep: {} });++ const response = await chatCompletionHandler(+ createRequest({ ...baseBody, projectId }),+ );++ expect(response.status).toBe(403);+ expect(mocks.findConnection).not.toHaveBeenCalled();+ expect(mocks.generate).not.toHaveBeenCalled();+ expect(mocks.stream).not.toHaveBeenCalled();+ });++ it("allows internal admins to run demo completions", async () => {+ const projectId = "demo-project";+ mocks.session.mockResolvedValue(createSession(projectId, true));+ mocks.generate.mockResolvedValue({ text: "Hello back", finalStep: {} });++ const response = await chatCompletionHandler(+ createRequest({ ...baseBody, projectId }),+ );++ expect(response.status).toBe(200);+ expect(mocks.generate).toHaveBeenCalledOnce();+ });+it("maps a native text result to the existing playground JSON response", async () => {mocks.generate.mockResolvedValue({text: "Hello back",diff --git a/web/src/features/rbac/utils/checkAccess.clienttest.ts b/web/src/features/rbac/utils/checkAccess.clienttest.tsindex 22394517e0..537527f742 100644--- a/web/src/features/rbac/utils/checkAccess.clienttest.ts+++ b/web/src/features/rbac/utils/checkAccess.clienttest.ts@@ -2,6 +2,22 @@import { hasOrganizationAccess } from "./checkOrganizationAccess";import { hasProjectAccess } from "./checkProjectAccess";+import { type ProjectScope } from "@langfuse/shared";+import { type Role } from "@langfuse/shared/src/db";+import { type Session } from "next-auth";++vi.mock("@/src/env.mjs", () => ({+ env: { NEXT_PUBLIC_DEMO_PROJECT_ID: "demo-project" },+}));++function projectSession(projectId: string, role: Role, admin = false) {+ return {+ user: {+ admin,+ organizations: [{ projects: [{ id: projectId, role }] }],+ },+ } as Session;+}describe("RBAC access checks", () => {it("ignores inherited role/admin properties for organization access", () => {@@ -94,4 +110,51 @@ describe("RBAC access checks", () => {}),).toBe(true);});++ it("reserves demo LLM operations for internal admins even with project OWNER role", () => {+ const projectId = "demo-project";+ const session = projectSession(projectId, "OWNER");+ const paidScopes: ProjectScope[] = [+ "playground:execute",+ "topics:CUD",+ "evaluator:CUD",+ "evaluationRule:CUD",+ "promptExperiments:CUD",+ "evalDefaultModel:CUD",+ "llmApiKeys:create",+ "llmApiKeys:update",+ ];+ for (const scope of paidScopes) {+ expect(hasProjectAccess({ session, projectId, scope }), scope).toBe(+ false,+ );+ expect(+ hasProjectAccess({+ session: projectSession(projectId, "OWNER", true),+ projectId,+ scope,+ }),+ scope,+ ).toBe(true);+ }+ });++ it("preserves demo viewer read access", () => {+ const projectId = "demo-project";+ const session = projectSession(projectId, "VIEWER");+ for (const scope of ["topics:read", "llmApiKeys:read"] as const) {+ expect(hasProjectAccess({ session, projectId, scope }), scope).toBe(true);+ }+ });++ it("preserves normal project LLM permissions", () => {+ const projectId = "project-1";+ expect(+ hasProjectAccess({+ session: projectSession(projectId, "MEMBER"),+ projectId,+ scope: "playground:execute",+ }),+ ).toBe(true);+ });});diff --git a/web/src/features/rbac/utils/checkProjectAccess.ts b/web/src/features/rbac/utils/checkProjectAccess.tsindex c4968bd0c9..d22c009e43 100644--- a/web/src/features/rbac/utils/checkProjectAccess.ts+++ b/web/src/features/rbac/utils/checkProjectAccess.ts@@ -4,6 +4,19 @@ import { TRPCError } from "@trpc/server";import { type Session } from "next-auth";import { useSession } from "next-auth/react";import { hasOwnRole } from "./hasOwnRole";+import { env } from "@/src/env.mjs";++/** LLM execution scopes require an internal admin in the demo project. */+const demoProjectLlmScopes = new Set<ProjectScope>([+ "playground:execute",+ "topics:CUD",+ "evaluator:CUD",+ "evaluationRule:CUD",+ "promptExperiments:CUD",+ "evalDefaultModel:CUD",+ "llmApiKeys:create",+ "llmApiKeys:update",+]);type HasProjectAccessParams = (| {@@ -54,6 +67,8 @@ export function hasProjectAccess(p: HasProjectAccessParams): boolean {const isAdmin = hasOwnRole(p) ? p.admin : p.session?.user?.admin;if (isAdmin) return true;+ if (isDemoProjectLlmScope(p)) return false;+const projectRole: Role | undefined = hasOwnRole(p)? p.role: p.session?.user?.organizations@@ -63,3 +78,11 @@ export function hasProjectAccess(p: HasProjectAccessParams): boolean {return projectRoleAccessRights[projectRole].includes(p.scope);}++function isDemoProjectLlmScope(p: HasProjectAccessParams): boolean {+ return (+ !hasOwnRole(p) &&+ p.projectId === env.NEXT_PUBLIC_DEMO_PROJECT_ID &&+ demoProjectLlmScopes.has(p.scope)+ );+}