🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
test(topics): cover project isolation and permissions with real storage (#19640)
Lysander Kiesel8dd75978ea1e
diff --git a/web/src/__tests__/server/topics-trpc-access.servertest.ts b/web/src/__tests__/server/topics-trpc-access.servertest.tsnew file mode 100644index 0000000000..9acd4b3f4e--- /dev/null+++ b/web/src/__tests__/server/topics-trpc-access.servertest.ts@@ -0,0 +1,309 @@+import { randomUUID } from "node:crypto";+import type { Session } from "next-auth";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { LLMAdapter } from "@langfuse/shared";+import { env as sharedEnv } from "@langfuse/shared/src/env";+import { prisma } from "@langfuse/shared/src/db";+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";+import {+ DEFAULT_TOPIC_FACETS,+ topicEmbeddingConfigSchema,+ topicProcessingConfigSchema,+} from "@langfuse/shared/topics";+import {+ createTopicExecution,+ createTopicFacet,+ createTopicRun,+ getTopicsModels,+} from "@langfuse/shared/topics/server";+import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";+import { appRouter } from "@/src/server/api/root";+import { createInnerTRPCContext } from "@/src/server/api/trpc";++type Role = "OWNER" | "ADMIN" | "MEMBER" | "VIEWER";++function caller(userId: string, memberships: [string, string, Role][]) {+ const session = {+ expires: "1",+ user: {+ id: userId,+ admin: false,+ featureFlags: testFeatureFlags({ langfuseTopics: true }),+ organizations: memberships.map(([orgId, projectId, role]) => ({+ id: orgId,+ role,+ projects: [{ id: projectId, role }],+ })),+ },+ } as unknown as Session;+ return appRouter.createCaller({+ ...createInnerTRPCContext({ session, headers: {} }),+ prisma,+ }).topics;+}++function llmConnection(+ owner: { projectId: string } | { organizationId: string },+) {+ return prisma.llmApiKeys.create({+ data: {+ ...owner,+ provider: `openai-${randomUUID()}`,+ adapter: LLMAdapter.OpenAI,+ secretKey: "encrypted",+ displaySecretKey: "...key",+ },+ });+}++/** A project with a project-level LLM connection, configured models, a custom facet, a run and an execution. */+async function topicsProject(userId: string) {+ const { projectId, orgId } = await createOrgProjectAndApiKey();+ const llmKey = await llmConnection({ projectId });+ await prisma.topicConfig.create({+ data: {+ projectId,+ enabled: false,+ summaryLlmApiKeyId: llmKey.id,+ summaryModel: "summary-model",+ embeddingLlmApiKeyId: llmKey.id,+ embeddingModel: "embedding-model",+ embeddingDimensions: 1024,+ clusteringLlmApiKeyId: llmKey.id,+ clusteringModel: "clustering-model",+ },+ });+ const facet = await createTopicFacet({+ projectId,+ name: `Facet ${randomUUID()}`,+ description: "",+ prompt: "Summarize the trace.",+ });+ const run = await createTopicRun({+ projectId,+ facetId: facet.id,+ facetVersion: 1,+ config: {},+ });+ const execution = await createTopicExecution(+ {+ projectId,+ requestId: randomUUID(),+ operation: "process",+ facets: [{ facetId: facet.id, version: 1 }],+ traceIds: [randomUUID()],+ reuseExistingSummaries: false,+ processingConfig: topicProcessingConfigSchema.parse({}),+ embeddingConfig: topicEmbeddingConfigSchema.parse({}),+ },+ undefined,+ userId,+ );+ return {+ projectId,+ orgId,+ llmKeyId: llmKey.id,+ facetId: facet.id,+ runId: run.id,+ executionId: execution.id,+ };+}++describe("Topics tRPC access", () => {+ const original = {+ enabled: sharedEnv.LANGFUSE_TOPICS_ENABLED,+ projectIds: sharedEnv.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS,+ region: sharedEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ };+ let userId: string;+ let a: Awaited<ReturnType<typeof topicsProject>>;+ let b: Awaited<ReturnType<typeof topicsProject>>;++ beforeAll(async () => {+ userId = (+ await prisma.user.create({+ data: { email: `topics-access-${randomUUID()}@example.com` },+ })+ ).id;+ a = await topicsProject(userId);+ b = await topicsProject(userId);+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = "true";+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS = [+ a.projectId,+ b.projectId,+ ];+ });++ afterAll(() => {+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = original.enabled;+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS =+ original.projectIds;+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = original.region;+ });++ it("lets VIEWER read but rejects every write without changing state", async () => {+ const viewer = caller(userId, [[a.orgId, a.projectId, "VIEWER"]]);+ const projectId = a.projectId;+ const before = await prisma.evaluator.count({+ where: { projectId, type: "FACET" },+ });++ await expect(viewer.facets({ projectId })).resolves.toHaveLength(before);+ // Inputs carry only projectId: the topics:CUD check must run before any handler work.+ for (const write of [+ viewer.saveFacet,+ viewer.saveRule,+ viewer.testModelSettings,+ viewer.saveSetup,+ viewer.setAutomaticProcessing,+ viewer.startSetup,+ viewer.trigger,+ viewer.retry,+ ])+ await expect(write({ projectId } as never)).rejects.toMatchObject({+ code: "FORBIDDEN",+ });++ expect(+ await prisma.evaluator.count({ where: { projectId, type: "FACET" } }),+ ).toBe(before);+ expect(+ await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),+ ).toMatchObject({ enabled: false });++ // Positive control: MEMBER holds topics:CUD.+ const member = caller(userId, [[a.orgId, a.projectId, "MEMBER"]]);+ await expect(+ member.saveFacet({+ projectId,+ name: `Member facet ${randomUUID()}`,+ prompt: "Summarize the trace.",+ }),+ ).resolves.toMatchObject({ projectId });+ });++ it("rejects another project's resources even when the caller administers both", async () => {+ const admin = caller(userId, [+ [a.orgId, a.projectId, "ADMIN"],+ [b.orgId, b.projectId, "ADMIN"],+ ]);+ const projectId = a.projectId;+ const timeRange = {+ from: new Date(Date.now() - 86_400_000),+ to: new Date(),+ };++ // Positive control: the same ids resolve within their own project.+ await expect(+ admin.execution({ projectId: b.projectId, executionId: b.executionId }),+ ).resolves.toMatchObject({ id: b.executionId });++ await expect(+ admin.execution({ projectId, executionId: b.executionId }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ await expect(+ admin.retry({ projectId, executionId: b.executionId }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ await expect(+ admin.map({ projectId, runId: b.runId, timeRange }),+ ).rejects.toMatchObject({ code: "NOT_FOUND" });+ await expect(+ admin.saveFacet({+ projectId,+ facetId: b.facetId,+ name: "Hijacked",+ prompt: "Leak the other project's traces.",+ }),+ ).rejects.toThrow("Facet not found in this project.");+ await expect(+ admin.saveRule({+ projectId,+ name: "Foreign rule",+ filter: [],+ facetIds: [b.facetId],+ }),+ ).rejects.toThrow("Select facets from this project.");+ await expect(+ admin.trigger({+ projectId,+ requestId: randomUUID(),+ operation: "process",+ facets: [{ facetId: b.facetId, version: 1 }],+ traceIds: [randomUUID()],+ reuseExistingSummaries: false,+ processingConfig: topicProcessingConfigSchema.parse({}),+ embeddingConfig: topicEmbeddingConfigSchema.parse({}),+ }),+ ).rejects.toThrow("Facet version not found in this project.");++ // Project B is untouched.+ expect(+ await prisma.evaluatorVersion.count({+ where: { evaluatorId: b.facetId },+ }),+ ).toBe(1);+ expect(+ await prisma.evaluator.count({+ where: { projectId: a.projectId, name: "Hijacked" },+ }),+ ).toBe(0);+ });++ it("uses only LLM connections of the project or its organization", async () => {+ const admin = caller(userId, [+ [a.orgId, a.projectId, "ADMIN"],+ [b.orgId, b.projectId, "ADMIN"],+ ]);+ const projectId = a.projectId;+ const ownOrgKey = await llmConnection({ organizationId: a.orgId });+ const otherOrgKey = await llmConnection({ organizationId: b.orgId });+ const config = await prisma.topicConfig.findUniqueOrThrow({+ where: { projectId },+ });+ const settings = (llmApiKeyId: string) => ({+ projectId,+ summary: { llmApiKeyId, model: "summary-model" },+ embedding: null,+ embeddingDimensions: 1024,+ clustering: null,+ });++ // Saving or testing: another project's or another org's connection is refused.+ for (const foreignKeyId of [b.llmKeyId, otherOrgKey.id]) {+ await expect(+ admin.testModelSettings(settings(foreignKeyId)),+ ).rejects.toThrow("LLM connection not found in this project.");+ await expect(+ admin.saveSetup({+ ...settings(foreignKeyId),+ facets: DEFAULT_TOPIC_FACETS.map(({ name }) => ({+ name,+ enabled: true,+ })),+ filter: [],+ traceIdleMs: 600_000,+ }),+ ).rejects.toThrow("LLM connection not found in this project.");+ }+ expect(+ await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),+ ).toEqual(config);++ // Running: a stored foreign connection resolves to no models; the own org's does.+ const resolveWithSummaryKey = async (summaryLlmApiKeyId: string) => {+ await prisma.topicConfig.update({+ where: { projectId },+ data: { summaryLlmApiKeyId },+ });+ return getTopicsModels(projectId);+ };+ expect(await resolveWithSummaryKey(ownOrgKey.id)).not.toBeNull();+ expect(await resolveWithSummaryKey(otherOrgKey.id)).toBeNull();+ expect(await resolveWithSummaryKey(b.llmKeyId)).toBeNull();+ await prisma.topicConfig.update({+ where: { projectId },+ data: { summaryLlmApiKeyId: config.summaryLlmApiKeyId },+ });+ });+});diff --git a/web/src/__tests__/server/unstable-topics-public-api-access.servertest.ts b/web/src/__tests__/server/unstable-topics-public-api-access.servertest.tsnew file mode 100644index 0000000000..960c880250--- /dev/null+++ b/web/src/__tests__/server/unstable-topics-public-api-access.servertest.ts@@ -0,0 +1,190 @@+import { randomUUID } from "node:crypto";+import type { NextApiRequest, NextApiResponse } from "next";+import { createMocks } from "node-mocks-http";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { env as sharedEnv } from "@langfuse/shared/src/env";+import { prisma } from "@langfuse/shared/src/db";+import {+ createApiKey,+ createBasicAuthHeader,+ createOrgProjectAndApiKey,+} from "@langfuse/shared/src/server";+import {+ OrganizationId,+ ProjectId,+ SystemRoleId,+ UserId,+} from "@langfuse/shared/rbac";+import {+ createTopicFacet,+ createTopicRun,+ saveTopicRun,+ writeTopicSummaries,+} from "@langfuse/shared/topics/server";+import topicsHandler from "@/src/pages/api/public/unstable/topics/index";+import topicHandler from "@/src/pages/api/public/unstable/topics/[topicId]";+import facetsHandler from "@/src/pages/api/public/unstable/topic-facets/index";+import facetHandler from "@/src/pages/api/public/unstable/topic-facets/[facetId]";++type Handler = (req: NextApiRequest, res: NextApiResponse) => Promise<void>;++// In-process calls, since the Topics gate reads shared env in this process.+async function get(+ handler: Handler,+ authorization: string,+ query: Record<string, string> = {},+) {+ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({+ method: "GET",+ query,+ headers: { authorization },+ });+ await handler(req, res);+ return { status: res._getStatusCode(), body: res._getJSONData() };+}++/** A Topics-enabled project with one facet whose published run holds one topic. */+async function topicsProject() {+ const project = await createOrgProjectAndApiKey();+ const facet = await createTopicFacet({+ projectId: project.projectId,+ name: `Facet ${randomUUID()}`,+ description: "Private facet",+ prompt: "Summarize the trace.",+ });+ const run = await createTopicRun({+ projectId: project.projectId,+ facetId: facet.id,+ facetVersion: 1,+ config: {},+ });+ const topicId = randomUUID();+ const now = new Date().toISOString();+ await saveTopicRun({+ ...run,+ status: "completed",+ finishedAt: now,+ topics: [+ {+ topicVersionId: randomUUID(),+ topicId,+ projectId: project.projectId,+ createdByRunId: run.id,+ createdAt: now,+ tags: [],+ name: "Private topic",+ description: "Private topic description",+ centroid: [1, 0],+ radius: 0.5,+ representativeSummaries: [],+ metadata: {},+ },+ ],+ });+ // Topics are served only once the project has a stored summary.+ await writeTopicSummaries([+ {+ projectId: project.projectId,+ facetId: facet.id,+ facetVersion: 1,+ traceId: randomUUID(),+ sessionId: null,+ triggerType: "manual_poc",+ environment: "production",+ traceName: "assistant",+ unitStartTime: now,+ state: "complete",+ summary: "Private summary",+ embedding: [1, 0],+ transcriptId: "test",+ transcriptVersion: "test",+ summaryModel: "summary-model",+ embeddingModel: "embedding-model",+ providedUsageDetails: {},+ usageDetails: {},+ providedCostDetails: {},+ costDetails: {},+ processedAt: now,+ metadata: {},+ },+ ]);+ return { ...project, facetId: facet.id, topicId };+}++describe("unstable Topics public API access", () => {+ const original = {+ enabled: sharedEnv.LANGFUSE_TOPICS_ENABLED,+ projectIds: sharedEnv.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS,+ region: sharedEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ };+ let owner: Awaited<ReturnType<typeof topicsProject>>;+ let other: Awaited<ReturnType<typeof topicsProject>>;++ beforeAll(async () => {+ owner = await topicsProject();+ other = await topicsProject();+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = "true";+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS = [+ owner.projectId,+ other.projectId,+ ];+ });++ afterAll(() => {+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = original.enabled;+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS =+ original.projectIds;+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = original.region;+ });++ it("does not expose a project's facets or topics to another project's key", async () => {+ const ownerAuth = createBasicAuthHeader(owner.publicKey, owner.secretKey);+ const otherAuth = createBasicAuthHeader(other.publicKey, other.secretKey);++ // Positive control: the owner reads both, so the 404s below are not vacuous.+ for (const [auth, status] of [+ [ownerAuth, 200],+ [otherAuth, 404],+ ] as const) {+ expect(+ (await get(facetHandler, auth, { facetId: owner.facetId })).status,+ ).toBe(status);+ expect(+ (await get(topicHandler, auth, { topicId: owner.topicId })).status,+ ).toBe(status);+ }++ // No Topics rule exists, so every facet counts as active and is listed.+ const facets = await get(facetsHandler, otherAuth);+ const topics = await get(topicsHandler, otherAuth);+ expect(facets.status).toBe(200);+ expect(topics.status).toBe(200);+ expect(JSON.stringify(facets.body)).not.toContain(owner.facetId);+ expect(JSON.stringify(topics.body)).not.toContain(owner.topicId);+ });++ it("denies a project key without topics:read", async () => {+ const creator = await prisma.user.create({+ data: { email: `topics-ingest-${randomUUID()}@example.com` },+ });+ const ingestKey = await createApiKey(prisma, {+ tenant: OrganizationId(owner.orgId),+ scope: ProjectId(owner.projectId),+ role: SystemRoleId("INGEST"),+ createdBy: UserId(creator.id),+ });+ const auth = createBasicAuthHeader(+ ingestKey.publicKey,+ ingestKey.secretKey,+ );++ for (const [handler, query] of [+ [facetsHandler, {}],+ [facetHandler, { facetId: owner.facetId }],+ [topicsHandler, {}],+ [topicHandler, { topicId: owner.topicId }],+ ] as const)+ expect((await get(handler, auth, query)).status).toBe(403);+ });+});