Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

test(topics): cover project isolation and permissions with real storage (#19640)

Lysander Kiesel8dd75978ea1e
diff --git a/web/src/__tests__/server/topics-trpc-access.servertest.ts b/web/src/__tests__/server/topics-trpc-access.servertest.ts
new file mode 100644
index 0000000000..9acd4b3f4e
--- /dev/null
+++ b/web/src/__tests__/server/topics-trpc-access.servertest.ts
@@ -0,0 +1,309 @@
+import { randomUUID } from "node:crypto";
+import type { Session } from "next-auth";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { LLMAdapter } from "@langfuse/shared";
+import { env as sharedEnv } from "@langfuse/shared/src/env";
+import { prisma } from "@langfuse/shared/src/db";
+import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
+import {
+ DEFAULT_TOPIC_FACETS,
+ topicEmbeddingConfigSchema,
+ topicProcessingConfigSchema,
+} from "@langfuse/shared/topics";
+import {
+ createTopicExecution,
+ createTopicFacet,
+ createTopicRun,
+ getTopicsModels,
+} from "@langfuse/shared/topics/server";
+import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+
+type Role = "OWNER" | "ADMIN" | "MEMBER" | "VIEWER";
+
+function caller(userId: string, memberships: [string, string, Role][]) {
+ const session = {
+ expires: "1",
+ user: {
+ id: userId,
+ admin: false,
+ featureFlags: testFeatureFlags({ langfuseTopics: true }),
+ organizations: memberships.map(([orgId, projectId, role]) => ({
+ id: orgId,
+ role,
+ projects: [{ id: projectId, role }],
+ })),
+ },
+ } as unknown as Session;
+ return appRouter.createCaller({
+ ...createInnerTRPCContext({ session, headers: {} }),
+ prisma,
+ }).topics;
+}
+
+function llmConnection(
+ owner: { projectId: string } | { organizationId: string },
+) {
+ return prisma.llmApiKeys.create({
+ data: {
+ ...owner,
+ provider: `openai-${randomUUID()}`,
+ adapter: LLMAdapter.OpenAI,
+ secretKey: "encrypted",
+ displaySecretKey: "...key",
+ },
+ });
+}
+
+/** A project with a project-level LLM connection, configured models, a custom facet, a run and an execution. */
+async function topicsProject(userId: string) {
+ const { projectId, orgId } = await createOrgProjectAndApiKey();
+ const llmKey = await llmConnection({ projectId });
+ await prisma.topicConfig.create({
+ data: {
+ projectId,
+ enabled: false,
+ summaryLlmApiKeyId: llmKey.id,
+ summaryModel: "summary-model",
+ embeddingLlmApiKeyId: llmKey.id,
+ embeddingModel: "embedding-model",
+ embeddingDimensions: 1024,
+ clusteringLlmApiKeyId: llmKey.id,
+ clusteringModel: "clustering-model",
+ },
+ });
+ const facet = await createTopicFacet({
+ projectId,
+ name: `Facet ${randomUUID()}`,
+ description: "",
+ prompt: "Summarize the trace.",
+ });
+ const run = await createTopicRun({
+ projectId,
+ facetId: facet.id,
+ facetVersion: 1,
+ config: {},
+ });
+ const execution = await createTopicExecution(
+ {
+ projectId,
+ requestId: randomUUID(),
+ operation: "process",
+ facets: [{ facetId: facet.id, version: 1 }],
+ traceIds: [randomUUID()],
+ reuseExistingSummaries: false,
+ processingConfig: topicProcessingConfigSchema.parse({}),
+ embeddingConfig: topicEmbeddingConfigSchema.parse({}),
+ },
+ undefined,
+ userId,
+ );
+ return {
+ projectId,
+ orgId,
+ llmKeyId: llmKey.id,
+ facetId: facet.id,
+ runId: run.id,
+ executionId: execution.id,
+ };
+}
+
+describe("Topics tRPC access", () => {
+ const original = {
+ enabled: sharedEnv.LANGFUSE_TOPICS_ENABLED,
+ projectIds: sharedEnv.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS,
+ region: sharedEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ };
+ let userId: string;
+ let a: Awaited<ReturnType<typeof topicsProject>>;
+ let b: Awaited<ReturnType<typeof topicsProject>>;
+
+ beforeAll(async () => {
+ userId = (
+ await prisma.user.create({
+ data: { email: `topics-access-${randomUUID()}@example.com` },
+ })
+ ).id;
+ a = await topicsProject(userId);
+ b = await topicsProject(userId);
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = "true";
+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS = [
+ a.projectId,
+ b.projectId,
+ ];
+ });
+
+ afterAll(() => {
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = original.enabled;
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS =
+ original.projectIds;
+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = original.region;
+ });
+
+ it("lets VIEWER read but rejects every write without changing state", async () => {
+ const viewer = caller(userId, [[a.orgId, a.projectId, "VIEWER"]]);
+ const projectId = a.projectId;
+ const before = await prisma.evaluator.count({
+ where: { projectId, type: "FACET" },
+ });
+
+ await expect(viewer.facets({ projectId })).resolves.toHaveLength(before);
+ // Inputs carry only projectId: the topics:CUD check must run before any handler work.
+ for (const write of [
+ viewer.saveFacet,
+ viewer.saveRule,
+ viewer.testModelSettings,
+ viewer.saveSetup,
+ viewer.setAutomaticProcessing,
+ viewer.startSetup,
+ viewer.trigger,
+ viewer.retry,
+ ])
+ await expect(write({ projectId } as never)).rejects.toMatchObject({
+ code: "FORBIDDEN",
+ });
+
+ expect(
+ await prisma.evaluator.count({ where: { projectId, type: "FACET" } }),
+ ).toBe(before);
+ expect(
+ await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),
+ ).toMatchObject({ enabled: false });
+
+ // Positive control: MEMBER holds topics:CUD.
+ const member = caller(userId, [[a.orgId, a.projectId, "MEMBER"]]);
+ await expect(
+ member.saveFacet({
+ projectId,
+ name: `Member facet ${randomUUID()}`,
+ prompt: "Summarize the trace.",
+ }),
+ ).resolves.toMatchObject({ projectId });
+ });
+
+ it("rejects another project's resources even when the caller administers both", async () => {
+ const admin = caller(userId, [
+ [a.orgId, a.projectId, "ADMIN"],
+ [b.orgId, b.projectId, "ADMIN"],
+ ]);
+ const projectId = a.projectId;
+ const timeRange = {
+ from: new Date(Date.now() - 86_400_000),
+ to: new Date(),
+ };
+
+ // Positive control: the same ids resolve within their own project.
+ await expect(
+ admin.execution({ projectId: b.projectId, executionId: b.executionId }),
+ ).resolves.toMatchObject({ id: b.executionId });
+
+ await expect(
+ admin.execution({ projectId, executionId: b.executionId }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ await expect(
+ admin.retry({ projectId, executionId: b.executionId }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ await expect(
+ admin.map({ projectId, runId: b.runId, timeRange }),
+ ).rejects.toMatchObject({ code: "NOT_FOUND" });
+ await expect(
+ admin.saveFacet({
+ projectId,
+ facetId: b.facetId,
+ name: "Hijacked",
+ prompt: "Leak the other project's traces.",
+ }),
+ ).rejects.toThrow("Facet not found in this project.");
+ await expect(
+ admin.saveRule({
+ projectId,
+ name: "Foreign rule",
+ filter: [],
+ facetIds: [b.facetId],
+ }),
+ ).rejects.toThrow("Select facets from this project.");
+ await expect(
+ admin.trigger({
+ projectId,
+ requestId: randomUUID(),
+ operation: "process",
+ facets: [{ facetId: b.facetId, version: 1 }],
+ traceIds: [randomUUID()],
+ reuseExistingSummaries: false,
+ processingConfig: topicProcessingConfigSchema.parse({}),
+ embeddingConfig: topicEmbeddingConfigSchema.parse({}),
+ }),
+ ).rejects.toThrow("Facet version not found in this project.");
+
+ // Project B is untouched.
+ expect(
+ await prisma.evaluatorVersion.count({
+ where: { evaluatorId: b.facetId },
+ }),
+ ).toBe(1);
+ expect(
+ await prisma.evaluator.count({
+ where: { projectId: a.projectId, name: "Hijacked" },
+ }),
+ ).toBe(0);
+ });
+
+ it("uses only LLM connections of the project or its organization", async () => {
+ const admin = caller(userId, [
+ [a.orgId, a.projectId, "ADMIN"],
+ [b.orgId, b.projectId, "ADMIN"],
+ ]);
+ const projectId = a.projectId;
+ const ownOrgKey = await llmConnection({ organizationId: a.orgId });
+ const otherOrgKey = await llmConnection({ organizationId: b.orgId });
+ const config = await prisma.topicConfig.findUniqueOrThrow({
+ where: { projectId },
+ });
+ const settings = (llmApiKeyId: string) => ({
+ projectId,
+ summary: { llmApiKeyId, model: "summary-model" },
+ embedding: null,
+ embeddingDimensions: 1024,
+ clustering: null,
+ });
+
+ // Saving or testing: another project's or another org's connection is refused.
+ for (const foreignKeyId of [b.llmKeyId, otherOrgKey.id]) {
+ await expect(
+ admin.testModelSettings(settings(foreignKeyId)),
+ ).rejects.toThrow("LLM connection not found in this project.");
+ await expect(
+ admin.saveSetup({
+ ...settings(foreignKeyId),
+ facets: DEFAULT_TOPIC_FACETS.map(({ name }) => ({
+ name,
+ enabled: true,
+ })),
+ filter: [],
+ traceIdleMs: 600_000,
+ }),
+ ).rejects.toThrow("LLM connection not found in this project.");
+ }
+ expect(
+ await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),
+ ).toEqual(config);
+
+ // Running: a stored foreign connection resolves to no models; the own org's does.
+ const resolveWithSummaryKey = async (summaryLlmApiKeyId: string) => {
+ await prisma.topicConfig.update({
+ where: { projectId },
+ data: { summaryLlmApiKeyId },
+ });
+ return getTopicsModels(projectId);
+ };
+ expect(await resolveWithSummaryKey(ownOrgKey.id)).not.toBeNull();
+ expect(await resolveWithSummaryKey(otherOrgKey.id)).toBeNull();
+ expect(await resolveWithSummaryKey(b.llmKeyId)).toBeNull();
+ await prisma.topicConfig.update({
+ where: { projectId },
+ data: { summaryLlmApiKeyId: config.summaryLlmApiKeyId },
+ });
+ });
+});
diff --git a/web/src/__tests__/server/unstable-topics-public-api-access.servertest.ts b/web/src/__tests__/server/unstable-topics-public-api-access.servertest.ts
new file mode 100644
index 0000000000..960c880250
--- /dev/null
+++ b/web/src/__tests__/server/unstable-topics-public-api-access.servertest.ts
@@ -0,0 +1,190 @@
+import { randomUUID } from "node:crypto";
+import type { NextApiRequest, NextApiResponse } from "next";
+import { createMocks } from "node-mocks-http";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { env as sharedEnv } from "@langfuse/shared/src/env";
+import { prisma } from "@langfuse/shared/src/db";
+import {
+ createApiKey,
+ createBasicAuthHeader,
+ createOrgProjectAndApiKey,
+} from "@langfuse/shared/src/server";
+import {
+ OrganizationId,
+ ProjectId,
+ SystemRoleId,
+ UserId,
+} from "@langfuse/shared/rbac";
+import {
+ createTopicFacet,
+ createTopicRun,
+ saveTopicRun,
+ writeTopicSummaries,
+} from "@langfuse/shared/topics/server";
+import topicsHandler from "@/src/pages/api/public/unstable/topics/index";
+import topicHandler from "@/src/pages/api/public/unstable/topics/[topicId]";
+import facetsHandler from "@/src/pages/api/public/unstable/topic-facets/index";
+import facetHandler from "@/src/pages/api/public/unstable/topic-facets/[facetId]";
+
+type Handler = (req: NextApiRequest, res: NextApiResponse) => Promise<void>;
+
+// In-process calls, since the Topics gate reads shared env in this process.
+async function get(
+ handler: Handler,
+ authorization: string,
+ query: Record<string, string> = {},
+) {
+ const { req, res } = createMocks<NextApiRequest, NextApiResponse>({
+ method: "GET",
+ query,
+ headers: { authorization },
+ });
+ await handler(req, res);
+ return { status: res._getStatusCode(), body: res._getJSONData() };
+}
+
+/** A Topics-enabled project with one facet whose published run holds one topic. */
+async function topicsProject() {
+ const project = await createOrgProjectAndApiKey();
+ const facet = await createTopicFacet({
+ projectId: project.projectId,
+ name: `Facet ${randomUUID()}`,
+ description: "Private facet",
+ prompt: "Summarize the trace.",
+ });
+ const run = await createTopicRun({
+ projectId: project.projectId,
+ facetId: facet.id,
+ facetVersion: 1,
+ config: {},
+ });
+ const topicId = randomUUID();
+ const now = new Date().toISOString();
+ await saveTopicRun({
+ ...run,
+ status: "completed",
+ finishedAt: now,
+ topics: [
+ {
+ topicVersionId: randomUUID(),
+ topicId,
+ projectId: project.projectId,
+ createdByRunId: run.id,
+ createdAt: now,
+ tags: [],
+ name: "Private topic",
+ description: "Private topic description",
+ centroid: [1, 0],
+ radius: 0.5,
+ representativeSummaries: [],
+ metadata: {},
+ },
+ ],
+ });
+ // Topics are served only once the project has a stored summary.
+ await writeTopicSummaries([
+ {
+ projectId: project.projectId,
+ facetId: facet.id,
+ facetVersion: 1,
+ traceId: randomUUID(),
+ sessionId: null,
+ triggerType: "manual_poc",
+ environment: "production",
+ traceName: "assistant",
+ unitStartTime: now,
+ state: "complete",
+ summary: "Private summary",
+ embedding: [1, 0],
+ transcriptId: "test",
+ transcriptVersion: "test",
+ summaryModel: "summary-model",
+ embeddingModel: "embedding-model",
+ providedUsageDetails: {},
+ usageDetails: {},
+ providedCostDetails: {},
+ costDetails: {},
+ processedAt: now,
+ metadata: {},
+ },
+ ]);
+ return { ...project, facetId: facet.id, topicId };
+}
+
+describe("unstable Topics public API access", () => {
+ const original = {
+ enabled: sharedEnv.LANGFUSE_TOPICS_ENABLED,
+ projectIds: sharedEnv.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS,
+ region: sharedEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ };
+ let owner: Awaited<ReturnType<typeof topicsProject>>;
+ let other: Awaited<ReturnType<typeof topicsProject>>;
+
+ beforeAll(async () => {
+ owner = await topicsProject();
+ other = await topicsProject();
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = "true";
+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS = [
+ owner.projectId,
+ other.projectId,
+ ];
+ });
+
+ afterAll(() => {
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED = original.enabled;
+ (sharedEnv as any).LANGFUSE_TOPICS_ENABLED_PROJECT_IDS =
+ original.projectIds;
+ (sharedEnv as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = original.region;
+ });
+
+ it("does not expose a project's facets or topics to another project's key", async () => {
+ const ownerAuth = createBasicAuthHeader(owner.publicKey, owner.secretKey);
+ const otherAuth = createBasicAuthHeader(other.publicKey, other.secretKey);
+
+ // Positive control: the owner reads both, so the 404s below are not vacuous.
+ for (const [auth, status] of [
+ [ownerAuth, 200],
+ [otherAuth, 404],
+ ] as const) {
+ expect(
+ (await get(facetHandler, auth, { facetId: owner.facetId })).status,
+ ).toBe(status);
+ expect(
+ (await get(topicHandler, auth, { topicId: owner.topicId })).status,
+ ).toBe(status);
+ }
+
+ // No Topics rule exists, so every facet counts as active and is listed.
+ const facets = await get(facetsHandler, otherAuth);
+ const topics = await get(topicsHandler, otherAuth);
+ expect(facets.status).toBe(200);
+ expect(topics.status).toBe(200);
+ expect(JSON.stringify(facets.body)).not.toContain(owner.facetId);
+ expect(JSON.stringify(topics.body)).not.toContain(owner.topicId);
+ });
+
+ it("denies a project key without topics:read", async () => {
+ const creator = await prisma.user.create({
+ data: { email: `topics-ingest-${randomUUID()}@example.com` },
+ });
+ const ingestKey = await createApiKey(prisma, {
+ tenant: OrganizationId(owner.orgId),
+ scope: ProjectId(owner.projectId),
+ role: SystemRoleId("INGEST"),
+ createdBy: UserId(creator.id),
+ });
+ const auth = createBasicAuthHeader(
+ ingestKey.publicKey,
+ ingestKey.secretKey,
+ );
+
+ for (const [handler, query] of [
+ [facetsHandler, {}],
+ [facetHandler, { facetId: owner.facetId }],
+ [topicsHandler, {}],
+ [topicHandler, { topicId: owner.topicId }],
+ ] as const)
+ expect((await get(handler, auth, query)).status).toBe(403);
+ });
+});