Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

19.2 KB
Raw Blame
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import { prisma, type Role } from "@langfuse/shared/src/db";
import {
  BatchExportQueue,
  createOrgProjectAndApiKey,
} from "@langfuse/shared/src/server";
import { vi } from "vitest";
import { randomUUID } from "crypto";
import type { Session } from "next-auth";
import {
  BatchExportFileFormat,
  BatchExportStatus,
  BatchTableNames,
  type Plan,
} from "@langfuse/shared";

const __orgIds: string[] = [];

function makeSession(
  orgId: string,
  orgName: string,
  projectId: string,
  projectName: string,
  opts: {
    plan?: Plan;
    projectRole?: Role;
    v4BetaEnabled?: boolean;
  } = {},
): Session {
  const {
    plan = "cloud:hobby",
    projectRole = "MEMBER",
    v4BetaEnabled = false,
  } = opts;
  return {
    expires: "1",
    user: {
      id: "user-test",
      canCreateOrganizations: true,
      name: "Test User",
      v4BetaEnabled,
      organizations: [
        {
          id: orgId,
          name: orgName,
          role: "MEMBER",
          plan,
          cloudConfig: undefined,
          metadata: {},
          aiFeaturesEnabled: false,
          aiTelemetryEnabled: false,
          projects: [
            {
              id: projectId,
              role: projectRole,
              retentionDays: null,
              deletedAt: null,
              hasTraces: false,
              name: projectName,
              metadata: {},
              createdAt: new Date().toISOString(),
            },
          ],
        },
      ],
      featureFlags: testFeatureFlags({ templateFlag: false }),
      admin: false,
    },
    environment: {
      enableExperimentalFeatures: false,
      selfHostedInstancePlan: plan,
    },
  };
}

const exportInput = (projectId: string) => ({
  projectId,
  name: "audit log export attempt",
  query: {
    tableName: BatchTableNames.AuditLogs,
    filter: null,
    orderBy: null,
  },
  format: BatchExportFileFormat.CSV,
});

describe("batchExport tRPC – audit_logs table authorization", () => {
  afterAll(async () => {
    await prisma.organization.deleteMany({
      where: { id: { in: __orgIds } },
    });
  });

  it("blocks a MEMBER (no projectAuditLogs:read) from exporting audit_logs even on cloud:enterprise plan", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:enterprise",
          projectRole: "MEMBER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.create(exportInput(project.id)),
    ).rejects.toMatchObject({ code: "FORBIDDEN" });
  });

  it.each<Plan>(["cloud:hobby", "cloud:team"])(
    "blocks an OWNER on %s (no audit-logs entitlement) from exporting audit_logs",
    async (plan) => {
      const { project, org } = await createOrgProjectAndApiKey();
      __orgIds.push(org.id);

      const caller = appRouter.createCaller({
        ...createInnerTRPCContext({
          session: makeSession(org.id, org.name, project.id, project.name, {
            plan,
            projectRole: "OWNER",
          }),
          headers: {},
        }),
        prisma,
      });

      await expect(
        caller.batchExport.create(exportInput(project.id)),
      ).rejects.toMatchObject({ code: "FORBIDDEN" });
    },
  );

  it.each<[Plan, Role]>([
    ["cloud:enterprise", "OWNER"],
    ["cloud:enterprise", "ADMIN"],
    ["self-hosted:enterprise", "OWNER"],
  ])(
    "allows plan=%s role=%s (both entitlement and projectAuditLogs:read) to export audit_logs",
    async (plan, projectRole) => {
      const { project, org } = await createOrgProjectAndApiKey();
      __orgIds.push(org.id);

      const caller = appRouter.createCaller({
        ...createInnerTRPCContext({
          session: makeSession(org.id, org.name, project.id, project.name, {
            plan,
            projectRole,
          }),
          headers: {},
        }),
        prisma,
      });

      await caller.batchExport.create(exportInput(project.id));

      const job = await prisma.batchExport.findFirst({
        where: { projectId: project.id, name: "audit log export attempt" },
      });
      expect(job).not.toBeNull();
      expect(job?.query).toMatchObject({ tableName: "audit_logs" });
    },
  );
});

const seedCompletedExport = (opts: {
  projectId: string;
  name: string;
  tableName: BatchTableNames;
}) =>
  prisma.batchExport.create({
    data: {
      projectId: opts.projectId,
      userId: "user-test",
      status: BatchExportStatus.COMPLETED,
      name: opts.name,
      format: BatchExportFileFormat.CSV,
      // Non-URL so downloadUrl cannot parse an object key and must fall back
      // to the stored value instead of signing against the test bucket.
      url: "not-a-valid-url",
      finishedAt: new Date(),
      expiresAt: new Date(Date.now() + 60 * 60 * 1000),
      query: {
        tableName: opts.tableName,
        filter: null,
        orderBy: null,
      },
    },
  });

describe("batchExport tRPC – audit_logs download and list authorization", () => {
  afterAll(async () => {
    await prisma.organization.deleteMany({
      where: { id: { in: __orgIds } },
    });
  });

  it("blocks a MEMBER from downloadUrl of an OWNER-created audit_logs export", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const exportJob = await seedCompletedExport({
      projectId: project.id,
      name: "owner audit export",
      tableName: BatchTableNames.AuditLogs,
    });

    const memberCaller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:enterprise",
          projectRole: "MEMBER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      memberCaller.batchExport.downloadUrl({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).rejects.toMatchObject({ code: "FORBIDDEN" });
  });

  it("blocks an OWNER on cloud:hobby from downloadUrl of an audit_logs export", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const exportJob = await seedCompletedExport({
      projectId: project.id,
      name: "hobby audit export",
      tableName: BatchTableNames.AuditLogs,
    });

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:hobby",
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.downloadUrl({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).rejects.toMatchObject({ code: "FORBIDDEN" });
  });

  it("allows an OWNER with audit-logs entitlement to downloadUrl an audit_logs export", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const exportJob = await seedCompletedExport({
      projectId: project.id,
      name: "entitled audit export",
      tableName: BatchTableNames.AuditLogs,
    });

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:enterprise",
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.downloadUrl({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).resolves.toEqual({ url: "not-a-valid-url" });
  });

  it("still allows a MEMBER to downloadUrl a traces export", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const exportJob = await seedCompletedExport({
      projectId: project.id,
      name: "traces export",
      tableName: BatchTableNames.Traces,
    });

    const memberCaller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:team",
          projectRole: "MEMBER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      memberCaller.batchExport.downloadUrl({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).resolves.toEqual({ url: "not-a-valid-url" });
  });

  it("hides audit_logs exports from all for a MEMBER without projectAuditLogs:read", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    await seedCompletedExport({
      projectId: project.id,
      name: "audit export hidden",
      tableName: BatchTableNames.AuditLogs,
    });
    await seedCompletedExport({
      projectId: project.id,
      name: "traces export visible",
      tableName: BatchTableNames.Traces,
    });

    const memberCaller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:enterprise",
          projectRole: "MEMBER",
        }),
        headers: {},
      }),
      prisma,
    });

    const result = await memberCaller.batchExport.all({
      projectId: project.id,
      page: 0,
      limit: 50,
    });

    expect(result.exports.map((e) => e.name)).toEqual([
      "traces export visible",
    ]);
    expect(result.totalCount).toBe(1);
  });

  it("includes audit_logs exports in all for an OWNER with entitlement", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    await seedCompletedExport({
      projectId: project.id,
      name: "audit export listed",
      tableName: BatchTableNames.AuditLogs,
    });
    await seedCompletedExport({
      projectId: project.id,
      name: "traces export listed",
      tableName: BatchTableNames.Traces,
    });

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          plan: "cloud:enterprise",
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    const result = await caller.batchExport.all({
      projectId: project.id,
      page: 0,
      limit: 50,
    });

    expect(result.exports.map((e) => e.name).sort()).toEqual([
      "audit export listed",
      "traces export listed",
    ]);
    expect(result.totalCount).toBe(2);
  });
});

describe("batchExport tRPC – useEventsTable snapshot", () => {
  afterAll(async () => {
    await prisma.organization.deleteMany({
      where: { id: { in: __orgIds } },
    });
  });

  const sessionsExportInput = (projectId: string, name: string) => ({
    projectId,
    name,
    query: {
      tableName: BatchTableNames.Sessions,
      filter: null,
      orderBy: null,
    },
    format: BatchExportFileFormat.CSV,
  });

  it("snapshots useEventsTable=true into the persisted query when v4 beta is enabled", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
          v4BetaEnabled: true,
        }),
        headers: {},
      }),
      prisma,
    });

    await caller.batchExport.create(
      sessionsExportInput(project.id, "sessions export v4 on"),
    );

    const job = await prisma.batchExport.findFirst({
      where: { projectId: project.id, name: "sessions export v4 on" },
    });
    expect(job?.query).toMatchObject({
      tableName: "sessions",
      useEventsTable: true,
    });
  });

  it("snapshots useEventsTable=false when v4 beta is disabled", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
          v4BetaEnabled: false,
        }),
        headers: {},
      }),
      prisma,
    });

    await caller.batchExport.create(
      sessionsExportInput(project.id, "sessions export v4 off"),
    );

    const job = await prisma.batchExport.findFirst({
      where: { projectId: project.id, name: "sessions export v4 off" },
    });
    expect(job?.query).toMatchObject({
      tableName: "sessions",
      useEventsTable: false,
    });
  });
});

describe("batchExport tRPC – cancel", () => {
  afterAll(async () => {
    await prisma.organization.deleteMany({
      where: { id: { in: __orgIds } },
    });
  });

  it("returns NOT_FOUND when the export is missing instead of a Prisma update error", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.cancel({
        projectId: project.id,
        batchExportId: randomUUID(),
      }),
    ).rejects.toMatchObject({ code: "NOT_FOUND" });
  });

  it("returns NOT_FOUND when the export belongs to another project", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    const other = await createOrgProjectAndApiKey();
    __orgIds.push(org.id, other.org.id);

    const exportJob = await prisma.batchExport.create({
      data: {
        projectId: other.project.id,
        userId: "user-test",
        status: BatchExportStatus.QUEUED,
        name: "other project export",
        format: BatchExportFileFormat.CSV,
        query: {
          tableName: BatchTableNames.Traces,
          filter: null,
          orderBy: null,
        },
      },
    });

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.cancel({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).rejects.toMatchObject({ code: "NOT_FOUND" });

    const stillQueued = await prisma.batchExport.findUnique({
      where: { id: exportJob.id },
    });
    expect(stillQueued?.status).toBe(BatchExportStatus.QUEUED);
    expect(stillQueued?.projectId).toBe(other.project.id);
  });

  it("cancels an existing queued export", async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);

    const exportJob = await prisma.batchExport.create({
      data: {
        projectId: project.id,
        userId: "user-test",
        status: BatchExportStatus.QUEUED,
        name: "queued export",
        format: BatchExportFileFormat.CSV,
        query: {
          tableName: BatchTableNames.Traces,
          filter: null,
          orderBy: null,
        },
      },
    });

    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });

    await expect(
      caller.batchExport.cancel({
        projectId: project.id,
        batchExportId: exportJob.id,
      }),
    ).resolves.toBeUndefined();

    const cancelled = await prisma.batchExport.findUnique({
      where: { id: exportJob.id },
    });
    expect(cancelled?.status).toBe(BatchExportStatus.CANCELLED);
  });
});

describe("batchExport tRPC – one active export per user and project", () => {
  afterAll(async () => {
    await prisma.organization.deleteMany({
      where: { id: { in: __orgIds } },
    });
  });

  const tracesQuery = {
    tableName: BatchTableNames.Traces,
    filter: null,
    orderBy: null,
  };

  const setup = async () => {
    const { project, org } = await createOrgProjectAndApiKey();
    __orgIds.push(org.id);
    const caller = appRouter.createCaller({
      ...createInnerTRPCContext({
        session: makeSession(org.id, org.name, project.id, project.name, {
          projectRole: "OWNER",
        }),
        headers: {},
      }),
      prisma,
    });
    const createExport = () =>
      caller.batchExport.create({
        projectId: project.id,
        name: "new export",
        query: tracesQuery,
        format: BatchExportFileFormat.CSV,
      });
    const seedExport = (data: {
      status: BatchExportStatus;
      userId?: string;
      createdAt?: Date;
      updatedAt?: Date;
    }) =>
      prisma.batchExport.create({
        data: {
          projectId: project.id,
          userId: "user-test",
          name: "existing export",
          format: BatchExportFileFormat.CSV,
          query: tracesQuery,
          ...data,
        },
      });
    return { project, caller, createExport, seedExport };
  };

  it("rejects a second export and names when the running one started", async () => {
    const { project, createExport, seedExport } = await setup();
    await seedExport({
      status: BatchExportStatus.PROCESSING,
      createdAt: new Date("2026-01-02T03:04:05Z"),
    });

    await expect(createExport()).rejects.toMatchObject({
      code: "CONFLICT",
      message: expect.stringContaining("2026-01-02 03:04 UTC"),
    });
    expect(
      await prisma.batchExport.count({ where: { projectId: project.id } }),
    ).toBe(1);
  });

  it("accepts a new export once the running one is cancelled", async () => {
    const { project, caller, createExport, seedExport } = await setup();
    const running = await seedExport({ status: BatchExportStatus.QUEUED });

    await caller.batchExport.cancel({
      projectId: project.id,
      batchExportId: running.id,
    });

    await expect(createExport()).resolves.toBeUndefined();
  });

  it("fails an export that could not be queued so it does not block a retry", async () => {
    const { project, createExport } = await setup();
    const queue = BatchExportQueue.getInstance();
    const add = vi
      .spyOn(queue!, "add")
      .mockRejectedValueOnce(new Error("Redis unavailable"));

    await expect(createExport()).rejects.toMatchObject({
      code: "INTERNAL_SERVER_ERROR",
    });
    add.mockRestore();

    const [failed] = await prisma.batchExport.findMany({
      where: { projectId: project.id },
    });
    expect(failed.status).toBe(BatchExportStatus.FAILED);
    await expect(createExport()).resolves.toBeUndefined();
  });

  it("ignores other users' exports and abandoned ones", async () => {
    const { createExport, seedExport } = await setup();
    await seedExport({
      status: BatchExportStatus.PROCESSING,
      userId: "other-user",
    });
    await seedExport({
      status: BatchExportStatus.PROCESSING,
      updatedAt: new Date(Date.now() - 2 * 60 * 60 * 1000),
    });

    await expect(createExport()).resolves.toBeUndefined();
  });
});