Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

20.0 KB
Raw Blame
import { randomUUID } from "crypto";
import {
  makeAPICall,
  makeZodVerifiedAPICall,
} from "@/src/__tests__/test-utils";
import {
  GetCommentsV1Response,
  GetCommentV1Response,
  PostCommentsV1Response,
} from "@/src/features/public-api/types/comments";
import { prisma } from "@langfuse/shared/src/db";
import { z } from "zod";
import {
  createObservationsCh,
  createTracesCh,
  createObservation,
  createTrace,
  createEvent,
  createEventsCh,
} from "@langfuse/shared/src/server";

const seedProjectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";

// POST /api/public/comments only accepts an authorUserId that belongs to a
// member of the project's organization. CI provisions the seed project through
// LANGFUSE_INIT_* instead of the Postgres seeder, so seeded ids such as
// "user-1" do not exist there. Create a member of the project's organization
// and use its id wherever a valid comment author is needed.
let orgMemberUserId: string;

beforeAll(async () => {
  const project = await prisma.project.findUniqueOrThrow({
    where: { id: seedProjectId },
    select: { orgId: true },
  });

  const user = await prisma.user.create({
    data: {
      name: "Comments API Author",
      email: `comments-api-author-${randomUUID()}@langfuse.com`,
    },
  });
  orgMemberUserId = user.id;

  await prisma.organizationMembership.create({
    data: { orgId: project.orgId, userId: orgMemberUserId, role: "MEMBER" },
  });
});

afterAll(async () => {
  await prisma.organizationMembership.deleteMany({
    where: { userId: orgMemberUserId },
  });
  await prisma.user.delete({ where: { id: orgMemberUserId } });
});

describe("Create and get comments", () => {
  beforeAll(async () => {
    const traces = [
      createTrace({
        name: "trace-name",
        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        id: "1234",
      }),
    ];

    await createTracesCh(traces);
  });

  it("should create and get comment", async () => {
    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content: "hello",
        objectId: "1234",
        objectType: "TRACE",
        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        authorUserId: orgMemberUserId,
      },
    );

    const { id: commentId } = commentResponse.body;

    const response = await makeZodVerifiedAPICall(
      GetCommentV1Response,
      "GET",
      `/api/public/comments/${commentId}`,
    );

    expect(response.status).toBe(200);
    expect(response.body).toMatchObject({
      id: commentId,
      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      objectId: "1234",
      objectType: "TRACE",
      content: "hello",
      authorUserId: orgMemberUserId,
    });
  });

  it("should create an observation comment when objectStartTime is in the observation's minute", async () => {
    const startTime = new Date("2024-05-15T12:00:00.000Z");
    // Same minute, different second: the lookup floors to the minute, so this
    // still resolves the observation.
    const sameMinuteStartTime = new Date("2024-05-15T12:00:45.000Z");
    const observationId = randomUUID();
    // Seed both tables so the lookup resolves regardless of the v4 write-mode
    // routing the test environment happens to use.
    await Promise.all([
      createEventsCh([
        createEvent({
          id: observationId,
          span_id: observationId,
          project_id: seedProjectId,
          start_time: startTime,
          type: "GENERATION",
        }),
      ]),
      createObservationsCh([
        createObservation({
          id: observationId,
          project_id: seedProjectId,
          start_time: startTime,
          type: "GENERATION",
        }),
      ]),
    ]);

    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content: "bounded observation comment",
        objectId: observationId,
        objectType: "OBSERVATION",
        projectId: seedProjectId,
        objectStartTime: sameMinuteStartTime.toISOString(),
        authorUserId: orgMemberUserId,
      },
    );

    const { id: commentId } = commentResponse.body;

    const response = await makeZodVerifiedAPICall(
      GetCommentV1Response,
      "GET",
      `/api/public/comments/${commentId}`,
    );

    expect(response.status).toBe(200);
    expect(response.body).toMatchObject({
      id: commentId,
      projectId: seedProjectId,
      objectId: observationId,
      objectType: "OBSERVATION",
      content: "bounded observation comment",
    });
  });

  it("should still create an observation comment when objectStartTime is a wrong/stale hint", async () => {
    const actualStartTime = new Date("2024-05-15T12:00:00.000Z");
    // A different minute than the observation: the bounded lookup misses, but the
    // hint falls back to an unbounded lookup, so the comment is still created.
    const wrongStartTime = new Date("2024-05-15T12:02:00.000Z");
    const observationId = randomUUID();
    // Seed both tables so the lookup resolves regardless of the v4 write-mode
    // routing the test environment happens to use.
    await Promise.all([
      createEventsCh([
        createEvent({
          id: observationId,
          span_id: observationId,
          project_id: seedProjectId,
          start_time: actualStartTime,
          type: "GENERATION",
        }),
      ]),
      createObservationsCh([
        createObservation({
          id: observationId,
          project_id: seedProjectId,
          start_time: actualStartTime,
          type: "GENERATION",
        }),
      ]),
    ]);

    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content: "wrong-hint observation comment",
        objectId: observationId,
        objectType: "OBSERVATION",
        projectId: seedProjectId,
        objectStartTime: wrongStartTime.toISOString(),
        authorUserId: orgMemberUserId,
      },
    );

    const { id: commentId } = commentResponse.body;

    const response = await makeZodVerifiedAPICall(
      GetCommentV1Response,
      "GET",
      `/api/public/comments/${commentId}`,
    );

    expect(response.status).toBe(200);
    expect(response.body).toMatchObject({
      id: commentId,
      projectId: seedProjectId,
      objectId: observationId,
      objectType: "OBSERVATION",
      content: "wrong-hint observation comment",
    });
  });

  it("should fail to create comment if reference object does not exist", async () => {
    expect.assertions(2); // Ensure that we confirm two things
    try {
      await makeZodVerifiedAPICall(
        z.object({
          message: z.string(),
          error: z.array(z.object({})),
        }),
        "POST",
        "/api/public/comments",
        {
          content: "hello",
          objectId: "invalid-trace-id",
          objectType: "TRACE",
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        },
      );
    } catch (error) {
      expect((error as Error).message).toContain(
        `API call did not return 200, returned status 404`,
      );
      expect((error as Error).message).toContain(
        `TRACE: invalid-trace-id not found`,
      );
    }
  });

  it("should fail to create comment if content is empty", async () => {
    try {
      await makeZodVerifiedAPICall(
        z.object({
          message: z.string(),
          error: z.array(z.object({})),
        }),
        "POST",
        "/api/public/comments",
        {
          content: "",
          objectId: "1234",
          objectType: "TRACE",
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        },
      );
    } catch (error) {
      expect((error as Error).message).toBe(
        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"origin\":\"string\",\"code\":\"too_small\",\"minimum\":1,\"inclusive\":true,\"path\":[\"content\"],\"message\":\"Too small: expected string to have >=1 characters\"}]}`,
      );
    }
  });

  it("should fail to create comment if content is larger than 5000 characters", async () => {
    try {
      await makeZodVerifiedAPICall(
        z.object({
          message: z.string(),
          error: z.array(z.object({})),
        }),
        "POST",
        "/api/public/comments",
        {
          content: "a".repeat(5001),
          objectId: "1234",
          objectType: "TRACE",
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        },
      );
    } catch (error) {
      expect((error as Error).message).toBe(
        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"origin\":\"string\",\"code\":\"too_big\",\"maximum\":5000,\"inclusive\":true,\"path\":[\"content\"],\"message\":\"Too big: expected string to have <=5000 characters\"}]}`,
      );
    }
  });
});

describe("GET /api/public/comments API Endpoint", () => {
  beforeAll(async () => {
    const traces = [
      createTrace({
        name: "trace-1",
        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        id: "1234",
      }),
    ];

    await createTracesCh(traces);

    const observation = createObservation({
      name: "generation-1",
      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      id: "5678",
      type: "GENERATION",
      trace_id: "1234",
    });

    await createObservationsCh([observation]);

    await prisma.comment.deleteMany({
      where: { projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
    });
    await prisma.comment.createMany({
      data: [
        {
          id: "comment-2021-01-01",
          createdAt: new Date("2021-01-01T00:00:00Z"),
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
          content: "comment-1",
          objectId: "1234",
          objectType: "TRACE",
          authorUserId: "user-1",
        },
        {
          id: "comment-2021-02-01",
          createdAt: new Date("2021-02-01T00:00:00Z"),
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
          content: "comment-2",
          objectId: "5678",
          objectType: "OBSERVATION",
          authorUserId: "user-1",
        },
        {
          id: "comment-2021-03-01",
          createdAt: new Date("2021-03-01T00:00:00Z"),
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
          content: "comment-3",
          objectId: "1234",
          objectType: "TRACE",
          authorUserId: "user-1",
        },
        {
          id: "comment-2021-04-01",
          createdAt: new Date("2021-04-01T00:00:00Z"),
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
          content: "comment-4",
          objectId: "1234",
          objectType: "TRACE",
        },
        {
          id: "comment-2021-05-01",
          createdAt: new Date("2021-05-01T00:00:00Z"),
          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
          content: "comment-5",
          objectId: "1234",
          objectType: "TRACE",
        },
      ],
    });
  });

  it("should return all comments", async () => {
    const comments = await makeZodVerifiedAPICall(
      GetCommentsV1Response,
      "GET",
      "/api/public/comments",
    );
    expect(comments.body.data).toHaveLength(5);
    expect(comments.body.meta).toMatchObject({
      page: 1,
      limit: 50,
      totalItems: 5,
      totalPages: 1,
    });
  });

  it("should return comments for a specific objectId and objectType", async () => {
    const objectId = "1234";
    const objectType = "TRACE";

    const comments = await makeZodVerifiedAPICall(
      GetCommentsV1Response,
      "GET",
      `/api/public/comments?objectType=${objectType}&objectId=${objectId}`,
    );

    expect(comments.body.data).toHaveLength(4);
    expect(comments.body.meta).toMatchObject({
      page: 1,
      limit: 50,
      totalItems: 4,
      totalPages: 1,
    });
    expect(comments.body.data.map((comment) => comment.id)).toEqual([
      "comment-2021-01-01",
      "comment-2021-03-01",
      "comment-2021-04-01",
      "comment-2021-05-01",
    ]);
  });

  it("should return comments linked to a specific object and by a specific author", async () => {
    const authorUserId = "user-1";
    const objectId = "1234";
    const objectType = "TRACE";

    const comments = await makeZodVerifiedAPICall(
      GetCommentsV1Response,
      "GET",
      `/api/public/comments?objectType=${objectType}&objectId=${objectId}&authorUserId=${authorUserId}`,
    );

    expect(comments.body.data).toHaveLength(2);
    expect(comments.body.meta).toMatchObject({
      page: 1,
      limit: 50,
      totalItems: 2,
      totalPages: 1,
    });
    expect(comments.body.data.map((comment) => comment.id)).toEqual([
      "comment-2021-01-01",
      "comment-2021-03-01",
    ]);
  });

  it("should return an empty array when no comments match the criteria", async () => {
    const comments = await makeZodVerifiedAPICall(
      GetCommentsV1Response,
      "GET",
      "/api/public/comments?authorUserId=non-existent-user",
    );

    expect(comments.body.data).toHaveLength(0);
    expect(comments.body.meta).toMatchObject({
      page: 1,
      limit: 50,
      totalItems: 0,
      totalPages: 0,
    });
  });

  it("should throw 400 error with descriptive error message if objectType is provided but invalid", async () => {
    try {
      await makeZodVerifiedAPICall(
        z.object({
          message: z.string(),
          error: z.array(z.object({})),
        }),
        "GET",
        "/api/public/comments?objectType=INVALID_TYPE",
      );
    } catch (error) {
      expect((error as Error).message).toContain(
        "API call did not return 200, returned status 400",
      );
    }
  });

  it("should return all trace comments if objectType is provided and objectId is not", async () => {
    const comments = await makeZodVerifiedAPICall(
      GetCommentsV1Response,
      "GET",
      "/api/public/comments?objectType=TRACE",
    );
    expect(comments.body.data).toHaveLength(4);
    expect(comments.body.meta).toMatchObject({
      page: 1,
      limit: 50,
      totalItems: 4,
      totalPages: 1,
    });
    expect(comments.body.data.map((comment) => comment.id)).toEqual([
      "comment-2021-01-01",
      "comment-2021-03-01",
      "comment-2021-04-01",
      "comment-2021-05-01",
    ]);
  });

  it("should throw 400 error with descriptive error message if objectId is provided but objectType is not", async () => {
    try {
      await makeZodVerifiedAPICall(
        z.object({
          message: z.string(),
          error: z.array(z.object({})),
        }),
        "GET",
        "/api/public/comments?objectId=trace-2021-01-01",
      );
    } catch (error) {
      expect((error as Error).message).toBe(
        `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"code\":\"custom\",\"path\":[\"objectType\"],\"message\":\"objectType is required when objectId is provided\"}]}`,
      );
    }
  });
});

describe("Public API does NOT process mentions", () => {
  beforeAll(async () => {
    const traces = [
      createTrace({
        name: "trace-for-no-mention-processing",
        project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        id: "no-mention-processing-trace",
      }),
    ];

    await createTracesCh(traces);
  });

  it("should preserve mention markdown as-is without processing", async () => {
    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content:
          "Hey @[FakeAdmin](user:user-1) and @[InvalidUser](user:invalid-id), check this!",
        objectId: "no-mention-processing-trace",
        objectType: "TRACE",
        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
        authorUserId: orgMemberUserId,
      },
    );

    const { id: commentId } = commentResponse.body;

    const response = await makeZodVerifiedAPICall(
      GetCommentV1Response,
      "GET",
      `/api/public/comments/${commentId}`,
    );

    expect(response.status).toBe(200);
    // Content should be stored exactly as provided - NO sanitization or normalization
    expect(response.body.content).toBe(
      "Hey @[FakeAdmin](user:user-1) and @[InvalidUser](user:invalid-id), check this!",
    );
  });
});

describe("POST /api/public/comments authorUserId scoping", () => {
  const projectId = seedProjectId;
  const objectId = "author-scoping-trace";
  let otherOrgId: string;
  let otherOrgUserId: string;

  beforeAll(async () => {
    await createTracesCh([
      createTrace({
        name: "trace-for-author-scoping",
        project_id: projectId,
        id: objectId,
      }),
    ]);

    const otherOrg = await prisma.organization.create({
      data: { name: `Comment Author Scoping Org ${randomUUID()}` },
    });
    otherOrgId = otherOrg.id;

    const otherOrgUser = await prisma.user.create({
      data: {
        name: "Other Org User",
        email: `comment-author-scoping-${randomUUID()}@langfuse.com`,
      },
    });
    otherOrgUserId = otherOrgUser.id;

    await prisma.organizationMembership.create({
      data: { orgId: otherOrgId, userId: otherOrgUserId, role: "MEMBER" },
    });
  });

  afterAll(async () => {
    await prisma.comment.deleteMany({ where: { projectId, objectId } });
    await prisma.organizationMembership.deleteMany({
      where: { userId: otherOrgUserId },
    });
    await prisma.organization.delete({ where: { id: otherOrgId } });
    await prisma.user.delete({ where: { id: otherOrgUserId } });
  });

  it("should reject an authorUserId that is not a member of the project's organization", async () => {
    const response = await makeAPICall<{ message: string; error: string }>(
      "POST",
      "/api/public/comments",
      {
        content: "spoofed comment",
        objectId,
        objectType: "TRACE",
        projectId,
        authorUserId: otherOrgUserId,
      },
    );

    expect(response.status).toBe(400);
    expect(response.body.error).toBe("InvalidRequestError");

    const comments = await prisma.comment.findMany({
      where: { projectId, authorUserId: otherOrgUserId },
    });
    expect(comments).toHaveLength(0);
  });

  it("should not disclose whether a rejected authorUserId exists", async () => {
    const crossOrgResponse = await makeAPICall<{ message: string }>(
      "POST",
      "/api/public/comments",
      {
        content: "spoofed comment",
        objectId,
        objectType: "TRACE",
        projectId,
        authorUserId: otherOrgUserId,
      },
    );

    const unknownUserResponse = await makeAPICall<{ message: string }>(
      "POST",
      "/api/public/comments",
      {
        content: "spoofed comment",
        objectId,
        objectType: "TRACE",
        projectId,
        authorUserId: `does-not-exist-${randomUUID()}`,
      },
    );

    expect(crossOrgResponse.status).toBe(400);
    expect(unknownUserResponse.status).toBe(400);
    expect(crossOrgResponse.body.message).toBe(
      unknownUserResponse.body.message,
    );
  });

  it("should accept an authorUserId of an organization member without project-level ownership", async () => {
    // orgMemberUserId has an organization membership but no project membership.
    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content: "comment by org member",
        objectId,
        objectType: "TRACE",
        projectId,
        authorUserId: orgMemberUserId,
      },
    );

    const comment = await prisma.comment.findUnique({
      where: { id: commentResponse.body.id },
    });
    expect(comment?.authorUserId).toBe(orgMemberUserId);
  });

  it("should still create comments without an authorUserId", async () => {
    const commentResponse = await makeZodVerifiedAPICall(
      PostCommentsV1Response,
      "POST",
      "/api/public/comments",
      {
        content: "comment without author",
        objectId,
        objectType: "TRACE",
        projectId,
      },
    );

    const comment = await prisma.comment.findUnique({
      where: { id: commentResponse.body.id },
    });
    expect(comment?.authorUserId).toBeNull();
  });
});