Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

4.7 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import { test, expect, type Page } from "@playwright/test";
import { prisma } from "@langfuse/shared/src/db";
import { signIn } from "./utils/auth";

// const checkConsoleErrors = async (page: Page) => {
//   const errors: string[] = [];
//   page.on("pageerror", (err) => {
//     errors.push(err.message);
//   });
//   page.on("console", (msg) => {
//     if (msg.type() === "error") {
//       errors.push(msg.text());
//     }
//   });
//
//   page.on("response", async (response) => {
//     if (response.status() === 500) {
//       console.error(
//         "Network request error: ",
//         response.url,
//         await response.text(),
//       );
//     }
//   });
//
//   return errors;
// };

const cleanUpConsoleEventListeners = (page: Page) => {
  page.removeAllListeners("pageerror");
  page.removeAllListeners("console");
};

test.describe("Create project", () => {
  test("Sign in, create an organization, create a project", async ({
    page,
  }) => {
    test.setTimeout(60000);
    // const errors = await checkConsoleErrors(page);

    // Sign in
    await page.goto("/auth/sign-in");
    await signIn(page);

    // Start create org flow. `page.isVisible` does not wait and its result
    // was discarded, so the click below carried the whole wait on its 10s
    // action timeout; assert it instead.
    await expect(
      page.locator('[data-testid="create-organization-btn"]'),
    ).toBeVisible();
    await page.click('[data-testid="create-organization-btn"]');
    await expect(page).toHaveURL("/setup");

    // Create an organization
    await expect(page.locator("data-testid=new-org-form")).toBeVisible();
    await page.fill('[data-testid="new-org-name-input"]', "e2e test org");
    await page.click('button[type="submit"]');
    await expect(page).toHaveURL(
      /\/organization\/.*\/setup\?orgstep=create-project/,
      {
        timeout: 15000,
      },
    );

    // Create project
    await expect(page.locator("data-testid=new-project-form")).toBeVisible();
    await page.fill(
      '[data-testid="new-project-name-input"]',
      "e2e test project",
    );
    await page.click('button[type="submit"]');

    // should be redirected to the traces page of the new project
    await expect(page).toHaveURL(/\/project\/.*\/traces/, { timeout: 15000 });

    const projectUrl = new URL(page.url());
    const projectId = projectUrl.pathname.split("/")[2];

    // check that the project exists by navigating to its home screen
    await page.goto("/project/" + projectId);
    await expect(page).toHaveURL(
      (url) => url.pathname === `/project/${projectId}`,
    );

    await page.waitForTimeout(10000);
    await checkPageHeaderTitle(page, "Home");

    // Check for console errors
    // expect(errors).toHaveLength(0);
  });

  test("Sign in", async ({ page }) => {
    // const errors = await checkConsoleErrors(page);
    await signin(page);
    // expect(errors).toHaveLength(0);
    cleanUpConsoleEventListeners(page);
  });

  [
    { title: "Tracing", url: "/traces", subTitle: "Traces" },
    { title: "Sessions", url: "/sessions" },
    { title: "Tracing", url: "/observations", subTitle: "Observations" },
    { title: "Scores", url: "/scores" },
  ].forEach(({ title, url, subTitle }) => {
    test(`Check ${title} ${subTitle ? `- ${subTitle}` : ""} page`, async ({
      page,
    }) => {
      // const errors = await checkConsoleErrors(page);
      await signin(page);

      const projectUrl = await getProjectUrlForEmail("[email protected]");
      await page.goto(projectUrl + url, { waitUntil: "networkidle" });
      await expect(page).toHaveURL(projectUrl + url);
      await checkPageHeaderTitle(page, title);

      // Check that each error contains the expected string

      // errors.forEach((error) => {
      //   expect(error).toContain(
      //     "Document policy violation: js-profiling is not allowed in this document.",
      //   );
      // });
      cleanUpConsoleEventListeners(page);
    });
  });
});

const signin = async (page: Page) => {
  await page.goto("/auth/sign-in");
  await signIn(page);
};

const checkPageHeaderTitle = async (page: Page, title: string) => {
  const pageHeaderTitle = await page
    .locator('[data-testid="page-header-title"]')
    .textContent();
  expect(pageHeaderTitle).toContain(title);
};

const getProjectUrlForEmail = async (email: string) => {
  const user = await prisma.user.findUnique({
    where: { email },
    include: {
      organizationMemberships: {
        include: {
          organization: {
            include: {
              projects: true,
            },
          },
        },
      },
    },
  });

  return `/project/${user?.organizationMemberships[0].organization.projects[0].id}`;
};