🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import { test, expect, type Page } from "@playwright/test";
import { prisma } from "@langfuse/shared/src/db";
import { signIn } from "./utils/auth";
// const checkConsoleErrors = async (page: Page) => {
// const errors: string[] = [];
// page.on("pageerror", (err) => {
// errors.push(err.message);
// });
// page.on("console", (msg) => {
// if (msg.type() === "error") {
// errors.push(msg.text());
// }
// });
//
// page.on("response", async (response) => {
// if (response.status() === 500) {
// console.error(
// "Network request error: ",
// response.url,
// await response.text(),
// );
// }
// });
//
// return errors;
// };
const cleanUpConsoleEventListeners = (page: Page) => {
page.removeAllListeners("pageerror");
page.removeAllListeners("console");
};
test.describe("Create project", () => {
test("Sign in, create an organization, create a project", async ({
page,
}) => {
test.setTimeout(60000);
// const errors = await checkConsoleErrors(page);
// Sign in
await page.goto("/auth/sign-in");
await signIn(page);
// Start create org flow. `page.isVisible` does not wait and its result
// was discarded, so the click below carried the whole wait on its 10s
// action timeout; assert it instead.
await expect(
page.locator('[data-testid="create-organization-btn"]'),
).toBeVisible();
await page.click('[data-testid="create-organization-btn"]');
await expect(page).toHaveURL("/setup");
// Create an organization
await expect(page.locator("data-testid=new-org-form")).toBeVisible();
await page.fill('[data-testid="new-org-name-input"]', "e2e test org");
await page.click('button[type="submit"]');
await expect(page).toHaveURL(
/\/organization\/.*\/setup\?orgstep=create-project/,
{
timeout: 15000,
},
);
// Create project
await expect(page.locator("data-testid=new-project-form")).toBeVisible();
await page.fill(
'[data-testid="new-project-name-input"]',
"e2e test project",
);
await page.click('button[type="submit"]');
// should be redirected to the traces page of the new project
await expect(page).toHaveURL(/\/project\/.*\/traces/, { timeout: 15000 });
const projectUrl = new URL(page.url());
const projectId = projectUrl.pathname.split("/")[2];
// check that the project exists by navigating to its home screen
await page.goto("/project/" + projectId);
await expect(page).toHaveURL(
(url) => url.pathname === `/project/${projectId}`,
);
await page.waitForTimeout(10000);
await checkPageHeaderTitle(page, "Home");
// Check for console errors
// expect(errors).toHaveLength(0);
});
test("Sign in", async ({ page }) => {
// const errors = await checkConsoleErrors(page);
await signin(page);
// expect(errors).toHaveLength(0);
cleanUpConsoleEventListeners(page);
});
[
{ title: "Tracing", url: "/traces", subTitle: "Traces" },
{ title: "Sessions", url: "/sessions" },
{ title: "Tracing", url: "/observations", subTitle: "Observations" },
{ title: "Scores", url: "/scores" },
].forEach(({ title, url, subTitle }) => {
test(`Check ${title} ${subTitle ? `- ${subTitle}` : ""} page`, async ({
page,
}) => {
// const errors = await checkConsoleErrors(page);
await signin(page);
const projectUrl = await getProjectUrlForEmail("[email protected]");
await page.goto(projectUrl + url, { waitUntil: "networkidle" });
await expect(page).toHaveURL(projectUrl + url);
await checkPageHeaderTitle(page, title);
// Check that each error contains the expected string
// errors.forEach((error) => {
// expect(error).toContain(
// "Document policy violation: js-profiling is not allowed in this document.",
// );
// });
cleanUpConsoleEventListeners(page);
});
});
});
const signin = async (page: Page) => {
await page.goto("/auth/sign-in");
await signIn(page);
};
const checkPageHeaderTitle = async (page: Page, title: string) => {
const pageHeaderTitle = await page
.locator('[data-testid="page-header-title"]')
.textContent();
expect(pageHeaderTitle).toContain(title);
};
const getProjectUrlForEmail = async (email: string) => {
const user = await prisma.user.findUnique({
where: { email },
include: {
organizationMemberships: {
include: {
organization: {
include: {
projects: true,
},
},
},
},
},
});
return `/project/${user?.organizationMemberships[0].organization.projects[0].id}`;
};