🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
test(e2e): stop the sign-in helper from failing silently (#18286)
Der Gerätef983a07f981
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex 64cf16d61d..23db39b61e 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -1385,6 +1385,18 @@ jobs:exit "$(cat /tmp/playwright-install.exit)"- name: Run e2e testsrun: pnpm --filter=web run test:e2e+ # Playwright writes a screenshot, an error context and a trace for every+ # failure, and the reporter prints the command to open them — but they+ # die with the runner, so a flake here is reported with no way to see+ # what the browser was actually showing.+ - name: Upload Playwright failure artifacts+ if: failure()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: playwright-results-${{ github.run_id }}-${{ github.run_attempt }}+ path: web/test-results/+ retention-days: 7+ if-no-files-found: ignoree2e-server-tests:runs-on: blacksmith-8vcpu-ubuntu-2404diff --git a/web/playwright.config.ts b/web/playwright.config.tsindex 3bda799baa..15a04c0c87 100644--- a/web/playwright.config.ts+++ b/web/playwright.config.ts@@ -2,6 +2,8 @@ import { defineConfig } from "@playwright/test";export default defineConfig({testDir: "./src/__e2e__",+ // Takes the server's cold start off whichever test happens to run first.+ globalSetup: "./src/__e2e__/utils/warm-up.ts",timeout: 180000, // test timeout 180s (3 minutes)expect: {timeout: 60000, // assertion timeout 60s (increased for CI)diff --git a/web/src/__e2e__/app-shell-overflow.spec.ts b/web/src/__e2e__/app-shell-overflow.spec.tsindex 41631f5016..0ca6d2648c 100644--- a/web/src/__e2e__/app-shell-overflow.spec.ts+++ b/web/src/__e2e__/app-shell-overflow.spec.ts@@ -1,4 +1,5 @@import { test, expect, type Page } from "@playwright/test";+import { signIn } from "./utils/auth";/*** The app shell is fully height- and width-constrained: page content scrolls@@ -8,14 +9,6 @@ import { test, expect, type Page } from "@playwright/test";const SHORT_VIEWPORT = { width: 1190, height: 560 };-async function signIn(page: Page) {- await page.goto("/auth/sign-in");- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");- await page.click('button[data-testid="submit-email-password-sign-in-form"]');- await expect(page).toHaveURL("/");-}-/** Document overflow in CSS px, per axis. */function documentOverflow(page: Page) {return page.evaluate(() => {@@ -31,6 +24,7 @@ test("a right drawer's absolutely-positioned content does not scroll the documenpage,}) => {await page.setViewportSize(SHORT_VIEWPORT);+ await page.goto("/auth/sign-in");await signIn(page);await page.getByRole("button", { name: "Support" }).click();const drawerBody = page.locator("#secondary div.overflow-y-auto").first();@@ -57,6 +51,7 @@ test("a vertical document scrollbar does not spawn a horizontal one", async ({page,}) => {await page.setViewportSize(SHORT_VIEWPORT);+ await page.goto("/auth/sign-in");await signIn(page);// Viewport units ignore scrollbars, so a shell sized off `100vw` overshoots// the space beside the sidebar by the scrollbar's width once one is showing.diff --git a/web/src/__e2e__/auth.spec.ts b/web/src/__e2e__/auth.spec.tsindex 6178f19346..f141380f55 100644--- a/web/src/__e2e__/auth.spec.ts+++ b/web/src/__e2e__/auth.spec.ts@@ -1,4 +1,5 @@import { test, expect } from "@playwright/test";+import { signIn } from "./utils/auth";test("should redirect to sign-in if not signed in", async ({ page }) => {await page.goto("/");@@ -7,25 +8,7 @@ test("should redirect to sign-in if not signed in", async ({ page }) => {test("should redirect to home if signed in", async ({ page }) => {await page.goto("/auth/sign-in");- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- await expect(page).toHaveURL("/");+ await signIn(page);});test("Successfully sign up & able to go to homepage", async ({ page }) => {@@ -74,25 +57,7 @@ test("Unauthenticated user should be redirected to target URL after login", asynpage,}) => {await page.goto("/auth/sign-in");- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- await expect(page).toHaveURL("/");+ await signIn(page);// project id and prompt from seed.tsconst promptUrl =@@ -108,25 +73,7 @@ test("Unauthenticated user should be redirected to target URL after login", asynawait expect(page).toHaveURL(/targetPath/);- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement2 = page.locator(".text-destructive");- const hasError2 = await errorElement2.isVisible().catch(() => false);- if (hasError2) {- const errorText = await errorElement2.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- await expect(page).toHaveURL(promptUrl);+ await signIn(page, { expectedUrl: promptUrl });});test("Unauthenticated user should not be redirected to non-relative URLs after login", async ({@@ -137,26 +84,8 @@ test("Unauthenticated user should not be redirected to non-relative URLs after l`/auth/sign-in?targetPath=${encodeURIComponent(nonRelativeUrl)}`,);- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- // Expect to be redirected to the home page, not the non-relative URL- await expect(page).toHaveURL("/");+ // Lands on the home page, not on the non-relative targetPath+ await signIn(page, { expectedUrl: "/" });// Verify we're logged inawait expect(page.getByRole("button", { name: /Demo User/ })).toBeVisible();@@ -170,24 +99,6 @@ test("Unauthenticated user should be redirected to relative URL after login", as`/auth/sign-in?targetPath=${encodeURIComponent(relativeUrl)}`,);- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-// Expect to be redirected to the relative URL- await expect(page).toHaveURL(relativeUrl);+ await signIn(page, { expectedUrl: relativeUrl });});diff --git a/web/src/__e2e__/create-project.spec.ts b/web/src/__e2e__/create-project.spec.tsindex 46e1a1eabd..9f9b84ded2 100644--- a/web/src/__e2e__/create-project.spec.ts+++ b/web/src/__e2e__/create-project.spec.ts@@ -1,5 +1,6 @@import { test, expect, type Page } from "@playwright/test";import { prisma } from "@langfuse/shared/src/db";+import { signIn } from "./utils/auth";// const checkConsoleErrors = async (page: Page) => {// const errors: string[] = [];@@ -39,30 +40,14 @@ test.describe("Create project", () => {// Sign inawait page.goto("/auth/sign-in");- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");+ await signIn(page);+ // Start create org flow. `page.isVisible` does not wait and its result+ // was discarded, so the click below carried the whole wait on its 10s+ // action timeout; assert it instead.await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click(- 'button[data-testid="submit-email-password-sign-in-form"]',- );-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- await expect(page).toHaveURL("/");-- // Start create org flow- await page.isVisible('[data-testid="create-organization-btn"]');+ page.locator('[data-testid="create-organization-btn"]'),+ ).toBeVisible();await page.click('[data-testid="create-organization-btn"]');await expect(page).toHaveURL("/setup");@@ -142,25 +127,7 @@ test.describe("Create project", () => {const signin = async (page: Page) => {await page.goto("/auth/sign-in");- await page.fill('input[name="email"]', "[email protected]");- await page.fill('input[type="password"]', "password");-- await expect(- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),- ).toBeEnabled();-- await page.click('button[data-testid="submit-email-password-sign-in-form"]');-- await page.waitForTimeout(2000);-- const errorElement = page.locator(".text-destructive");- const hasError = await errorElement.isVisible().catch(() => false);- if (hasError) {- const errorText = await errorElement.textContent();- throw new Error(`Sign-in failed with error: ${errorText}`);- }-- await expect(page).toHaveURL("/");+ await signIn(page);};const checkPageHeaderTitle = async (page: Page, title: string) => {diff --git a/web/src/__e2e__/utils/auth.ts b/web/src/__e2e__/utils/auth.tsnew file mode 100644index 0000000000..574929f831--- /dev/null+++ b/web/src/__e2e__/utils/auth.ts@@ -0,0 +1,71 @@+import { expect, type Page } from "@playwright/test";++/** The account created by `pnpm --filter=shared run db:seed`. */+const DEMO_USER = {+ email: "[email protected]",+ password: "password",+} as const;++/** Matched to the `expect` timeout in playwright.config.ts. */+const SIGN_IN_TIMEOUT_MS = 60_000;++/**+ * Signs in through the credentials form and waits for the app to land on+ * `expectedUrl`.+ *+ * Navigation is the caller's job: specs reach this form from `/auth/sign-in`,+ * from a `?targetPath=` redirect, and from signing out mid-test.+ */+export async function signIn(+ page: Page,+ {+ email = DEMO_USER.email,+ password = DEMO_USER.password,+ expectedUrl = "/",+ }: { email?: string; password?: string; expectedUrl?: string } = {},+): Promise<void> {+ const submitButton = page.getByTestId("submit-email-password-sign-in-form");++ // "Sign in" stays disabled until react-hook-form has seen both fields, and+ // it only sees them once the page has hydrated: React keeps the text an+ // input already holds while hydrating but fires no change event for it, so+ // anything typed into the server-rendered form before that point lives in+ // the DOM and nowhere else. Retrying the fills until the button enables+ // turns the race into a wait instead of a form that can never be submitted.+ await expect(async () => {+ await page.fill('input[name="email"]', email);+ await page.fill('input[type="password"]', password);+ await expect(submitButton).toBeEnabled({ timeout: 1_000 });+ }).toPass({ timeout: SIGN_IN_TIMEOUT_MS });++ await submitButton.click();++ // A rejected sign-in parks the browser on exactly the URL a slow one does,+ // so waiting only for the navigation reports "still on /auth/sign-in"+ // either way — true in both cases and an explanation in neither. Race the+ // form's own error against the navigation and report whichever arrives.+ //+ // `FormLabel` also turns `text-destructive` when its own field is invalid+ // and sits ahead of the form-level error in the DOM, so excluding labels+ // keeps this from reporting a rejection as the single word "Email".+ const formError = page.locator(".text-destructive:not(label)").first();+ const rejection = await Promise.race([+ page+ .waitForURL(expectedUrl, { timeout: SIGN_IN_TIMEOUT_MS })+ .then(() => null),+ formError+ .waitFor({ state: "visible", timeout: SIGN_IN_TIMEOUT_MS })+ .then(async () => (await formError.textContent())?.trim() || "(no text)"),+ ]);++ if (rejection !== null) {+ throw new Error(`Sign-in was rejected by the form: ${rejection}`);+ }++ // `waitForURL` resolves on the first navigation that matches and never+ // looks again, so by itself it only proves the app passed *through*+ // `expectedUrl`. Pin where it came to rest too — that is what the+ // `toHaveURL` assertions this replaces were actually checking, and for the+ // targetPath tests it is the whole point.+ await expect(page).toHaveURL(expectedUrl);+}diff --git a/web/src/__e2e__/utils/warm-up.ts b/web/src/__e2e__/utils/warm-up.tsnew file mode 100644index 0000000000..a78ed151e3--- /dev/null+++ b/web/src/__e2e__/utils/warm-up.ts@@ -0,0 +1,60 @@+import { request, type FullConfig } from "@playwright/test";++/**+ * Playwright's `webServer.url` check only proves the port answers. `next start`+ * still resolves a route's bundle, its data clients and the NextAuth handlers+ * on the first request that reaches each one, and on a 4-vCPU runner that+ * lands while two browsers are also starting. Whichever test runs first pays+ * the entire cold start, and it is the only test in the run that does.+ *+ * Pay it here instead, where nothing is asserting.+ */+const WARM_UP_PATHS = [+ "/auth/sign-in",+ "/api/auth/providers",+ "/api/auth/csrf",+ "/api/auth/session",+];++const TOTAL_BUDGET_MS = 120_000;+const REQUEST_TIMEOUT_MS = 30_000;+const RETRY_DELAY_MS = 500;++export default async function warmUp(config: FullConfig) {+ const baseURL = config.projects[0]?.use.baseURL;+ if (!baseURL) return;++ const deadline = Date.now() + TOTAL_BUDGET_MS;+ const context = await request.newContext({ baseURL });++ try {+ for (const path of WARM_UP_PATHS) {+ // Playwright has moved `globalSetup` relative to `webServer` across+ // versions, so retry rather than depend on an ordering.+ let warmed = false;+ while (!warmed && Date.now() < deadline) {+ try {+ // A 5xx is the server telling us it is not ready yet — the exact+ // state being warmed away — so it is worth another go. Anything+ // below that means the route resolved and ran, which is all this+ // needs; a 404 would mean the path is simply wrong, and retrying+ // that would burn the whole budget for nothing.+ const response = await context.get(path, {+ timeout: REQUEST_TIMEOUT_MS,+ });+ if (response.status() >= 500) {+ throw new Error(`${path} answered ${response.status()}`);+ }+ warmed = true;+ } catch {+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));+ }+ }+ // Warming is an optimisation, not a gate: a run that would have passed+ // must not go red over a request no test is making. Say so and move on.+ if (!warmed) console.warn(`[warm-up] gave up on ${path}`);+ }+ } finally {+ await context.dispose();+ }+}