Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

test(e2e): stop the sign-in helper from failing silently (#18286)

Der Gerätef983a07f981
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 64cf16d61d..23db39b61e 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -1385,6 +1385,18 @@ jobs:
exit "$(cat /tmp/playwright-install.exit)"
- name: Run e2e tests
run: pnpm --filter=web run test:e2e
+ # Playwright writes a screenshot, an error context and a trace for every
+ # failure, and the reporter prints the command to open them — but they
+ # die with the runner, so a flake here is reported with no way to see
+ # what the browser was actually showing.
+ - name: Upload Playwright failure artifacts
+ if: failure()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: playwright-results-${{ github.run_id }}-${{ github.run_attempt }}
+ path: web/test-results/
+ retention-days: 7
+ if-no-files-found: ignore
e2e-server-tests:
runs-on: blacksmith-8vcpu-ubuntu-2404
diff --git a/web/playwright.config.ts b/web/playwright.config.ts
index 3bda799baa..15a04c0c87 100644
--- a/web/playwright.config.ts
+++ b/web/playwright.config.ts
@@ -2,6 +2,8 @@ import { defineConfig } from "@playwright/test";
export default defineConfig({
testDir: "./src/__e2e__",
+ // Takes the server's cold start off whichever test happens to run first.
+ globalSetup: "./src/__e2e__/utils/warm-up.ts",
timeout: 180000, // test timeout 180s (3 minutes)
expect: {
timeout: 60000, // assertion timeout 60s (increased for CI)
diff --git a/web/src/__e2e__/app-shell-overflow.spec.ts b/web/src/__e2e__/app-shell-overflow.spec.ts
index 41631f5016..0ca6d2648c 100644
--- a/web/src/__e2e__/app-shell-overflow.spec.ts
+++ b/web/src/__e2e__/app-shell-overflow.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, type Page } from "@playwright/test";
+import { signIn } from "./utils/auth";
/**
* The app shell is fully height- and width-constrained: page content scrolls
@@ -8,14 +9,6 @@ import { test, expect, type Page } from "@playwright/test";
const SHORT_VIEWPORT = { width: 1190, height: 560 };
-async function signIn(page: Page) {
- await page.goto("/auth/sign-in");
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
- await expect(page).toHaveURL("/");
-}
-
/** Document overflow in CSS px, per axis. */
function documentOverflow(page: Page) {
return page.evaluate(() => {
@@ -31,6 +24,7 @@ test("a right drawer's absolutely-positioned content does not scroll the documen
page,
}) => {
await page.setViewportSize(SHORT_VIEWPORT);
+ await page.goto("/auth/sign-in");
await signIn(page);
await page.getByRole("button", { name: "Support" }).click();
const drawerBody = page.locator("#secondary div.overflow-y-auto").first();
@@ -57,6 +51,7 @@ test("a vertical document scrollbar does not spawn a horizontal one", async ({
page,
}) => {
await page.setViewportSize(SHORT_VIEWPORT);
+ await page.goto("/auth/sign-in");
await signIn(page);
// Viewport units ignore scrollbars, so a shell sized off `100vw` overshoots
// the space beside the sidebar by the scrollbar's width once one is showing.
diff --git a/web/src/__e2e__/auth.spec.ts b/web/src/__e2e__/auth.spec.ts
index 6178f19346..f141380f55 100644
--- a/web/src/__e2e__/auth.spec.ts
+++ b/web/src/__e2e__/auth.spec.ts
@@ -1,4 +1,5 @@
import { test, expect } from "@playwright/test";
+import { signIn } from "./utils/auth";
test("should redirect to sign-in if not signed in", async ({ page }) => {
await page.goto("/");
@@ -7,25 +8,7 @@ test("should redirect to sign-in if not signed in", async ({ page }) => {
test("should redirect to home if signed in", async ({ page }) => {
await page.goto("/auth/sign-in");
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- await expect(page).toHaveURL("/");
+ await signIn(page);
});
test("Successfully sign up & able to go to homepage", async ({ page }) => {
@@ -74,25 +57,7 @@ test("Unauthenticated user should be redirected to target URL after login", asyn
page,
}) => {
await page.goto("/auth/sign-in");
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- await expect(page).toHaveURL("/");
+ await signIn(page);
// project id and prompt from seed.ts
const promptUrl =
@@ -108,25 +73,7 @@ test("Unauthenticated user should be redirected to target URL after login", asyn
await expect(page).toHaveURL(/targetPath/);
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement2 = page.locator(".text-destructive");
- const hasError2 = await errorElement2.isVisible().catch(() => false);
- if (hasError2) {
- const errorText = await errorElement2.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- await expect(page).toHaveURL(promptUrl);
+ await signIn(page, { expectedUrl: promptUrl });
});
test("Unauthenticated user should not be redirected to non-relative URLs after login", async ({
@@ -137,26 +84,8 @@ test("Unauthenticated user should not be redirected to non-relative URLs after l
`/auth/sign-in?targetPath=${encodeURIComponent(nonRelativeUrl)}`,
);
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- // Expect to be redirected to the home page, not the non-relative URL
- await expect(page).toHaveURL("/");
+ // Lands on the home page, not on the non-relative targetPath
+ await signIn(page, { expectedUrl: "/" });
// Verify we're logged in
await expect(page.getByRole("button", { name: /Demo User/ })).toBeVisible();
@@ -170,24 +99,6 @@ test("Unauthenticated user should be redirected to relative URL after login", as
`/auth/sign-in?targetPath=${encodeURIComponent(relativeUrl)}`,
);
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
// Expect to be redirected to the relative URL
- await expect(page).toHaveURL(relativeUrl);
+ await signIn(page, { expectedUrl: relativeUrl });
});
diff --git a/web/src/__e2e__/create-project.spec.ts b/web/src/__e2e__/create-project.spec.ts
index 46e1a1eabd..9f9b84ded2 100644
--- a/web/src/__e2e__/create-project.spec.ts
+++ b/web/src/__e2e__/create-project.spec.ts
@@ -1,5 +1,6 @@
import { test, expect, type Page } from "@playwright/test";
import { prisma } from "@langfuse/shared/src/db";
+import { signIn } from "./utils/auth";
// const checkConsoleErrors = async (page: Page) => {
// const errors: string[] = [];
@@ -39,30 +40,14 @@ test.describe("Create project", () => {
// Sign in
await page.goto("/auth/sign-in");
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
+ await signIn(page);
+ // Start create org flow. `page.isVisible` does not wait and its result
+ // was discarded, so the click below carried the whole wait on its 10s
+ // action timeout; assert it instead.
await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click(
- 'button[data-testid="submit-email-password-sign-in-form"]',
- );
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- await expect(page).toHaveURL("/");
-
- // Start create org flow
- await page.isVisible('[data-testid="create-organization-btn"]');
+ page.locator('[data-testid="create-organization-btn"]'),
+ ).toBeVisible();
await page.click('[data-testid="create-organization-btn"]');
await expect(page).toHaveURL("/setup");
@@ -142,25 +127,7 @@ test.describe("Create project", () => {
const signin = async (page: Page) => {
await page.goto("/auth/sign-in");
- await page.fill('input[name="email"]', "[email protected]");
- await page.fill('input[type="password"]', "password");
-
- await expect(
- page.locator('button[data-testid="submit-email-password-sign-in-form"]'),
- ).toBeEnabled();
-
- await page.click('button[data-testid="submit-email-password-sign-in-form"]');
-
- await page.waitForTimeout(2000);
-
- const errorElement = page.locator(".text-destructive");
- const hasError = await errorElement.isVisible().catch(() => false);
- if (hasError) {
- const errorText = await errorElement.textContent();
- throw new Error(`Sign-in failed with error: ${errorText}`);
- }
-
- await expect(page).toHaveURL("/");
+ await signIn(page);
};
const checkPageHeaderTitle = async (page: Page, title: string) => {
diff --git a/web/src/__e2e__/utils/auth.ts b/web/src/__e2e__/utils/auth.ts
new file mode 100644
index 0000000000..574929f831
--- /dev/null
+++ b/web/src/__e2e__/utils/auth.ts
@@ -0,0 +1,71 @@
+import { expect, type Page } from "@playwright/test";
+
+/** The account created by `pnpm --filter=shared run db:seed`. */
+const DEMO_USER = {
+ email: "[email protected]",
+ password: "password",
+} as const;
+
+/** Matched to the `expect` timeout in playwright.config.ts. */
+const SIGN_IN_TIMEOUT_MS = 60_000;
+
+/**
+ * Signs in through the credentials form and waits for the app to land on
+ * `expectedUrl`.
+ *
+ * Navigation is the caller's job: specs reach this form from `/auth/sign-in`,
+ * from a `?targetPath=` redirect, and from signing out mid-test.
+ */
+export async function signIn(
+ page: Page,
+ {
+ email = DEMO_USER.email,
+ password = DEMO_USER.password,
+ expectedUrl = "/",
+ }: { email?: string; password?: string; expectedUrl?: string } = {},
+): Promise<void> {
+ const submitButton = page.getByTestId("submit-email-password-sign-in-form");
+
+ // "Sign in" stays disabled until react-hook-form has seen both fields, and
+ // it only sees them once the page has hydrated: React keeps the text an
+ // input already holds while hydrating but fires no change event for it, so
+ // anything typed into the server-rendered form before that point lives in
+ // the DOM and nowhere else. Retrying the fills until the button enables
+ // turns the race into a wait instead of a form that can never be submitted.
+ await expect(async () => {
+ await page.fill('input[name="email"]', email);
+ await page.fill('input[type="password"]', password);
+ await expect(submitButton).toBeEnabled({ timeout: 1_000 });
+ }).toPass({ timeout: SIGN_IN_TIMEOUT_MS });
+
+ await submitButton.click();
+
+ // A rejected sign-in parks the browser on exactly the URL a slow one does,
+ // so waiting only for the navigation reports "still on /auth/sign-in"
+ // either way — true in both cases and an explanation in neither. Race the
+ // form's own error against the navigation and report whichever arrives.
+ //
+ // `FormLabel` also turns `text-destructive` when its own field is invalid
+ // and sits ahead of the form-level error in the DOM, so excluding labels
+ // keeps this from reporting a rejection as the single word "Email".
+ const formError = page.locator(".text-destructive:not(label)").first();
+ const rejection = await Promise.race([
+ page
+ .waitForURL(expectedUrl, { timeout: SIGN_IN_TIMEOUT_MS })
+ .then(() => null),
+ formError
+ .waitFor({ state: "visible", timeout: SIGN_IN_TIMEOUT_MS })
+ .then(async () => (await formError.textContent())?.trim() || "(no text)"),
+ ]);
+
+ if (rejection !== null) {
+ throw new Error(`Sign-in was rejected by the form: ${rejection}`);
+ }
+
+ // `waitForURL` resolves on the first navigation that matches and never
+ // looks again, so by itself it only proves the app passed *through*
+ // `expectedUrl`. Pin where it came to rest too — that is what the
+ // `toHaveURL` assertions this replaces were actually checking, and for the
+ // targetPath tests it is the whole point.
+ await expect(page).toHaveURL(expectedUrl);
+}
diff --git a/web/src/__e2e__/utils/warm-up.ts b/web/src/__e2e__/utils/warm-up.ts
new file mode 100644
index 0000000000..a78ed151e3
--- /dev/null
+++ b/web/src/__e2e__/utils/warm-up.ts
@@ -0,0 +1,60 @@
+import { request, type FullConfig } from "@playwright/test";
+
+/**
+ * Playwright's `webServer.url` check only proves the port answers. `next start`
+ * still resolves a route's bundle, its data clients and the NextAuth handlers
+ * on the first request that reaches each one, and on a 4-vCPU runner that
+ * lands while two browsers are also starting. Whichever test runs first pays
+ * the entire cold start, and it is the only test in the run that does.
+ *
+ * Pay it here instead, where nothing is asserting.
+ */
+const WARM_UP_PATHS = [
+ "/auth/sign-in",
+ "/api/auth/providers",
+ "/api/auth/csrf",
+ "/api/auth/session",
+];
+
+const TOTAL_BUDGET_MS = 120_000;
+const REQUEST_TIMEOUT_MS = 30_000;
+const RETRY_DELAY_MS = 500;
+
+export default async function warmUp(config: FullConfig) {
+ const baseURL = config.projects[0]?.use.baseURL;
+ if (!baseURL) return;
+
+ const deadline = Date.now() + TOTAL_BUDGET_MS;
+ const context = await request.newContext({ baseURL });
+
+ try {
+ for (const path of WARM_UP_PATHS) {
+ // Playwright has moved `globalSetup` relative to `webServer` across
+ // versions, so retry rather than depend on an ordering.
+ let warmed = false;
+ while (!warmed && Date.now() < deadline) {
+ try {
+ // A 5xx is the server telling us it is not ready yet — the exact
+ // state being warmed away — so it is worth another go. Anything
+ // below that means the route resolved and ran, which is all this
+ // needs; a 404 would mean the path is simply wrong, and retrying
+ // that would burn the whole budget for nothing.
+ const response = await context.get(path, {
+ timeout: REQUEST_TIMEOUT_MS,
+ });
+ if (response.status() >= 500) {
+ throw new Error(`${path} answered ${response.status()}`);
+ }
+ warmed = true;
+ } catch {
+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));
+ }
+ }
+ // Warming is an optimisation, not a gate: a run that would have passed
+ // must not go red over a request no test is making. Say so and move on.
+ if (!warmed) console.warn(`[warm-up] gave up on ${path}`);
+ }
+ } finally {
+ await context.dispose();
+ }
+}