π OpenHands: AI-Driven Development
feat: add Playwright visual snapshot testing infrastructure (#392)
Rohit Malhotradd18f8aee0bc
diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.ymlnew file mode 100644index 0000000000..a4e71687d4--- /dev/null+++ b/.github/workflows/snapshot-tests.yml@@ -0,0 +1,71 @@+name: Snapshot Tests++on:+ pull_request:+ push:+ branches:+ - main+ workflow_dispatch:+ inputs:+ update_snapshots:+ description: "Update baseline snapshots"+ type: boolean+ default: false++concurrency:+ group: snapshot-tests-${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++permissions:+ contents: write++jobs:+ snapshot-tests:+ name: Visual Snapshot Tests+ runs-on: ubuntu-24.04+ timeout-minutes: 15++ steps:+ - name: Check out repository+ uses: actions/checkout@v6+ with:+ ref: ${{ github.head_ref || github.ref_name }}++ - name: Set up Node.js with npm cache+ uses: actions/setup-node@v6+ with:+ node-version: "24"+ cache: npm++ - name: Install dependencies+ run: npm ci++ - name: Install Playwright Chromium+ run: npx playwright install --with-deps chromium++ - name: Run snapshot tests+ if: ${{ github.event.inputs.update_snapshots != 'true' }}+ run: npm run test:e2e:snapshots++ - name: Update snapshots+ if: ${{ github.event.inputs.update_snapshots == 'true' }}+ run: npm run test:e2e:snapshots:update++ - name: Commit updated snapshots+ if: ${{ github.event.inputs.update_snapshots == 'true' }}+ run: |+ git config user.name "github-actions[bot]"+ git config user.email "github-actions[bot]@users.noreply.github.com"+ git add tests/e2e/__snapshots__/+ git diff --staged --quiet || git commit -m "chore: update baseline snapshots [skip ci]"+ git push++ - name: Upload snapshot diff artifacts+ if: failure()+ uses: actions/upload-artifact@v4+ with:+ name: snapshot-test-results+ path: |+ test-results/+ playwright-report/+ retention-days: 7diff --git a/AGENTS.md b/AGENTS.mdindex fccd7afeab..963f07f0ca 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -19,6 +19,21 @@- Verification command: `npm run typecheck && npm run build`.- GitHub automation now includes `.github/workflows/ci.yml` for `npm ci`, `npm test`, and `npm run build`, plus `.github/dependabot.yml` with weekly npm/github-actions updates gated by a 7-day cooldown.+## Visual Snapshot Testing++- Snapshot tests live in `tests/e2e/snapshots/` and compare screenshots against baseline images in `tests/e2e/__snapshots__/`.+- Run with `npm run test:e2e:snapshots`; update baselines after intentional UI changes with `npm run test:e2e:snapshots:update`.+- CI runs snapshots via `.github/workflows/snapshot-tests.yml` on every PR/push to main; failures upload diff artifacts.+- **Updating snapshots in CI**: Snapshots must be generated in CI to avoid local/CI rendering differences. Run the workflow manually from Actions tab with "Update baseline snapshots" checked, or via CLI: `gh workflow run snapshot-tests.yml --ref <branch> -f update_snapshots=true`.+- **Viewing diffs**: On failure, Playwright generates `*-actual.png`, `*-expected.png`, and `*-diff.png` in `test-results/`. Run `npx playwright show-report` to view the HTML report with visual diffs.+- Key patterns for writing snapshot tests:+ - Use `setupMocks(page, showConsentModal)` helper to configure API mocks consistently.+ - Use `dismissConsentModal(page)` after navigation to dismiss the analytics modal.+ - Use `animations: "disabled"` and `maxDiffPixelRatio: 0.01` in `toHaveScreenshot()` to reduce flakiness.+ - Target specific elements via `page.getByTestId()` rather than full-page screenshots when possible.+- **Composing tests**: Use `test.step()` for iterative snapshots within a single test, or extract helper functions (like `navigateToSettings(page)`) to share setup across tests. For heavier reuse, use Playwright fixtures via `test.extend()`.+- Snapshots are organized by `{snapshotDir}/{testFilePath}/{projectName}/{arg}.png` (configured in `playwright.config.ts`).+## Live End-to-End Test Framework- The live QA path is intentionally separate from ordinary mocked Playwright coverage. If ordinary browser tests are added, keep them outside `tests/e2e/live/` so `playwright.config.ts` can run them while ignoring `**/live/**`; live LLM-backed tests must never run as part of `npm run test:e2e`.diff --git a/package.json b/package.jsonindex 5742877212..be07b37577 100644--- a/package.json+++ b/package.json@@ -81,6 +81,8 @@"test": "npm run make-i18n && vitest run","test:e2e": "playwright test --pass-with-no-tests","test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",+ "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium",+ "test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots","test:coverage": "npm run make-i18n && vitest run --coverage","dev_wsl": "VITE_WATCH_USE_POLLING=true vite","preview": "vite preview",diff --git a/playwright.config.ts b/playwright.config.tsindex b60c6e6a60..a1bb6fd0d5 100644--- a/playwright.config.ts+++ b/playwright.config.ts@@ -24,6 +24,20 @@ export default defineConfig({workers: process.env.CI ? 1 : undefined,/* Reporter to use. See https://playwright.dev/docs/test-reporters */reporter: "html",++ /* Snapshot configuration for visual regression tests */+ snapshotDir: "./tests/e2e/__snapshots__",+ snapshotPathTemplate:+ "{snapshotDir}/{testFilePath}/{projectName}/{arg}{ext}",+ expect: {+ toHaveScreenshot: {+ /* Threshold for pixel comparison (0-1, lower = stricter) */+ threshold: 0.01,+ /* Maximum allowed different pixels */+ maxDiffPixels: 100,+ },+ },+/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */use: {/* Base URL to use in actions like `await page.goto('/')`. */diff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.pngnew file mode 100644index 0000000000..6a13343566Binary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.pngnew file mode 100644index 0000000000..4811de5a5eBinary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.pngnew file mode 100644index 0000000000..43f0f8f94aBinary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.pngnew file mode 100644index 0000000000..9be1a9f02bBinary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png differdiff --git a/tests/e2e/snapshots/settings-page.snapshot.spec.ts b/tests/e2e/snapshots/settings-page.snapshot.spec.tsnew file mode 100644index 0000000000..cb5f0c9f23--- /dev/null+++ b/tests/e2e/snapshots/settings-page.snapshot.spec.ts@@ -0,0 +1,193 @@+import { test, expect, Page } from "@playwright/test";++/**+ * Visual snapshot tests for UI pages.+ *+ * These tests capture screenshots of pages and compare them against+ * baseline images to detect unintended visual regressions.+ *+ * To update baselines after intentional UI changes:+ * npm run test:e2e:snapshots:update+ */++/** Mock settings response with analytics consent already given */+const SETTINGS_WITH_CONSENT = {+ llm_model: "anthropic/claude-sonnet-4-20250514",+ llm_base_url: "",+ agent: "CodeActAgent",+ language: "en",+ llm_api_key: null,+ llm_api_key_set: true,+ search_api_key_set: false,+ confirmation_mode: false,+ security_analyzer: "llm",+ remote_runtime_resource_factor: 1,+ provider_tokens_set: { github: "" },+ enable_default_condenser: true,+ condenser_max_size: 240,+ enable_sound_notifications: false,+ // Analytics consent already given - modal won't show+ user_consents_to_analytics: false,+ enable_proactive_conversation_starters: false,+ enable_solvability_analysis: false,+ max_budget_per_task: null,+};++/** Mock settings response with analytics consent pending (null = show modal) */+const SETTINGS_WITHOUT_CONSENT = {+ ...SETTINGS_WITH_CONSENT,+ // null means user hasn't made a choice yet - modal will show+ user_consents_to_analytics: null,+};++/**+ * Sets up common API mocks for snapshot tests.+ * @param page - Playwright page+ * @param showConsentModal - Whether to show the analytics consent modal+ */+async function setupMocks(page: Page, showConsentModal = false) {+ // Pre-set localStorage to skip onboarding+ await page.addInitScript(() => {+ window.localStorage.setItem("openhands-onboarded", "true");+ });++ // Mock settings API - consent modal appears when user_consents_to_analytics is null+ const settingsResponse = showConsentModal+ ? SETTINGS_WITHOUT_CONSENT+ : SETTINGS_WITH_CONSENT;++ await page.route("**/api/settings", async (route) => {+ if (route.request().method() === "GET") {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify(settingsResponse),+ });+ } else {+ await route.continue();+ }+ });++ // Mock settings schemas+ await page.route("**/api/settings/agent-schema", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({}),+ });+ });++ await page.route("**/api/settings/conversation-schema", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({}),+ });+ });++ // Mock conversations search for home page+ await page.route("**/api/conversations/search**", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({ results: [] }),+ });+ });++ // Mock file APIs to prevent proxy errors+ await page.route("**/api/file/**", async (route) => {+ await route.fulfill({+ status: 200,+ contentType: "application/json",+ body: JSON.stringify({ path: "/home", subdirs: [] }),+ });+ });+}++/**+ * Dismisses the analytics consent modal if it appears.+ */+async function dismissConsentModal(page: Page) {+ const consentDialog = page.getByRole("dialog", {+ name: "Help improve OpenHands",+ });+ await page+ .getByRole("button", { name: "Confirm preferences" })+ .click({ timeout: 3000 })+ .catch(() => undefined);+ await expect(consentDialog).toHaveCount(0, { timeout: 3000 });+}++test.describe("UI Visual Snapshots", () => {+ // Increase timeout for this test - modal loading can be slow+ test.setTimeout(60000);++ test("Analytics consent modal renders correctly", async ({ page }) => {+ // Use setupMocks with showConsentModal=true to guarantee modal appears+ await setupMocks(page, true);++ await page.goto("/conversations", { waitUntil: "networkidle" });++ // Wait for the page to stabilize+ const rootLayout = page.getByTestId("root-layout");+ await expect(rootLayout).toBeVisible({ timeout: 15000 });++ // Wait for the consent modal (lazy-loaded) with extended timeout+ const consentModal = page.getByRole("dialog", {+ name: "Help improve OpenHands",+ });+ await expect(consentModal).toBeVisible({ timeout: 15000 });++ // Snapshot the full page with the consent modal+ await expect(rootLayout).toHaveScreenshot("analytics-consent-modal.png", {+ maxDiffPixelRatio: 0.01,+ animations: "disabled",+ });+ });++ test("Home page renders correctly", async ({ page }) => {+ await setupMocks(page, false);+ await page.goto("/conversations");+ await dismissConsentModal(page);++ const homeScreen = page.getByTestId("home-screen");+ await expect(homeScreen).toBeVisible();+ await page.waitForLoadState("networkidle");++ const rootLayout = page.getByTestId("root-layout");+ await expect(rootLayout).toHaveScreenshot("home-screen.png", {+ maxDiffPixelRatio: 0.01,+ animations: "disabled",+ });+ });++ test("Settings page renders correctly", async ({ page }) => {+ await setupMocks(page, false);+ await page.goto("/settings");+ await dismissConsentModal(page);++ const rootLayout = page.getByTestId("root-layout");+ await expect(rootLayout).toBeVisible();+ await page.waitForLoadState("networkidle");++ await expect(rootLayout).toHaveScreenshot("settings-page.png", {+ maxDiffPixelRatio: 0.01,+ animations: "disabled",+ });+ });++ test("Settings app page renders correctly", async ({ page }) => {+ await setupMocks(page, false);+ await page.goto("/settings/app");+ await dismissConsentModal(page);++ const rootLayout = page.getByTestId("root-layout");+ await expect(rootLayout).toBeVisible();+ await page.waitForLoadState("networkidle");++ await expect(rootLayout).toHaveScreenshot("settings-app-page.png", {+ maxDiffPixelRatio: 0.01,+ animations: "disabled",+ });+ });+});