Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: add Playwright visual snapshot testing infrastructure (#392)

Rohit Malhotradd18f8aee0bc
diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml
new file mode 100644
index 0000000000..a4e71687d4
--- /dev/null
+++ b/.github/workflows/snapshot-tests.yml
@@ -0,0 +1,71 @@
+name: Snapshot Tests
+
+on:
+ pull_request:
+ push:
+ branches:
+ - main
+ workflow_dispatch:
+ inputs:
+ update_snapshots:
+ description: "Update baseline snapshots"
+ type: boolean
+ default: false
+
+concurrency:
+ group: snapshot-tests-${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: write
+
+jobs:
+ snapshot-tests:
+ name: Visual Snapshot Tests
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+
+ steps:
+ - name: Check out repository
+ uses: actions/checkout@v6
+ with:
+ ref: ${{ github.head_ref || github.ref_name }}
+
+ - name: Set up Node.js with npm cache
+ uses: actions/setup-node@v6
+ with:
+ node-version: "24"
+ cache: npm
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Install Playwright Chromium
+ run: npx playwright install --with-deps chromium
+
+ - name: Run snapshot tests
+ if: ${{ github.event.inputs.update_snapshots != 'true' }}
+ run: npm run test:e2e:snapshots
+
+ - name: Update snapshots
+ if: ${{ github.event.inputs.update_snapshots == 'true' }}
+ run: npm run test:e2e:snapshots:update
+
+ - name: Commit updated snapshots
+ if: ${{ github.event.inputs.update_snapshots == 'true' }}
+ run: |
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git add tests/e2e/__snapshots__/
+ git diff --staged --quiet || git commit -m "chore: update baseline snapshots [skip ci]"
+ git push
+
+ - name: Upload snapshot diff artifacts
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: snapshot-test-results
+ path: |
+ test-results/
+ playwright-report/
+ retention-days: 7
diff --git a/AGENTS.md b/AGENTS.md
index fccd7afeab..963f07f0ca 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -19,6 +19,21 @@
- Verification command: `npm run typecheck && npm run build`.
- GitHub automation now includes `.github/workflows/ci.yml` for `npm ci`, `npm test`, and `npm run build`, plus `.github/dependabot.yml` with weekly npm/github-actions updates gated by a 7-day cooldown.
+## Visual Snapshot Testing
+
+- Snapshot tests live in `tests/e2e/snapshots/` and compare screenshots against baseline images in `tests/e2e/__snapshots__/`.
+- Run with `npm run test:e2e:snapshots`; update baselines after intentional UI changes with `npm run test:e2e:snapshots:update`.
+- CI runs snapshots via `.github/workflows/snapshot-tests.yml` on every PR/push to main; failures upload diff artifacts.
+- **Updating snapshots in CI**: Snapshots must be generated in CI to avoid local/CI rendering differences. Run the workflow manually from Actions tab with "Update baseline snapshots" checked, or via CLI: `gh workflow run snapshot-tests.yml --ref <branch> -f update_snapshots=true`.
+- **Viewing diffs**: On failure, Playwright generates `*-actual.png`, `*-expected.png`, and `*-diff.png` in `test-results/`. Run `npx playwright show-report` to view the HTML report with visual diffs.
+- Key patterns for writing snapshot tests:
+ - Use `setupMocks(page, showConsentModal)` helper to configure API mocks consistently.
+ - Use `dismissConsentModal(page)` after navigation to dismiss the analytics modal.
+ - Use `animations: "disabled"` and `maxDiffPixelRatio: 0.01` in `toHaveScreenshot()` to reduce flakiness.
+ - Target specific elements via `page.getByTestId()` rather than full-page screenshots when possible.
+- **Composing tests**: Use `test.step()` for iterative snapshots within a single test, or extract helper functions (like `navigateToSettings(page)`) to share setup across tests. For heavier reuse, use Playwright fixtures via `test.extend()`.
+- Snapshots are organized by `{snapshotDir}/{testFilePath}/{projectName}/{arg}.png` (configured in `playwright.config.ts`).
+
## Live End-to-End Test Framework
- The live QA path is intentionally separate from ordinary mocked Playwright coverage. If ordinary browser tests are added, keep them outside `tests/e2e/live/` so `playwright.config.ts` can run them while ignoring `**/live/**`; live LLM-backed tests must never run as part of `npm run test:e2e`.
diff --git a/package.json b/package.json
index 5742877212..be07b37577 100644
--- a/package.json
+++ b/package.json
@@ -81,6 +81,8 @@
"test": "npm run make-i18n && vitest run",
"test:e2e": "playwright test --pass-with-no-tests",
"test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",
+ "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium",
+ "test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots",
"test:coverage": "npm run make-i18n && vitest run --coverage",
"dev_wsl": "VITE_WATCH_USE_POLLING=true vite",
"preview": "vite preview",
diff --git a/playwright.config.ts b/playwright.config.ts
index b60c6e6a60..a1bb6fd0d5 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -24,6 +24,20 @@ export default defineConfig({
workers: process.env.CI ? 1 : undefined,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: "html",
+
+ /* Snapshot configuration for visual regression tests */
+ snapshotDir: "./tests/e2e/__snapshots__",
+ snapshotPathTemplate:
+ "{snapshotDir}/{testFilePath}/{projectName}/{arg}{ext}",
+ expect: {
+ toHaveScreenshot: {
+ /* Threshold for pixel comparison (0-1, lower = stricter) */
+ threshold: 0.01,
+ /* Maximum allowed different pixels */
+ maxDiffPixels: 100,
+ },
+ },
+
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
diff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png
new file mode 100644
index 0000000000..6a13343566
Binary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png differ
diff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png
new file mode 100644
index 0000000000..4811de5a5e
Binary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png differ
diff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png
new file mode 100644
index 0000000000..43f0f8f94a
Binary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png differ
diff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png
new file mode 100644
index 0000000000..9be1a9f02b
Binary files /dev/null and b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png differ
diff --git a/tests/e2e/snapshots/settings-page.snapshot.spec.ts b/tests/e2e/snapshots/settings-page.snapshot.spec.ts
new file mode 100644
index 0000000000..cb5f0c9f23
--- /dev/null
+++ b/tests/e2e/snapshots/settings-page.snapshot.spec.ts
@@ -0,0 +1,193 @@
+import { test, expect, Page } from "@playwright/test";
+
+/**
+ * Visual snapshot tests for UI pages.
+ *
+ * These tests capture screenshots of pages and compare them against
+ * baseline images to detect unintended visual regressions.
+ *
+ * To update baselines after intentional UI changes:
+ * npm run test:e2e:snapshots:update
+ */
+
+/** Mock settings response with analytics consent already given */
+const SETTINGS_WITH_CONSENT = {
+ llm_model: "anthropic/claude-sonnet-4-20250514",
+ llm_base_url: "",
+ agent: "CodeActAgent",
+ language: "en",
+ llm_api_key: null,
+ llm_api_key_set: true,
+ search_api_key_set: false,
+ confirmation_mode: false,
+ security_analyzer: "llm",
+ remote_runtime_resource_factor: 1,
+ provider_tokens_set: { github: "" },
+ enable_default_condenser: true,
+ condenser_max_size: 240,
+ enable_sound_notifications: false,
+ // Analytics consent already given - modal won't show
+ user_consents_to_analytics: false,
+ enable_proactive_conversation_starters: false,
+ enable_solvability_analysis: false,
+ max_budget_per_task: null,
+};
+
+/** Mock settings response with analytics consent pending (null = show modal) */
+const SETTINGS_WITHOUT_CONSENT = {
+ ...SETTINGS_WITH_CONSENT,
+ // null means user hasn't made a choice yet - modal will show
+ user_consents_to_analytics: null,
+};
+
+/**
+ * Sets up common API mocks for snapshot tests.
+ * @param page - Playwright page
+ * @param showConsentModal - Whether to show the analytics consent modal
+ */
+async function setupMocks(page: Page, showConsentModal = false) {
+ // Pre-set localStorage to skip onboarding
+ await page.addInitScript(() => {
+ window.localStorage.setItem("openhands-onboarded", "true");
+ });
+
+ // Mock settings API - consent modal appears when user_consents_to_analytics is null
+ const settingsResponse = showConsentModal
+ ? SETTINGS_WITHOUT_CONSENT
+ : SETTINGS_WITH_CONSENT;
+
+ await page.route("**/api/settings", async (route) => {
+ if (route.request().method() === "GET") {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify(settingsResponse),
+ });
+ } else {
+ await route.continue();
+ }
+ });
+
+ // Mock settings schemas
+ await page.route("**/api/settings/agent-schema", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({}),
+ });
+ });
+
+ await page.route("**/api/settings/conversation-schema", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({}),
+ });
+ });
+
+ // Mock conversations search for home page
+ await page.route("**/api/conversations/search**", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({ results: [] }),
+ });
+ });
+
+ // Mock file APIs to prevent proxy errors
+ await page.route("**/api/file/**", async (route) => {
+ await route.fulfill({
+ status: 200,
+ contentType: "application/json",
+ body: JSON.stringify({ path: "/home", subdirs: [] }),
+ });
+ });
+}
+
+/**
+ * Dismisses the analytics consent modal if it appears.
+ */
+async function dismissConsentModal(page: Page) {
+ const consentDialog = page.getByRole("dialog", {
+ name: "Help improve OpenHands",
+ });
+ await page
+ .getByRole("button", { name: "Confirm preferences" })
+ .click({ timeout: 3000 })
+ .catch(() => undefined);
+ await expect(consentDialog).toHaveCount(0, { timeout: 3000 });
+}
+
+test.describe("UI Visual Snapshots", () => {
+ // Increase timeout for this test - modal loading can be slow
+ test.setTimeout(60000);
+
+ test("Analytics consent modal renders correctly", async ({ page }) => {
+ // Use setupMocks with showConsentModal=true to guarantee modal appears
+ await setupMocks(page, true);
+
+ await page.goto("/conversations", { waitUntil: "networkidle" });
+
+ // Wait for the page to stabilize
+ const rootLayout = page.getByTestId("root-layout");
+ await expect(rootLayout).toBeVisible({ timeout: 15000 });
+
+ // Wait for the consent modal (lazy-loaded) with extended timeout
+ const consentModal = page.getByRole("dialog", {
+ name: "Help improve OpenHands",
+ });
+ await expect(consentModal).toBeVisible({ timeout: 15000 });
+
+ // Snapshot the full page with the consent modal
+ await expect(rootLayout).toHaveScreenshot("analytics-consent-modal.png", {
+ maxDiffPixelRatio: 0.01,
+ animations: "disabled",
+ });
+ });
+
+ test("Home page renders correctly", async ({ page }) => {
+ await setupMocks(page, false);
+ await page.goto("/conversations");
+ await dismissConsentModal(page);
+
+ const homeScreen = page.getByTestId("home-screen");
+ await expect(homeScreen).toBeVisible();
+ await page.waitForLoadState("networkidle");
+
+ const rootLayout = page.getByTestId("root-layout");
+ await expect(rootLayout).toHaveScreenshot("home-screen.png", {
+ maxDiffPixelRatio: 0.01,
+ animations: "disabled",
+ });
+ });
+
+ test("Settings page renders correctly", async ({ page }) => {
+ await setupMocks(page, false);
+ await page.goto("/settings");
+ await dismissConsentModal(page);
+
+ const rootLayout = page.getByTestId("root-layout");
+ await expect(rootLayout).toBeVisible();
+ await page.waitForLoadState("networkidle");
+
+ await expect(rootLayout).toHaveScreenshot("settings-page.png", {
+ maxDiffPixelRatio: 0.01,
+ animations: "disabled",
+ });
+ });
+
+ test("Settings app page renders correctly", async ({ page }) => {
+ await setupMocks(page, false);
+ await page.goto("/settings/app");
+ await dismissConsentModal(page);
+
+ const rootLayout = page.getByTestId("root-layout");
+ await expect(rootLayout).toBeVisible();
+ await page.waitForLoadState("networkidle");
+
+ await expect(rootLayout).toHaveScreenshot("settings-app-page.png", {
+ maxDiffPixelRatio: 0.01,
+ animations: "disabled",
+ });
+ });
+});