π OpenHands: AI-Driven Development
perf(snapshots): skip retries on comparison pass, suppress consent modal (#505)
Rohit Malhotraacb04fd9ba42
diff --git a/package.json b/package.jsonindex 60f97ef392..e2a6ee8a18 100644--- a/package.json+++ b/package.json@@ -82,7 +82,7 @@"test": "npm run make-i18n && vitest run","test:e2e": "playwright test --pass-with-no-tests","test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",- "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium",+ "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium --retries=0","test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots","test:coverage": "npm run make-i18n && vitest run --coverage","dev_wsl": "VITE_WATCH_USE_POLLING=true vite",diff --git a/tests/e2e/snapshots/automations.snapshot.spec.ts b/tests/e2e/snapshots/automations.snapshot.spec.tsindex 7e206af0a7..698d5c2ead 100644--- a/tests/e2e/snapshots/automations.snapshot.spec.ts+++ b/tests/e2e/snapshots/automations.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the Automations pages.@@ -39,9 +40,7 @@ async function dismissConsentModal(page: Page) {* here with INLINE_AUTOMATIONS so all tests have a consistent baseline.*/async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);await page.route("**/api/conversations/search**", async (route) => {await route.fulfill({diff --git a/tests/e2e/snapshots/backends-extended.snapshot.spec.ts b/tests/e2e/snapshots/backends-extended.snapshot.spec.tsindex 46e3fc1496..cffdb729fb 100644--- a/tests/e2e/snapshots/backends-extended.snapshot.spec.ts+++ b/tests/e2e/snapshots/backends-extended.snapshot.spec.ts@@ -1,5 +1,6 @@import { test, expect, type Page } from "@playwright/test";import type { Backend } from "../../../src/api/backend-registry/types";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Extended visual snapshot tests for the backend management UI.@@ -77,21 +78,16 @@ async function setupPage(activeBackendId,}: { backends?: Backend[]; activeBackendId?: string } = {},) {- await page.addInitScript(- ({ backendsJson, activeJson }: { backendsJson: string; activeJson: string | null }) => {- window.localStorage.setItem("openhands-onboarded", "true");- window.localStorage.setItem("openhands-backends", backendsJson);- if (activeJson) {- window.localStorage.setItem("openhands-active-backend", activeJson);- }- },- {- backendsJson: JSON.stringify(backends),- activeJson: activeBackendId- ? JSON.stringify({ backendId: activeBackendId, orgId: null })- : null,- },- );+ const extra: [string, string][] = [+ ["openhands-backends", JSON.stringify(backends)],+ ];+ if (activeBackendId) {+ extra.push([+ "openhands-active-backend",+ JSON.stringify({ backendId: activeBackendId, orgId: null }),+ ]);+ }+ await seedLocalStorage(page, { extra });// Prevent workspace-scan 404s in the sidebar from cluttering timing.await page.route("**/api/file/**", (route) =>diff --git a/tests/e2e/snapshots/backends.snapshot.spec.ts b/tests/e2e/snapshots/backends.snapshot.spec.tsindex 5fb6819ca8..2b8dc063c3 100644--- a/tests/e2e/snapshots/backends.snapshot.spec.ts+++ b/tests/e2e/snapshots/backends.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the backend management UI.@@ -27,9 +28,7 @@ async function dismissConsentModal(page: Page) {}async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);// Suppress file-API proxy errors emitted when the home page scans the// workspace directory (same suppression used in sidebar.snapshot.spec.ts).diff --git a/tests/e2e/snapshots/changes-tab.snapshot.spec.ts b/tests/e2e/snapshots/changes-tab.snapshot.spec.tsindex 57bb98e062..eda595d221 100644--- a/tests/e2e/snapshots/changes-tab.snapshot.spec.ts+++ b/tests/e2e/snapshots/changes-tab.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the Changes (diff viewer) UI.@@ -52,13 +53,9 @@ const CONVERSATION_STATE_VALUE = JSON.stringify({* Skip onboarding and pre-enable the diff view for conversation 1.*/async function setupMocks(page: Page) {- await page.addInitScript(- ([key, value]) => {- window.localStorage.setItem("openhands-onboarded", "true");- window.localStorage.setItem(key, value);- },- [CONVERSATION_STATE_KEY, CONVERSATION_STATE_VALUE] as [string, string],- );+ await seedLocalStorage(page, {+ extra: [[CONVERSATION_STATE_KEY, CONVERSATION_STATE_VALUE]],+ });// Stub WebSocket so the conversation page doesn't hang waiting for a real// socket connection. Copied from collapsible-thinking.snapshot.spec.ts.diff --git a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.tsindex ce76c03ef4..f4ab61f3d0 100644--- a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts+++ b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the CollapsibleThinking component.@@ -173,10 +174,7 @@ async function injectEvents(page: Page, events: unknown[]) {* Uses mock conversation "1" which exists in the MSW handlers.*/async function navigateToConversation(page: Page, events: unknown[]) {- // Skip onboarding- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);await page.route("**/api/bash/execute_bash_command", async (route) => {await route.fulfill({diff --git a/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts b/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.tsindex 0e5fd6c2b7..5fcf296f23 100644--- a/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts+++ b/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts@@ -1,13 +1,14 @@import { expect, test } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";test("captures Docker /projects workspace browser state", async ({ page }) => {test.setTimeout(60_000);- await page.addInitScript(() => {- window.localStorage.setItem("analytics-consent", "true");- window.localStorage.setItem("openhands-telemetry-consent", "denied");- window.localStorage.setItem("openhands-telemetry-first-use", "true");- window.localStorage.setItem("openhands-onboarded", "1");+ await seedLocalStorage(page, {+ extra: [+ ["analytics-consent", "true"],+ ["openhands-telemetry-first-use", "true"],+ ],});await page.goto("/conversations", { waitUntil: "domcontentloaded" });diff --git a/tests/e2e/snapshots/mcp-page.snapshot.spec.ts b/tests/e2e/snapshots/mcp-page.snapshot.spec.tsindex 763704b6ea..bc39a23821 100644--- a/tests/e2e/snapshots/mcp-page.snapshot.spec.ts+++ b/tests/e2e/snapshots/mcp-page.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the MCP page (/mcp).@@ -32,9 +33,7 @@ async function dismissConsentModal(page: Page) {* instead of trying to suppress it here.*/async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);await page.route("**/api/conversations/search**", async (route) => {await route.fulfill({diff --git a/tests/e2e/snapshots/onboarding.snapshot.spec.ts b/tests/e2e/snapshots/onboarding.snapshot.spec.tsindex 762ef2e20c..c897e67009 100644--- a/tests/e2e/snapshots/onboarding.snapshot.spec.ts+++ b/tests/e2e/snapshots/onboarding.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the 4-step onboarding modal.@@ -26,10 +27,9 @@ import { test, expect, Page } from "@playwright/test";test.describe.configure({ mode: "serial" });async function setupMocks(page: Page) {- // Intentionally do NOT set openhands-onboarded so the modal appears- await page.addInitScript(() => {- window.localStorage.removeItem("openhands-onboarded");- });+ // removeOnboarded: true ensures the onboarding modal appears.+ // Analytics consent modal is suppressed (separate concern).+ await seedLocalStorage(page, { removeOnboarded: true });}async function dismissConsentModal(page: Page) {diff --git a/tests/e2e/snapshots/settings-page.snapshot.spec.ts b/tests/e2e/snapshots/settings-page.snapshot.spec.tsindex 3a343c7287..85805ae8bd 100644--- a/tests/e2e/snapshots/settings-page.snapshot.spec.ts+++ b/tests/e2e/snapshots/settings-page.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for UI pages.@@ -46,10 +47,7 @@ const SETTINGS_WITHOUT_CONSENT = {* @param showConsentModal - Whether to show the analytics consent modal*/async function setupMocks(page: Page, showConsentModal = false) {- // Pre-set localStorage to skip onboarding- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page, { showConsentModal });// Mock settings API - consent modal appears when user_consents_to_analytics is nullconst settingsResponse = showConsentModaldiff --git a/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts b/tests/e2e/snapshots/settings-secrets.snapshot.spec.tsindex 69245bd42d..f3f7335419 100644--- a/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts+++ b/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the Secrets Settings page (/settings/secrets).@@ -23,9 +24,7 @@ async function dismissConsentModal(page: Page) {}async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);// Keep conversations sidebar quiet (page.route wins for this cross-origin path// only; for same-origin MSW takes precedence and we rely on MSW data).await page.route("**/api/conversations/search**", async (route) => {diff --git a/tests/e2e/snapshots/settings-verification.snapshot.spec.ts b/tests/e2e/snapshots/settings-verification.snapshot.spec.tsindex 53848c6d17..73b828172d 100644--- a/tests/e2e/snapshots/settings-verification.snapshot.spec.ts+++ b/tests/e2e/snapshots/settings-verification.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for:@@ -23,9 +24,7 @@ async function dismissConsentModal(page: Page) {}async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);}test.describe("Settings β Verification & Condenser Visual Snapshots", () => {diff --git a/tests/e2e/snapshots/sidebar.snapshot.spec.ts b/tests/e2e/snapshots/sidebar.snapshot.spec.tsindex 395a59a12f..1f4d358c61 100644--- a/tests/e2e/snapshots/sidebar.snapshot.spec.ts+++ b/tests/e2e/snapshots/sidebar.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the sidebar / conversation panel.@@ -22,9 +23,7 @@ async function dismissConsentModal(page: Page) {}async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);// Suppress proxy errors for file API (home page workspace scan)await page.route("**/api/file/**", async (route) => {await route.fulfill({diff --git a/tests/e2e/snapshots/skills-page.snapshot.spec.ts b/tests/e2e/snapshots/skills-page.snapshot.spec.tsindex f38524c897..79997d5571 100644--- a/tests/e2e/snapshots/skills-page.snapshot.spec.ts+++ b/tests/e2e/snapshots/skills-page.snapshot.spec.ts@@ -1,4 +1,5 @@import { test, expect, Page } from "@playwright/test";+import { seedLocalStorage } from "./support/seed-local-storage";/*** Visual snapshot tests for the Skills page (/skills).@@ -78,9 +79,7 @@ async function dismissConsentModal(page: Page) {* Wire up the base routes every skills test needs.*/async function setupMocks(page: Page) {- await page.addInitScript(() => {- window.localStorage.setItem("openhands-onboarded", "true");- });+ await seedLocalStorage(page);}/**diff --git a/tests/e2e/snapshots/support/seed-local-storage.ts b/tests/e2e/snapshots/support/seed-local-storage.tsnew file mode 100644index 0000000000..9f72f68011--- /dev/null+++ b/tests/e2e/snapshots/support/seed-local-storage.ts@@ -0,0 +1,60 @@+import type { Page } from "@playwright/test";++export interface SeedLocalStorageOptions {+ /**+ * Remove openhands-onboarded instead of setting it.+ * Use in onboarding modal tests that need the modal to appear.+ */+ removeOnboarded?: boolean;+ /**+ * Skip suppressing the analytics consent modal.+ * Use in tests that specifically snapshot the consent modal UI.+ */+ showConsentModal?: boolean;+ /**+ * Additional [key, value] pairs to seed alongside the standard keys.+ */+ extra?: [string, string][];+}++/**+ * Seeds the standard localStorage keys required by snapshot tests via+ * `page.addInitScript`, so the values are present before any app code runs.+ *+ * Defaults (overridable via options):+ * - openhands-onboarded = "1" (suppresses onboarding modal)+ * - openhands-telemetry-consent = "denied" (suppresses analytics consent modal)+ */+export async function seedLocalStorage(+ page: Page,+ {+ removeOnboarded = false,+ showConsentModal = false,+ extra = [],+ }: SeedLocalStorageOptions = {},+) {+ await page.addInitScript(+ ({+ removeOnboarded,+ showConsentModal,+ extra,+ }: {+ removeOnboarded: boolean;+ showConsentModal: boolean;+ extra: [string, string][];+ }) => {+ if (removeOnboarded) {+ window.localStorage.removeItem("openhands-onboarded");+ } else {+ window.localStorage.setItem("openhands-onboarded", "1");+ }+ if (!showConsentModal) {+ window.localStorage.setItem("openhands-telemetry-consent", "denied");+ }+ for (const [key, value] of extra) {+ window.localStorage.setItem(key, value);+ }+ },+ { removeOnboarded, showConsentModal, extra },+ );+}