Skip to content

πŸ™Œ OpenHands: AI-Driven Development

perf(snapshots): skip retries on comparison pass, suppress consent modal (#505)

Rohit Malhotraacb04fd9ba42
diff --git a/package.json b/package.json
index 60f97ef392..e2a6ee8a18 100644
--- a/package.json
+++ b/package.json
@@ -82,7 +82,7 @@
"test": "npm run make-i18n && vitest run",
"test:e2e": "playwright test --pass-with-no-tests",
"test:e2e:live": "node --env-file-if-exists=.env tests/e2e/live/scripts/run-live-e2e.mjs",
- "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium",
+ "test:e2e:snapshots": "playwright test tests/e2e/snapshots --project=chromium --retries=0",
"test:e2e:snapshots:update": "playwright test tests/e2e/snapshots --project=chromium --update-snapshots",
"test:coverage": "npm run make-i18n && vitest run --coverage",
"dev_wsl": "VITE_WATCH_USE_POLLING=true vite",
diff --git a/tests/e2e/snapshots/automations.snapshot.spec.ts b/tests/e2e/snapshots/automations.snapshot.spec.ts
index 7e206af0a7..698d5c2ead 100644
--- a/tests/e2e/snapshots/automations.snapshot.spec.ts
+++ b/tests/e2e/snapshots/automations.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the Automations pages.
@@ -39,9 +40,7 @@ async function dismissConsentModal(page: Page) {
* here with INLINE_AUTOMATIONS so all tests have a consistent baseline.
*/
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
await page.route("**/api/conversations/search**", async (route) => {
await route.fulfill({
diff --git a/tests/e2e/snapshots/backends-extended.snapshot.spec.ts b/tests/e2e/snapshots/backends-extended.snapshot.spec.ts
index 46e3fc1496..cffdb729fb 100644
--- a/tests/e2e/snapshots/backends-extended.snapshot.spec.ts
+++ b/tests/e2e/snapshots/backends-extended.snapshot.spec.ts
@@ -1,5 +1,6 @@
import { test, expect, type Page } from "@playwright/test";
import type { Backend } from "../../../src/api/backend-registry/types";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Extended visual snapshot tests for the backend management UI.
@@ -77,21 +78,16 @@ async function setupPage(
activeBackendId,
}: { backends?: Backend[]; activeBackendId?: string } = {},
) {
- await page.addInitScript(
- ({ backendsJson, activeJson }: { backendsJson: string; activeJson: string | null }) => {
- window.localStorage.setItem("openhands-onboarded", "true");
- window.localStorage.setItem("openhands-backends", backendsJson);
- if (activeJson) {
- window.localStorage.setItem("openhands-active-backend", activeJson);
- }
- },
- {
- backendsJson: JSON.stringify(backends),
- activeJson: activeBackendId
- ? JSON.stringify({ backendId: activeBackendId, orgId: null })
- : null,
- },
- );
+ const extra: [string, string][] = [
+ ["openhands-backends", JSON.stringify(backends)],
+ ];
+ if (activeBackendId) {
+ extra.push([
+ "openhands-active-backend",
+ JSON.stringify({ backendId: activeBackendId, orgId: null }),
+ ]);
+ }
+ await seedLocalStorage(page, { extra });
// Prevent workspace-scan 404s in the sidebar from cluttering timing.
await page.route("**/api/file/**", (route) =>
diff --git a/tests/e2e/snapshots/backends.snapshot.spec.ts b/tests/e2e/snapshots/backends.snapshot.spec.ts
index 5fb6819ca8..2b8dc063c3 100644
--- a/tests/e2e/snapshots/backends.snapshot.spec.ts
+++ b/tests/e2e/snapshots/backends.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the backend management UI.
@@ -27,9 +28,7 @@ async function dismissConsentModal(page: Page) {
}
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
// Suppress file-API proxy errors emitted when the home page scans the
// workspace directory (same suppression used in sidebar.snapshot.spec.ts).
diff --git a/tests/e2e/snapshots/changes-tab.snapshot.spec.ts b/tests/e2e/snapshots/changes-tab.snapshot.spec.ts
index 57bb98e062..eda595d221 100644
--- a/tests/e2e/snapshots/changes-tab.snapshot.spec.ts
+++ b/tests/e2e/snapshots/changes-tab.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the Changes (diff viewer) UI.
@@ -52,13 +53,9 @@ const CONVERSATION_STATE_VALUE = JSON.stringify({
* Skip onboarding and pre-enable the diff view for conversation 1.
*/
async function setupMocks(page: Page) {
- await page.addInitScript(
- ([key, value]) => {
- window.localStorage.setItem("openhands-onboarded", "true");
- window.localStorage.setItem(key, value);
- },
- [CONVERSATION_STATE_KEY, CONVERSATION_STATE_VALUE] as [string, string],
- );
+ await seedLocalStorage(page, {
+ extra: [[CONVERSATION_STATE_KEY, CONVERSATION_STATE_VALUE]],
+ });
// Stub WebSocket so the conversation page doesn't hang waiting for a real
// socket connection. Copied from collapsible-thinking.snapshot.spec.ts.
diff --git a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
index ce76c03ef4..f4ab61f3d0 100644
--- a/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
+++ b/tests/e2e/snapshots/collapsible-thinking.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the CollapsibleThinking component.
@@ -173,10 +174,7 @@ async function injectEvents(page: Page, events: unknown[]) {
* Uses mock conversation "1" which exists in the MSW handlers.
*/
async function navigateToConversation(page: Page, events: unknown[]) {
- // Skip onboarding
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
await page.route("**/api/bash/execute_bash_command", async (route) => {
await route.fulfill({
diff --git a/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts b/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts
index 0e5fd6c2b7..5fcf296f23 100644
--- a/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts
+++ b/tests/e2e/snapshots/docker-workspace-browser.snapshot.spec.ts
@@ -1,13 +1,14 @@
import { expect, test } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
test("captures Docker /projects workspace browser state", async ({ page }) => {
test.setTimeout(60_000);
- await page.addInitScript(() => {
- window.localStorage.setItem("analytics-consent", "true");
- window.localStorage.setItem("openhands-telemetry-consent", "denied");
- window.localStorage.setItem("openhands-telemetry-first-use", "true");
- window.localStorage.setItem("openhands-onboarded", "1");
+ await seedLocalStorage(page, {
+ extra: [
+ ["analytics-consent", "true"],
+ ["openhands-telemetry-first-use", "true"],
+ ],
});
await page.goto("/conversations", { waitUntil: "domcontentloaded" });
diff --git a/tests/e2e/snapshots/mcp-page.snapshot.spec.ts b/tests/e2e/snapshots/mcp-page.snapshot.spec.ts
index 763704b6ea..bc39a23821 100644
--- a/tests/e2e/snapshots/mcp-page.snapshot.spec.ts
+++ b/tests/e2e/snapshots/mcp-page.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the MCP page (/mcp).
@@ -32,9 +33,7 @@ async function dismissConsentModal(page: Page) {
* instead of trying to suppress it here.
*/
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
await page.route("**/api/conversations/search**", async (route) => {
await route.fulfill({
diff --git a/tests/e2e/snapshots/onboarding.snapshot.spec.ts b/tests/e2e/snapshots/onboarding.snapshot.spec.ts
index 762ef2e20c..c897e67009 100644
--- a/tests/e2e/snapshots/onboarding.snapshot.spec.ts
+++ b/tests/e2e/snapshots/onboarding.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the 4-step onboarding modal.
@@ -26,10 +27,9 @@ import { test, expect, Page } from "@playwright/test";
test.describe.configure({ mode: "serial" });
async function setupMocks(page: Page) {
- // Intentionally do NOT set openhands-onboarded so the modal appears
- await page.addInitScript(() => {
- window.localStorage.removeItem("openhands-onboarded");
- });
+ // removeOnboarded: true ensures the onboarding modal appears.
+ // Analytics consent modal is suppressed (separate concern).
+ await seedLocalStorage(page, { removeOnboarded: true });
}
async function dismissConsentModal(page: Page) {
diff --git a/tests/e2e/snapshots/settings-page.snapshot.spec.ts b/tests/e2e/snapshots/settings-page.snapshot.spec.ts
index 3a343c7287..85805ae8bd 100644
--- a/tests/e2e/snapshots/settings-page.snapshot.spec.ts
+++ b/tests/e2e/snapshots/settings-page.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for UI pages.
@@ -46,10 +47,7 @@ const SETTINGS_WITHOUT_CONSENT = {
* @param showConsentModal - Whether to show the analytics consent modal
*/
async function setupMocks(page: Page, showConsentModal = false) {
- // Pre-set localStorage to skip onboarding
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page, { showConsentModal });
// Mock settings API - consent modal appears when user_consents_to_analytics is null
const settingsResponse = showConsentModal
diff --git a/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts b/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts
index 69245bd42d..f3f7335419 100644
--- a/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts
+++ b/tests/e2e/snapshots/settings-secrets.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the Secrets Settings page (/settings/secrets).
@@ -23,9 +24,7 @@ async function dismissConsentModal(page: Page) {
}
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
// Keep conversations sidebar quiet (page.route wins for this cross-origin path
// only; for same-origin MSW takes precedence and we rely on MSW data).
await page.route("**/api/conversations/search**", async (route) => {
diff --git a/tests/e2e/snapshots/settings-verification.snapshot.spec.ts b/tests/e2e/snapshots/settings-verification.snapshot.spec.ts
index 53848c6d17..73b828172d 100644
--- a/tests/e2e/snapshots/settings-verification.snapshot.spec.ts
+++ b/tests/e2e/snapshots/settings-verification.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for:
@@ -23,9 +24,7 @@ async function dismissConsentModal(page: Page) {
}
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
}
test.describe("Settings – Verification & Condenser Visual Snapshots", () => {
diff --git a/tests/e2e/snapshots/sidebar.snapshot.spec.ts b/tests/e2e/snapshots/sidebar.snapshot.spec.ts
index 395a59a12f..1f4d358c61 100644
--- a/tests/e2e/snapshots/sidebar.snapshot.spec.ts
+++ b/tests/e2e/snapshots/sidebar.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the sidebar / conversation panel.
@@ -22,9 +23,7 @@ async function dismissConsentModal(page: Page) {
}
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
// Suppress proxy errors for file API (home page workspace scan)
await page.route("**/api/file/**", async (route) => {
await route.fulfill({
diff --git a/tests/e2e/snapshots/skills-page.snapshot.spec.ts b/tests/e2e/snapshots/skills-page.snapshot.spec.ts
index f38524c897..79997d5571 100644
--- a/tests/e2e/snapshots/skills-page.snapshot.spec.ts
+++ b/tests/e2e/snapshots/skills-page.snapshot.spec.ts
@@ -1,4 +1,5 @@
import { test, expect, Page } from "@playwright/test";
+import { seedLocalStorage } from "./support/seed-local-storage";
/**
* Visual snapshot tests for the Skills page (/skills).
@@ -78,9 +79,7 @@ async function dismissConsentModal(page: Page) {
* Wire up the base routes every skills test needs.
*/
async function setupMocks(page: Page) {
- await page.addInitScript(() => {
- window.localStorage.setItem("openhands-onboarded", "true");
- });
+ await seedLocalStorage(page);
}
/**
diff --git a/tests/e2e/snapshots/support/seed-local-storage.ts b/tests/e2e/snapshots/support/seed-local-storage.ts
new file mode 100644
index 0000000000..9f72f68011
--- /dev/null
+++ b/tests/e2e/snapshots/support/seed-local-storage.ts
@@ -0,0 +1,60 @@
+import type { Page } from "@playwright/test";
+
+export interface SeedLocalStorageOptions {
+ /**
+ * Remove openhands-onboarded instead of setting it.
+ * Use in onboarding modal tests that need the modal to appear.
+ */
+ removeOnboarded?: boolean;
+ /**
+ * Skip suppressing the analytics consent modal.
+ * Use in tests that specifically snapshot the consent modal UI.
+ */
+ showConsentModal?: boolean;
+ /**
+ * Additional [key, value] pairs to seed alongside the standard keys.
+ */
+ extra?: [string, string][];
+}
+
+/**
+ * Seeds the standard localStorage keys required by snapshot tests via
+ * `page.addInitScript`, so the values are present before any app code runs.
+ *
+ * Defaults (overridable via options):
+ * - openhands-onboarded = "1" (suppresses onboarding modal)
+ * - openhands-telemetry-consent = "denied" (suppresses analytics consent modal)
+ */
+export async function seedLocalStorage(
+ page: Page,
+ {
+ removeOnboarded = false,
+ showConsentModal = false,
+ extra = [],
+ }: SeedLocalStorageOptions = {},
+) {
+ await page.addInitScript(
+ ({
+ removeOnboarded,
+ showConsentModal,
+ extra,
+ }: {
+ removeOnboarded: boolean;
+ showConsentModal: boolean;
+ extra: [string, string][];
+ }) => {
+ if (removeOnboarded) {
+ window.localStorage.removeItem("openhands-onboarded");
+ } else {
+ window.localStorage.setItem("openhands-onboarded", "1");
+ }
+ if (!showConsentModal) {
+ window.localStorage.setItem("openhands-telemetry-consent", "denied");
+ }
+ for (const [key, value] of extra) {
+ window.localStorage.setItem(key, value);
+ }
+ },
+ { removeOnboarded, showConsentModal, extra },
+ );
+}