Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

test: add visual regression harness (#10670)

Kiran Gadhave59ef0a9b6939
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
index 57f19baada..353dc45416 100644
--- a/.github/workflows/playwright.yml
+++ b/.github/workflows/playwright.yml
@@ -35,7 +35,7 @@ jobs:
needs: changes
if: ${{ needs.changes.outputs.playwright == 'true' }}
timeout-minutes: 18 # 2024-01-18 avg: 5.0m max: 7.0m
- runs-on: ubuntu-latest
+ runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write # required for Doppler OIDC
@@ -108,7 +108,7 @@ jobs:
- name: 📥 Install Playwright Browsers
working-directory: ./frontend
if: steps.playwright-cache.outputs.cache-hit != 'true'
- run: pnpm playwright install chromium webkit --with-deps
+ run: pnpm playwright install chromium --with-deps
- name: 🎭 Run Playwright tests
working-directory: ./frontend
diff --git a/frontend/e2e-tests/README.md b/frontend/e2e-tests/README.md
index dce709022a..a9fdd17eec 100644
--- a/frontend/e2e-tests/README.md
+++ b/frontend/e2e-tests/README.md
@@ -18,3 +18,49 @@ Rebuild the frontend for tests that take screenshots to view the latest changes.
make fe
pnpm playwright test
```
+
+## Visual Regression Tests
+
+The focused visual suite protects shared theme, typography, chrome, and layout behavior. It uses the deterministic fixture in `py/visual_tokens.py`.
+
+### Run the visual suite
+
+If frontend files changed, rebuild the frontend from the repository root:
+
+```bash
+make fe
+```
+
+Run the focused suite from `frontend/`:
+
+```bash
+pnpm playwright test e2e-tests/visual-regression.spec.ts --project=chromium
+```
+
+### Update visual baselines
+
+CAUTION: Generate committed baselines on Ubuntu 24.04. Native macOS images use a different platform suffix.
+
+Run this command from `frontend/`:
+
+```bash
+pnpm playwright test e2e-tests/visual-regression.spec.ts --project=chromium --update-snapshots
+```
+
+Review every changed PNG. Then run the suite again without `--update-snapshots`.
+
+### Add a visual assertion
+
+Add a visual assertion for a shared semantic token, theme behavior, typography, product chrome, or responsive layout.
+
+If pixels do not express the requirement, use a DOM or behavior assertion. Do not add screenshots to every functional E2E test.
+
+If the existing fixture contains the affected semantic role, reuse it. Add a fixture state only for a named planned change.
+
+### Review a baseline change
+
+Use the Playwright report to compare the expected, actual, and difference images. Make sure that each changed region matches the stated product change.
+
+Never use `--update-snapshots` to silence an unexplained failure. Correct the fault or unstable state before a baseline update.
+
+Make sure that unrelated baselines stay unchanged. State the visual reason for each baseline change in the pull-request summary.
diff --git a/frontend/e2e-tests/py/visual_tokens.py b/frontend/e2e-tests/py/visual_tokens.py
new file mode 100644
index 0000000000..8992475dd6
--- /dev/null
+++ b/frontend/e2e-tests/py/visual_tokens.py
@@ -0,0 +1,81 @@
+# /// script
+# [tool.marimo.runtime]
+# auto_instantiate = true
+# ///
+import marimo
+
+__generated_with = "0.24.0"
+app = marimo.App()
+
+
+def _():
+ import marimo as mo
+
+ return (mo,)
+
+
+def _(mo):
+ focused_input = mo.ui.text(
+ value="Semantic input",
+ label="Focused input",
+ )
+
+ controls = mo.hstack(
+ [
+ focused_input,
+ mo.ui.checkbox(value=True, label="Checkbox"),
+ mo.ui.dropdown(
+ options=["Alpha", "Beta"],
+ value="Alpha",
+ label="Dropdown",
+ ),
+ ],
+ justify="start",
+ wrap=True,
+ gap=1,
+ )
+
+ buttons = mo.hstack(
+ [
+ mo.ui.button(label="Neutral", kind="neutral"),
+ mo.ui.button(label="Success", kind="success"),
+ mo.ui.button(label="Warning", kind="warn"),
+ mo.ui.button(label="Danger", kind="danger"),
+ ],
+ justify="start",
+ wrap=True,
+ gap=1,
+ )
+
+ callouts = mo.hstack(
+ [
+ mo.md("Neutral state").callout(kind="neutral", title="Neutral"),
+ mo.md("Information state").callout(kind="info", title="Info"),
+ mo.md("Success state").callout(kind="success", title="Success"),
+ mo.md("Warning state").callout(kind="warn", title="Warning"),
+ mo.md("Danger state").callout(kind="danger", title="Danger"),
+ ],
+ justify="start",
+ wrap=True,
+ gap=0.5,
+ )
+
+ mo.vstack(
+ [
+ mo.md(
+ "# Semantic token fixture\n"
+ "Body text with a [link](#) and `inline code`."
+ ),
+ controls,
+ buttons,
+ callouts,
+ ],
+ gap=1,
+ )
+ return (focused_input,)
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/frontend/e2e-tests/visual-regression.spec.ts b/frontend/e2e-tests/visual-regression.spec.ts
new file mode 100644
index 0000000000..5f50478a1f
--- /dev/null
+++ b/frontend/e2e-tests/visual-regression.spec.ts
@@ -0,0 +1,130 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { expect, type Page, test } from "@playwright/test";
+import { type ApplicationNames, getAppUrl } from "../playwright.config";
+import { maybeRestartKernel } from "./helper";
+import { waitForMarimoApp } from "./test-utils";
+
+type VisualTheme = "light" | "dark";
+type VisualView = "edit" | "read";
+
+interface VisualCase {
+ readonly app: ApplicationNames;
+ readonly name: `${VisualView}-${VisualTheme}`;
+ readonly theme: VisualTheme;
+ readonly view: VisualView;
+}
+
+const DESKTOP_VIEWPORT = { width: 1280, height: 720 } as const;
+const NARROW_VIEWPORT = { width: 390, height: 720 } as const;
+
+const VISUAL_CASES = [
+ {
+ app: "visual_tokens.py",
+ name: "edit-light",
+ theme: "light",
+ view: "edit",
+ },
+ {
+ app: "visual_tokens.py",
+ name: "edit-dark",
+ theme: "dark",
+ view: "edit",
+ },
+ {
+ app: "visual_tokens.py//run",
+ name: "read-light",
+ theme: "light",
+ view: "read",
+ },
+ {
+ app: "visual_tokens.py//run",
+ name: "read-dark",
+ theme: "dark",
+ view: "read",
+ },
+] satisfies readonly VisualCase[];
+
+function getThemedUrl(app: ApplicationNames, theme: VisualTheme): string {
+ const url = new URL(getAppUrl(app));
+ url.searchParams.set("theme", theme);
+ return url.toString();
+}
+
+async function prepareVisualCase(
+ page: Page,
+ visualCase: VisualCase,
+): Promise<void> {
+ await page.setViewportSize(DESKTOP_VIEWPORT);
+ await page.goto(getThemedUrl(visualCase.app, visualCase.theme));
+
+ if (visualCase.view === "edit") {
+ await maybeRestartKernel(page);
+ } else {
+ await waitForMarimoApp(page);
+ }
+
+ const heading = page.getByRole("heading", {
+ name: "Semantic token fixture",
+ });
+ await expect(heading).toBeVisible();
+ await heading.scrollIntoViewIfNeeded();
+
+ await expect(page.locator("body")).toHaveAttribute(
+ "data-theme",
+ visualCase.theme,
+ );
+ await expect(page.locator("html")).toHaveCSS(
+ "color-scheme",
+ visualCase.theme,
+ );
+ await expect(page.locator("#App")).toHaveAttribute(
+ "data-connection-state",
+ "OPEN",
+ );
+
+ const saveButton = page.locator("#save-button");
+ if (visualCase.view === "edit") {
+ await expect(saveButton).toBeVisible();
+ } else {
+ await expect(saveButton).toHaveCount(0);
+ }
+
+ await page.evaluate(async () => {
+ await document.fonts.ready;
+ });
+ await expect(page.locator(".marimo-output-loading")).toHaveCount(0);
+
+ const focusedInput = page.getByTestId("marimo-plugin-text-input");
+ await focusedInput.focus();
+ await expect(focusedInput).toBeFocused();
+}
+
+for (const visualCase of VISUAL_CASES) {
+ test(`${visualCase.view} view uses the ${visualCase.theme} theme`, async ({
+ page,
+ }) => {
+ await prepareVisualCase(page, visualCase);
+
+ const volatileMeters = page.locator(
+ "[data-testid='memory-usage-bar'], " +
+ "[data-testid='cpu-bar'], " +
+ "[data-testid='gpu-bar']",
+ );
+ const mask = [volatileMeters];
+
+ await expect(page).toHaveScreenshot(
+ `${visualCase.name}-1280x720.png`,
+ { mask },
+ );
+
+ await page.setViewportSize(NARROW_VIEWPORT);
+ await page
+ .getByRole("heading", { name: "Semantic token fixture" })
+ .scrollIntoViewIfNeeded();
+ await expect(page).toHaveScreenshot(
+ `${visualCase.name}-390x720.png`,
+ { mask },
+ );
+ });
+}
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.png
new file mode 100644
index 0000000000..a361968bf8
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.png
new file mode 100644
index 0000000000..4774a9b92b
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.png
new file mode 100644
index 0000000000..00bb1313da
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.png
new file mode 100644
index 0000000000..d9a475c5d4
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.png
new file mode 100644
index 0000000000..6f5511882c
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.png
new file mode 100644
index 0000000000..8fe26840e0
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.png
new file mode 100644
index 0000000000..4a678570fd
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.png differ
diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.png
new file mode 100644
index 0000000000..3b3e0801fc
Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.png differ
diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts
index 0db92c843a..ef5f022f2c 100644
--- a/frontend/playwright.config.ts
+++ b/frontend/playwright.config.ts
@@ -45,6 +45,7 @@ const appToOptions = {
"stdin.py": { command: "edit" },
"layout_grid_with_sidebar.py": { command: "edit" },
"slides.py": { command: "edit" },
+ "visual_tokens.py": { command: "edit" },
// Custom server for shutdown
"shutdown.py": { command: "edit", port: port() },
// Run
@@ -53,6 +54,7 @@ const appToOptions = {
"layout_grid.py//run": { port: port(), command: "run" },
"layout_grid_max_width.py//run": { port: port(), command: "run" },
"layout_grid_with_sidebar.py//run": { port: port(), command: "run" },
+ "visual_tokens.py//run": { port: port(), command: "run" },
"output.py//run": {
port: port(),
command: "run",
diff --git a/frontend/src/css/globals.css b/frontend/src/css/globals.css
index deea37a319..c11cbf44b3 100644
--- a/frontend/src/css/globals.css
+++ b/frontend/src/css/globals.css
@@ -117,6 +117,7 @@
--cm-search-match-selected-background-dark: #6199ff88;
}
+ :root:has(body.dark),
.dark,
.marimo:is(.dark *) {
color-scheme: dark;