A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
test: add visual regression harness (#10670)
Kiran Gadhave59ef0a9b6939
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.ymlindex 57f19baada..353dc45416 100644--- a/.github/workflows/playwright.yml+++ b/.github/workflows/playwright.yml@@ -35,7 +35,7 @@ jobs:needs: changesif: ${{ needs.changes.outputs.playwright == 'true' }}timeout-minutes: 18 # 2024-01-18 avg: 5.0m max: 7.0m- runs-on: ubuntu-latest+ runs-on: ubuntu-24.04permissions:contents: readid-token: write # required for Doppler OIDC@@ -108,7 +108,7 @@ jobs:- name: 📥 Install Playwright Browsersworking-directory: ./frontendif: steps.playwright-cache.outputs.cache-hit != 'true'- run: pnpm playwright install chromium webkit --with-deps+ run: pnpm playwright install chromium --with-deps- name: 🎭 Run Playwright testsworking-directory: ./frontenddiff --git a/frontend/e2e-tests/README.md b/frontend/e2e-tests/README.mdindex dce709022a..a9fdd17eec 100644--- a/frontend/e2e-tests/README.md+++ b/frontend/e2e-tests/README.md@@ -18,3 +18,49 @@ Rebuild the frontend for tests that take screenshots to view the latest changes.make fepnpm playwright test```++## Visual Regression Tests++The focused visual suite protects shared theme, typography, chrome, and layout behavior. It uses the deterministic fixture in `py/visual_tokens.py`.++### Run the visual suite++If frontend files changed, rebuild the frontend from the repository root:++```bash+make fe+```++Run the focused suite from `frontend/`:++```bash+pnpm playwright test e2e-tests/visual-regression.spec.ts --project=chromium+```++### Update visual baselines++CAUTION: Generate committed baselines on Ubuntu 24.04. Native macOS images use a different platform suffix.++Run this command from `frontend/`:++```bash+pnpm playwright test e2e-tests/visual-regression.spec.ts --project=chromium --update-snapshots+```++Review every changed PNG. Then run the suite again without `--update-snapshots`.++### Add a visual assertion++Add a visual assertion for a shared semantic token, theme behavior, typography, product chrome, or responsive layout.++If pixels do not express the requirement, use a DOM or behavior assertion. Do not add screenshots to every functional E2E test.++If the existing fixture contains the affected semantic role, reuse it. Add a fixture state only for a named planned change.++### Review a baseline change++Use the Playwright report to compare the expected, actual, and difference images. Make sure that each changed region matches the stated product change.++Never use `--update-snapshots` to silence an unexplained failure. Correct the fault or unstable state before a baseline update.++Make sure that unrelated baselines stay unchanged. State the visual reason for each baseline change in the pull-request summary.diff --git a/frontend/e2e-tests/py/visual_tokens.py b/frontend/e2e-tests/py/visual_tokens.pynew file mode 100644index 0000000000..8992475dd6--- /dev/null+++ b/frontend/e2e-tests/py/visual_tokens.py@@ -0,0 +1,81 @@+# /// script+# [tool.marimo.runtime]+# auto_instantiate = true+# ///+import marimo++__generated_with = "0.24.0"+app = marimo.App()+++def _():+ import marimo as mo++ return (mo,)+++def _(mo):+ focused_input = mo.ui.text(+ value="Semantic input",+ label="Focused input",+ )++ controls = mo.hstack(+ [+ focused_input,+ mo.ui.checkbox(value=True, label="Checkbox"),+ mo.ui.dropdown(+ options=["Alpha", "Beta"],+ value="Alpha",+ label="Dropdown",+ ),+ ],+ justify="start",+ wrap=True,+ gap=1,+ )++ buttons = mo.hstack(+ [+ mo.ui.button(label="Neutral", kind="neutral"),+ mo.ui.button(label="Success", kind="success"),+ mo.ui.button(label="Warning", kind="warn"),+ mo.ui.button(label="Danger", kind="danger"),+ ],+ justify="start",+ wrap=True,+ gap=1,+ )++ callouts = mo.hstack(+ [+ mo.md("Neutral state").callout(kind="neutral", title="Neutral"),+ mo.md("Information state").callout(kind="info", title="Info"),+ mo.md("Success state").callout(kind="success", title="Success"),+ mo.md("Warning state").callout(kind="warn", title="Warning"),+ mo.md("Danger state").callout(kind="danger", title="Danger"),+ ],+ justify="start",+ wrap=True,+ gap=0.5,+ )++ mo.vstack(+ [+ mo.md(+ "# Semantic token fixture\n"+ "Body text with a [link](#) and `inline code`."+ ),+ controls,+ buttons,+ callouts,+ ],+ gap=1,+ )+ return (focused_input,)+++if __name__ == "__main__":+ app.run()diff --git a/frontend/e2e-tests/visual-regression.spec.ts b/frontend/e2e-tests/visual-regression.spec.tsnew file mode 100644index 0000000000..5f50478a1f--- /dev/null+++ b/frontend/e2e-tests/visual-regression.spec.ts@@ -0,0 +1,130 @@+/* Copyright 2026 Marimo. All rights reserved. */++import { expect, type Page, test } from "@playwright/test";+import { type ApplicationNames, getAppUrl } from "../playwright.config";+import { maybeRestartKernel } from "./helper";+import { waitForMarimoApp } from "./test-utils";++type VisualTheme = "light" | "dark";+type VisualView = "edit" | "read";++interface VisualCase {+ readonly app: ApplicationNames;+ readonly name: `${VisualView}-${VisualTheme}`;+ readonly theme: VisualTheme;+ readonly view: VisualView;+}++const DESKTOP_VIEWPORT = { width: 1280, height: 720 } as const;+const NARROW_VIEWPORT = { width: 390, height: 720 } as const;++const VISUAL_CASES = [+ {+ app: "visual_tokens.py",+ name: "edit-light",+ theme: "light",+ view: "edit",+ },+ {+ app: "visual_tokens.py",+ name: "edit-dark",+ theme: "dark",+ view: "edit",+ },+ {+ app: "visual_tokens.py//run",+ name: "read-light",+ theme: "light",+ view: "read",+ },+ {+ app: "visual_tokens.py//run",+ name: "read-dark",+ theme: "dark",+ view: "read",+ },+] satisfies readonly VisualCase[];++function getThemedUrl(app: ApplicationNames, theme: VisualTheme): string {+ const url = new URL(getAppUrl(app));+ url.searchParams.set("theme", theme);+ return url.toString();+}++async function prepareVisualCase(+ page: Page,+ visualCase: VisualCase,+): Promise<void> {+ await page.setViewportSize(DESKTOP_VIEWPORT);+ await page.goto(getThemedUrl(visualCase.app, visualCase.theme));++ if (visualCase.view === "edit") {+ await maybeRestartKernel(page);+ } else {+ await waitForMarimoApp(page);+ }++ const heading = page.getByRole("heading", {+ name: "Semantic token fixture",+ });+ await expect(heading).toBeVisible();+ await heading.scrollIntoViewIfNeeded();++ await expect(page.locator("body")).toHaveAttribute(+ "data-theme",+ visualCase.theme,+ );+ await expect(page.locator("html")).toHaveCSS(+ "color-scheme",+ visualCase.theme,+ );+ await expect(page.locator("#App")).toHaveAttribute(+ "data-connection-state",+ "OPEN",+ );++ const saveButton = page.locator("#save-button");+ if (visualCase.view === "edit") {+ await expect(saveButton).toBeVisible();+ } else {+ await expect(saveButton).toHaveCount(0);+ }++ await page.evaluate(async () => {+ await document.fonts.ready;+ });+ await expect(page.locator(".marimo-output-loading")).toHaveCount(0);++ const focusedInput = page.getByTestId("marimo-plugin-text-input");+ await focusedInput.focus();+ await expect(focusedInput).toBeFocused();+}++for (const visualCase of VISUAL_CASES) {+ test(`${visualCase.view} view uses the ${visualCase.theme} theme`, async ({+ page,+ }) => {+ await prepareVisualCase(page, visualCase);++ const volatileMeters = page.locator(+ "[data-testid='memory-usage-bar'], " ++ "[data-testid='cpu-bar'], " ++ "[data-testid='gpu-bar']",+ );+ const mask = [volatileMeters];++ await expect(page).toHaveScreenshot(+ `${visualCase.name}-1280x720.png`,+ { mask },+ );++ await page.setViewportSize(NARROW_VIEWPORT);+ await page+ .getByRole("heading", { name: "Semantic token fixture" })+ .scrollIntoViewIfNeeded();+ await expect(page).toHaveScreenshot(+ `${visualCase.name}-390x720.png`,+ { mask },+ );+ });+}diff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.pngnew file mode 100644index 0000000000..a361968bf8Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-1280x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.pngnew file mode 100644index 0000000000..4774a9b92bBinary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-dark-390x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.pngnew file mode 100644index 0000000000..00bb1313daBinary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-1280x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.pngnew file mode 100644index 0000000000..d9a475c5d4Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/edit-light-390x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.pngnew file mode 100644index 0000000000..6f5511882cBinary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-1280x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.pngnew file mode 100644index 0000000000..8fe26840e0Binary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-dark-390x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.pngnew file mode 100644index 0000000000..4a678570fdBinary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-1280x720-chromium-linux.png differdiff --git a/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.png b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.pngnew file mode 100644index 0000000000..3b3e0801fcBinary files /dev/null and b/frontend/e2e-tests/visual-regression.spec.ts-snapshots/read-light-390x720-chromium-linux.png differdiff --git a/frontend/playwright.config.ts b/frontend/playwright.config.tsindex 0db92c843a..ef5f022f2c 100644--- a/frontend/playwright.config.ts+++ b/frontend/playwright.config.ts@@ -45,6 +45,7 @@ const appToOptions = {"stdin.py": { command: "edit" },"layout_grid_with_sidebar.py": { command: "edit" },"slides.py": { command: "edit" },+ "visual_tokens.py": { command: "edit" },// Custom server for shutdown"shutdown.py": { command: "edit", port: port() },// Run@@ -53,6 +54,7 @@ const appToOptions = {"layout_grid.py//run": { port: port(), command: "run" },"layout_grid_max_width.py//run": { port: port(), command: "run" },"layout_grid_with_sidebar.py//run": { port: port(), command: "run" },+ "visual_tokens.py//run": { port: port(), command: "run" },"output.py//run": {port: port(),command: "run",diff --git a/frontend/src/css/globals.css b/frontend/src/css/globals.cssindex deea37a319..c11cbf44b3 100644--- a/frontend/src/css/globals.css+++ b/frontend/src/css/globals.css@@ -117,6 +117,7 @@--cm-search-match-selected-background-dark: #6199ff88;}+ :root:has(body.dark),.dark,.marimo:is(.dark *) {color-scheme: dark;