Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
ci: replace Lost Pixel with self-hosted visual regression (#6086)
Oleg Isonendb2d092268e7
diff --git a/.github/workflows/chromatic.yml b/.github/workflows/chromatic.ymldeleted file mode 100644index 8a050895ee..0000000000--- a/.github/workflows/chromatic.yml+++ /dev/null@@ -1,53 +0,0 @@-name: Chromatic--on:- push:- branches:- - main- pull_request:- pull_request_target:- types: [labeled]--# cancel in-progress runs on new commits to same PR (gitub.event.number)-concurrency:- group: ${{ github.workflow }}-${{ github.event.number || github.sha }}- cancel-in-progress: true--permissions:- contents: read # to fetch code (actions/checkout)--jobs:- chromatic:- # Run on push, regular PR (for repo branches), or labeled PR with safe-to-deploy (for forks)- if: |- github.event_name == 'push' ||- github.event_name == 'pull_request' ||- (github.event_name == 'pull_request_target' && github.event.label.name == 'safe-to-deploy')-- timeout-minutes: 20-- runs-on: ubuntu-latest-- environment:- name: development-- steps:- - uses: actions/checkout@v5- with:- fetch-depth: 2 # we need to fetch at least parent commit to satisfy Chromatic- ref: ${{ github.event.pull_request.head.sha || github.sha }} # HEAD commit instead of merge commit- allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' && github.event.label.name == 'safe-to-deploy' }}-- # Storybook with submodules- - uses: ./.github/actions/submodules-checkout- with:- submodules-ssh-key: ${{ secrets.PRIVATE_GITHUB_DEPLOY_TOKEN }}-- - uses: ./.github/actions/ci-setup-- - name: Chromatic- id: chromatic- uses: chromaui/action@v16- with:- projectToken: bea8dc1981d4- buildScriptName: storybook:builddiff --git a/.github/workflows/vis-reg-tests.yml b/.github/workflows/vis-reg-tests.ymldeleted file mode 100644index 11e28346c6..0000000000--- a/.github/workflows/vis-reg-tests.yml+++ /dev/null@@ -1,50 +0,0 @@-name: Visual Regression Tests--on:- push:- branches:- - main- pull_request:- pull_request_target:- types: [labeled]--# cancel in-progress runs on new commits to same PR (gitub.event.number)-concurrency:- group: ${{ github.workflow }}-${{ github.event.number || github.sha }}- cancel-in-progress: true--permissions:- contents: read # to fetch code (actions/checkout)--jobs:- lost-pixel:- # Run on push, regular PR (for repo branches), or labeled PR with safe-to-deploy (for forks)- # Release branches are assembled from already-reviewed changes, and visual- # diffs should not block publishing the release branch itself.- if: |- (- github.event_name == 'push' ||- github.event_name == 'pull_request' ||- (github.event_name == 'pull_request_target' && github.event.label.name == 'safe-to-deploy')- ) &&- !startsWith(github.head_ref || github.ref_name, 'release-')-- timeout-minutes: 20- runs-on: ubuntu-latest- env:- DATABASE_URL: postgres://- AUTH_SECRET: test- steps:- - uses: actions/checkout@v4- with:- ref: ${{ github.event.pull_request.head.sha || github.sha }} # HEAD commit instead of merge commit- allow-unsafe-pr-checkout: ${{ github.event_name == 'pull_request_target' && github.event.label.name == 'safe-to-deploy' }}-- - uses: ./.github/actions/ci-setup-- - run: VISUAL_TESTING=true pnpm storybook:build-- - name: Lost Pixel- uses: lost-pixel/[email protected]- env:- LOST_PIXEL_API_KEY: 8b76db6c-b9f0-46d1-982f-70900a02690adiff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.ymlnew file mode 100644index 0000000000..6f169794ff--- /dev/null+++ b/.github/workflows/visual-regression.yml@@ -0,0 +1,152 @@+name: Visual regression++on:+ push:+ branches:+ - main+ pull_request:+ types: [opened, synchronize, reopened, labeled]+ pull_request_target:+ types: [labeled, synchronize]++concurrency:+ # A synchronize action emits both pull_request and pull_request_target. Keep+ # their runs separate so approval cleanup cannot cancel the visual check.+ group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.sha }}+ cancel-in-progress: true++permissions:+ contents: read++jobs:+ remove-approval:+ if: github.event_name == 'pull_request_target' && github.event.action == 'synchronize'+ permissions:+ contents: read+ pull-requests: write+ runs-on: ubuntu-latest+ steps:+ - name: Remove visual change approval+ uses: actions/github-script@v8+ with:+ script: |+ const labels = context.payload.pull_request.labels.map(({ name }) => name);+ if (labels.includes('visual-change-approved')) {+ await github.rest.issues.removeLabel({+ owner: context.repo.owner,+ repo: context.repo.repo,+ issue_number: context.issue.number,+ name: 'visual-change-approved',+ });+ }++ visual-regression:+ if: |+ github.event_name == 'push' ||+ (+ github.event_name == 'pull_request' &&+ github.event.pull_request.head.repo.full_name == github.repository &&+ (+ github.event.action != 'labeled' ||+ github.event.label.name == 'visual-change-approved'+ )+ ) ||+ (+ github.event_name == 'pull_request_target' &&+ github.event.action == 'labeled' &&+ (+ github.event.label.name == 'safe-to-deploy' ||+ (+ github.event.label.name == 'visual-change-approved' &&+ github.event.pull_request.head.repo.full_name != github.repository &&+ contains(github.event.pull_request.labels.*.name, 'safe-to-deploy')+ )+ )+ )+ timeout-minutes: 30+ runs-on: ubuntu-latest+ environment:+ name: development+ steps:+ - uses: actions/checkout@v5+ with:+ fetch-depth: 0+ ref: ${{ github.event.pull_request.head.sha || github.sha }}++ - uses: ./.github/actions/submodules-checkout+ with:+ submodules-ssh-key: ${{ secrets.PRIVATE_GITHUB_DEPLOY_TOKEN }}++ - uses: ./.github/actions/ci-setup++ - name: Cache pinned visual browser+ uses: actions/cache@v5+ with:+ path: ~/.cache/ms-playwright+ key: visual-playwright-${{ runner.os }}-${{ hashFiles('packages/sdk-components-animation/package.json', 'pnpm-lock.yaml') }}++ - name: Install pinned visual browser+ run: |+ pnpm --filter @webstudio-is/sdk-components-animation exec playwright install chromium+ playwright_browser_path="$(pnpm --filter @webstudio-is/sdk-components-animation exec node -e 'console.log(require("playwright").chromium.executablePath())')"+ test -x "$playwright_browser_path"+ echo "WEBSTUDIO_BROWSER_PATH=$playwright_browser_path" >> "$GITHUB_ENV"++ - name: Install visual comparison dependencies+ run: sudo apt-get install --no-install-recommends -y tesseract-ocr++ - name: Resolve visual baseline cache key+ id: visual-cache+ env:+ BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before || 'origin/main' }}+ run: |+ echo "base=$(git merge-base HEAD "$BASE_REF")" >> "$GITHUB_OUTPUT"+ echo "head=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"+ browser_version="$("$WEBSTUDIO_BROWSER_PATH" --version)"+ runtime_hash="$(pnpm tsx scripts/visual-regression/runtime-hash.ts "$browser_version")"+ echo "runtime=$runtime_hash" >> "$GITHUB_OUTPUT"++ - name: Restore visual baseline screenshots+ uses: actions/cache/restore@v5+ with:+ path: .visual-regression-cache/${{ steps.visual-cache.outputs.base }}-*+ key: visual-baseline-v1-${{ runner.os }}-${{ steps.visual-cache.outputs.base }}-${{ steps.visual-cache.outputs.runtime }}++ - name: Compare portable stories+ env:+ BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before || 'origin/main' }}+ APPROVE_VISUAL_CHANGES: ${{ github.event_name == 'push' || (github.event.action == 'labeled' && github.event.label.name == 'visual-change-approved') }}+ run: |+ args=(--base "$BASE_REF")+ if [[ "$APPROVE_VISUAL_CHANGES" == "true" ]]; then+ args+=(--approve-visual-changes)+ fi+ pnpm visual-regression -- "${args[@]}"++ - name: Validate visual comparison report+ if: always()+ run: |+ node -e 'const report = require("./.visual-regression/report/report.json"); if (!Array.isArray(report.comparisons) || report.comparisons.length === 0) throw new Error("Visual comparison report has no story results.")'++ - name: Upload visual comparison report+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: visual-regression-report+ path: .visual-regression/report+ if-no-files-found: error+ retention-days: 14++ - name: Cache visual screenshots for dependent revisions+ if: always()+ uses: actions/cache/save@v5+ with:+ path: .visual-regression-cache/${{ steps.visual-cache.outputs.head }}-*+ key: visual-baseline-v1-${{ runner.os }}-${{ steps.visual-cache.outputs.head }}-${{ steps.visual-cache.outputs.runtime }}++ - name: Cache generated baseline screenshots+ if: always()+ uses: actions/cache/save@v5+ with:+ path: .visual-regression-cache/${{ steps.visual-cache.outputs.base }}-*+ key: visual-baseline-v1-${{ runner.os }}-${{ steps.visual-cache.outputs.base }}-${{ steps.visual-cache.outputs.runtime }}diff --git a/.gitignore b/.gitignoreindex eea1d5784c..df2fd3a014 100644--- a/.gitignore+++ b/.gitignore@@ -51,6 +51,8 @@ yarn-error.log*packages/**/libgeneratedstorybook-static+.visual-regression+.visual-regression-cachetsconfig.tsbuildinfo# to save thunder https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client filesdiff --git a/.storybook/main.ts b/.storybook/main.tsindex fa64ddf898..7a5aecb14f 100644--- a/.storybook/main.ts+++ b/.storybook/main.ts@@ -1,70 +1,13 @@import * as path from "node:path";-import { existsSync, readdirSync } from "node:fs";import { defaultClientConditions } from "vite";import type { StorybookConfig } from "@storybook/react-vite";+import storySources from "./story-sources.json" with { type: "json" };+import { hasPrivateStorySources } from "./story-settings";-const isFolderEmpty = (folderPath: string) => {- if (!existsSync(folderPath)) {- return true; // Folder does not exist- }- const contents = readdirSync(folderPath);-- return contents.length === 0;-};--const hasPrivateFolders = !isFolderEmpty(- path.join(__dirname, "../../packages/sdk-components-animation/private-src")-);--const visualTestingStories: StorybookConfig["stories"] = [- {- directory: "../apps/builder",- titlePrefix: "Builder",- files: "**/*.stories.tsx",- },- {- directory: "../packages/design-system/src/components",- titlePrefix: "Design system",- files: "**/*.stories.tsx",- },-];+const repositoryRoot = path.resolve(__dirname, "..");export default {- stories: process.env.VISUAL_TESTING- ? visualTestingStories- : [- ...visualTestingStories,- {- directory: "../packages/css-engine/src",- titlePrefix: "CSS engine",- files: "**/*.stories.tsx",- },- {- directory: "../packages/image/src",- titlePrefix: "Image",- files: "**/*.stories.tsx",- },- {- directory: "../packages/icons",- titlePrefix: "Icons",- files: "**/*.stories.tsx",- },- {- directory: "../packages/sdk-components-react",- titlePrefix: "SDK components React",- files: "**/*.stories.tsx",- },- {- directory: "../packages/sdk-components-react-radix",- titlePrefix: "SDK components React Radix",- files: "**/*.stories.tsx",- },- {- directory: "../packages/sdk-components-animation",- titlePrefix: "SDK components animation",- files: "**/*.stories.tsx",- },- ],+ stories: storySources,framework: {name: "@storybook/react-vite",options: {},@@ -90,7 +33,7 @@ export default {},resolve: {...config.resolve,- conditions: hasPrivateFolders+ conditions: hasPrivateStorySources(repositoryRoot)? ["webstudio-private", "webstudio", ...defaultClientConditions]: ["webstudio", ...defaultClientConditions],diff --git a/.storybook/story-settings.ts b/.storybook/story-settings.tsnew file mode 100644index 0000000000..e9eb3bab90--- /dev/null+++ b/.storybook/story-settings.ts@@ -0,0 +1,10 @@+import { existsSync, readdirSync } from "node:fs";+import path from "node:path";++export const hasPrivateStorySources = (repositoryRoot: string) => {+ const directory = path.join(+ repositoryRoot,+ "packages/sdk-components-animation/private-src"+ );+ return existsSync(directory) && readdirSync(directory).length > 0;+};diff --git a/.storybook/story-sources.json b/.storybook/story-sources.jsonnew file mode 100644index 0000000000..548ef584d3--- /dev/null+++ b/.storybook/story-sources.json@@ -0,0 +1,42 @@+[+ {+ "directory": "../apps/builder",+ "titlePrefix": "Builder",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/design-system/src/components",+ "titlePrefix": "Design system",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/css-engine/src",+ "titlePrefix": "CSS engine",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/image/src",+ "titlePrefix": "Image",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/icons",+ "titlePrefix": "Icons",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/sdk-components-react",+ "titlePrefix": "SDK components React",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/sdk-components-react-radix",+ "titlePrefix": "SDK components React Radix",+ "files": "**/*.stories.tsx"+ },+ {+ "directory": "../packages/sdk-components-animation",+ "titlePrefix": "SDK components animation",+ "files": "**/*.stories.tsx"+ }+]diff --git a/README.md b/README.mdindex 04b77ad7b2..f179a7d9ce 100644--- a/README.md+++ b/README.md@@ -21,12 +21,6 @@- [Youtube](https://www.youtube.com/@getwebstudio)- [Discord](https://wstd.us/community)-## Thanks--<a href="https://www.lost-pixel.com/"><img src="https://user-images.githubusercontent.com/29632358/168112844-77e76a0d-b96f-4bc8-b753-cd39f4afd428.png" width="50" height="50" alt="Lost Pixel" /></a>--Thanks to [Lost Pixel](https://www.lost-pixel.com/) for providing the visual testing platform that helps us review UI changes and catch visual regressions.-## License- **Webstudio core** (all functionality in this repository) is free/open-source under AGPL-3.0-or-later.diff --git a/apps/builder/app/builder/features/pages/page-settings.stories.tsx b/apps/builder/app/builder/features/pages/page-settings.stories.tsxindex 8a5fb36b64..3e043e8544 100644--- a/apps/builder/app/builder/features/pages/page-settings.stories.tsx+++ b/apps/builder/app/builder/features/pages/page-settings.stories.tsx@@ -13,12 +13,6 @@ import { createDefaultPages } from "@webstudio-is/project-build";export default {title: "Pages/Page Settings",component: PageSettingsComponent,- parameters: {- lostpixel: {- // this is to fix cutting off the after scroll area in the screenshot- waitBeforeScreenshot: 3000,- },- },};$assets.set(diff --git a/apps/builder/app/builder/features/settings-panel/props-section/props-section.stories.tsx b/apps/builder/app/builder/features/settings-panel/props-section/props-section.stories.tsxindex 58ffb33498..86fe201523 100644--- a/apps/builder/app/builder/features/settings-panel/props-section/props-section.stories.tsx+++ b/apps/builder/app/builder/features/settings-panel/props-section/props-section.stories.tsx@@ -442,10 +442,4 @@ export const PropsSection = () => {export default {title: "Settings panel/Props Section",component: PropsSection,- parameters: {- lostpixel: {- // this is to fix cutting off the after scroll area in the screenshot- waitBeforeScreenshot: 5000,- },- },};diff --git a/lostpixel.config.js b/lostpixel.config.jsdeleted file mode 100644index b0b39d1262..0000000000--- a/lostpixel.config.js+++ /dev/null@@ -1,7 +0,0 @@-import * as process from "process";--export const config = {- lostPixelProjectId: "cleiive6c0gchi40em3uzx1xv",- apiKey: process.env.LOST_PIXEL_API_KEY,- waitBeforeScreenshot: 2000,-};diff --git a/package.json b/package.jsonindex c7de39cd13..de0fdd9856 100644--- a/package.json+++ b/package.json@@ -32,6 +32,11 @@"format": "oxfmt","storybook:dev": "storybook dev -p 6006","storybook:build": "storybook build",+ "visual-regression": "tsx --conditions=webstudio scripts/visual-regression/run.ts",+ "visual-regression:ui": "tsx --conditions=webstudio scripts/visual-regression/run.ts --open-report",+ "visual-regression:report": "tsx scripts/visual-regression/open-report.ts",+ "visual-regression:test": "tsx --test scripts/visual-regression/*.test.ts",+ "visual-regression:typecheck": "tsc -p scripts/visual-regression/tsconfig.json","migrations": "pnpm --filter=@webstudio-is/prisma-client generate && pnpm --filter=./packages/prisma-client migrations --dev --cwd ../../apps/builder","prepare": "which git && git config core.hooksPath .git/hooks/ && simple-git-hooks || echo git not installed","local:version-snapshot": "pnpm -r exec pnpm version prepatch --preid $(cat /dev/urandom | LC_ALL=C tr -dc 'a-z' | fold -w 8 | head -n 1)",@@ -56,16 +61,21 @@"@storybook/react-vite": "^8.6.4","@types/node": "^22.13.10","@types/react": "^18.2.70",+ "@types/react-dom": "^18.2.25",+ "@webstudio-is/project-build": "workspace:^",+ "@webstudio-is/vision": "workspace:^","esbuild": "^0.25.3","nano-staged": "^0.8.0","oxfmt": "0.58.0","oxlint": "^1.73.0",+ "parse5": "7.3.0","react": "18.3.0-canary-14898b6a9-20240318","react-dom": "18.3.0-canary-14898b6a9-20240318","simple-git-hooks": "^2.11.1","storybook": "^8.6.4","tsx": "^4.19.3","typescript": "7.0.2",+ "webstudio": "workspace:^","vite": "^6.3.4"},"engines": {diff --git a/packages/cli/package.json b/packages/cli/package.jsonindex 1d5a9faa1b..756c2f2ecb 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -9,6 +9,11 @@"webstudio-cli": "./bin.js","webstudio": "./bin.js"},+ "exports": {+ "./vision": {+ "webstudio": "./src/vision.ts"+ }+ },"files": ["lib/*","templates/*",@@ -105,6 +110,7 @@"@webstudio-is/sdk-components-react-router": "workspace:*","@webstudio-is/sdk-components-registry": "workspace:*","@webstudio-is/tsconfig": "workspace:*",+ "@webstudio-is/vision": "workspace:*","@webstudio-is/wsauth": "workspace:*","h3": "^1.15.1","ipx": "^3.0.3",diff --git a/packages/cli/src/commands/mcp-preview.ts b/packages/cli/src/commands/mcp-preview.tsindex 3746591202..63d30b5659 100644--- a/packages/cli/src/commands/mcp-preview.ts+++ b/packages/cli/src/commands/mcp-preview.ts@@ -11,7 +11,7 @@ import {} from "../screenshot";import { inspectGeneratedBuildMetrics } from "../generated-build-metrics";import { createExclusiveAsyncRunner, withTimeout } from "../async-utils";-import { defaultScreenshotTimeout } from "@webstudio-is/project-build/visual";+import { defaultScreenshotTimeout } from "@webstudio-is/vision/browser";import {preparePreviewProject,previewDefaultTemplate,diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.tsindex 5f3ffe7113..3ac89d6b88 100644--- a/packages/cli/src/commands/mcp.ts+++ b/packages/cli/src/commands/mcp.ts@@ -11,7 +11,7 @@ import {getProjectBasicAuthCredentials,type BuilderNamespace,} from "@webstudio-is/project-build/contracts";-import { diffPngFiles } from "@webstudio-is/project-build/visual";+import { diffPngFiles } from "@webstudio-is/vision/diff";import {publicApiOperationRequiresServerSupport,publicApiOperations,diff --git a/packages/cli/src/commands/preview.ts b/packages/cli/src/commands/preview.tsindex b852f012a4..9e654fecab 100644--- a/packages/cli/src/commands/preview.ts+++ b/packages/cli/src/commands/preview.ts@@ -20,7 +20,7 @@ import { log } from "@clack/prompts";import {projectPreviewSources,type ProjectPreviewSource,-} from "@webstudio-is/project-build/visual";+} from "@webstudio-is/project-build/preview";import { sanitizeValidationDetail } from "@webstudio-is/project-build/runtime";import { generatedFilesManifest, prebuild } from "../prebuild";import { LOCAL_CONFIG_FILE, LOCAL_DATA_FILE } from "../config";diff --git a/packages/cli/src/commands/screenshot.ts b/packages/cli/src/commands/screenshot.tsindex 59a04648f1..c2a0090482 100644--- a/packages/cli/src/commands/screenshot.ts+++ b/packages/cli/src/commands/screenshot.ts@@ -8,7 +8,7 @@ import {screenshotWaitUntilValues,type ScreenshotBrowser,type ScreenshotWaitUntil,-} from "@webstudio-is/project-build/visual";+} from "@webstudio-is/vision/browser";import { printJson } from "../json-output";import { HandledCliError } from "../errors";import {diff --git a/packages/cli/src/preview-server.ts b/packages/cli/src/preview-server.tsindex b1e4f9943a..9ff294148c 100644--- a/packages/cli/src/preview-server.ts+++ b/packages/cli/src/preview-server.ts@@ -10,7 +10,7 @@ import detectPort from "detect-port";import getPort from "get-port";import pathKey from "path-key";import { parse as parseHtml, type DefaultTreeAdapterMap } from "parse5";-import type { ProjectPreviewMode } from "@webstudio-is/project-build/visual";+import type { ProjectPreviewMode } from "@webstudio-is/project-build/preview";export type PreviewMode = ProjectPreviewMode;diff --git a/packages/cli/src/screenshot.test.ts b/packages/cli/src/screenshot.test.tsindex b30fd3b785..d3c01bc1a8 100644--- a/packages/cli/src/screenshot.test.ts+++ b/packages/cli/src/screenshot.test.ts@@ -10,6 +10,7 @@ import {captureScreenshot,captureScreenshotWithBrowserInstall,createScreenshotCaptureSession,+ defaultScreenshotDependencies,getChromiumInstallCommand,getNoBrowserFoundMessage,getPlaywrightInstallations,@@ -43,6 +44,8 @@ const createDependencies = (fetch: vi.fn(),createWebSocket: vi.fn(),captureBrowserScreenshot: vi.fn(async () => undefined),+ createBrowserScreenshotSession:+ defaultScreenshotDependencies.createBrowserScreenshotSession,installCommand: vi.fn(async () => undefined),readArtifactByte: vi.fn(async () => 1),getuid: vi.fn(() => 1000),@@ -50,6 +53,30 @@ const createDependencies = (...overrides,});+const genericNavigation = {+ requestedUrl: "https://example.com",+ finalUrl: "https://example.com/home",+ redirects: [],+ documentReadyState: "complete",+ pageMetadata: {+ rootMarkerPresent: true,+ id: "project-1",+ version: 7,+ },+ layoutStable: true,+};++const webstudioNavigation = {+ requestedUrl: "https://example.com",+ finalUrl: "https://example.com/home",+ redirects: [],+ documentReadyState: "complete",+ generatedSiteRootPresent: true,+ projectId: "project-1",+ projectVersion: 7,+ layoutStable: true,+};+describe("resolveScreenshotBrowser", () => {test("finds Playwright Chromium executables in the configured cache", async () => {const cacheDirectory = await mkdtemp(@@ -317,6 +344,7 @@ test("formats actionable browser installation guidance", () => {describe("captureScreenshot", () => {test("captures with browser readiness defaults", async () => {const captureBrowserScreenshot = vi.fn(async () => ({+ navigation: genericNavigation,viewportWidth: 1440,viewportHeight: 900,contentWidth: 1440,@@ -350,7 +378,9 @@ describe("captureScreenshot", () => {fullPage: false,elapsedMs: 0,warnings: [],+ navigation: webstudioNavigation,layout: {+ navigation: webstudioNavigation,viewportWidth: 1440,viewportHeight: 900,contentWidth: 1440,@@ -366,8 +396,19 @@ describe("captureScreenshot", () => {width: 1440,height: 900,fullPage: undefined,+ format: undefined,+ httpCredentials: undefined,+ includeContrastMetrics: undefined,includeImageMetrics: undefined,includeResourceMetrics: undefined,+ instanceIdAttribute: "data-ws-id",+ pageMetadata: {+ rootMarkerAttribute: "data-ws-project",+ idAttribute: "data-ws-project",+ versionAttribute: "data-ws-version",+ },+ quality: undefined,+ scale: undefined,url: "https://example.com",uid: 1000,waitUntil: "load",@@ -501,6 +542,43 @@ describe("captureScreenshot", () => {});});+test("maps page metadata from reusable capture sessions", async () => {+ const genericLayout = {+ navigation: genericNavigation,+ viewportWidth: 800,+ viewportHeight: 600,+ contentWidth: 800,+ contentHeight: 600,+ horizontalOverflow: false,+ };+ const browserSession = {+ capture: vi.fn(async () => genericLayout),+ capturePage: vi.fn(async () => [genericLayout]),+ close: vi.fn(async () => undefined),+ };+ const dependencies = createDependencies({+ which: vi.fn(async (command) =>+ command === "chromium" ? "/usr/bin/chromium" : undefined+ ),+ createBrowserScreenshotSession: vi.fn(async () => browserSession),+ });+ const session = createScreenshotCaptureSession(dependencies);++ const [capture] = await session.capturePage([+ {+ url: "https://example.com",+ output: "page.png",+ width: 800,+ height: 600,+ browser: "auto",+ },+ ]);++ expect(capture?.navigation).toEqual(webstudioNavigation);+ expect(capture?.layout.navigation).toEqual(capture?.navigation);+ await session.close();+});+test("resets a reusable capture session after browser startup rejects", async () => {const spawnBrowser = vi.fn(() => ({kill: vi.fn(() => true),@@ -535,6 +613,43 @@ test("resets a reusable capture session after browser startup rejects", async ()expect(spawnBrowser).toHaveBeenCalledTimes(4);});+test("does not create a browser session after cleanup during discovery", async () => {+ let resolveBrowser: ((path: string) => void) | undefined;+ const browserPath = new Promise<string>((resolve) => {+ resolveBrowser = resolve;+ });+ const createBrowserScreenshotSession = vi.fn();+ const dependencies = createDependencies({+ which: vi.fn(async (command) =>+ command === "chromium" ? await browserPath : undefined+ ),+ createBrowserScreenshotSession,+ });+ const session = createScreenshotCaptureSession(dependencies);+ const capture = session.capture({+ url: "https://example.com",+ width: 800,+ height: 600,+ browser: "auto",+ });++ await session.close();+ resolveBrowser?.("/usr/bin/chromium");++ await expect(capture).rejects.toThrow(+ "Screenshot capture session is closed."+ );+ await expect(+ session.capture({+ url: "https://example.com",+ width: 800,+ height: 600,+ browser: "auto",+ })+ ).rejects.toThrow("Screenshot capture session is closed.");+ expect(createBrowserScreenshotSession).not.toHaveBeenCalled();+});+describe("browser installation", () => {test("does not offer install in json, mcp, ci, or non-interactive contexts", () => {expect(diff --git a/packages/cli/src/screenshot.ts b/packages/cli/src/screenshot.tsindex 1dcc7cba57..ffc23e8ff5 100644--- a/packages/cli/src/screenshot.ts+++ b/packages/cli/src/screenshot.ts@@ -11,16 +11,51 @@ import {defaultScreenshotWaitUntil,type ScreenshotBrowser,type ScreenshotWaitUntil,-} from "@webstudio-is/project-build/visual";+} from "@webstudio-is/vision/browser";import {captureBrowserScreenshot,- createBrowserScreenshotSession,+ createBrowserScreenshotSession as createVisionBrowserScreenshotSession,defaultBrowserScreenshotDependencies,type BrowserScreenshotSession,type BrowserScreenshotDependencies,type BrowserScreenshotLayout,+ type BrowserScreenshotNavigation,type BrowserScreenshotOptions,-} from "./screenshot-browser-cdp";+} from "@webstudio-is/vision/browser";++const webstudioPageInstrumentation = {+ pageMetadata: {+ rootMarkerAttribute: "data-ws-project",+ idAttribute: "data-ws-project",+ versionAttribute: "data-ws-version",+ },+ instanceIdAttribute: "data-ws-id",+} satisfies Pick<+ BrowserScreenshotOptions,+ "pageMetadata" | "instanceIdAttribute"+>;++const toWebstudioNavigation = ({+ pageMetadata,+ ...navigation+}: BrowserScreenshotNavigation) => ({+ ...navigation,+ generatedSiteRootPresent: pageMetadata?.rootMarkerPresent ?? false,+ ...(pageMetadata?.id === undefined ? {} : { projectId: pageMetadata.id }),+ ...(pageMetadata?.version === undefined+ ? {}+ : { projectVersion: pageMetadata.version }),+});++const toWebstudioLayout = ({+ navigation,+ ...layout+}: BrowserScreenshotLayout) => ({+ ...layout,+ ...(navigation === undefined+ ? {}+ : { navigation: toWebstudioNavigation(navigation) }),+});export type BrowserCandidate = {path: string;@@ -63,6 +98,7 @@ export type ScreenshotDependencies = {captureBrowserScreenshot?: (options: BrowserScreenshotOptions) => Promise<BrowserScreenshotLayout | undefined>;+ createBrowserScreenshotSession: typeof createVisionBrowserScreenshotSession;installCommand: (file: string, args: readonly string[]) => Promise<void>;readArtifactByte: (path: string) => Promise<number>;getuid: () => number | undefined;@@ -85,6 +121,7 @@ export const defaultScreenshotDependencies: ScreenshotDependencies = {},getPlaywrightInstallations: () => getPlaywrightInstallations(),...defaultBrowserScreenshotDependencies,+ createBrowserScreenshotSession: createVisionBrowserScreenshotSession,async installCommand(file, args) {await new Promise<void>((resolve, reject) => {const child = spawn(file, [...args], { stdio: "inherit" });@@ -620,6 +657,7 @@ const getBrowserScreenshotOptions = (includeImageMetrics: options.includeImageMetrics,includeResourceMetrics: options.includeResourceMetrics,includeContrastMetrics: options.includeContrastMetrics,+ ...webstudioPageInstrumentation,url: options.url,httpCredentials: options.httpCredentials,uid: dependencies.getuid(),@@ -671,7 +709,7 @@ const captureResolvedScreenshot = async (output,dependencies);- const layout =+ const capturedLayout =browserSession !== undefined? await browserSession.capture(browserScreenshotOptions): dependencies.captureBrowserScreenshot !== undefined@@ -680,6 +718,10 @@ const captureResolvedScreenshot = async (browserScreenshotOptions,dependencies);+ const layout =+ capturedLayout === undefined+ ? undefined+ : toWebstudioLayout(capturedLayout);await validateScreenshotArtifact(output, dependencies);return {output,@@ -713,19 +755,48 @@ export const createScreenshotCaptureSession = (let browserPromise: Promise<BrowserCandidate> | undefined;let browserSession: BrowserScreenshotSession | undefined;let browserSessionPromise: Promise<BrowserScreenshotSession> | undefined;+ let closePromise: Promise<void> | undefined;+ let closed = false;+ const assertOpen = () => {+ if (closed) {+ throw new Error("Screenshot capture session is closed.");+ }+ };+ const getBrowser = async (options: CaptureScreenshotOptions) => {+ assertOpen();+ const pendingBrowser =+ browserPromise ?? resolveScreenshotBrowser(options, dependencies);+ browserPromise = pendingBrowser;+ try {+ const browser = await pendingBrowser;+ assertOpen();+ return browser;+ } catch (error) {+ if (browserPromise === pendingBrowser) {+ browserPromise = undefined;+ }+ throw error;+ }+ };const getBrowserSession = async (options: BrowserScreenshotOptions): Promise<BrowserScreenshotSession> => {+ assertOpen();if (browserSession !== undefined) {return browserSession;}const pendingSession =browserSessionPromise ??- createBrowserScreenshotSession(options, dependencies);+ dependencies.createBrowserScreenshotSession(options, dependencies);browserSessionPromise = pendingSession;try {- browserSession = await pendingSession;- return browserSession;+ const session = await pendingSession;+ if (closed) {+ await session.close();+ throw new Error("Screenshot capture session is closed.");+ }+ browserSession = session;+ return session;} catch (error) {if (browserSessionPromise === pendingSession) {browserSessionPromise = undefined;@@ -735,8 +806,7 @@ export const createScreenshotCaptureSession = (};return {async capture(options: CaptureScreenshotOptions) {- browserPromise ??= resolveScreenshotBrowser(options, dependencies);- const resolvedBrowser = await browserPromise;+ const resolvedBrowser = await getBrowser(options);if ((options.browserPath !== undefined &&options.browserPath !== resolvedBrowser.path) ||@@ -763,12 +833,12 @@ export const createScreenshotCaptureSession = ();},async capturePage(optionsList: readonly CaptureScreenshotOptions[]) {+ assertOpen();const firstOptions = optionsList[0];if (firstOptions === undefined) {return [];}- browserPromise ??= resolveScreenshotBrowser(firstOptions, dependencies);- const resolvedBrowser = await browserPromise;+ const resolvedBrowser = await getBrowser(firstOptions);if (optionsList.some((options) =>@@ -818,10 +888,11 @@ export const createScreenshotCaptureSession = ());return captures.map(({ options, output }, index) => {- const layout = layouts[index];- if (layout === undefined) {+ const capturedLayout = layouts[index];+ if (capturedLayout === undefined) {throw new Error("Browser omitted a resized screenshot result.");}+ const layout = toWebstudioLayout(capturedLayout);return {output,browser: resolvedBrowser,@@ -838,14 +909,12 @@ export const createScreenshotCaptureSession = (});},async close() {- try {+ closed = true;+ closePromise ??= (async () => {browserSession ??= await browserSessionPromise?.catch(() => undefined);await browserSession?.close();- } finally {- browserSession = undefined;- browserSessionPromise = undefined;- browserPromise = undefined;- }+ })();+ await closePromise;},};};diff --git a/packages/cli/src/vision.ts b/packages/cli/src/vision.tsnew file mode 100644index 0000000000..271f5d6d5d--- /dev/null+++ b/packages/cli/src/vision.ts@@ -0,0 +1,4 @@+export {+ defaultScreenshotDependencies,+ resolveScreenshotBrowser,+} from "./screenshot";diff --git a/packages/project-build/package.json b/packages/project-build/package.jsonindex 99ab87a8fb..d072f9baeb 100644--- a/packages/project-build/package.json+++ b/packages/project-build/package.json@@ -43,8 +43,8 @@"./mcp": {"webstudio": "./src/mcp.ts"},- "./visual": {- "webstudio": "./src/visual.ts"+ "./preview": {+ "webstudio": "./src/preview.ts"},"./persistence": {"webstudio": "./src/persistence.ts",@@ -83,6 +83,7 @@"@webstudio-is/sdk-components-registry": "workspace:*","@webstudio-is/template": "workspace:*","@webstudio-is/trpc-interface": "workspace:*",+ "@webstudio-is/vision": "workspace:*","@webstudio-is/wsauth": "workspace:*","acorn": "^8.14.1","acorn-jsx": "^5.3.2",@@ -92,11 +93,9 @@"esbuild": "^0.25.3","fast-deep-equal": "^3.1.3","fastest-levenshtein": "^1.0.16",- "fast-png": "8.0.0","immer": "^10.1.1","nanoid": "^5.1.5","papaparse": "^5.5.3",- "parse5": "7.3.0","slugify": "^1.6.6","title-case": "^4.3.2","urlpattern-polyfill": "^10.1.0",diff --git a/packages/project-build/src/entrypoints.test.ts b/packages/project-build/src/entrypoints.test.tsindex 1f46c6cfd1..2a8daf0e07 100644--- a/packages/project-build/src/entrypoints.test.ts+++ b/packages/project-build/src/entrypoints.test.ts@@ -20,7 +20,7 @@ describe("package entrypoints", () => {test("does not expose Node-only surfaces through generic conditions", () => {for (const entrypoint of ["./mcp",- "./visual",+ "./preview","./transfer/server","./server",]) {diff --git a/packages/project-build/src/mcp-rendered-audit.test.ts b/packages/project-build/src/mcp-rendered-audit.test.tsindex 58a27e500e..43be113feb 100644--- a/packages/project-build/src/mcp-rendered-audit.test.ts+++ b/packages/project-build/src/mcp-rendered-audit.test.ts@@ -357,7 +357,7 @@ describe("rendered audit evidence", () => {verbose: true,},} as never,- input: { rendered: true, verbose: true },+ input: { rendered: true },executeRead,startPreview: createStartPreview(),stopPreview: vi.fn(),@@ -394,7 +394,7 @@ describe("rendered audit evidence", () => {committed: false,result: { scopes: ["accessibility"], manualCheckCount: 0 },} as never,- input: { rendered: true },+ input: { rendered: true, verbose: true },executeRead: async (command) =>({result: command === "list-pages" ? { pages } : { breakpoints: [] },@@ -1240,6 +1240,44 @@ describe("rendered audit evidence", () => {expect(stopPreview).toHaveBeenCalledOnce();});+ test("stops a preview that resolves to an unowned origin", async () => {+ const stopPreview = vi.fn();+ const captureScreenshot = vi.fn();+ const result = await augmentAuditWithRenderedChecks({+ envelope: {+ operationId: "project.audit",+ source: "remote",+ projectId: "project",+ buildId: "build",+ version: 1,+ committed: false,+ result: { scopes: ["accessibility"], manualCheckCount: 0 },+ } as never,+ input: { rendered: true, verbose: true },+ executeRead: async (command) =>+ ({+ result:+ command === "list-pages"+ ? { pages: [{ id: "home", path: "/" }] }+ : { breakpoints: [] },+ }) as never,+ startPreview: vi.fn(async () => ({ url: "http://localhost:5177" })),+ stopPreview,+ captureScreenshot,+ });++ expect(result.result).toMatchObject({+ renderedState: "failed",+ renderedFailures: [+ expect.objectContaining({+ code: "RENDERED_AUDIT_PREVIEW_START_FAILED",+ }),+ ],+ });+ expect(captureScreenshot).not.toHaveBeenCalled();+ expect(stopPreview).toHaveBeenCalledOnce();+ });+test("accepts a generated non-HTML document without an HTML marker", async () => {const result = await augmentAuditWithRenderedChecks({envelope: {diff --git a/packages/project-build/src/mcp-rendered-audit.ts b/packages/project-build/src/mcp-rendered-audit.tsindex 70161c9257..169541dfb8 100644--- a/packages/project-build/src/mcp-rendered-audit.ts+++ b/packages/project-build/src/mcp-rendered-audit.ts@@ -21,6 +21,18 @@ import {createConfirmationToken,validateConfirmationToken,} from "./confirmation-token";+import {+ findContrastIssues,+ findCrossViewportGeometryIssues,+ findGeometryIssues,+ findImageIssues,+ findResourceIssues,+ type ScreenshotGeometryIssue,+} from "@webstudio-is/vision/audit";+import {+ selectRepresentativeViewports,+ type ViewportRange,+} from "@webstudio-is/vision/viewports";type Viewport = { width: number; height: number; purposes: string[] };type RenderedAuditPlan = z.infer<typeof renderedAuditPlan>;@@ -285,49 +297,6 @@ const validateRenderedAuditConfirmation = () => validateConfirmationToken(token, signature);export const getRenderedAuditViewports = (breakpoints: unknown): Viewport[] => {- const popularViewports = [- { width: 390, height: 844 },- { width: 667, height: 375 },- { width: 820, height: 1180 },- { width: 1024, height: 768 },- { width: 1366, height: 768 },- { width: 1440, height: 900 },- { width: 1920, height: 1080 },- ];- const widths = new Map<number, { height: number; purposes: Set<string> }>();- const addViewport = (- viewport: { width: number; height: number },- purpose: string- ) => {- const purposes = widths.get(viewport.width)?.purposes ?? new Set<string>();- purposes.add(purpose);- widths.set(viewport.width, { height: viewport.height, purposes });- };- const addRepresentative = (- minimum: number,- maximum: number | undefined,- purpose: string- ) => {- const candidates = popularViewports.filter(- ({ width }) =>- width >= minimum && (maximum === undefined || width <= maximum)- );- const viewport =- maximum === undefined- ? candidates[0]- : candidates.sort(- (left, right) =>- Math.abs(left.width - (minimum + maximum) / 2) -- Math.abs(right.width - (minimum + maximum) / 2)- )[0];- addViewport(- viewport ?? {- width: maximum ?? minimum,- height: (maximum ?? minimum) <= 480 ? 844 : 900,- },- purpose- );- };const records =isRecord(breakpoints) && Array.isArray(breakpoints.breakpoints)? breakpoints.breakpoints.flatMap((breakpoint, index) => {@@ -366,15 +335,22 @@ export const getRenderedAuditViewports = (breakpoints: unknown): Viewport[] => {).sort((left, right) => left.maxWidth - right.maxWidth);if (minimumBreakpoints.length === 0 && maximumBreakpoints.length === 0) {- addViewport(popularViewports[5], "base breakpoint: desktop");+ return [+ {+ width: 1440,+ height: 900,+ purposes: ["base breakpoint: desktop"],+ },+ ];}+ const ranges: ViewportRange<string>[] = [];let previousMaximum = 0;for (const breakpoint of maximumBreakpoints) {- addRepresentative(- previousMaximum + 1,- breakpoint.maxWidth,- `${breakpoint.label}: representative`- );+ ranges.push({+ minimumWidth: previousMaximum + 1,+ maximumWidth: breakpoint.maxWidth,+ purpose: `${breakpoint.label}: representative`,+ });previousMaximum = breakpoint.maxWidth;}const firstMinimum = minimumBreakpoints[0]?.minWidth;@@ -382,29 +358,24 @@ export const getRenderedAuditViewports = (breakpoints: unknown): Viewport[] => {const baseMaximum =firstMinimum === undefined ? undefined : firstMinimum - 1;if (baseMaximum === undefined || previousMaximum < baseMaximum) {- addRepresentative(- previousMaximum + 1,- baseMaximum,- "base breakpoint: representative"- );+ ranges.push({+ minimumWidth: previousMaximum + 1,+ maximumWidth: baseMaximum,+ purpose: "base breakpoint: representative",+ });}}for (const [index, breakpoint] of minimumBreakpoints.entries()) {const nextMinimum = minimumBreakpoints[index + 1]?.minWidth;- addRepresentative(- breakpoint.minWidth,- breakpoint.maxWidth ??+ ranges.push({+ minimumWidth: breakpoint.minWidth,+ maximumWidth:+ breakpoint.maxWidth ??(nextMinimum === undefined ? undefined : nextMinimum - 1),- `${breakpoint.label}: representative`- );+ purpose: `${breakpoint.label}: representative`,+ });}- return [...widths.entries()]- .sort(([left], [right]) => left - right)- .map(([width, { height, purposes }]) => ({- width,- height,- purposes: [...purposes],- }));+ return selectRepresentativeViewports({ ranges });};type Layout = NonNullable<ProjectSessionScreenshotResult["layout"]> & {@@ -432,37 +403,62 @@ const emptyElementGeometry: GeometryLayout["elementGeometry"] = {export const getRenderedGeometryIssues = (layout: GeometryLayout-): RenderedAuditCheck["geometryIssues"] => {- const issues: RenderedAuditCheck["geometryIssues"] = [];- for (const element of layout.elementGeometry.elements) {- if (element.clippedX || element.clippedY) {- issues.push({+): RenderedAuditCheck["geometryIssues"] =>+ findGeometryIssues(layout.elementGeometry).map(toRenderedGeometryIssue);++const toRenderedGeometryIssue = (+ issue: ScreenshotGeometryIssue+): RenderedAuditCheck["geometryIssues"][number] => {+ switch (issue.kind) {+ case "clipped-content":+ return {kind: "clipped-content",confidence: "exact",- instanceId: element.instanceId,+ instanceId: issue.instanceId,message: "Rendered content exceeds a clipping boundary.",evidence: {- clippedX: element.clippedX,- clippedY: element.clippedY,+ clippedX: issue.clippedX,+ clippedY: issue.clippedY,},- });- }- for (const relatedInstanceId of element.overlapsWith) {- if (element.instanceId.localeCompare(relatedInstanceId) >= 0) {- continue;- }- issues.push({+ };+ case "overlapping-elements":+ return {kind: "overlapping-elements",confidence: "advisory",- instanceId: element.instanceId,- relatedInstanceId,+ instanceId: issue.instanceId,+ relatedInstanceId: issue.relatedInstanceId,message:"Rendered element bounds overlap; inspect the screenshot to determine whether this is intentional.",evidence: {},- });- }+ };+ case "hidden-content":+ return {+ kind: "hidden-content",+ confidence: "advisory",+ instanceId: issue.instanceId,+ message:+ "This element is hidden at every audited viewport; verify that it is intentionally unreachable.",+ evidence: { hiddenReason: issue.hiddenReason },+ };+ case "cross-viewport-layout-change":+ return {+ kind: "cross-breakpoint-layout-change",+ confidence: "advisory",+ instanceId: issue.instanceId,+ message:+ "This element changes visibility, position, size, or overlap state between audited breakpoints; inspect both screenshots.",+ evidence: {+ sourceViewportWidth: issue.sourceViewportWidth,+ targetViewportWidth: issue.targetViewportWidth,+ sourceVisible: issue.sourceVisible,+ targetVisible: issue.targetVisible,+ sourceX: issue.sourceX,+ targetX: issue.targetX,+ sourceWidth: issue.sourceWidth,+ targetWidth: issue.targetWidth,+ },+ };}- return issues;};const addCrossBreakpointGeometryIssues = (@@ -474,97 +470,21 @@ const addCrossBreakpointGeometryIssues = (pages.set(check.pageId, pageChecks);return pages;}, new Map<string, RenderedAuditCheck[]>());- const additions = new Map<- RenderedAuditCheck,- RenderedAuditCheck["geometryIssues"]- >();+ const additions = new Map<RenderedAuditCheck, ScreenshotGeometryIssue[]>();for (const pageChecks of checksByPage.values()) {- pageChecks.sort(- (left, right) => left.viewport.width - right.viewport.width- );- const elementsByCheck = pageChecks.map(- (check) =>- new Map(- check.layout.elementGeometry.elements.map((element) => [- element.instanceId,- element,- ])- )- );- const firstCheck = pageChecks[0];- const firstElements = elementsByCheck[0];- if (firstCheck !== undefined && firstElements !== undefined) {- const hiddenIssues = additions.get(firstCheck) ?? [];- for (const [instanceId, element] of firstElements) {- if (- elementsByCheck.every(- (elements) => elements.get(instanceId)?.visible === false- )- ) {- hiddenIssues.push({- kind: "hidden-content",- confidence: "advisory",- instanceId,- message:- "This element is hidden at every audited viewport; verify that it is intentionally unreachable.",- evidence: { hiddenReason: element.hiddenReason },- });- }- }- additions.set(firstCheck, hiddenIssues);- }- for (let index = 1; index < pageChecks.length; index++) {- const previous = pageChecks[index - 1]!;- const check = pageChecks[index]!;- const previousElements = elementsByCheck[index - 1]!;- const geometryIssues = additions.get(check) ?? [];- for (const element of check.layout.elementGeometry.elements) {- const previousElement = previousElements.get(element.instanceId);- if (previousElement === undefined) {- continue;- }- const normalizedXChange = Math.abs(- previousElement.x / previous.viewport.width -- element.x / check.viewport.width- );- const normalizedWidthChange = Math.abs(- previousElement.width / previous.viewport.width -- element.width / check.viewport.width- );- const overlapChanged =- [...previousElement.overlapsWith].sort().join("\n") !==- [...element.overlapsWith].sort().join("\n");- if (- previousElement.visible === element.visible &&- normalizedXChange <= 0.35 &&- normalizedWidthChange <= 0.6 &&- overlapChanged === false- ) {- continue;- }- geometryIssues.push({- kind: "cross-breakpoint-layout-change",- confidence: "advisory",- instanceId: element.instanceId,- message:- "This element changes visibility, position, size, or overlap state between audited breakpoints; inspect both screenshots.",- evidence: {- sourceViewportWidth: previous.viewport.width,- targetViewportWidth: check.viewport.width,- sourceVisible: previousElement.visible,- targetVisible: element.visible,- sourceX: previousElement.x,- targetX: element.x,- sourceWidth: previousElement.width,- targetWidth: element.width,- },- });- }- additions.set(check, geometryIssues);+ const captures = pageChecks.map((check) => ({+ check,+ viewportWidth: check.viewport.width,+ elementGeometry: check.layout.elementGeometry,+ }));+ for (const [capture, issues] of findCrossViewportGeometryIssues(captures)) {+ additions.set(capture.check, issues);}}return checks.map((check) => {- const addedIssues = additions.get(check) ?? [];+ const addedIssues = (additions.get(check) ?? []).map(+ toRenderedGeometryIssue+ );return {...check,geometryIssues: [...check.geometryIssues, ...addedIssues],@@ -575,80 +495,27 @@ const addCrossBreakpointGeometryIssues = (export const getRenderedImageIssues = (layout: Layout-): RenderedAuditCheck["imageIssues"] => {- const eagerSourcesAboveFold = new Set(- layout.images.flatMap((image) =>- image.loading !== "lazy" &&- image.top < layout.viewportHeight &&- image.sourcePathname !== undefined- ? [image.sourcePathname]- : []- )- );- return layout.images.flatMap((image) => {- const sourceWidth = image.selectedSourceWidth ?? image.naturalWidth;- const sourceHeight = image.selectedSourceHeight ?? image.naturalHeight;- let kind: RenderedAuditCheck["imageIssues"][number]["kind"] | undefined;- if (- image.complete &&- image.naturalWidth === 0 &&- image.renderedWidth > 0 &&- image.renderedHeight > 0- ) {- kind = "broken-image";- } else if (- image.top >= layout.viewportHeight &&- image.loading !== "lazy" &&- (image.sourcePathname === undefined ||- eagerSourcesAboveFold.has(image.sourcePathname) === false)- ) {- kind = "eager-below-fold-image";- } else if (- image.sourcePathname?.toLowerCase().endsWith(".svg") !== true &&- image.sourcePathname?.toLowerCase().startsWith("data:image/svg+xml") !==- true &&- image.renderedWidth > 0 &&- image.renderedHeight > 0 &&- sourceWidth > image.renderedWidth * 2 &&- sourceHeight > image.renderedHeight * 2- ) {- kind = "oversized-image";- }- return kind === undefined ? [] : [{ kind, ...image }];+): RenderedAuditCheck["imageIssues"] =>+ findImageIssues({+ images: layout.images,+ viewportHeight: layout.viewportHeight,});-};export const getRenderedResourceIssues = (layout: Layout): RenderedAuditCheck["resourceIssues"] =>- layout.resources.flatMap((resource) => {- const issues: RenderedAuditCheck["resourceIssues"] = [];- if (- resource.renderBlockingStatus === "blocking" &&- /^\/assets\/index-[a-zA-Z0-9_-]+\.css$/.test(resource.pathname) === false- ) {- issues.push({ kind: "render-blocking-resource", ...resource });- }- if (/\.(?:ttf|otf|woff)$/i.test(resource.pathname)) {- issues.push({ kind: "legacy-font-format", ...resource });- }- return issues;+ findResourceIssues(layout.resources, {+ ignoreRenderBlocking: ({ pathname }) =>+ /^\/assets\/index-[a-zA-Z0-9_-]+\.css$/.test(pathname),});export const getRenderedContrastIssues = (layout: Layout): NonNullable<RenderedAuditCheck["contrastIssues"]> =>- layout.contrasts.flatMap((contrast) =>- contrast.ratio < contrast.requiredRatio- ? [- {- kind: "low-text-contrast" as const,- confidence: "exact" as const,- ...contrast,- },- ]- : []- );+ findContrastIssues(layout.contrasts).map((issue) => ({+ ...issue,+ confidence: "exact",+ }));const getRenderedAuditPlan = (pagesResult: unknown,@@ -1011,6 +878,7 @@ export const augmentAuditWithRenderedChecks = async ({};let generatedBuildMetrics: GeneratedBuildMetrics | undefined;let previewOrigin: string | undefined;+ let previewRunning = false;const captureTimeout = timeouts?.capture ?? renderedAuditCaptureTimeout;const pageTimeout = timeouts?.page ?? renderedAuditPageTimeout;let overallTimeout = getRenderedAuditOverallTimeout(1, timeouts?.overall);@@ -1046,6 +914,27 @@ export const augmentAuditWithRenderedChecks = async ({message: "Rendered audit was cancelled by the caller.",});};+ const stopRenderedPreview = async () => {+ if (previewRunning === false) {+ return;+ }+ previewRunning = false;+ const previewStopStartedAt = Date.now();+ try {+ await stopPreview();+ } catch (error) {+ failures.push({+ code: "RENDERED_AUDIT_PREVIEW_STOP_FAILED",+ phase: "preview-stop",+ retryable: true,+ remediation:+ "Run preview.status and preview.stop in the same MCP session before retrying.",+ message: `Rendered audit could not stop its preview: ${error instanceof Error ? error.message : String(error)}`,+ });+ } finally {+ performance.previewStopMs += Date.now() - previewStopStartedAt;+ }+ };const finish = async () => {const completedChecks = addCrossBreakpointGeometryIssues(checks);let merged = mergeRenderedAudit({@@ -1229,6 +1118,7 @@ export const augmentAuditWithRenderedChecks = async ({},{ report: (message) => reportProgress?.(message) });+ previewRunning = true;const previewUrl = new URL(previewResult.url);if (previewUrl.hostname !== "127.0.0.1") {throw new Error(@@ -1253,6 +1143,7 @@ export const augmentAuditWithRenderedChecks = async ({"Run preview.start separately to inspect the generated build failure, fix it, then retry the rendered audit.",message: `Rendered audit could not start: ${error instanceof Error ? error.message : String(error)}`,});+ await stopRenderedPreview();return await finish();}@@ -1692,21 +1583,7 @@ export const augmentAuditWithRenderedChecks = async ({);} finally {performance.captureWallMs = Date.now() - captureStartedAt;- const previewStopStartedAt = Date.now();- try {- await stopPreview();- } catch (error) {- failures.push({- code: "RENDERED_AUDIT_PREVIEW_STOP_FAILED",- phase: "preview-stop",- retryable: true,- remediation:- "Run preview.status and preview.stop in the same MCP session before retrying.",- message: `Rendered audit could not stop its preview: ${error instanceof Error ? error.message : String(error)}`,- });- } finally {- performance.previewStopMs = Date.now() - previewStopStartedAt;- }+ await stopRenderedPreview();}return await finish();};diff --git a/packages/project-build/src/mcp.test.ts b/packages/project-build/src/mcp.test.tsindex 05aa0f703f..c23d23dfbf 100644--- a/packages/project-build/src/mcp.test.ts+++ b/packages/project-build/src/mcp.test.ts@@ -34,8 +34,12 @@ import {projectSessionBusyMessage,type ProjectSessionEnvelope,} from "./project-session";-import { diffPngFiles } from "./visual/screenshot-diff";-import { createPng, paintRect, writePng } from "./visual/screenshot.test-utils";+import { diffPngFiles } from "@webstudio-is/vision/diff";+import {+ createPng,+ paintRect,+ writePng,+} from "@webstudio-is/vision/test-utils";type CreateProjectSession = NonNullable<Parameters<typeof createProjectSessionMcpCore>[0]["createProjectSession"]diff --git a/packages/project-build/src/mcp.ts b/packages/project-build/src/mcp.tsindex 97ad957c01..03a165e819 100644--- a/packages/project-build/src/mcp.ts+++ b/packages/project-build/src/mcp.ts@@ -27,7 +27,11 @@ import {type ProjectSessionEnvelope,type ProjectSessionRestorePointSummary,} from "./project-session";-import type { ScreenshotVisualExpectation } from "./visual/screenshot-diff";+import type {+ ScreenshotDiffResult,+ ScreenshotVisualExpectation,+} from "@webstudio-is/vision/diff";+import { isScreenshotVisualExpectation } from "@webstudio-is/vision/diff";import { isPlainRecord, isRecord } from "./shared/type-utils";import {augmentAuditWithRenderedChecks,@@ -37,19 +41,21 @@ import {defaultScreenshotTimeout,defaultScreenshotWaitForTimeout,defaultScreenshotWaitUntil,+ isScreenshotBrowser,isScreenshotWaitUntil,screenshotBrowserChoices,- type ScreenshotBrowser,screenshotWaitUntilValues,- type ScreenshotWaitUntil,-} from "./visual/screenshot-browser";-import type { ScreenshotDiffResult } from "./visual/screenshot-diff";+ type BrowserScreenshotLayout,+ type BrowserScreenshotNavigation,+ type BrowserScreenshotTimings,+ type ScreenshotCaptureOptions,+} from "@webstudio-is/vision/browser";import {projectPreviewModes,projectPreviewSources,type ProjectPreviewMode,type ProjectPreviewSource,-} from "./visual/preview";+} from "./preview";import { Server } from "@modelcontextprotocol/sdk/server/index.js";import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";@@ -144,47 +150,31 @@ export type ProjectSessionPreviewSource = ProjectPreviewSource;export const projectSessionPreviewModes = projectPreviewModes;export type ProjectSessionPreviewMode = ProjectPreviewMode;-type ProjectSessionScreenshotNavigation = {- requestedUrl: string;- finalUrl: string;- status?: number;- statusText?: string;- mimeType?: string;- redirects: string[];- documentReadyState: string;+type ProjectSessionScreenshotNavigation = Omit<+ BrowserScreenshotNavigation,+ "pageMetadata"+> & {generatedSiteRootPresent: boolean;projectId?: string;projectVersion?: number;- layoutStable: boolean;};-export type ProjectSessionScreenshotInput = {+export type ProjectSessionScreenshotInput = ScreenshotCaptureOptions & {url?: string;baseUrl?: string;path?: string;- output?: string;host?: string;port?: number;imageDomains?: string[];source?: ProjectSessionPreviewSource;mode?: ProjectSessionPreviewMode;- viewport: {- width: number;- height: number;- };- fullPage?: boolean;- includeImageMetrics?: boolean;- includeResourceMetrics?: boolean;- includeContrastMetrics?: boolean;- browser: ScreenshotBrowser;- browserPath?: string;- waitUntil?: ScreenshotWaitUntil;- waitForSelector?: string;- waitForTimeout?: number;- timeout?: number;- format?: "png" | "jpeg" | "webp";- quality?: number;- scale?: number;+};++type ProjectSessionScreenshotLayout = Omit<+ BrowserScreenshotLayout,+ "navigation"+> & {+ navigation?: ProjectSessionScreenshotNavigation;};export type ProjectSessionScreenshotResult = {@@ -206,59 +196,9 @@ export type ProjectSessionScreenshotResult = {captureMs: number;totalMs: number;};- timings?: {- wallMs: number;- targetSetupMs: number;- navigationMs: number;- readinessMs: number;- imageInspectionMs: number;- resourceInspectionMs: number;- screenshotMs: number;- artifactWriteMs: number;- targetCleanupMs: number;- };+ timings?: BrowserScreenshotTimings;navigation?: ProjectSessionScreenshotNavigation;- layout?: {- navigation?: ProjectSessionScreenshotNavigation;- documentType?: string;- viewportWidth: number;- viewportHeight: number;- contentWidth: number;- contentHeight: number;- horizontalOverflow: boolean;- images?: Array<{- instanceId?: string;- sourcePathname?: string;- loading: string;- complete: boolean;- naturalWidth: number;- naturalHeight: number;- selectedSourceWidth?: number;- selectedSourceHeight?: number;- renderedWidth: number;- renderedHeight: number;- top: number;- }>;- resources?: Array<{- pathname: string;- initiatorType: string;- transferSize: number;- encodedBodySize: number;- decodedBodySize: number;- duration: number;- renderBlockingStatus?: string;- }>;- contrasts?: Array<{- instanceId: string;- tagName: string;- foreground: string;- background: string;- ratio: number;- requiredRatio: 3 | 4.5;- fontSize: number;- fontWeight: number;- }>;- };+ layout?: ProjectSessionScreenshotLayout;};type McpToolProgress = {@@ -366,10 +306,6 @@ export type ProjectSessionMcpGuidance = {getVisionWorkflowSummary: (options: { includeDiff: boolean }) => string;};-const isScreenshotBrowser = (value: unknown): value is ScreenshotBrowser =>- typeof value === "string" &&- screenshotBrowserChoices.includes(value as ScreenshotBrowser);-const isProjectSessionPreviewSource = (value: unknown): value is ProjectSessionPreviewSource =>@@ -7026,68 +6962,6 @@ const getResponsiveScreenshotInputs = (});};-const isValidScreenshotDiffVisualExpectation = (- value: unknown-): value is ScreenshotVisualExpectation => {- if (isRecord(value) === false || Object.keys(value).length === 0) {- return false;- }- const numericFieldValid = (- key: "maxMismatchPercentage" | "minChangedRegions" | "maxChangedRegions"- ) => {- const field = value[key];- if (field === undefined) {- return true;- }- if (typeof field !== "number" || Number.isFinite(field) === false) {- return false;- }- if (key === "maxMismatchPercentage") {- return field >= 0 && field <= 100;- }- return Number.isInteger(field) && field >= 0;- };- if (- numericFieldValid("maxMismatchPercentage") === false ||- numericFieldValid("minChangedRegions") === false ||- numericFieldValid("maxChangedRegions") === false- ) {- return false;- }- if (- typeof value.minChangedRegions === "number" &&- typeof value.maxChangedRegions === "number" &&- value.minChangedRegions > value.maxChangedRegions- ) {- return false;- }- const dominantColorChange = value.dominantColorChange;- if (dominantColorChange !== undefined) {- if (- isRecord(dominantColorChange) === false ||- (dominantColorChange.channel !== "red" &&- dominantColorChange.channel !== "green" &&- dominantColorChange.channel !== "blue" &&- dominantColorChange.channel !== "luminance") ||- (dominantColorChange.direction !== "increase" &&- dominantColorChange.direction !== "decrease") ||- (dominantColorChange.minMagnitude !== undefined &&- (typeof dominantColorChange.minMagnitude !== "number" ||- Number.isFinite(dominantColorChange.minMagnitude) === false ||- dominantColorChange.minMagnitude < 0))- ) {- return false;- }- }- return Object.keys(value).every(- (key) =>- key === "maxMismatchPercentage" ||- key === "minChangedRegions" ||- key === "maxChangedRegions" ||- key === "dominantColorChange"- );-};-const getScreenshotDiffInput = (input: unknown): ProjectSessionScreenshotDiffInput => {@@ -7143,7 +7017,7 @@ const getScreenshotDiffInput = (const expectedVisual = input.expectedVisual;if (expectedVisual !== undefined &&- isValidScreenshotDiffVisualExpectation(expectedVisual) === false+ isScreenshotVisualExpectation(expectedVisual) === false) {throw new Error("screenshot.diff expectedVisual must include valid maxMismatchPercentage (0-100), minChangedRegions, or maxChangedRegions values."diff --git a/packages/project-build/src/visual/preview.ts b/packages/project-build/src/preview.tssimilarity index 100%rename from packages/project-build/src/visual/preview.tsrename to packages/project-build/src/preview.tsdiff --git a/packages/project-build/src/visual.ts b/packages/project-build/src/visual.tsdeleted file mode 100644index 2246bd753a..0000000000--- a/packages/project-build/src/visual.ts+++ /dev/null@@ -1,3 +0,0 @@-export * from "./visual/preview";-export * from "./visual/screenshot-browser";-export * from "./visual/screenshot-diff";diff --git a/packages/vision/README.md b/packages/vision/README.mdnew file mode 100644index 0000000000..ea5d05ea60--- /dev/null+++ b/packages/vision/README.md@@ -0,0 +1,21 @@+# Vision++`@webstudio-is/vision` provides source-agnostic browser capture, screenshot+comparison, OCR analysis, caching primitives, and portable visual reports.++Import the smallest relevant entry point:++```ts+import { createBrowserScreenshotSession } from "@webstudio-is/vision/browser";+import { captureVisualEntries } from "@webstudio-is/vision/capture";+import { diffPngFiles } from "@webstudio-is/vision/diff";+import {+ classifyScreenshotComparisonReport,+ writeScreenshotComparisonReport,+} from "@webstudio-is/vision/report";+import { selectRepresentativeViewports } from "@webstudio-is/vision/viewports";+```++Consumers own their source-specific orchestration. The visual regression+runner, for example, handles Git revisions and Storybook CSF rendering while+this package handles capture batches, image comparison, and reports.diff --git a/packages/vision/package.json b/packages/vision/package.jsonnew file mode 100644index 0000000000..69fae50b47--- /dev/null+++ b/packages/vision/package.json@@ -0,0 +1,78 @@+{+ "name": "@webstudio-is/vision",+ "version": "0.0.0",+ "description": "Browser capture and visual comparison tooling",+ "author": "Webstudio <[email protected]>",+ "homepage": "https://webstudio.is",+ "license": "AGPL-3.0-or-later",+ "private": true,+ "type": "module",+ "sideEffects": false,+ "exports": {+ "./audit": {+ "webstudio": "./src/audit.ts",+ "import": "./src/audit.ts",+ "default": "./src/audit.ts"+ },+ "./browser": {+ "webstudio": "./src/browser.ts",+ "import": "./src/browser.ts",+ "default": "./src/browser.ts"+ },+ "./capture": {+ "webstudio": "./src/capture.ts",+ "import": "./src/capture.ts",+ "default": "./src/capture.ts"+ },+ "./comparison": {+ "webstudio": "./src/comparison.ts",+ "import": "./src/comparison.ts",+ "default": "./src/comparison.ts"+ },+ "./diff": {+ "webstudio": "./src/diff.ts",+ "import": "./src/diff.ts",+ "default": "./src/diff.ts"+ },+ "./diff-pool": {+ "webstudio": "./src/diff-pool.ts",+ "import": "./src/diff-pool.ts",+ "default": "./src/diff-pool.ts"+ },+ "./report": {+ "webstudio": "./src/report.ts",+ "import": "./src/report.ts",+ "default": "./src/report.ts"+ },+ "./screenshot-cache": {+ "webstudio": "./src/screenshot-cache.ts",+ "import": "./src/screenshot-cache.ts",+ "default": "./src/screenshot-cache.ts"+ },+ "./test-utils": {+ "webstudio": "./src/screenshot.test-utils.ts",+ "import": "./src/screenshot.test-utils.ts",+ "default": "./src/screenshot.test-utils.ts"+ },+ "./viewports": {+ "webstudio": "./src/viewports.ts",+ "import": "./src/viewports.ts",+ "default": "./src/viewports.ts"+ }+ },+ "scripts": {+ "typecheck": "tsc --noEmit",+ "test": "vitest run"+ },+ "dependencies": {+ "fast-png": "8.0.0",+ "fastest-levenshtein": "^1.0.16",+ "tsx": "^4.19.3"+ },+ "devDependencies": {+ "@webstudio-is/tsconfig": "workspace:*",+ "esbuild": "^0.25.3",+ "parse5": "7.3.0",+ "vitest": "^3.1.2"+ }+}diff --git a/packages/vision/src/audit.test.ts b/packages/vision/src/audit.test.tsnew file mode 100644index 0000000000..7f500c1c76--- /dev/null+++ b/packages/vision/src/audit.test.ts@@ -0,0 +1,177 @@+import { expect, test } from "vitest";+import {+ findContrastIssues,+ findCrossViewportGeometryIssues,+ findGeometryIssues,+ findImageIssues,+ findResourceIssues,+} from "./audit";++test("finds measurable layout, image, resource, and contrast issues", () => {+ expect(+ findGeometryIssues({+ total: 2,+ sampled: 2,+ truncated: false,+ elements: [+ {+ instanceId: "card",+ tagName: "article",+ x: 0,+ y: 0,+ width: 300,+ height: 200,+ visible: true,+ clippedX: true,+ clippedY: false,+ overlapsWith: ["badge"],+ },+ {+ instanceId: "badge",+ tagName: "div",+ x: 250,+ y: 0,+ width: 100,+ height: 40,+ visible: true,+ clippedX: false,+ clippedY: false,+ overlapsWith: ["card"],+ },+ ],+ }).map(({ kind }) => kind)+ ).toEqual(["clipped-content", "overlapping-elements"]);++ expect(+ findImageIssues({+ viewportHeight: 800,+ images: [+ {+ loading: "eager",+ complete: true,+ naturalWidth: 0,+ naturalHeight: 0,+ renderedWidth: 100,+ renderedHeight: 50,+ top: 900,+ },+ {+ loading: "lazy",+ complete: true,+ naturalWidth: 1000,+ naturalHeight: 800,+ renderedWidth: 200,+ renderedHeight: 150,+ top: 0,+ },+ ],+ }).map(({ kind }) => kind)+ ).toEqual(["broken-image", "oversized-image"]);++ expect(+ findResourceIssues([+ {+ pathname: "/fonts/brand.woff",+ initiatorType: "link",+ transferSize: 10,+ encodedBodySize: 8,+ decodedBodySize: 12,+ duration: 5,+ renderBlockingStatus: "blocking",+ },+ ]).map(({ kind }) => kind)+ ).toEqual(["render-blocking-resource", "legacy-font-format"]);++ expect(+ findContrastIssues([+ {+ instanceId: "copy",+ tagName: "p",+ foreground: "rgb(120, 120, 120)",+ background: "rgb(255, 255, 255)",+ ratio: 4.2,+ requiredRatio: 4.5,+ fontSize: 16,+ fontWeight: 400,+ },+ ])+ ).toHaveLength(1);+});++test("finds persistent hidden content and cross-viewport layout changes", () => {+ const createGeometry = (visible: boolean, x: number, width: number) => ({+ total: 1,+ sampled: 1,+ truncated: false,+ elements: [+ {+ instanceId: "navigation",+ tagName: "nav",+ x,+ y: 0,+ width,+ height: 100,+ visible,+ ...(visible ? {} : { hiddenReason: "display" as const }),+ clippedX: false,+ clippedY: false,+ overlapsWith: [],+ },+ ],+ });+ const mobile = {+ viewportWidth: 390,+ elementGeometry: createGeometry(false, 0, 0),+ };+ const desktop = {+ viewportWidth: 1440,+ elementGeometry: createGeometry(true, 900, 400),+ };+ const issues = findCrossViewportGeometryIssues([mobile, desktop]);++ expect(issues.get(desktop)?.map(({ kind }) => kind)).toEqual([+ "cross-viewport-layout-change",+ ]);+});++test("finds overlaps reported by only one element", () => {+ expect(+ findGeometryIssues({+ total: 2,+ sampled: 2,+ truncated: false,+ elements: [+ {+ instanceId: "z-card",+ tagName: "article",+ x: 0,+ y: 0,+ width: 100,+ height: 100,+ visible: true,+ clippedX: false,+ clippedY: false,+ overlapsWith: ["a-badge"],+ },+ {+ instanceId: "a-badge",+ tagName: "div",+ x: 50,+ y: 0,+ width: 100,+ height: 100,+ visible: true,+ clippedX: false,+ clippedY: false,+ overlapsWith: [],+ },+ ],+ })+ ).toEqual([+ {+ kind: "overlapping-elements",+ instanceId: "a-badge",+ relatedInstanceId: "z-card",+ },+ ]);+});diff --git a/packages/vision/src/audit.ts b/packages/vision/src/audit.tsnew file mode 100644index 0000000000..cb7c770074--- /dev/null+++ b/packages/vision/src/audit.ts@@ -0,0 +1,272 @@+import type {+ BrowserScreenshotContrast,+ BrowserScreenshotElementGeometry,+ BrowserScreenshotImage,+ BrowserScreenshotResource,+} from "./screenshot-browser-cdp";++export type ScreenshotGeometryIssue =+ | {+ kind: "clipped-content";+ instanceId: string;+ clippedX: boolean;+ clippedY: boolean;+ }+ | {+ kind: "overlapping-elements";+ instanceId: string;+ relatedInstanceId: string;+ }+ | {+ kind: "hidden-content";+ instanceId: string;+ hiddenReason?: "display" | "visibility" | "opacity" | "hidden";+ }+ | {+ kind: "cross-viewport-layout-change";+ instanceId: string;+ sourceViewportWidth: number;+ targetViewportWidth: number;+ sourceVisible: boolean;+ targetVisible: boolean;+ sourceX: number;+ targetX: number;+ sourceWidth: number;+ targetWidth: number;+ };++export type ScreenshotImageIssue = BrowserScreenshotImage & {+ kind: "broken-image" | "eager-below-fold-image" | "oversized-image";+};++export type ScreenshotResourceIssue = BrowserScreenshotResource & {+ kind: "render-blocking-resource" | "legacy-font-format";+};++export type ScreenshotContrastIssue = BrowserScreenshotContrast & {+ kind: "low-text-contrast";+};++export const findGeometryIssues = (+ geometry: BrowserScreenshotElementGeometry+): ScreenshotGeometryIssue[] => {+ const issues: ScreenshotGeometryIssue[] = [];+ const overlapPairs = new Set<string>();+ for (const element of geometry.elements) {+ if (element.clippedX || element.clippedY) {+ issues.push({+ kind: "clipped-content",+ instanceId: element.instanceId,+ clippedX: element.clippedX,+ clippedY: element.clippedY,+ });+ }+ for (const relatedInstanceId of element.overlapsWith) {+ const [instanceId, pairedInstanceId] = [+ element.instanceId,+ relatedInstanceId,+ ].sort((left, right) => left.localeCompare(right));+ if (instanceId === pairedInstanceId) {+ continue;+ }+ const pair = JSON.stringify([instanceId, pairedInstanceId]);+ if (overlapPairs.has(pair)) {+ continue;+ }+ overlapPairs.add(pair);+ issues.push({+ kind: "overlapping-elements",+ instanceId,+ relatedInstanceId: pairedInstanceId,+ });+ }+ }+ return issues;+};++export const findCrossViewportGeometryIssues = <+ Capture extends {+ viewportWidth: number;+ elementGeometry: BrowserScreenshotElementGeometry;+ },+>(+ captures: readonly Capture[],+ {+ maximumNormalizedXChange = 0.35,+ maximumNormalizedWidthChange = 0.6,+ }: {+ maximumNormalizedXChange?: number;+ maximumNormalizedWidthChange?: number;+ } = {}+) => {+ const issues = new Map<Capture, ScreenshotGeometryIssue[]>(+ captures.map((capture) => [capture, []])+ );+ const ordered = [...captures].sort(+ (left, right) => left.viewportWidth - right.viewportWidth+ );+ const elementsByCapture = ordered.map(+ ({ elementGeometry }) =>+ new Map(+ elementGeometry.elements.map((element) => [element.instanceId, element])+ )+ );+ const firstCapture = ordered[0];+ const firstElements = elementsByCapture[0];+ if (firstCapture !== undefined && firstElements !== undefined) {+ const firstIssues = issues.get(firstCapture) ?? [];+ for (const [instanceId, element] of firstElements) {+ if (+ elementsByCapture.every(+ (elements) => elements.get(instanceId)?.visible === false+ )+ ) {+ firstIssues.push({+ kind: "hidden-content",+ instanceId,+ hiddenReason: element.hiddenReason,+ });+ }+ }+ issues.set(firstCapture, firstIssues);+ }+ for (let index = 1; index < ordered.length; index += 1) {+ const previous = ordered[index - 1];+ const current = ordered[index];+ const previousElements = elementsByCapture[index - 1];+ if (+ previous === undefined ||+ current === undefined ||+ previousElements === undefined+ ) {+ continue;+ }+ const currentIssues = issues.get(current) ?? [];+ for (const element of current.elementGeometry.elements) {+ const previousElement = previousElements.get(element.instanceId);+ if (previousElement === undefined) {+ continue;+ }+ const normalizedXChange = Math.abs(+ previousElement.x / previous.viewportWidth -+ element.x / current.viewportWidth+ );+ const normalizedWidthChange = Math.abs(+ previousElement.width / previous.viewportWidth -+ element.width / current.viewportWidth+ );+ const overlapChanged =+ [...previousElement.overlapsWith].sort().join("\n") !==+ [...element.overlapsWith].sort().join("\n");+ if (+ previousElement.visible === element.visible &&+ normalizedXChange <= maximumNormalizedXChange &&+ normalizedWidthChange <= maximumNormalizedWidthChange &&+ overlapChanged === false+ ) {+ continue;+ }+ currentIssues.push({+ kind: "cross-viewport-layout-change",+ instanceId: element.instanceId,+ sourceViewportWidth: previous.viewportWidth,+ targetViewportWidth: current.viewportWidth,+ sourceVisible: previousElement.visible,+ targetVisible: element.visible,+ sourceX: previousElement.x,+ targetX: element.x,+ sourceWidth: previousElement.width,+ targetWidth: element.width,+ });+ }+ issues.set(current, currentIssues);+ }+ return issues;+};++export const findImageIssues = ({+ images,+ viewportHeight,+ oversizedScale = 2,+}: {+ images: readonly BrowserScreenshotImage[];+ viewportHeight: number;+ oversizedScale?: number;+}): ScreenshotImageIssue[] => {+ const eagerSourcesAboveFold = new Set(+ images.flatMap((image) =>+ image.loading !== "lazy" &&+ image.top < viewportHeight &&+ image.sourcePathname !== undefined+ ? [image.sourcePathname]+ : []+ )+ );+ return images.flatMap((image) => {+ const sourceWidth = image.selectedSourceWidth ?? image.naturalWidth;+ const sourceHeight = image.selectedSourceHeight ?? image.naturalHeight;+ let kind: ScreenshotImageIssue["kind"] | undefined;+ if (+ image.complete &&+ image.naturalWidth === 0 &&+ image.renderedWidth > 0 &&+ image.renderedHeight > 0+ ) {+ kind = "broken-image";+ } else if (+ image.top >= viewportHeight &&+ image.loading !== "lazy" &&+ (image.sourcePathname === undefined ||+ eagerSourcesAboveFold.has(image.sourcePathname) === false)+ ) {+ kind = "eager-below-fold-image";+ } else if (+ image.sourcePathname?.toLowerCase().endsWith(".svg") !== true &&+ image.sourcePathname?.toLowerCase().startsWith("data:image/svg+xml") !==+ true &&+ image.renderedWidth > 0 &&+ image.renderedHeight > 0 &&+ sourceWidth > image.renderedWidth * oversizedScale &&+ sourceHeight > image.renderedHeight * oversizedScale+ ) {+ kind = "oversized-image";+ }+ return kind === undefined ? [] : [{ kind, ...image }];+ });+};++export const findResourceIssues = (+ resources: readonly BrowserScreenshotResource[],+ {+ ignoreRenderBlocking,+ }: {+ ignoreRenderBlocking?: (resource: BrowserScreenshotResource) => boolean;+ } = {}+): ScreenshotResourceIssue[] =>+ resources.flatMap((resource) => {+ const issues: ScreenshotResourceIssue[] = [];+ if (+ resource.renderBlockingStatus === "blocking" &&+ ignoreRenderBlocking?.(resource) !== true+ ) {+ issues.push({ kind: "render-blocking-resource", ...resource });+ }+ const pathname = resource.pathname.toLowerCase();+ if (+ pathname.endsWith(".ttf") ||+ pathname.endsWith(".otf") ||+ pathname.endsWith(".woff")+ ) {+ issues.push({ kind: "legacy-font-format", ...resource });+ }+ return issues;+ });++export const findContrastIssues = (+ contrasts: readonly BrowserScreenshotContrast[]+): ScreenshotContrastIssue[] =>+ contrasts.flatMap((contrast) =>+ contrast.ratio < contrast.requiredRatio+ ? [{ kind: "low-text-contrast" as const, ...contrast }]+ : []+ );diff --git a/packages/vision/src/browser.ts b/packages/vision/src/browser.tsnew file mode 100644index 0000000000..4f4c702d2c--- /dev/null+++ b/packages/vision/src/browser.ts@@ -0,0 +1,2 @@+export * from "./screenshot-browser";+export * from "./screenshot-browser-cdp";diff --git a/packages/vision/src/capture.test.ts b/packages/vision/src/capture.test.tsnew file mode 100644index 0000000000..f6452292e7--- /dev/null+++ b/packages/vision/src/capture.test.ts@@ -0,0 +1,134 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, test } from "vitest";+import type { BrowserScreenshotOptions } from "./screenshot-browser-cdp";+import { captureVisualEntries, orderForGroupedConcurrency } from "./capture";++const createCapture = (directory: string, id: string) => ({+ id,+ options: {+ browserPath: "/usr/bin/chromium",+ output: path.join(directory, `${id}.png`),+ width: 1280,+ height: 800,+ url: "http://127.0.0.1",+ waitUntil: "load",+ waitForTimeout: 0,+ timeout: 1000,+ } satisfies BrowserScreenshotOptions,+});++test("orders contiguous stories into the same concurrent browser page", () => {+ expect(orderForGroupedConcurrency([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3)).toEqual(+ [0, 4, 7, 1, 5, 8, 2, 6, 9, 3]+ );+});++test("rejects invalid capture concurrency", () => {+ expect(() => orderForGroupedConcurrency([0, 1], 0)).toThrow(+ "Capture concurrency must be a positive integer."+ );+ expect(() => orderForGroupedConcurrency([0, 1], 1.5)).toThrow(+ "Capture concurrency must be a positive integer."+ );+});++test("rejects duplicate capture identifiers", async () => {+ const assetDirectory = await mkdtemp(path.join(tmpdir(), "visual-capture-"));+ try {+ await expect(+ captureVisualEntries({+ captures: [+ createCapture(assetDirectory, "duplicate"),+ createCapture(assetDirectory, "duplicate"),+ ],+ concurrency: 1,+ session: {+ async capturePage() {+ throw new Error("Browser capture must not start");+ },+ },+ })+ ).rejects.toThrow('Duplicate visual capture id "duplicate".');+ } finally {+ await rm(assetDirectory, { recursive: true, force: true });+ }+});++test("captures entries independently and preserves individual errors", async () => {+ const assetDirectory = await mkdtemp(path.join(tmpdir(), "visual-capture-"));+ try {+ const result = await captureVisualEntries({+ captures: [+ createCapture(assetDirectory, "success"),+ createCapture(assetDirectory, "failure"),+ ],+ concurrency: 2,+ session: {+ async capturePage(options) {+ if (options.some(({ output }) => output.includes("failure"))) {+ throw new Error("Story failed");+ }+ await Promise.all(+ options.map(({ output }) => writeFile(output, "screenshot"))+ );+ return {};+ },+ },+ });++ expect(result.paths.has("success")).toEqual(true);+ expect(result.paths.has("failure")).toEqual(false);+ expect(result.errors.get("failure") ?? "").toMatch(/Story failed/);+ } finally {+ await rm(assetDirectory, { recursive: true, force: true });+ }+});++test("fails captures that do not produce screenshots", async () => {+ const assetDirectory = await mkdtemp(path.join(tmpdir(), "visual-capture-"));+ try {+ const result = await captureVisualEntries({+ captures: [createCapture(assetDirectory, "missing")],+ concurrency: 1,+ session: {+ async capturePage() {+ return {};+ },+ },+ });++ expect(result.paths.size).toEqual(0);+ expect(result.errors.get("missing") ?? "").toMatch(/ENOENT/);+ } finally {+ await rm(assetDirectory, { recursive: true, force: true });+ }+});++test("uses the browser output as the only capture output path", async () => {+ const assetDirectory = await mkdtemp(path.join(tmpdir(), "visual-capture-"));+ try {+ const capture = {+ ...createCapture(assetDirectory, "expected"),+ output: path.join(assetDirectory, "stale.png"),+ };+ const result = await captureVisualEntries({+ captures: [capture],+ concurrency: 1,+ session: {+ async capturePage(options) {+ await writeFile(options[0]?.output ?? "", "screenshot");+ return {};+ },+ },+ });++ expect(result.paths).toEqual(+ new Map([["expected", capture.options.output]])+ );+ expect(result.errors.size).toEqual(0);+ } finally {+ await rm(assetDirectory, { recursive: true, force: true });+ }+});diff --git a/packages/vision/src/capture.ts b/packages/vision/src/capture.tsnew file mode 100644index 0000000000..bb03f05a65--- /dev/null+++ b/packages/vision/src/capture.ts@@ -0,0 +1,124 @@+import { access, mkdir } from "node:fs/promises";+import path from "node:path";+import type { BrowserScreenshotOptions } from "./screenshot-browser-cdp";++export type VisualCapture = {+ id: string;+ options: BrowserScreenshotOptions;+};++export type VisualCaptureSession = {+ capturePage: (+ options: readonly BrowserScreenshotOptions[],+ sessionOptions: { concurrency: number }+ ) => Promise<unknown>;+};++const assertCaptureConcurrency = (concurrency: number) => {+ if (Number.isInteger(concurrency) === false || concurrency < 1) {+ throw new Error("Capture concurrency must be a positive integer.");+ }+};++export const orderForGroupedConcurrency = <Value>(+ values: readonly Value[],+ concurrency: number+) => {+ assertCaptureConcurrency(concurrency);+ // Browser sessions assign options to pages round-robin. Interleave contiguous+ // groups so captures that share resources reuse the same page and its cache.+ const groupCount = Math.min(concurrency, values.length);+ if (groupCount < 2) {+ return [...values];+ }+ const minimumGroupSize = Math.floor(values.length / groupCount);+ const maximumGroupSize = Math.ceil(values.length / groupCount);+ const largerGroupCount = values.length % groupCount;+ let start = 0;+ const groups = Array.from({ length: groupCount }, (_, index) => {+ const size = minimumGroupSize + (index < largerGroupCount ? 1 : 0);+ const group = values.slice(start, start + size);+ start += size;+ return group;+ });+ const ordered: Value[] = [];+ for (let index = 0; index < maximumGroupSize; index += 1) {+ for (const group of groups) {+ const value = group[index];+ if (value !== undefined) {+ ordered.push(value);+ }+ }+ }+ return ordered;+};++export const captureVisualEntries = async ({+ captures,+ concurrency,+ session,+ onBatchFailure,+}: {+ captures: readonly VisualCapture[];+ concurrency: number;+ session: VisualCaptureSession | undefined;+ onBatchFailure?: (error: unknown, count: number) => void;+}) => {+ assertCaptureConcurrency(concurrency);+ const captureIds = new Set<string>();+ for (const { id } of captures) {+ if (captureIds.has(id)) {+ throw new Error(`Duplicate visual capture id ${JSON.stringify(id)}.`);+ }+ captureIds.add(id);+ }+ const paths = new Map<string, string>();+ const errors = new Map<string, string>();+ if (captures.length === 0) {+ return { paths, errors };+ }+ if (session === undefined) {+ throw new Error("A browser session is required to capture visual entries.");+ }+ const ordered = orderForGroupedConcurrency(+ await Promise.all(+ captures.map(async (capture) => {+ await mkdir(path.dirname(capture.options.output), { recursive: true });+ return capture;+ })+ ),+ concurrency+ );+ const captureBatch = async (batch: typeof ordered): Promise<void> => {+ try {+ await session.capturePage(+ batch.map(({ options }) => options),+ { concurrency }+ );+ await Promise.all(batch.map(({ options }) => access(options.output)));+ for (const { id, options } of batch) {+ paths.set(id, options.output);+ }+ return;+ } catch (error) {+ onBatchFailure?.(error, batch.length);+ if (batch.length === 1) {+ const capture = batch[0];+ if (capture !== undefined) {+ errors.set(+ capture.id,+ error instanceof Error+ ? (error.stack ?? error.message)+ : String(error)+ );+ }+ return;+ }+ }+ const middle = Math.ceil(batch.length / 2);+ await captureBatch(batch.slice(0, middle));+ await captureBatch(batch.slice(middle));+ };+ await captureBatch(ordered);+ return { paths, errors };+};diff --git a/packages/vision/src/comparison.test.ts b/packages/vision/src/comparison.test.tsnew file mode 100644index 0000000000..fce532e06e--- /dev/null+++ b/packages/vision/src/comparison.test.ts@@ -0,0 +1,142 @@+import { expect, test } from "vitest";+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { compareVisualEntries, getVisualComparisons } from "./comparison";+import { createPng, writePng } from "./screenshot.test-utils";++const createEntry = (id: string) => ({ id, title: id, name: id });++test("classifies common, added, and removed visual entries", () => {+ expect(+ getVisualComparisons({+ baselineEntries: [createEntry("shared"), createEntry("removed")],+ currentEntries: [createEntry("shared"), createEntry("added")],+ })+ ).toEqual([+ {+ baseline: undefined,+ current: createEntry("added"),+ id: "added",+ status: "added",+ },+ {+ baseline: createEntry("removed"),+ current: undefined,+ id: "removed",+ status: "removed",+ },+ {+ baseline: createEntry("shared"),+ current: createEntry("shared"),+ id: "shared",+ status: "comparable",+ },+ ]);+});++test("rejects duplicate visual entry ids", () => {+ expect(() =>+ getVisualComparisons({+ baselineEntries: [createEntry("duplicate"), createEntry("duplicate")],+ currentEntries: [],+ })+ ).toThrowError('Duplicate baseline visual entry id: "duplicate"');+});++test("rejects invalid comparison concurrency without image differences", async () => {+ await expect(+ compareVisualEntries({+ baselineEntries: [],+ currentEntries: [],+ baselinePaths: new Map(),+ currentPaths: new Map(),+ baselineErrors: new Map(),+ currentErrors: new Map(),+ artifactDirectory: "/unused",+ pixelThreshold: 0.1,+ maxMismatchPercentage: 0,+ concurrency: 0,+ })+ ).rejects.toThrow("Visual comparison concurrency must be a positive integer");+});++test("compares captured entries and preserves capture failures", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "visual-comparison-"));+ try {+ const shared = path.join(directory, "shared.png");+ await writeFile(+ shared,+ Buffer.from(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",+ "base64"+ )+ );+ const comparisons = await compareVisualEntries({+ baselineEntries: [createEntry("shared"), createEntry("removed")],+ currentEntries: [+ createEntry("shared"),+ createEntry("added"),+ createEntry("failed"),+ ],+ baselinePaths: new Map([+ ["shared", shared],+ ["removed", shared],+ ]),+ currentPaths: new Map([+ ["shared", shared],+ ["added", shared],+ ]),+ baselineErrors: new Map(),+ currentErrors: new Map([["failed", "Story failed"]]),+ artifactDirectory: directory,+ pixelThreshold: 0.1,+ maxMismatchPercentage: 0.001,+ concurrency: 1,+ });++ expect(comparisons.map(({ id, status }) => ({ id, status }))).toEqual([+ { id: "added", status: "added" },+ { id: "failed", status: "error" },+ { id: "removed", status: "removed" },+ { id: "shared", status: "unchanged" },+ ]);+ expect(comparisons[1]?.error).toEqual("Story failed");+ } finally {+ await rm(directory, { recursive: true, force: true });+ }+});++test("keeps diff artifacts inside the artifact directory", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "visual-comparison-"));+ const artifactDirectory = path.join(directory, "artifacts");+ try {+ const baselinePath = path.join(directory, "baseline.png");+ const currentPath = path.join(directory, "current.png");+ await Promise.all([+ writePng(baselinePath, createPng(10, 10, { r: 0, g: 0, b: 0 })),+ writePng(currentPath, createPng(10, 10, { r: 255, g: 255, b: 255 })),+ ]);+ const [comparison] = await compareVisualEntries({+ baselineEntries: [createEntry("../outside")],+ currentEntries: [createEntry("../outside")],+ baselinePaths: new Map([["../outside", baselinePath]]),+ currentPaths: new Map([["../outside", currentPath]]),+ baselineErrors: new Map(),+ currentErrors: new Map(),+ artifactDirectory,+ pixelThreshold: 0.1,+ maxMismatchPercentage: 0,+ concurrency: 1,+ });++ const relativeDiffPath = path.relative(+ artifactDirectory,+ comparison?.diffPath ?? ""+ );+ expect(relativeDiffPath.startsWith(`..${path.sep}`)).toBe(false);+ expect(path.isAbsolute(relativeDiffPath)).toBe(false);+ } finally {+ await rm(directory, { recursive: true, force: true });+ }+});diff --git a/packages/vision/src/comparison.ts b/packages/vision/src/comparison.tsnew file mode 100644index 0000000000..79658ae7b9--- /dev/null+++ b/packages/vision/src/comparison.ts@@ -0,0 +1,215 @@+import { createHash } from "node:crypto";+import path from "node:path";+import { createScreenshotDiffPool } from "./diff-pool";+import { arePngFilesIdentical, diffPngFiles } from "./screenshot-diff";+import type { ScreenshotComparisonReportItem } from "./screenshot-report";++export type VisualEntry = {+ id: string;+};++export type VisualComparison<Entry extends VisualEntry = VisualEntry> = {+ id: string;+ status: "added" | "removed" | "comparable";+ baseline: Entry | undefined;+ current: Entry | undefined;+};++const indexVisualEntries = <Entry extends VisualEntry>(+ entries: readonly Entry[],+ label: "baseline" | "current"+) => {+ const indexed = new Map<string, Entry>();+ for (const entry of entries) {+ if (indexed.has(entry.id)) {+ throw new Error(+ `Duplicate ${label} visual entry id: ${JSON.stringify(entry.id)}`+ );+ }+ indexed.set(entry.id, entry);+ }+ return indexed;+};++export const getVisualComparisons = <Entry extends VisualEntry>({+ baselineEntries,+ currentEntries,+}: {+ baselineEntries: readonly Entry[];+ currentEntries: readonly Entry[];+}): VisualComparison<Entry>[] => {+ const baselineById = indexVisualEntries(baselineEntries, "baseline");+ const currentById = indexVisualEntries(currentEntries, "current");+ const ids = new Set([...baselineById.keys(), ...currentById.keys()]);++ return [...ids].sort().map((id) => {+ const baseline = baselineById.get(id);+ const current = currentById.get(id);+ if (baseline === undefined) {+ return { id, status: "added", baseline, current };+ }+ if (current === undefined) {+ return { id, status: "removed", baseline, current };+ }+ return { id, status: "comparable", baseline, current };+ });+};++export type VisualReportEntry = VisualEntry & {+ title: string;+ name: string;+};++export const compareVisualEntries = async <Entry extends VisualReportEntry>({+ baselineEntries,+ currentEntries,+ baselinePaths,+ currentPaths,+ baselineErrors,+ currentErrors,+ artifactDirectory,+ pixelThreshold,+ maxMismatchPercentage,+ concurrency,+ analyzeText = "when-different",+ writeArtifacts = "when-different",+}: {+ baselineEntries: readonly Entry[];+ currentEntries: readonly Entry[];+ baselinePaths: ReadonlyMap<string, string>;+ currentPaths: ReadonlyMap<string, string>;+ baselineErrors: ReadonlyMap<string, string>;+ currentErrors: ReadonlyMap<string, string>;+ artifactDirectory: string;+ pixelThreshold: number;+ maxMismatchPercentage: number;+ concurrency: number;+ analyzeText?: boolean | "when-different";+ writeArtifacts?: boolean | "when-different";+}): Promise<ScreenshotComparisonReportItem[]> => {+ if (Number.isInteger(concurrency) === false || concurrency < 1) {+ throw new Error(+ "Visual comparison concurrency must be a positive integer."+ );+ }+ const comparisons = getVisualComparisons({+ baselineEntries,+ currentEntries,+ });+ const identical = new Map(+ await Promise.all(+ comparisons.flatMap((comparison) => {+ if (+ comparison.status !== "comparable" ||+ baselineErrors.has(comparison.id) ||+ currentErrors.has(comparison.id)+ ) {+ return [];+ }+ const baselinePath = baselinePaths.get(comparison.id);+ const currentPath = currentPaths.get(comparison.id);+ if (baselinePath === undefined || currentPath === undefined) {+ return [];+ }+ return [+ arePngFilesIdentical(baselinePath, currentPath).then(+ (equal) => [comparison.id, equal] as const+ ),+ ];+ })+ )+ );+ const differentCount = [...identical.values()].filter(+ (equal) => equal === false+ ).length;+ const pool =+ differentCount > 1+ ? createScreenshotDiffPool(Math.min(concurrency, differentCount))+ : undefined;+ const runDiff =+ pool === undefined+ ? diffPngFiles+ : (options: Parameters<typeof diffPngFiles>[0]) => pool.run(options);+ try {+ return await Promise.all(+ comparisons.map(+ async (comparison): Promise<ScreenshotComparisonReportItem> => {+ const entry = comparison.current ?? comparison.baseline;+ if (entry === undefined) {+ throw new Error(`Visual comparison has no entry: ${comparison.id}`);+ }+ const baselinePath = baselinePaths.get(comparison.id);+ const currentPath = currentPaths.get(comparison.id);+ const captureErrors = [+ baselineErrors.get(comparison.id),+ currentErrors.get(comparison.id),+ ].filter((error): error is string => error !== undefined);+ if (captureErrors.length > 0) {+ return {+ ...entry,+ status: "error",+ baselinePath,+ currentPath,+ error: captureErrors.join("\n\n"),+ };+ }+ if (comparison.status === "added") {+ return { ...entry, status: "added", currentPath };+ }+ if (comparison.status === "removed") {+ return { ...entry, status: "removed", baselinePath };+ }+ if (baselinePath === undefined || currentPath === undefined) {+ throw new Error(`Screenshot is missing for ${comparison.id}`);+ }+ if (identical.get(comparison.id) === true) {+ return {+ ...entry,+ status: "unchanged",+ baselinePath,+ currentPath,+ differentPixels: 0,+ mismatchPercentage: 0,+ regions: [],+ textAnalysis: {+ status: "skipped",+ provider: "tesseract",+ changes: [],+ },+ warnings: [],+ };+ }+ const diff = await runDiff({+ baselinePath,+ currentPath,+ outputDir: path.join(+ artifactDirectory,+ createHash("sha256").update(comparison.id).digest("hex")+ ),+ threshold: pixelThreshold,+ analyzeText,+ writeArtifacts,+ });+ const changed =+ diff.dimensionMismatch !== undefined ||+ diff.mismatchPercentage > maxMismatchPercentage;+ return {+ ...entry,+ status: changed ? "changed" : "unchanged",+ baselinePath,+ currentPath,+ diffPath: diff.diffPath,+ contextDiffPath: diff.contextDiffPath,+ differentPixels: diff.differentPixels,+ mismatchPercentage: diff.mismatchPercentage,+ regions: diff.regions,+ textAnalysis: diff.textAnalysis,+ warnings: diff.warnings,+ };+ }+ )+ );+ } finally {+ await pool?.close();+ }+};diff --git a/packages/vision/src/diff-pool.test.ts b/packages/vision/src/diff-pool.test.tsnew file mode 100644index 0000000000..d5eed86adc--- /dev/null+++ b/packages/vision/src/diff-pool.test.ts@@ -0,0 +1,88 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { Worker } from "node:worker_threads";+import { expect, test, vi } from "vitest";+import { createScreenshotDiffPool } from "./diff-pool";++const onePixelPng = Buffer.from(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",+ "base64"+);++test("compares images in reusable workers", async () => {+ const directory = await mkdtemp(path.join(tmpdir(), "visual-diff-pool-"));+ const pool = createScreenshotDiffPool(2);+ try {+ const baselinePath = path.join(directory, "baseline.png");+ const currentPath = path.join(directory, "current.png");+ await Promise.all([+ writeFile(baselinePath, onePixelPng),+ writeFile(currentPath, onePixelPng),+ ]);+ const results = await Promise.all(+ Array.from({ length: 4 }, () =>+ pool.run({+ baselinePath,+ currentPath,+ outputDir: directory,+ analyzeText: false,+ writeArtifacts: false,+ })+ )+ );+ expect(results.map(({ differentPixels }) => differentPixels)).toEqual([+ 0, 0, 0, 0,+ ]);+ } finally {+ await pool.close();+ await rm(directory, { recursive: true, force: true });+ }+});++test("rejects invalid worker pool sizes", () => {+ expect(() => createScreenshotDiffPool(0)).toThrowError(+ "Screenshot diff pool size must be a positive integer."+ );+});++test("rejects work after the pool closes", async () => {+ const pool = createScreenshotDiffPool(1);+ await pool.close();+ await expect(+ pool.run({+ baselinePath: "baseline.png",+ currentPath: "current.png",+ outputDir: ".",+ })+ ).rejects.toThrowError("Screenshot diff pool is closed.");+});++test("waits for worker termination from every concurrent close", async () => {+ let releaseTermination: () => void = () => undefined;+ const terminationReleased = new Promise<void>((resolve) => {+ releaseTermination = resolve;+ });+ const terminate = Worker.prototype.terminate;+ const terminateSpy = vi+ .spyOn(Worker.prototype, "terminate")+ .mockImplementation(function (this: Worker) {+ const terminated = terminate.call(this);+ return terminationReleased.then(async () => await terminated);+ });+ const pool = createScreenshotDiffPool(1);+ try {+ const firstClose = pool.close();+ let secondCloseFinished = false;+ const secondClose = pool.close().then(() => {+ secondCloseFinished = true;+ });+ await Promise.resolve();+ expect(secondCloseFinished).toBe(false);+ releaseTermination();+ await Promise.all([firstClose, secondClose]);+ } finally {+ releaseTermination();+ terminateSpy.mockRestore();+ }+});diff --git a/packages/vision/src/diff-pool.ts b/packages/vision/src/diff-pool.tsnew file mode 100644index 0000000000..5451b94f38--- /dev/null+++ b/packages/vision/src/diff-pool.ts@@ -0,0 +1,137 @@+import { Worker } from "node:worker_threads";+import { diffPngFiles, type ScreenshotDiffResult } from "./screenshot-diff";+import type {+ ScreenshotDiffWorkerRequest,+ ScreenshotDiffWorkerResponse,+} from "./diff-worker";++type Task = {+ id: number;+ options: Parameters<typeof diffPngFiles>[0];+ resolve: (result: ScreenshotDiffResult) => void;+ reject: (error: Error) => void;+};++export const createScreenshotDiffPool = (size: number) => {+ if (Number.isInteger(size) === false || size < 1) {+ throw new Error("Screenshot diff pool size must be a positive integer.");+ }+ const workers = Array.from(+ { length: size },+ () =>+ new Worker(new URL("./diff-worker-entry.mjs", import.meta.url), {+ execArgv: ["--conditions=webstudio"],+ })+ );+ const idleWorkers = [...workers];+ const queue: Task[] = [];+ const activeTasks = new Map<number, Task>();+ const taskIds = new Map<Worker, number>();+ let nextTaskId = 0;+ let failure: Error | undefined;+ let closing = false;+ let closePromise: Promise<void> | undefined;++ const dispatch = () => {+ while (closing === false && idleWorkers.length > 0 && queue.length > 0) {+ const worker = idleWorkers.shift();+ const task = queue.shift();+ if (worker === undefined || task === undefined) {+ return;+ }+ activeTasks.set(task.id, task);+ taskIds.set(worker, task.id);+ worker.postMessage({+ id: task.id,+ options: task.options,+ } satisfies ScreenshotDiffWorkerRequest);+ }+ };++ const rejectQueuedTasks = (error: Error) => {+ for (const task of queue.splice(0)) {+ task.reject(error);+ }+ };++ const fail = (error: Error) => {+ failure ??= error;+ rejectQueuedTasks(error);+ };++ const handleWorkerFailure = (worker: Worker, error: Error) => {+ const taskId = taskIds.get(worker);+ if (taskId !== undefined) {+ activeTasks.get(taskId)?.reject(error);+ activeTasks.delete(taskId);+ taskIds.delete(worker);+ }+ const idleIndex = idleWorkers.indexOf(worker);+ if (idleIndex !== -1) {+ idleWorkers.splice(idleIndex, 1);+ }+ fail(error);+ };++ for (const worker of workers) {+ worker.on("message", (response: ScreenshotDiffWorkerResponse) => {+ const task = activeTasks.get(response.id);+ if (task === undefined) {+ return;+ }+ activeTasks.delete(response.id);+ taskIds.delete(worker);+ if ("error" in response) {+ task.reject(new Error(response.error));+ } else {+ task.resolve(response.result);+ }+ idleWorkers.push(worker);+ dispatch();+ });+ worker.on("error", (error) => {+ handleWorkerFailure(worker, error);+ });+ worker.on("exit", (code) => {+ if (closing === false) {+ handleWorkerFailure(+ worker,+ new Error(`Visual diff worker exited unexpectedly with code ${code}.`)+ );+ }+ });+ }++ return {+ run(options: Parameters<typeof diffPngFiles>[0]) {+ if (closing) {+ return Promise.reject(new Error("Screenshot diff pool is closed."));+ }+ if (failure !== undefined) {+ return Promise.reject(failure);+ }+ return new Promise<ScreenshotDiffResult>((resolve, reject) => {+ queue.push({ id: nextTaskId, options, resolve, reject });+ nextTaskId += 1;+ dispatch();+ });+ },+ close() {+ if (closePromise !== undefined) {+ return closePromise;+ }+ closing = true;+ const error = new Error("Screenshot diff pool is closed.");+ rejectQueuedTasks(error);+ for (const task of activeTasks.values()) {+ task.reject(error);+ }+ activeTasks.clear();+ taskIds.clear();+ closePromise = Promise.all(+ workers.map(async (worker) => worker.terminate())+ ).then(() => undefined);+ return closePromise;+ },+ };+};diff --git a/packages/vision/src/diff-worker-entry.mjs b/packages/vision/src/diff-worker-entry.mjsnew file mode 100644index 0000000000..203ecc8f8a--- /dev/null+++ b/packages/vision/src/diff-worker-entry.mjs@@ -0,0 +1,4 @@+import { register } from "tsx/esm/api";++register();+await import("./diff-worker.ts");diff --git a/packages/vision/src/diff-worker.ts b/packages/vision/src/diff-worker.tsnew file mode 100644index 0000000000..a6f551ef0e--- /dev/null+++ b/packages/vision/src/diff-worker.ts@@ -0,0 +1,31 @@+import { parentPort } from "node:worker_threads";+import { diffPngFiles, type ScreenshotDiffResult } from "./screenshot-diff";++export type ScreenshotDiffWorkerRequest = {+ id: number;+ options: Parameters<typeof diffPngFiles>[0];+};++export type ScreenshotDiffWorkerResponse =+ | { id: number; result: ScreenshotDiffResult }+ | { id: number; error: string };++parentPort?.on(+ "message",+ async ({ id, options }: ScreenshotDiffWorkerRequest) => {+ try {+ parentPort?.postMessage({+ id,+ result: await diffPngFiles(options),+ } satisfies ScreenshotDiffWorkerResponse);+ } catch (error) {+ parentPort?.postMessage({+ id,+ error:+ error instanceof Error+ ? (error.stack ?? error.message)+ : String(error),+ } satisfies ScreenshotDiffWorkerResponse);+ }+ }+);diff --git a/packages/vision/src/diff.ts b/packages/vision/src/diff.tsnew file mode 100644index 0000000000..4793d202be--- /dev/null+++ b/packages/vision/src/diff.ts@@ -0,0 +1,2 @@+export * from "./screenshot-diff";+export * from "./screenshot-text-diff";diff --git a/packages/vision/src/report.ts b/packages/vision/src/report.tsnew file mode 100644index 0000000000..df88a651d1--- /dev/null+++ b/packages/vision/src/report.ts@@ -0,0 +1 @@+export * from "./screenshot-report";diff --git a/packages/cli/src/screenshot-browser-cdp.test.ts b/packages/vision/src/screenshot-browser-cdp.test.tssimilarity index 76%rename from packages/cli/src/screenshot-browser-cdp.test.tsrename to packages/vision/src/screenshot-browser-cdp.test.tsindex f9055c032e..5b8d408f22 100644--- a/packages/cli/src/screenshot-browser-cdp.test.ts+++ b/packages/vision/src/screenshot-browser-cdp.test.ts@@ -4,11 +4,11 @@ import {captureBrowserScreenshot,createBrowserScreenshotSession,type BrowserScreenshotDependencies,-} from "./screenshot-browser-cdp";+} from "./browser";class FakeBrowserProcess {readonly listeners = new Map<string, Array<(value?: unknown) => void>>();- kill = vi.fn(() => {+ kill = vi.fn((_signal?: NodeJS.Signals | number) => {this.emit("exit", 0);return true;});@@ -89,15 +89,26 @@ class FakeWebSocket {if (message.method === "Runtime.evaluate" &&typeof message.params?.expression === "string" &&- message.params.expression.includes("generatedSiteRootPresent")+ message.params.expression.includes("rootMarkerPresent")) {+ const hasPageMetadata = message.params.expression.includes(+ '"rootMarkerAttribute":"data-site"'+ );setTimeout(() =>respond({result: {value: {documentReadyState: "complete",- generatedSiteRootPresent: true,+ ...(hasPageMetadata+ ? {+ pageMetadata: {+ rootMarkerPresent: true,+ id: "site",+ version: 2,+ },+ }+ : {}),width: this.readinessWidths?.shift() ?? 813,height: 2346,},@@ -165,9 +176,7 @@ class FakeWebSocket {if (message.method === "Runtime.evaluate" &&typeof message.params?.expression === "string" &&- message.params.expression.includes(- 'document.querySelectorAll("[data-ws-id]")'- )+ message.params.expression.includes("const overlapLimit")) {setTimeout(() =>@@ -421,6 +430,23 @@ class CrashingWebSocket extends FakeWebSocket {};}+class NeverOpeningWebSocket {+ readonly listeners = new Map<+ string,+ Array<(event: { data?: string }) => void>+ >();+ addEventListener = (+ event: string,+ listener: (event: { data?: string }) => void+ ) => {+ const listeners = this.listeners.get(event) ?? [];+ listeners.push(listener);+ this.listeners.set(event, listeners);+ };+ close = vi.fn();+ send = vi.fn();+}+class NavigationErrorWebSocket extends FakeWebSocket {send = (data: string) => {const message = JSON.parse(data) as {@@ -476,7 +502,7 @@ const createDependencies = ({() => browserProcess) as unknown as BrowserScreenshotDependencies["spawnBrowser"],mkdtemp: vi.fn(- async () => "/tmp/webstudio-browser-test"+ async () => "/tmp/vision-browser-test") as unknown as BrowserScreenshotDependencies["mkdtemp"],readFile: vi.fn(async () => "9222\n/devtools/browser/1\n"@@ -500,9 +526,16 @@ test("captures through DevTools with lifecycle and selector waits", async () =>width: 800,height: 600,browserPath: "/usr/bin/chromium",- uid: 0,+ uid: 1000,+ disableSandbox: true,waitUntil: "networkidle",waitForSelector: "#ready",+ pageMetadata: {+ rootMarkerAttribute: "data-site",+ idAttribute: "data-site",+ versionAttribute: "data-site-version",+ },+ instanceIdAttribute: "data-instance-id",waitForTimeout: 0,timeout: 1000,},@@ -517,7 +550,7 @@ test("captures through DevTools with lifecycle and selector waits", async () =>"--disable-backgrounding-occluded-windows","--disable-renderer-backgrounding","--remote-debugging-port=0",- "--user-data-dir=/tmp/webstudio-browser-test",+ "--user-data-dir=/tmp/vision-browser-test","--window-size=800,600","--no-sandbox","about:blank",@@ -546,7 +579,7 @@ test("captures through DevTools with lifecycle and selector waits", async () =>(message) =>message.method === "Runtime.evaluate" &&String(message.params?.expression).includes(- 'document.documentElement.hasAttribute("data-ws-project")'+ '"rootMarkerAttribute":"data-site"'))).toBe(true);@@ -556,7 +589,7 @@ test("captures through DevTools with lifecycle and selector waits", async () =>);expect(socket.close).toHaveBeenCalled();expect(browserProcess.kill).toHaveBeenCalled();- expect(dependencies.rm).toHaveBeenCalledWith("/tmp/webstudio-browser-test", {+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {recursive: true,force: true,});@@ -675,6 +708,7 @@ test("navigates once while resizing one page through multiple viewports", asyncurl: "https://example.com/responsive",browserPath: "/usr/bin/chromium",includeImageMetrics: true,+ includeElementGeometry: false,waitUntil: "networkidle" as const,waitForTimeout: 0,timeout: 1000,@@ -685,6 +719,8 @@ test("navigates once while resizing one page through multiple viewports", asyncoutput: "/tmp/mobile.png",width: 375,height: 812,+ prepareExpression: "window.renderStory('mobile')",+ finalizeExpression: "window.finishStory('mobile')",},dependencies);@@ -695,12 +731,16 @@ test("navigates once while resizing one page through multiple viewports", asyncoutput: "/tmp/mobile.png",width: 375,height: 812,+ prepareExpression: "window.renderStory('mobile')",+ finalizeExpression: "window.finishStory('mobile')",},{...baseOptions,output: "/tmp/desktop.png",width: 1440,height: 900,+ prepareExpression: "window.renderStory('desktop')",+ finalizeExpression: "window.finishStory('desktop')",},]);@@ -715,6 +755,11 @@ test("navigates once while resizing one page through multiple viewports", asyncexpect(socket.sentMessages.filter((message) => message.method === "Page.navigate")).toHaveLength(1);+ expect(+ socket.sentMessages.filter(+ (message) => message.method === "Emulation.setFocusEmulationEnabled"+ )+ ).toEqual([expect.objectContaining({ params: { enabled: true } })]);expect(socket.sentMessages.filter(@@ -724,12 +769,33 @@ test("navigates once while resizing one page through multiple viewports", async).toEqual([375, 1440]);expect(dependencies.createWebSocket).toHaveBeenCalledOnce();expect(dependencies.writeFile).toHaveBeenCalledTimes(2);+ expect(+ layouts.every(({ elementGeometry }) => elementGeometry === undefined)+ ).toBe(true);+ expect(+ socket.sentMessages+ .filter(+ (message) =>+ message.method === "Runtime.evaluate" &&+ String(message.params?.expression).startsWith("window.renderStory")+ )+ .map((message) => message.params?.expression)+ ).toEqual(["window.renderStory('mobile')", "window.renderStory('desktop')"]);+ expect(+ socket.sentMessages+ .filter(+ (message) =>+ message.method === "Runtime.evaluate" &&+ String(message.params?.expression).startsWith("window.finishStory")+ )+ .map((message) => message.params?.expression)+ ).toEqual(["window.finishStory('mobile')", "window.finishStory('desktop')"]);expect(socket.sentMessages.some((message) =>message.method === "Runtime.evaluate" &&String(message.params?.expression).includes(- "__webstudioImageDimensionCache"+ "__visionImageDimensionCache"))).toBe(true);@@ -853,6 +919,144 @@ test("restarts the shared browser once after an unexpected exit", async () => {expect(dependencies.rm).toHaveBeenCalledTimes(2);});+test("retries a browser restart after a transient startup failure", async () => {+ const firstProcess = new FakeBrowserProcess();+ const secondProcess = new FakeBrowserProcess();+ const dependencies = createDependencies();+ vi.mocked(dependencies.mkdtemp)+ .mockResolvedValueOnce("/tmp/vision-browser-first")+ .mockRejectedValueOnce(new Error("Temporary directory unavailable"))+ .mockResolvedValueOnce("/tmp/vision-browser-second");+ vi.mocked(dependencies.spawnBrowser)+ .mockReturnValueOnce(firstProcess as never)+ .mockReturnValueOnce(secondProcess as never);+ vi.mocked(dependencies.createWebSocket)+ .mockImplementationOnce(+ () => new CrashingWebSocket(firstProcess) as unknown as WebSocket+ )+ .mockImplementationOnce(() => new FakeWebSocket() as unknown as WebSocket);+ const options = {+ url: "https://example.com",+ output: "/tmp/restarted.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "networkidle" as const,+ waitForTimeout: 0,+ timeout: 1000,+ };+ const session = await createBrowserScreenshotSession(options, dependencies);++ await expect(session.capture(options)).rejects.toThrow(+ "Temporary directory unavailable"+ );+ await expect(session.capture(options)).resolves.toMatchObject({+ viewportWidth: 800,+ viewportHeight: 600,+ });+ await session.close();++ expect(dependencies.mkdtemp).toHaveBeenCalledTimes(3);+ expect(secondProcess.kill).toHaveBeenCalledOnce();+});++test("closes a browser that finishes restarting during session cleanup", async () => {+ const firstProcess = new FakeBrowserProcess();+ const secondProcess = new FakeBrowserProcess();+ const dependencies = createDependencies();+ let resolveRestartDirectory: (directory: string) => void = () => undefined;+ const restartDirectory = new Promise<string>((resolve) => {+ resolveRestartDirectory = resolve;+ });+ vi.mocked(dependencies.mkdtemp)+ .mockResolvedValueOnce("/tmp/vision-browser-first")+ .mockReturnValueOnce(restartDirectory);+ vi.mocked(dependencies.spawnBrowser)+ .mockReturnValueOnce(firstProcess as never)+ .mockReturnValueOnce(secondProcess as never);+ vi.mocked(dependencies.createWebSocket).mockImplementationOnce(+ () => new CrashingWebSocket(firstProcess) as unknown as WebSocket+ );+ const options = {+ url: "https://example.com",+ output: "/tmp/restarted.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "networkidle" as const,+ waitForTimeout: 0,+ timeout: 1000,+ };+ const session = await createBrowserScreenshotSession(options, dependencies);++ const capture = session.capture(options);+ await vi.waitFor(() => {+ expect(dependencies.mkdtemp).toHaveBeenCalledTimes(2);+ });+ const close = session.close();+ resolveRestartDirectory("/tmp/vision-browser-second");++ await expect(capture).rejects.toThrow(+ "Browser screenshot session was closed."+ );+ await close;+ expect(secondProcess.kill).toHaveBeenCalledOnce();+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-second", {+ recursive: true,+ force: true,+ });+});++test("removes the browser profile when spawning throws", async () => {+ const dependencies = createDependencies();+ vi.mocked(dependencies.spawnBrowser).mockImplementationOnce(() => {+ throw new Error("Browser spawn failed");+ });++ await expect(+ createBrowserScreenshotSession(+ {+ url: "https://example.com",+ output: "/tmp/current.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "load",+ waitForTimeout: 0,+ timeout: 1000,+ },+ dependencies+ )+ ).rejects.toThrow("Browser spawn failed");+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {+ recursive: true,+ force: true,+ });+});++test("closes a DevTools socket that does not open before the deadline", async () => {+ const socket = new NeverOpeningWebSocket();+ const dependencies = createDependencies();+ dependencies.createWebSocket = vi.fn(() => socket as unknown as WebSocket);++ await expect(+ captureBrowserScreenshot(+ {+ url: "https://example.com",+ output: "/tmp/current.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "load",+ waitForTimeout: 0,+ timeout: 10,+ },+ dependencies+ )+ ).rejects.toThrow("Browser DevTools connection did not open within 10ms.");+ expect(socket.close).toHaveBeenCalledOnce();+});+test("retries browser startup before capturing a viewport batch", async () => {const failedBrowser = new FakeBrowserProcess();const activeBrowser = new FakeBrowserProcess();@@ -920,7 +1124,7 @@ test("reports browser startup exit diagnostics without local paths", async () =>)).rejects.toMatchObject({message:- "Browser exited before its DevTools endpoint became ready (exit code 21). Check the browser installation or set WEBSTUDIO_BROWSER_PATH to a supported Chromium executable.",+ "Browser exited before its DevTools endpoint became ready (exit code 21). Check the browser installation or provide a supported Chromium executable.",});});@@ -961,6 +1165,12 @@ test("captures full page and returns DevTools layout metrics", async () => {includeImageMetrics: true,includeResourceMetrics: true,includeContrastMetrics: true,+ pageMetadata: {+ rootMarkerAttribute: "data-site",+ idAttribute: "data-site",+ versionAttribute: "data-site-version",+ },+ instanceIdAttribute: "data-instance-id",browserPath: "/usr/bin/chromium",uid: 1000,waitUntil: "networkidle",@@ -979,7 +1189,11 @@ test("captures full page and returns DevTools layout metrics", async () => {mimeType: "text/html",redirects: [],documentReadyState: "complete",- generatedSiteRootPresent: true,+ pageMetadata: {+ rootMarkerPresent: true,+ id: "site",+ version: 2,+ },layoutStable: true,},documentType: "text/html",@@ -1083,6 +1297,22 @@ test("captures full page and returns DevTools layout metrics", async () => {expect.arrayContaining(["Page.getLayoutMetrics", "Page.captureScreenshot"]));expect(socket.screenshotStartedBeforeImageInspectionFinished).toBe(true);+ const inspectionExpressions = socket.sentMessages.flatMap((message) =>+ message.method === "Runtime.evaluate" &&+ typeof message.params?.expression === "string" &&+ (message.params.expression.includes("const overlapLimit") ||+ message.params.expression.includes("const parseOpaqueRgb"))+ ? [message.params.expression]+ : []+ );+ expect(inspectionExpressions).toHaveLength(2);+ expect(inspectionExpressions).toEqual([+ expect.stringContaining("CSS.escape(instanceIdAttribute)"),+ expect.stringContaining("CSS.escape(instanceIdAttribute)"),+ ]);+ expect(inspectionExpressions.join("\n")).not.toContain(+ 'document.querySelectorAll("*")'+ );expect(socket.sentMessages.find((message) => message.method === "Page.captureScreenshot"@@ -1126,7 +1356,6 @@ test("returns final URL, redirects, and HTTP error status", async () => {mimeType: "text/html",redirects: ["https://example.com/old"],documentReadyState: "complete",- generatedSiteRootPresent: true,layoutStable: true,});});@@ -1199,7 +1428,7 @@ test("times out when browser DevTools commands stop responding", async () => {).rejects.toThrow("Browser command Page.enable did not finish within 10ms.");expect(browserProcess.kill).toHaveBeenCalled();- expect(dependencies.rm).toHaveBeenCalledWith("/tmp/webstudio-browser-test", {+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {recursive: true,force: true,});@@ -1257,7 +1486,7 @@ test("times out when browser page target creation does not respond", async () =>).rejects.toThrow("Browser page target could not be created within 10ms.");expect(browserProcess.kill).toHaveBeenCalled();- expect(dependencies.rm).toHaveBeenCalledWith("/tmp/webstudio-browser-test", {+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {recursive: true,force: true,});@@ -1292,6 +1521,90 @@ test("stops polling when the browser DevTools port is not created", async () =>expect(browserProcess.kill).toHaveBeenCalled();});+test("does not wait indefinitely for a failed browser startup to exit", async () => {+ const browserProcess = new FakeBrowserProcess();+ browserProcess.kill.mockImplementation((signal) => {+ if (signal === "SIGKILL") {+ browserProcess.emit("exit", null);+ }+ return true;+ });+ const dependencies = createDependencies({ browserProcess });+ vi.mocked(dependencies.readFile).mockRejectedValue(+ Object.assign(new Error("not ready"), { code: "ENOENT" })+ );++ const result = await Promise.race([+ captureBrowserScreenshot(+ {+ url: "https://example.com",+ output: "/tmp/current.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "load",+ waitForTimeout: 0,+ timeout: 10,+ },+ dependencies+ ).then(+ () => "resolved",+ (error: unknown) =>+ error instanceof Error ? error.message : String(error)+ ),+ new Promise<string>((resolve) => {+ setTimeout(() => resolve("startup cleanup hung"), 100).unref();+ }),+ ]);++ expect(result).toContain(+ "Browser DevTools endpoint was not created within 10ms."+ );+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {+ recursive: true,+ force: true,+ });+ expect(browserProcess.kill).toHaveBeenNthCalledWith(1);+ expect(browserProcess.kill).toHaveBeenNthCalledWith(2, "SIGKILL");+});++test("does not wait indefinitely for a page target to close", async () => {+ const fetch = vi.fn(async (url: string) => {+ if (url.includes("/json/new?")) {+ return {+ json: async () => ({+ id: "target-1",+ type: "page",+ webSocketDebuggerUrl: "ws://127.0.0.1:9222/page/created",+ }),+ } as Response;+ }+ return await new Promise<Response>(() => undefined);+ }) as unknown as BrowserScreenshotDependencies["fetch"];+ const dependencies = createDependencies({ fetch });++ const result = await Promise.race([+ captureBrowserScreenshot(+ {+ url: "https://example.com",+ output: "/tmp/current.png",+ width: 800,+ height: 600,+ browserPath: "/usr/bin/chromium",+ waitUntil: "networkidle",+ waitForTimeout: 0,+ timeout: 1000,+ },+ dependencies+ ).then(() => "captured"),+ new Promise<string>((resolve) => {+ setTimeout(() => resolve("target cleanup hung"), 2000).unref();+ }),+ ]);++ expect(result).toBe("captured");+});+test("reports browser navigation errors directly", async () => {const browserProcess = new FakeBrowserProcess();const socket = new NavigationErrorWebSocket();@@ -1316,7 +1629,7 @@ test("reports browser navigation errors directly", async () => {);expect(browserProcess.kill).toHaveBeenCalled();- expect(dependencies.rm).toHaveBeenCalledWith("/tmp/webstudio-browser-test", {+ expect(dependencies.rm).toHaveBeenCalledWith("/tmp/vision-browser-test", {recursive: true,force: true,});diff --git a/packages/cli/src/screenshot-browser-cdp.ts b/packages/vision/src/screenshot-browser-cdp.tssimilarity index 79%rename from packages/cli/src/screenshot-browser-cdp.tsrename to packages/vision/src/screenshot-browser-cdp.tsindex f77fc47fb4..85682e258e 100644--- a/packages/cli/src/screenshot-browser-cdp.ts+++ b/packages/vision/src/screenshot-browser-cdp.ts@@ -3,11 +3,34 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import { spawn, type ChildProcess } from "node:child_process";-import type { ScreenshotWaitUntil } from "@webstudio-is/project-build/visual";-import { withTimeout } from "./async-utils";+import type { ScreenshotWaitUntil } from "./screenshot-browser";++const withTimeout = async <Result>(+ operation: Promise<Result>,+ timeout: number,+ createTimeoutError: () => Error+) => {+ let timeoutId: NodeJS.Timeout | undefined;+ const timeoutPromise = new Promise<never>((_, reject) => {+ timeoutId = setTimeout(() => reject(createTimeoutError()), timeout);+ });+ try {+ return await Promise.race([operation, timeoutPromise]);+ } finally {+ if (timeoutId !== undefined) {+ clearTimeout(timeoutId);+ }+ }+};type BrowserProcess = Pick<ChildProcess, "kill" | "once">;+export type BrowserScreenshotPageMetadata = {+ rootMarkerPresent: boolean;+ id?: string;+ version?: number;+};+export type BrowserScreenshotOptions = {browserPath: string;output: string;@@ -17,12 +40,23 @@ export type BrowserScreenshotOptions = {includeImageMetrics?: boolean;includeResourceMetrics?: boolean;includeContrastMetrics?: boolean;+ includeElementGeometry?: boolean;+ pageMetadata?: {+ rootMarkerAttribute: string;+ idAttribute?: string;+ versionAttribute?: string;+ };+ instanceIdAttribute?: string;url: string;httpCredentials?: { username: string; password: string };uid?: number;+ disableSandbox?: boolean;waitUntil: ScreenshotWaitUntil;+ prepareExpression?: string;waitForSelector?: string;+ failForSelector?: string;waitForTimeout: number;+ finalizeExpression?: string;timeout: number;format?: "png" | "jpeg" | "webp";quality?: number;@@ -176,20 +210,29 @@ class CdpSession {timeout: number) => {const socket = dependencies.createWebSocket(url);- await withDeadline(- new Promise<void>((resolveConnection, rejectConnection) => {- socket.addEventListener("open", () => resolveConnection(), {- once: true,- });- socket.addEventListener(- "error",- () => rejectConnection(new Error("Could not connect to browser.")),- { once: true }- );- }),- "Browser DevTools connection did not open",- timeout- );+ try {+ await withDeadline(+ new Promise<void>((resolveConnection, rejectConnection) => {+ socket.addEventListener("open", () => resolveConnection(), {+ once: true,+ });+ socket.addEventListener(+ "error",+ () => rejectConnection(new Error("Could not connect to browser.")),+ { once: true }+ );+ }),+ "Browser DevTools connection did not open",+ timeout+ );+ } catch (error) {+ try {+ socket.close();+ } catch {+ // Preserve the connection failure when a pending socket cannot close.+ }+ throw error;+ }return new CdpSession(socket);};@@ -379,36 +422,58 @@ const waitForSelector = async (throw createTimeoutError(`Selector ${selector} was not found`, timeout);};+const getSelectorText = async (cdp: CdpSession, selector: string) => {+ const result = await cdp.send<{ result?: { value?: unknown } }>(+ "Runtime.evaluate",+ {+ expression: `document.querySelector(${JSON.stringify(selector)})?.textContent`,+ returnByValue: true,+ }+ );+ return typeof result.result?.value === "string"+ ? result.result.value.trim()+ : undefined;+};+type BrowserReadiness = {documentReadyState: string;- generatedSiteRootPresent: boolean;- projectId?: string;- projectVersion?: number;+ pageMetadata?: BrowserScreenshotPageMetadata;layoutStable: boolean;};const waitForFontsAndFrames = async (cdp: CdpSession,- timeout: number+ timeout: number,+ pageMetadata: BrowserScreenshotOptions["pageMetadata"]): Promise<BrowserReadiness> => {const measure = async () => {const response = await withDeadline(cdp.send<{ result?: { value?: unknown } }>("Runtime.evaluate", {- expression: `Promise.resolve(document.fonts?.ready).then(() => ({- documentReadyState: document.readyState,- generatedSiteRootPresent:- document.documentElement.hasAttribute("data-ws-project"),- projectId:- document.documentElement.getAttribute("data-ws-project") ?? undefined,- projectVersion: (() => {- const attribute = document.documentElement.getAttribute("data-ws-version");- if (attribute === null) return undefined;- const value = Number(attribute);- return Number.isFinite(value) ? value : undefined;- })(),- width: document.documentElement.scrollWidth,- height: document.documentElement.scrollHeight,- }))`,+ expression: `Promise.resolve(document.fonts?.ready).then(() => {+ const metadata = ${JSON.stringify(pageMetadata)};+ const root = document.documentElement;+ const id = metadata?.idAttribute === undefined+ ? undefined+ : root.getAttribute(metadata.idAttribute) ?? undefined;+ const versionAttribute = metadata?.versionAttribute === undefined+ ? undefined+ : root.getAttribute(metadata.versionAttribute);+ const version = versionAttribute === undefined || versionAttribute === null+ ? undefined+ : Number(versionAttribute);+ return {+ documentReadyState: document.readyState,+ pageMetadata: metadata === undefined+ ? undefined+ : {+ rootMarkerPresent: root.hasAttribute(metadata.rootMarkerAttribute),+ id,+ version: Number.isFinite(version) ? version : undefined,+ },+ width: root.scrollWidth,+ height: root.scrollHeight,+ };+ })`,awaitPromise: true,returnByValue: true,}),@@ -421,9 +486,7 @@ const waitForFontsAndFrames = async (value: unknown): value is {documentReadyState: string;- generatedSiteRootPresent: boolean;- projectId?: string;- projectVersion?: number;+ pageMetadata?: BrowserScreenshotPageMetadata;width: number;height: number;} =>@@ -431,23 +494,25 @@ const waitForFontsAndFrames = async (value !== null &&"documentReadyState" in value &&typeof value.documentReadyState === "string" &&- "generatedSiteRootPresent" in value &&- typeof value.generatedSiteRootPresent === "boolean" &&- ("projectId" in value === false ||- value.projectId === undefined ||- typeof value.projectId === "string") &&- ("projectVersion" in value === false ||- value.projectVersion === undefined ||- typeof value.projectVersion === "number") &&+ ("pageMetadata" in value === false ||+ value.pageMetadata === undefined ||+ (typeof value.pageMetadata === "object" &&+ value.pageMetadata !== null &&+ "rootMarkerPresent" in value.pageMetadata &&+ typeof value.pageMetadata.rootMarkerPresent === "boolean" &&+ ("id" in value.pageMetadata === false ||+ value.pageMetadata.id === undefined ||+ typeof value.pageMetadata.id === "string") &&+ ("version" in value.pageMetadata === false ||+ value.pageMetadata.version === undefined ||+ typeof value.pageMetadata.version === "number"))) &&"width" in value &&typeof value.width === "number" &&"height" in value &&typeof value.height === "number";const initial = await measure();if (isLayoutSample(initial) === false) {- throw new Error(- "Browser did not report generated page readiness evidence."- );+ throw new Error("Browser did not report page readiness evidence.");}let before = initial;const deadline = Date.now() + timeout;@@ -455,33 +520,23 @@ const waitForFontsAndFrames = async (await delay(32);const value = await measure();if (isLayoutSample(value) === false) {- throw new Error(- "Browser did not report generated page readiness evidence."- );+ throw new Error("Browser did not report page readiness evidence.");}if (before.width === value.width && before.height === value.height) {return {documentReadyState: value.documentReadyState,- generatedSiteRootPresent: value.generatedSiteRootPresent,- ...(value.projectId === undefined- ? {}- : { projectId: value.projectId }),- ...(value.projectVersion === undefined+ ...(value.pageMetadata === undefined? {}- : { projectVersion: value.projectVersion }),+ : { pageMetadata: value.pageMetadata }),layoutStable: true,};}if (Date.now() >= deadline) {return {documentReadyState: value.documentReadyState,- generatedSiteRootPresent: value.generatedSiteRootPresent,- ...(value.projectId === undefined+ ...(value.pageMetadata === undefined? {}- : { projectId: value.projectId }),- ...(value.projectVersion === undefined- ? {}- : { projectVersion: value.projectVersion }),+ : { pageMetadata: value.pageMetadata }),layoutStable: false,};}@@ -494,11 +549,13 @@ const createBrowserLaunchArgs = ({width,height,uid,+ disableSandbox,}: {userDataDir: string;width: number;height: number;uid?: number;+ disableSandbox?: boolean;}) => ["--headless=new","--disable-gpu",@@ -511,7 +568,7 @@ const createBrowserLaunchArgs = ({`--window-size=${width},${height}`,"--no-first-run","--no-default-browser-check",- ...(uid === 0 ? ["--no-sandbox"] : []),+ ...(uid === 0 || disableSandbox === true ? ["--no-sandbox"] : []),"about:blank",];@@ -579,9 +636,7 @@ export type BrowserScreenshotNavigation = {mimeType?: string;redirects: string[];documentReadyState: string;- generatedSiteRootPresent: boolean;- projectId?: string;- projectVersion?: number;+ pageMetadata?: BrowserScreenshotPageMetadata;layoutStable: boolean;};@@ -627,7 +682,8 @@ const getBrowserScreenshotElementGeometry = async (send: <Result = unknown>(method: string,params?: Record<string, unknown>- ) => Promise<Result>+ ) => Promise<Result>,+ instanceIdAttribute: string): Promise<BrowserScreenshotElementGeometry> => {const response = await send<{result?: { value?: unknown };@@ -635,7 +691,9 @@ const getBrowserScreenshotElementGeometry = async (expression: `(() => {const limit = ${maxBrowserScreenshotElements};const overlapLimit = ${maxBrowserScreenshotOverlapsPerElement};- const candidates = Array.from(document.querySelectorAll("[data-ws-id]"));+ const instanceIdAttribute = ${JSON.stringify(instanceIdAttribute)};+ const instanceSelector = "[" + CSS.escape(instanceIdAttribute) + "]";+ const candidates = Array.from(document.querySelectorAll(instanceSelector));const elements = candidates.slice(0, limit).map((element) => {const style = getComputedStyle(element);const rect = element.getBoundingClientRect();@@ -649,7 +707,7 @@ const getBrowserScreenshotElementGeometry = async (const clipsY = style.overflowY === "hidden" || style.overflowY === "clip";return {element,- instanceId: element.getAttribute("data-ws-id"),+ instanceId: element.getAttribute(instanceIdAttribute),tagName: element.tagName.toLowerCase(),x: rect.left + window.scrollX,y: rect.top + window.scrollY,@@ -816,13 +874,16 @@ const getBrowserScreenshotContrasts = async (send: <Result = unknown>(method: string,params?: Record<string, unknown>- ) => Promise<Result>+ ) => Promise<Result>,+ instanceIdAttribute: string): Promise<BrowserScreenshotContrast[]> => {const response = await send<{result?: { value?: unknown };}>("Runtime.evaluate", {expression: `(() => {const limit = ${maxBrowserScreenshotElements};+ const instanceIdAttribute = ${JSON.stringify(instanceIdAttribute)};+ const instanceSelector = "[" + CSS.escape(instanceIdAttribute) + "]";const parseOpaqueRgb = (value) => {const match = value.match(/^rgba?\\(\\s*(\\d+(?:\\.\\d+)?)\\D+(\\d+(?:\\.\\d+)?)\\D+(\\d+(?:\\.\\d+)?)(?:\\D+(\\d+(?:\\.\\d+)?))?\\s*\\)$/i);if (match === null || (match[4] !== undefined && Number(match[4]) !== 1)) return undefined;@@ -835,7 +896,7 @@ const getBrowserScreenshotContrasts = async (});return 0.2126 * channels[0] + 0.7152 * channels[1] + 0.0722 * channels[2];};- const candidates = Array.from(document.querySelectorAll("[data-ws-id]"))+ const candidates = Array.from(document.querySelectorAll(instanceSelector)).filter((element) => Array.from(element.childNodes).some((node) => node.nodeType === Node.TEXT_NODE && node.textContent.trim() !== "")).slice(0, limit);return candidates.flatMap((element) => {@@ -897,7 +958,7 @@ const getBrowserScreenshotContrasts = async (const fontWeight = Number.isFinite(parsedWeight) ? parsedWeight : style.fontWeight === "bold" ? 700 : 400;const requiredRatio = fontSize >= 24 || (fontSize >= 18.66 && fontWeight >= 700) ? 3 : 4.5;return [{- instanceId: element.getAttribute("data-ws-id"),+ instanceId: element.getAttribute(instanceIdAttribute),tagName: element.tagName.toLowerCase(),foreground: style.color,background: background.value,@@ -938,13 +999,15 @@ const getBrowserScreenshotImages = async (send: <Result = unknown>(method: string,params?: Record<string, unknown>- ) => Promise<Result>+ ) => Promise<Result>,+ instanceIdAttribute: string): Promise<BrowserScreenshotImage[]> => {const response = await send<{result?: { value?: unknown };}>("Runtime.evaluate", {expression: `Promise.all(Array.from(document.images, async (image) => {- globalThis.__webstudioImageDimensionCache ??= new Map();+ globalThis.__visionImageDimensionCache ??= new Map();+ const instanceIdAttribute = ${JSON.stringify(instanceIdAttribute)};const rect = image.getBoundingClientRect();let sourcePathname;let selectedSourceWidth;@@ -956,22 +1019,23 @@ const getBrowserScreenshotImages = async (: new URL(selectedSource, document.baseURI).pathname;} catch {}try {- let dimensions = globalThis.__webstudioImageDimensionCache.get(selectedSource);+ let dimensions = globalThis.__visionImageDimensionCache.get(selectedSource);if (dimensions === undefined) {const response = await fetch(selectedSource);const bitmap = await createImageBitmap(await response.blob());dimensions = { width: bitmap.width, height: bitmap.height };bitmap.close();- globalThis.__webstudioImageDimensionCache.set(selectedSource, dimensions);+ globalThis.__visionImageDimensionCache.set(selectedSource, dimensions);}selectedSourceWidth = dimensions.width;selectedSourceHeight = dimensions.height;} catch {}+ let identifiedElement = image;+ while (identifiedElement !== null && identifiedElement.hasAttribute(instanceIdAttribute) === false) {+ identifiedElement = identifiedElement.parentElement;+ }return {- instanceId:- image.getAttribute("data-ws-id") ||- image.closest("[data-ws-id]")?.getAttribute("data-ws-id") ||- undefined,+ instanceId: identifiedElement?.getAttribute(instanceIdAttribute) || undefined,sourcePathname,loading: image.loading || "eager",complete: image.complete,@@ -1178,7 +1242,7 @@ const getScreenshotCaptureParams = async ({class BrowserSessionClosedError extends Error {}const getBrowserExitMessage = (message: string, reason?: string) =>- `${message}${reason === undefined ? "" : ` (${reason})`}. Check the browser installation or set WEBSTUDIO_BROWSER_PATH to a supported Chromium executable.`;+ `${message}${reason === undefined ? "" : ` (${reason})`}. Check the browser installation or provide a supported Chromium executable.`;type BrowserRuntime = {userDataDir: string;@@ -1189,22 +1253,63 @@ type BrowserRuntime = {close: () => Promise<void>;};+const stopBrowserProcess = async ({+ browserProcess,+ browserClosed,+ running,+ gracePeriodMs,+}: {+ browserProcess: BrowserProcess;+ browserClosed: Promise<string | undefined>;+ running: boolean;+ gracePeriodMs: number;+}) => {+ if (running === false) {+ return;+ }+ browserProcess.kill();+ try {+ await withDeadline(+ browserClosed,+ "Browser process did not exit after termination",+ gracePeriodMs+ );+ return;+ } catch {+ browserProcess.kill("SIGKILL");+ }+ await withDeadline(+ browserClosed,+ "Browser process did not exit after forced termination",+ 2000+ ).catch(() => undefined);+};+const startBrowserRuntimeOnce = async (options: BrowserScreenshotOptions,dependencies: BrowserScreenshotDependencies): Promise<BrowserRuntime> => {const userDataDir = await dependencies.mkdtemp(- join(tmpdir(), "webstudio-browser-")- );- const browserProcess = dependencies.spawnBrowser(- options.browserPath,- createBrowserLaunchArgs({- userDataDir,- width: options.width,- height: options.height,- uid: options.uid,- })+ join(tmpdir(), "vision-browser-"));+ let browserProcess: BrowserProcess;+ try {+ browserProcess = dependencies.spawnBrowser(+ options.browserPath,+ createBrowserLaunchArgs({+ userDataDir,+ width: options.width,+ height: options.height,+ uid: options.uid,+ disableSandbox: options.disableSandbox,+ })+ );+ } catch (error) {+ await dependencies+ .rm(userDataDir, { recursive: true, force: true })+ .catch(() => undefined);+ throw error;+ }let running = true;const browserClosed = new Promise<string | undefined>((resolveClosed) => {const close = (reason?: string) => {@@ -1248,14 +1353,12 @@ const startBrowserRuntimeOnce = async (},close: async () => {closePromise ??= (async () => {- if (running) {- browserProcess.kill();- }- await withDeadline(+ await stopBrowserProcess({+ browserProcess,browserClosed,- "Browser process did not exit after screenshot capture",- 2000- ).catch(() => undefined);+ running,+ gracePeriodMs: 2000,+ });await dependencies.rm(userDataDir, {recursive: true,force: true,@@ -1266,9 +1369,15 @@ const startBrowserRuntimeOnce = async (};return runtime;} catch (error) {- browserProcess.kill();- await browserClosed;- await dependencies.rm(userDataDir, { recursive: true, force: true });+ await stopBrowserProcess({+ browserProcess,+ browserClosed,+ running,+ gracePeriodMs: Math.min(options.timeout, 2000),+ });+ await dependencies+ .rm(userDataDir, { recursive: true, force: true })+ .catch(() => undefined);throw error;}};@@ -1399,6 +1508,7 @@ const capturePageWithBrowserRuntime = async (send("Page.enable"),send("Network.enable"),send("Runtime.enable"),+ send("Emulation.setFocusEmulationEnabled", { enabled: true }),send("Page.setLifecycleEventsEnabled", { enabled: true }),]);const targetSetupMs = Date.now() - setupStartedAt;@@ -1451,18 +1561,71 @@ const capturePageWithBrowserRuntime = async (options.timeout);}+ if (options.prepareExpression !== undefined) {+ try {+ await send(+ "Runtime.evaluate",+ {+ expression: options.prepareExpression,+ awaitPromise: true,+ },+ options.timeout+ );+ } catch (cause) {+ throw new Error(+ `Screenshot target preparation failed: ${options.url}\nOutput: ${options.output}`,+ { cause }+ );+ }+ }const readinessStartedAt = Date.now();if (options.waitForSelector !== undefined) {- await waitForSelector(- cdp,- options.waitForSelector,- options.timeout- );+ try {+ await waitForSelector(+ cdp,+ options.waitForSelector,+ options.timeout+ );+ } catch (cause) {+ throw new Error(+ `Screenshot target did not become ready: ${options.url}\nOutput: ${options.output}`,+ { cause }+ );+ }+ }+ if (options.failForSelector !== undefined) {+ const failure = await getSelectorText(cdp, options.failForSelector);+ if (failure !== undefined) {+ throw new Error(+ `Screenshot target reported an error: ${failure}\nTarget: ${options.url}`+ );+ }}- const readiness = await waitForFontsAndFrames(cdp, options.timeout);+ const readiness = await waitForFontsAndFrames(+ cdp,+ options.timeout,+ options.pageMetadata+ );if (options.waitForTimeout > 0) {await delay(options.waitForTimeout);}+ if (options.finalizeExpression !== undefined) {+ try {+ await send(+ "Runtime.evaluate",+ {+ expression: options.finalizeExpression,+ awaitPromise: true,+ },+ options.timeout+ );+ } catch (cause) {+ throw new Error(+ `Screenshot target finalization failed: ${options.url}\nOutput: ${options.output}`,+ { cause }+ );+ }+ }const readinessMs = Date.now() - readinessStartedAt;const locationPromise = send<{result?: { value?: unknown };@@ -1481,10 +1644,23 @@ const capturePageWithBrowserRuntime = async ();const documentMetricsPromise = getBrowserDocumentMetrics(send);const elementGeometryPromise =- getBrowserScreenshotElementGeometry(send);+ options.includeElementGeometry === false+ ? Promise.resolve({+ total: 0,+ sampled: 0,+ truncated: false,+ elements: [],+ })+ : getBrowserScreenshotElementGeometry(+ send,+ options.instanceIdAttribute ?? "id"+ );const imageInspectionPromise = measureDuration(async () =>options.includeImageMetrics === true- ? await getBrowserScreenshotImages(send)+ ? await getBrowserScreenshotImages(+ send,+ options.instanceIdAttribute ?? "id"+ ): undefined);const resourceInspectionPromise = measureDuration(async () =>@@ -1494,7 +1670,10 @@ const capturePageWithBrowserRuntime = async ();const contrastInspectionPromise = measureDuration(async () =>options.includeContrastMetrics === true- ? await getBrowserScreenshotContrasts(send)+ ? await getBrowserScreenshotContrasts(+ send,+ options.instanceIdAttribute ?? "id"+ ): undefined);const screenshotPromise = (async () => {@@ -1597,6 +1776,9 @@ const capturePageWithBrowserRuntime = async (if (contrasts === undefined) {delete layout.contrasts;}+ if (options.includeElementGeometry === false) {+ delete layout.elementGeometry;+ }layout.timings = {wallMs: Date.now() - viewportStartedAt,targetSetupMs: index === 0 ? targetSetupMs : 0,@@ -1616,9 +1798,13 @@ const capturePageWithBrowserRuntime = async (const cleanupStartedAt = Date.now();cdp?.close();if (targetId !== undefined && runtime.running) {- await dependencies- .fetch(`http://127.0.0.1:${runtime.port}/json/close/${targetId}`)- .catch(() => undefined);+ await withDeadline(+ dependencies.fetch(+ `http://127.0.0.1:${runtime.port}/json/close/${targetId}`+ ),+ "Browser page target did not close",+ Math.min(firstOptions.timeout, 100)+ ).catch(() => undefined);}const lastLayout = layouts.at(-1);if (lastLayout?.timings !== undefined) {@@ -1652,7 +1838,8 @@ export type BrowserScreenshotSession = {options: BrowserScreenshotOptions) => Promise<BrowserScreenshotLayout>;capturePage: (- options: readonly BrowserScreenshotOptions[]+ options: readonly BrowserScreenshotOptions[],+ sessionOptions?: { concurrency?: number; staggerMs?: number }) => Promise<BrowserScreenshotLayout[]>;close: () => Promise<void>;};@@ -1663,24 +1850,39 @@ export const createBrowserScreenshotSession = async (): Promise<BrowserScreenshotSession> => {let runtime = await startBrowserRuntime(options, dependencies);let restartPromise: Promise<BrowserRuntime> | undefined;+ let closePromise: Promise<void> | undefined;let closed = false;const restart = async (failedRuntime: BrowserRuntime) => {if (runtime !== failedRuntime) {return runtime;}- restartPromise ??= (async () => {- await failedRuntime.close();- if (closed) {- throw new BrowserSessionClosedError(- "Browser screenshot session was closed."- );+ const pendingRestart =+ restartPromise ??+ (async () => {+ await failedRuntime.close();+ if (closed) {+ throw new BrowserSessionClosedError(+ "Browser screenshot session was closed."+ );+ }+ const next = await startBrowserRuntime(options, dependencies);+ if (closed) {+ await next.close();+ throw new BrowserSessionClosedError(+ "Browser screenshot session was closed."+ );+ }+ runtime = next;+ return next;+ })();+ restartPromise = pendingRestart;+ try {+ return await pendingRestart;+ } finally {+ if (restartPromise === pendingRestart) {+ restartPromise = undefined;}- const next = await startBrowserRuntime(options, dependencies);- runtime = next;- restartPromise = undefined;- return next;- })();- return await restartPromise;+ }};const captureWithRestart = async <Result>(capture: (activeRuntime: BrowserRuntime) => Promise<Result>@@ -1691,7 +1893,10 @@ export const createBrowserScreenshotSession = async ();}for (let attempt = 0; attempt < 2; attempt += 1) {- const activeRuntime = runtime;+ let activeRuntime = runtime;+ if (activeRuntime.running === false) {+ activeRuntime = await restart(activeRuntime);+ }try {return await capture(activeRuntime);} catch (error) {@@ -1719,19 +1924,68 @@ export const createBrowserScreenshotSession = async ());},- async capturePage(captureOptions) {- return await captureWithRestart(- async (activeRuntime) =>- await capturePageWithBrowserRuntime(+ async capturePage(captureOptions, sessionOptions) {+ return await captureWithRestart(async (activeRuntime) => {+ const concurrency = sessionOptions?.concurrency ?? 1;+ const staggerMs = sessionOptions?.staggerMs ?? 0;+ if (Number.isInteger(concurrency) === false || concurrency < 1) {+ throw new Error("Screenshot concurrency must be a positive integer.");+ }+ if (Number.isFinite(staggerMs) === false || staggerMs < 0) {+ throw new Error("Screenshot stagger must be a non-negative number.");+ }+ if (concurrency === 1 || captureOptions.length < 2) {+ return await capturePageWithBrowserRuntime(activeRuntime,captureOptions,dependencies- )- );+ );+ }++ const groups = Array.from(+ { length: Math.min(concurrency, captureOptions.length) },+ () =>+ [] as Array<{ index: number; options: BrowserScreenshotOptions }>+ );+ captureOptions.forEach((options, index) => {+ groups[index % groups.length].push({ index, options });+ });+ const captures = await Promise.all(+ groups.map(async (group, index) => {+ if (index > 0 && staggerMs > 0) {+ await delay(index * staggerMs);+ }+ return {+ group,+ layouts: await capturePageWithBrowserRuntime(+ activeRuntime,+ group.map(({ options }) => options),+ dependencies+ ),+ };+ })+ );+ const layouts = new Array<BrowserScreenshotLayout>(+ captureOptions.length+ );+ for (const capture of captures) {+ capture.group.forEach(({ index }, groupIndex) => {+ const layout = capture.layouts[groupIndex];+ if (layout !== undefined) {+ layouts[index] = layout;+ }+ });+ }+ return layouts;+ });},async close() {closed = true;- await runtime.close();+ closePromise ??= (async () => {+ await restartPromise?.catch(() => undefined);+ await runtime.close();+ })();+ await closePromise;},};};diff --git a/packages/project-build/src/visual/screenshot-browser.test.ts b/packages/vision/src/screenshot-browser.test.tssimilarity index 88%rename from packages/project-build/src/visual/screenshot-browser.test.tsrename to packages/vision/src/screenshot-browser.test.tsindex 280171d5d5..450c67efee 100644--- a/packages/project-build/src/visual/screenshot-browser.test.ts+++ b/packages/vision/src/screenshot-browser.test.ts@@ -3,6 +3,7 @@ import {defaultScreenshotTimeout,defaultScreenshotWaitForTimeout,defaultScreenshotWaitUntil,+ isScreenshotBrowser,isScreenshotWaitUntil,screenshotBrowserChoices,screenshotWaitUntilValues,@@ -16,6 +17,8 @@ test("defines supported screenshot browser choices in preference order", () => {"edge","brave",]);+ expect(isScreenshotBrowser("chromium")).toBe(true);+ expect(isScreenshotBrowser("firefox")).toBe(false);});test("defines supported screenshot readiness events in lifecycle order", () => {diff --git a/packages/project-build/src/visual/screenshot-browser.ts b/packages/vision/src/screenshot-browser.tssimilarity index 53%rename from packages/project-build/src/visual/screenshot-browser.tsrename to packages/vision/src/screenshot-browser.tsindex 50630ba8af..1759c604fd 100644--- a/packages/project-build/src/visual/screenshot-browser.ts+++ b/packages/vision/src/screenshot-browser.ts@@ -8,6 +8,12 @@ export const screenshotBrowserChoices = [export type ScreenshotBrowser = (typeof screenshotBrowserChoices)[number];+export const isScreenshotBrowser = (+ value: unknown+): value is ScreenshotBrowser =>+ typeof value === "string" &&+ screenshotBrowserChoices.includes(value as ScreenshotBrowser);+export const screenshotWaitUntilValues = ["commit","domcontentloaded",@@ -23,6 +29,24 @@ export const defaultScreenshotWaitForTimeout = 250;export const defaultScreenshotTimeout = 30000;+export type ScreenshotCaptureOptions = {+ output?: string;+ viewport: { width: number; height: number };+ fullPage?: boolean;+ includeImageMetrics?: boolean;+ includeResourceMetrics?: boolean;+ includeContrastMetrics?: boolean;+ browser: ScreenshotBrowser;+ browserPath?: string;+ waitUntil?: ScreenshotWaitUntil;+ waitForSelector?: string;+ waitForTimeout?: number;+ timeout?: number;+ format?: "png" | "jpeg" | "webp";+ quality?: number;+ scale?: number;+};+export const isScreenshotWaitUntil = (value: unknown): value is ScreenshotWaitUntil =>diff --git a/packages/vision/src/screenshot-cache.test.ts b/packages/vision/src/screenshot-cache.test.tsnew file mode 100644index 0000000000..a40af6a566--- /dev/null+++ b/packages/vision/src/screenshot-cache.test.ts@@ -0,0 +1,124 @@+import {+ access,+ mkdtemp,+ readFile,+ readdir,+ rm,+ writeFile,+} from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { expect, test } from "vitest";+import {+ restoreScreenshotCache,+ writeScreenshotCache,+} from "./screenshot-cache";++test("reuses screenshots only when the requested cache is complete", async () => {+ const temporaryDirectory = await mkdtemp(+ path.join(tmpdir(), "visual-screenshot-cache-")+ );+ try {+ const source = path.join(temporaryDirectory, "source.png");+ const cache = path.join(temporaryDirectory, "cache");+ const assets = path.join(temporaryDirectory, "assets");+ await writeFile(source, "screenshot");+ await writeScreenshotCache({+ directory: cache,+ paths: new Map([["button--default", source]]),+ });++ const [cacheFile] = await readdir(cache);+ expect(cacheFile).toBeDefined();+ expect(await readFile(path.join(cache, cacheFile ?? ""), "utf8")).toEqual(+ "screenshot"+ );+ const restoredPath = path.join(assets, "button--default", "baseline.png");+ expect(+ await restoreScreenshotCache({+ directory: cache,+ ids: ["button--default"],+ getOutputPath: (id) => path.join(assets, id, "baseline.png"),+ })+ ).toEqual(new Map([["button--default", restoredPath]]));+ expect(await readFile(restoredPath, "utf8")).toEqual("screenshot");+ expect(+ await restoreScreenshotCache({+ directory: cache,+ ids: ["button--default", "button--secondary"],+ getOutputPath: (id) => path.join(assets, id, "baseline.png"),+ })+ ).toBeUndefined();+ } finally {+ await rm(temporaryDirectory, { recursive: true, force: true });+ }+});++test("keeps arbitrary cache ids inside the cache directory", async () => {+ const temporaryDirectory = await mkdtemp(+ path.join(tmpdir(), "visual-screenshot-cache-")+ );+ try {+ const source = path.join(temporaryDirectory, "source.png");+ const cache = path.join(temporaryDirectory, "cache");+ await writeFile(source, "screenshot");++ await writeScreenshotCache({+ directory: cache,+ paths: new Map([+ ["../escape", source],+ ["pages/about", source],+ ]),+ });++ expect(await readdir(cache)).toHaveLength(2);+ await expect(+ access(path.join(temporaryDirectory, "escape.png"))+ ).rejects.toMatchObject({ code: "ENOENT" });+ expect(+ await restoreScreenshotCache({+ directory: cache,+ ids: ["../escape", "pages/about"],+ getOutputPath: (id) => path.join(temporaryDirectory, "restored", id),+ })+ ).toEqual(+ new Map([+ ["../escape", path.join(temporaryDirectory, "restored", "../escape")],+ [+ "pages/about",+ path.join(temporaryDirectory, "restored", "pages/about"),+ ],+ ])+ );+ } finally {+ await rm(temporaryDirectory, { recursive: true, force: true });+ }+});++test("rejects an incomplete cached screenshot", async () => {+ const temporaryDirectory = await mkdtemp(+ path.join(tmpdir(), "visual-screenshot-cache-")+ );+ try {+ const source = path.join(temporaryDirectory, "source.png");+ const cache = path.join(temporaryDirectory, "cache");+ await writeFile(source, "screenshot");+ await writeScreenshotCache({+ directory: cache,+ paths: new Map([["complete", source]]),+ });+ const [cacheFile] = await readdir(cache);+ expect(cacheFile).toBeDefined();+ await writeFile(path.join(cache, cacheFile ?? ""), "");++ expect(+ await restoreScreenshotCache({+ directory: cache,+ ids: ["complete"],+ getOutputPath: (id) => path.join(temporaryDirectory, "restored", id),+ })+ ).toBeUndefined();+ } finally {+ await rm(temporaryDirectory, { recursive: true, force: true });+ }+});diff --git a/packages/vision/src/screenshot-cache.ts b/packages/vision/src/screenshot-cache.tsnew file mode 100644index 0000000000..555fef4b8b--- /dev/null+++ b/packages/vision/src/screenshot-cache.ts@@ -0,0 +1,62 @@+import { createHash, randomUUID } from "node:crypto";+import { copyFile, mkdir, rename, rm, stat } from "node:fs/promises";+import path from "node:path";++const getCachedScreenshotPath = (directory: string, id: string) =>+ path.join(directory, `${createHash("sha256").update(id).digest("hex")}.png`);++export const restoreScreenshotCache = async ({+ directory,+ ids,+ getOutputPath,+}: {+ directory: string;+ ids: readonly string[];+ getOutputPath: (id: string) => string;+}) => {+ const cachedPaths = ids.map(+ (id) => [id, getCachedScreenshotPath(directory, id)] as const+ );+ const available = await Promise.all(+ cachedPaths.map(async ([, screenshotPath]) => {+ return await stat(screenshotPath)+ .then(({ size }) => size > 0)+ .catch(() => false);+ })+ );+ if (available.every(Boolean) === false) {+ return;+ }+ return new Map(+ await Promise.all(+ cachedPaths.map(async ([id, screenshotPath]) => {+ const output = getOutputPath(id);+ await mkdir(path.dirname(output), { recursive: true });+ await copyFile(screenshotPath, output);+ return [id, output] as const;+ })+ )+ );+};++export const writeScreenshotCache = async ({+ directory,+ paths,+}: {+ directory: string;+ paths: ReadonlyMap<string, string>;+}) => {+ await mkdir(directory, { recursive: true });+ await Promise.all(+ [...paths].map(async ([id, screenshotPath]) => {+ const destination = getCachedScreenshotPath(directory, id);+ const temporary = `${destination}.${randomUUID()}.tmp`;+ try {+ await copyFile(screenshotPath, temporary);+ await rename(temporary, destination);+ } finally {+ await rm(temporary, { force: true });+ }+ })+ );+};diff --git a/packages/project-build/src/visual/screenshot-diff.test.ts b/packages/vision/src/screenshot-diff.test.tssimilarity index 65%rename from packages/project-build/src/visual/screenshot-diff.test.tsrename to packages/vision/src/screenshot-diff.test.tsindex 08cfdf1be2..4be0b77ad2 100644--- a/packages/project-build/src/visual/screenshot-diff.test.ts+++ b/packages/vision/src/screenshot-diff.test.ts@@ -1,24 +1,63 @@-import { mkdtemp, readFile, rm } from "node:fs/promises";+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";import { afterEach, beforeEach, expect, test } from "vitest";import {+ arePngFilesIdentical,createScreenshotTextAssertions,createScreenshotVisualAssertions,diffPngFiles,+ isScreenshotVisualExpectation,} from "./screenshot-diff";import { createPng, paintRect, writePng } from "./screenshot.test-utils";let tempDir: string;beforeEach(async () => {- tempDir = await mkdtemp(path.join(tmpdir(), "webstudio-screenshot-diff-"));+ tempDir = await mkdtemp(path.join(tmpdir(), "vision-screenshot-diff-"));});afterEach(async () => {await rm(tempDir, { recursive: true, force: true });});+test("recognizes identical PNG files and rejects corrupt matches", async () => {+ const left = path.join(tempDir, "left.png");+ const right = path.join(tempDir, "right.png");+ await writePng(left, createPng(2, 2, { r: 20, g: 40, b: 60 }));+ await writePng(right, createPng(2, 2, { r: 20, g: 40, b: 60 }));++ await expect(arePngFilesIdentical(left, right)).resolves.toBe(true);++ const corrupted = Buffer.from(await readFile(left));+ corrupted[20] = (corrupted[20] ?? 0) ^ 1;+ await Promise.all([writeFile(left, corrupted), writeFile(right, corrupted)]);+ await expect(arePngFilesIdentical(left, right)).rejects.toThrow(+ "Invalid PNG chunk checksum"+ );+});++test("validates screenshot visual expectations", () => {+ expect(+ isScreenshotVisualExpectation({+ maxMismatchPercentage: 5,+ minChangedRegions: 1,+ dominantColorChange: {+ channel: "red",+ direction: "increase",+ minMagnitude: 2,+ },+ })+ ).toBe(true);+ expect(isScreenshotVisualExpectation({})).toBe(false);+ expect(+ isScreenshotVisualExpectation({+ minChangedRegions: 2,+ maxChangedRegions: 1,+ })+ ).toBe(false);+});+test("reports pass/fail expected text assertions from OCR text", () => {expect(createScreenshotTextAssertions(@@ -177,6 +216,80 @@ test("normalizes same-aspect screenshots before comparing", async () => {expect(result.summary).toContain("status: unchanged");});+test("can skip OCR while preserving pixel diff evidence", async () => {+ const baselinePath = path.join(tempDir, "baseline.png");+ const currentPath = path.join(tempDir, "current.png");++ await writePng(baselinePath, createPng(2, 2, { r: 255, g: 255, b: 255 }));+ await writePng(currentPath, createPng(2, 2, { r: 0, g: 0, b: 0 }));++ const result = await diffPngFiles({+ baselinePath,+ currentPath,+ outputDir: path.join(tempDir, "diff"),+ analyzeText: false,+ });++ expect(result.differentPixels).toBe(4);+ expect(result.textAnalysis).toEqual({+ status: "skipped",+ provider: "tesseract",+ changes: [],+ });+ expect(result.warnings).toEqual([]);+});++test("can skip writing diff artifacts during a detection pass", async () => {+ const baselinePath = path.join(tempDir, "baseline.png");+ const currentPath = path.join(tempDir, "current.png");++ await writePng(baselinePath, createPng(2, 2, { r: 255, g: 255, b: 255 }));+ await writePng(currentPath, createPng(2, 2, { r: 0, g: 0, b: 0 }));++ const result = await diffPngFiles({+ baselinePath,+ currentPath,+ outputDir: path.join(tempDir, "diff"),+ analyzeText: false,+ writeArtifacts: false,+ });++ expect(result.differentPixels).toBe(4);+ expect(result.diffPath).toBeUndefined();+ expect(result.contextDiffPath).toBeUndefined();+});++test("writes diff artifacts conditionally without a second comparison", async () => {+ const baselinePath = path.join(tempDir, "baseline.png");+ const unchangedPath = path.join(tempDir, "unchanged.png");+ const changedPath = path.join(tempDir, "changed.png");+ const white = createPng(2, 2, { r: 255, g: 255, b: 255 });++ await writePng(baselinePath, white);+ await writePng(unchangedPath, white);+ await writePng(changedPath, createPng(2, 2, { r: 0, g: 0, b: 0 }));++ const unchanged = await diffPngFiles({+ baselinePath,+ currentPath: unchangedPath,+ outputDir: path.join(tempDir, "unchanged-diff"),+ analyzeText: false,+ writeArtifacts: "when-different",+ });+ const changed = await diffPngFiles({+ baselinePath,+ currentPath: changedPath,+ outputDir: path.join(tempDir, "changed-diff"),+ analyzeText: false,+ writeArtifacts: "when-different",+ });++ expect(unchanged.diffPath).toBeUndefined();+ await expect(readFile(changed.diffPath ?? "")).resolves.toBeInstanceOf(+ Buffer+ );+});+test("reports dimension mismatch for different aspect ratios", async () => {const baselinePath = path.join(tempDir, "baseline.png");const currentPath = path.join(tempDir, "current.png");diff --git a/packages/project-build/src/visual/screenshot-diff.ts b/packages/vision/src/screenshot-diff.tssimilarity index 86%rename from packages/project-build/src/visual/screenshot-diff.tsrename to packages/vision/src/screenshot-diff.tsindex 0a2fef85ab..ed9a917eec 100644--- a/packages/project-build/src/visual/screenshot-diff.ts+++ b/packages/vision/src/screenshot-diff.ts@@ -6,6 +6,7 @@import fs from "node:fs/promises";import path from "node:path";+import { crc32 } from "node:zlib";import { decode, encode, type DecodedPng, type ImageData } from "fast-png";import {normalizeToCommonSize,@@ -17,6 +18,65 @@ import {type ScreenshotTextAnalysis,} from "./screenshot-text-diff";+const pngSignature = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);++const assertPngIntegrity = (contents: Buffer, file: string) => {+ if (+ contents.length < pngSignature.length ||+ contents.subarray(0, pngSignature.length).equals(pngSignature) === false+ ) {+ throw new Error(`Invalid PNG signature: ${file}`);+ }+ let offset = pngSignature.length;+ let chunkIndex = 0;+ let hasImageData = false;+ let complete = false;+ while (offset < contents.length) {+ if (contents.length - offset < 12) {+ throw new Error(`Truncated PNG chunk: ${file}`);+ }+ const length = contents.readUInt32BE(offset);+ if (length > contents.length - offset - 12) {+ throw new Error(`Truncated PNG chunk data: ${file}`);+ }+ const typeStart = offset + 4;+ const dataEnd = typeStart + 4 + length;+ const type = contents.toString("ascii", typeStart, typeStart + 4);+ const expectedCrc = contents.readUInt32BE(dataEnd);+ const actualCrc = crc32(contents.subarray(typeStart, dataEnd)) >>> 0;+ if (actualCrc !== expectedCrc) {+ throw new Error(`Invalid PNG chunk checksum: ${file}`);+ }+ if (chunkIndex === 0 && (type !== "IHDR" || length !== 13)) {+ throw new Error(`Invalid PNG header chunk: ${file}`);+ }+ if (type === "IDAT") {+ hasImageData = true;+ }+ offset = dataEnd + 4;+ chunkIndex += 1;+ if (type === "IEND") {+ complete = length === 0 && offset === contents.length;+ break;+ }+ }+ if (hasImageData === false || complete === false) {+ throw new Error(`Incomplete PNG image: ${file}`);+ }+};++export const arePngFilesIdentical = async (left: string, right: string) => {+ const [leftContents, rightContents] = await Promise.all([+ fs.readFile(left),+ fs.readFile(right),+ ]);+ if (leftContents.equals(rightContents) === false) {+ return false;+ }+ assertPngIntegrity(leftContents, left);+ return true;+};+export type ScreenshotDiffSize = {width: number;height: number;@@ -100,6 +160,81 @@ export type ScreenshotVisualExpectation = {};};+export const isScreenshotVisualExpectation = (+ value: unknown+): value is ScreenshotVisualExpectation => {+ if (+ typeof value !== "object" ||+ value === null ||+ Array.isArray(value) ||+ Object.keys(value).length === 0+ ) {+ return false;+ }+ const expectation = value as Record<string, unknown>;+ const numericFieldValid = (+ key: "maxMismatchPercentage" | "minChangedRegions" | "maxChangedRegions"+ ) => {+ const field = expectation[key];+ if (field === undefined) {+ return true;+ }+ if (typeof field !== "number" || Number.isFinite(field) === false) {+ return false;+ }+ if (key === "maxMismatchPercentage") {+ return field >= 0 && field <= 100;+ }+ return Number.isInteger(field) && field >= 0;+ };+ if (+ numericFieldValid("maxMismatchPercentage") === false ||+ numericFieldValid("minChangedRegions") === false ||+ numericFieldValid("maxChangedRegions") === false+ ) {+ return false;+ }+ if (+ typeof expectation.minChangedRegions === "number" &&+ typeof expectation.maxChangedRegions === "number" &&+ expectation.minChangedRegions > expectation.maxChangedRegions+ ) {+ return false;+ }+ const dominantColorChange = expectation.dominantColorChange;+ if (dominantColorChange !== undefined) {+ if (+ typeof dominantColorChange !== "object" ||+ dominantColorChange === null ||+ Array.isArray(dominantColorChange)+ ) {+ return false;+ }+ const colorChange = dominantColorChange as Record<string, unknown>;+ if (+ (colorChange.channel !== "red" &&+ colorChange.channel !== "green" &&+ colorChange.channel !== "blue" &&+ colorChange.channel !== "luminance") ||+ (colorChange.direction !== "increase" &&+ colorChange.direction !== "decrease") ||+ (colorChange.minMagnitude !== undefined &&+ (typeof colorChange.minMagnitude !== "number" ||+ Number.isFinite(colorChange.minMagnitude) === false ||+ colorChange.minMagnitude < 0))+ ) {+ return false;+ }+ }+ return Object.keys(expectation).every(+ (key) =>+ key === "maxMismatchPercentage" ||+ key === "minChangedRegions" ||+ key === "maxChangedRegions" ||+ key === "dominantColorChange"+ );+};+export type ScreenshotVisualAssertion = {expected: string;actual: number | string;@@ -150,6 +285,8 @@ export const diffPngFiles = async ({outputDir,threshold = DEFAULT_THRESHOLD,ignoreTopNormalizedY = DEFAULT_IGNORE_TOP_NORMALIZED_Y,+ analyzeText = true,+ writeArtifacts = true,expectedText,expectedVisual,}: {@@ -158,6 +295,8 @@ export const diffPngFiles = async ({outputDir: string;threshold?: number;ignoreTopNormalizedY?: number;+ analyzeText?: boolean | "when-different";+ writeArtifacts?: boolean | "when-different";expectedText?: readonly string[];expectedVisual?: ScreenshotVisualExpectation;}): Promise<ScreenshotDiffResult> => {@@ -234,29 +373,41 @@ export const diffPngFiles = async ({current,});const totalPixels = baseline.width * baseline.height;- const artifactPaths = getDiffArtifactPaths({ currentPath, outputDir });- await writeDiffArtifacts({- baseline,- current,- mask: pixelDiff.mask,- regions,- paths: artifactPaths,- contextDiffScale: DEFAULT_CONTEXT_DIFF_SCALE,- });+ const hasPixelDifference = pixelDiff.differentPixels > 0;+ const shouldWriteArtifacts =+ writeArtifacts === true ||+ (writeArtifacts === "when-different" && hasPixelDifference);+ const artifactPaths = shouldWriteArtifacts+ ? getDiffArtifactPaths({ currentPath, outputDir })+ : undefined;+ if (artifactPaths !== undefined) {+ await writeDiffArtifacts({+ baseline,+ current,+ mask: pixelDiff.mask,+ regions,+ paths: artifactPaths,+ contextDiffScale: DEFAULT_CONTEXT_DIFF_SCALE,+ });+ }const canAnalyzeText =decodedBaseline.width === decodedCurrent.width &&decodedBaseline.height === decodedCurrent.height;- const textAnalysis: ScreenshotTextAnalysis = canAnalyzeText- ? await analyzeScreenshotTextChanges({- baselinePath,- currentPath,- hasPixelDiff: pixelDiff.differentPixels > 0,- baselineImage: baseline,- currentImage: current,- ignoreTopPixels: Math.ceil(baseline.height * ignoreTopNormalizedY),- includeObservedText: expectedText !== undefined,- })- : createSkippedTextAnalysis();+ const shouldAnalyzeText =+ analyzeText === true ||+ (analyzeText === "when-different" && hasPixelDifference);+ const textAnalysis: ScreenshotTextAnalysis =+ canAnalyzeText && shouldAnalyzeText+ ? await analyzeScreenshotTextChanges({+ baselinePath,+ currentPath,+ hasPixelDiff: hasPixelDifference,+ baselineImage: baseline,+ currentImage: current,+ ignoreTopPixels: Math.ceil(baseline.height * ignoreTopNormalizedY),+ includeObservedText: expectedText !== undefined,+ })+ : createSkippedTextAnalysis();return summarizeResult({totalPixels,@@ -264,8 +415,8 @@ export const diffPngFiles = async ({mismatchPercentage:totalPixels === 0 ? 0 : (pixelDiff.differentPixels / totalPixels) * 100,imageSize: { width: baseline.width, height: baseline.height },- diffPath: artifactPaths.diffPath,- contextDiffPath: artifactPaths.contextDiffPath,+ diffPath: artifactPaths?.diffPath,+ contextDiffPath: artifactPaths?.contextDiffPath,regions,overallColorChange,textAnalysis,diff --git a/packages/project-build/src/visual/screenshot-ocr.test.ts b/packages/vision/src/screenshot-ocr.test.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-ocr.test.tsrename to packages/vision/src/screenshot-ocr.test.tsdiff --git a/packages/project-build/src/visual/screenshot-ocr.ts b/packages/vision/src/screenshot-ocr.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-ocr.tsrename to packages/vision/src/screenshot-ocr.tsdiff --git a/packages/vision/src/screenshot-report-assets.ts b/packages/vision/src/screenshot-report-assets.tsnew file mode 100644index 0000000000..11d3926ce3--- /dev/null+++ b/packages/vision/src/screenshot-report-assets.ts@@ -0,0 +1,252 @@+export const screenshotReportHtml = `<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />+ <title>Visual comparison</title>+ <link rel="stylesheet" href="report.css" />+ <script src="report.js" defer></script>+ </head>+ <body>+ <main>+ <h1>Visual comparison</h1>+ <p id="revisions"></p>+ <div class="summary" id="summary"></div>+ <div id="errors"></div>+ <div id="comparisons"></div>+ </main>+ <script id="visual-report-data" type="application/json">+ __VISUAL_REPORT_DATA__+ </script>+ </body>+</html>+`;++export const screenshotReportClient = `const report = JSON.parse(+ document.querySelector("#visual-report-data").textContent+);++const createElement = (tag, options = {}) => {+ const element = document.createElement(tag);+ if (options.className !== undefined) {+ element.className = options.className;+ }+ if (options.text !== undefined) {+ element.textContent = options.text;+ }+ return element;+};++const revisions = document.querySelector("#revisions");+revisions.append(+ createElement("code", { text: report.baselineLabel }),+ " → ",+ createElement("code", { text: report.currentLabel }),+ \` in \${(report.durationMs / 1000).toFixed(1)}s\`+);++const statuses = ["changed", "added", "removed", "error", "unchanged"];+const counts = Object.fromEntries(statuses.map((status) => [status, 0]));+for (const comparison of report.comparisons) {+ counts[comparison.status] += 1;+}+const summary = document.querySelector("#summary");+for (const status of statuses) {+ summary.append(+ createElement("span", { text: \`\${status}: \${counts[status]}\` })+ );+}++const errors = document.querySelector("#errors");+for (const error of report.errors) {+ errors.append(createElement("pre", { text: error }));+}++const comparisonRoot = document.querySelector("#comparisons");+const important = report.comparisons.filter(+ ({ status }) => status !== "unchanged"+);+if (important.length === 0) {+ comparisonRoot.append(+ createElement("p", { text: "No visual differences detected." })+ );+}++const imageLabels = [+ ["baselinePath", "Baseline", "Baseline for"],+ ["currentPath", "Current", "Current rendering for"],+ ["diffPath", "Diff", "Visual difference for"],+ ["contextDiffPath", "Context diff", "Contextual visual difference for"],+];+for (const comparison of important) {+ const article = createElement("article", {+ className: \`comparison \${comparison.status}\`,+ });+ const header = createElement("header");+ const identity = createElement("div");+ identity.append(+ createElement("h2", {+ text: \`\${comparison.title} › \${comparison.name}\`,+ }),+ createElement("code", { text: comparison.id })+ );+ const result = createElement("div");+ result.append(+ createElement("span", {+ className: "status",+ text: comparison.status,+ })+ );+ if (comparison.mismatchPercentage !== undefined) {+ result.append(+ createElement("span", {+ text: \`\${(comparison.differentPixels ?? 0).toLocaleString()} pixels · \${comparison.mismatchPercentage.toFixed(4)}%\`,+ })+ );+ }+ header.append(identity, result);+ article.append(header);+ if (comparison.error !== undefined) {+ article.append(createElement("pre", { text: comparison.error }));+ }++ const changes = comparison.textAnalysis?.changes ?? [];+ if (changes.length > 0) {+ const list = createElement("ul", { className: "text-changes" });+ for (const change of changes) {+ const item = createElement("li");+ const label = createElement("strong", {+ text: change.kind.replaceAll("_", " "),+ });+ const text =+ change.text ?? change.currentText ?? change.baselineText ?? "";+ item.append(label, text === "" ? "" : \`: \${text}\`);+ list.append(item);+ }+ article.append(list);+ }++ const images = createElement("div", { className: "images" });+ for (const [path, label, altPrefix] of imageLabels) {+ if (comparison[path] === undefined) {+ continue;+ }+ const figure = createElement("figure");+ const image = createElement("img");+ image.src = comparison[path];+ image.alt = \`\${altPrefix} \${comparison.title} \${comparison.name}\`;+ figure.append(createElement("figcaption", { text: label }), image);+ images.append(figure);+ }+ article.append(images);+ comparisonRoot.append(article);+}+`;++export const screenshotReportCss = `:root {+ color-scheme: light;+ font-family: Inter, ui-sans-serif, system-ui, sans-serif;+ background: #f6f6f6;+ color: #181818;+}++body {+ margin: 0;+ padding: 32px;+}++main {+ max-width: 1600px;+ margin: auto;+}++h1 {+ margin: 0 0 8px;+}++.summary {+ display: flex;+ gap: 12px;+ flex-wrap: wrap;+ margin: 24px 0;+}++.summary span,+.status {+ border-radius: 999px;+ background: #e7e7e7;+ padding: 4px 10px;+}++.comparison {+ background: white;+ border: 1px solid #ddd;+ border-radius: 12px;+ margin: 20px 0;+ overflow: hidden;+}++.comparison > header {+ align-items: flex-start;+ display: flex;+ justify-content: space-between;+ gap: 16px;+ padding: 18px;+}++.comparison h2 {+ font-size: 17px;+ margin: 0 0 5px;+}++.comparison header > div:last-child {+ align-items: flex-end;+ display: flex;+ flex-direction: column;+ gap: 8px;+}++.changed .status,+.added .status,+.removed .status {+ background: #ffe39c;+}++.error .status {+ background: #ffc4c4;+}++.images {+ display: grid;+ grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));+ border-top: 1px solid #ddd;+}++figure {+ margin: 0;+ min-width: 0;+ padding: 12px;+ border-right: 1px solid #ddd;+}++figcaption {+ font-weight: 600;+ margin-bottom: 8px;+}++img {+ display: block;+ max-width: 100%;+ border: 1px solid #eee;+}++pre {+ overflow: auto;+ padding: 18px;+ color: #a40000;+}++.text-changes {+ margin: 0 18px 18px;+}+`;diff --git a/packages/vision/src/screenshot-report.test.ts b/packages/vision/src/screenshot-report.test.tsnew file mode 100644index 0000000000..14c7366348--- /dev/null+++ b/packages/vision/src/screenshot-report.test.ts@@ -0,0 +1,192 @@+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";+import { parse, type DefaultTreeAdapterTypes as Html } from "parse5";+import { expect, test } from "vitest";+import {+ classifyScreenshotComparisonReport,+ openScreenshotComparisonReport,+ writeScreenshotComparisonReport,+ type ScreenshotComparisonReport,+} from "./screenshot-report";++test("reports when the platform cannot open the visual report", async () => {+ const originalPath = process.env.PATH;+ process.env.PATH = "";+ try {+ await expect(+ openScreenshotComparisonReport("/tmp/visual-report/index.html")+ ).rejects.toMatchObject({ code: "ENOENT" });+ } finally {+ if (originalPath === undefined) {+ delete process.env.PATH;+ } else {+ process.env.PATH = originalPath;+ }+ }+});++const createReport = (+ status: ScreenshotComparisonReport["comparisons"][number]["status"]+): ScreenshotComparisonReport => ({+ baselineLabel: "base",+ currentLabel: "current",+ durationMs: 100,+ comparisons: [{ id: "button", title: "Button", name: "Primary", status }],+ errors: [],+});++test("classifies visual differences separately from comparison failures", () => {+ expect(classifyScreenshotComparisonReport(createReport("unchanged"))).toBe(+ "passed"+ );+ expect(classifyScreenshotComparisonReport(createReport("changed"))).toBe(+ "differences"+ );+ expect(classifyScreenshotComparisonReport(createReport("added"))).toBe(+ "differences"+ );+ expect(classifyScreenshotComparisonReport(createReport("removed"))).toBe(+ "differences"+ );+ expect(classifyScreenshotComparisonReport(createReport("error"))).toBe(+ "failure"+ );+ expect(+ classifyScreenshotComparisonReport({+ ...createReport("unchanged"),+ errors: ["Browser crashed"],+ })+ ).toBe("failure");+ expect(+ classifyScreenshotComparisonReport({+ ...createReport("unchanged"),+ comparisons: [],+ })+ ).toBe("failure");+});++const findElement = (+ node: Html.Node,+ predicate: (element: Html.Element) => boolean+): Html.Element | undefined => {+ if ("tagName" in node && predicate(node)) {+ return node;+ }+ if ("childNodes" in node) {+ for (const child of node.childNodes) {+ const found = findElement(child, predicate);+ if (found !== undefined) {+ return found;+ }+ }+ }+};++test("writes portable JSON data for the static visual report", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-report-"));+ const reportDirectory = path.join(root, "report");+ const sourceImage = path.join(root, "captures", "baseline.png");+ try {+ await mkdir(path.dirname(sourceImage), { recursive: true });+ await writeFile(sourceImage, "baseline image");+ await writeScreenshotComparisonReport({+ reportDirectory,+ report: {+ baselineLabel: "baseline",+ currentLabel: "current",+ durationMs: 1200,+ errors: [],+ comparisons: [+ {+ id: "button",+ title: "Design system/Button </script><script>unsafe()</script>",+ name: "Primary",+ status: "changed",+ baselinePath: sourceImage,+ differentPixels: 12,+ mismatchPercentage: 0.01,+ },+ ],+ },+ });++ const json = JSON.parse(+ await readFile(path.join(reportDirectory, "report.json"), "utf8")+ );+ expect(json.comparisons[0].baselinePath).toEqual(+ "assets/comparisons/0/baseline.png"+ );+ expect(+ await readFile(+ path.join(reportDirectory, json.comparisons[0].baselinePath),+ "utf8"+ )+ ).toEqual("baseline image");+ expect(json.comparisons[0].currentPath).toBeUndefined();+ expect(json.comparisons[0].diffPath).toBeUndefined();++ const html = await readFile(+ path.join(reportDirectory, "index.html"),+ "utf8"+ );+ const document = parse(html);+ const data = findElement(+ document,+ (element) =>+ element.tagName === "script" &&+ element.attrs.some(+ ({ name, value }) => name === "id" && value === "visual-report-data"+ )+ );+ expect(data).not.toBeUndefined();+ const embeddedReport = JSON.parse(+ data?.childNodes+ .filter((node): node is Html.TextNode => node.nodeName === "#text")+ .map((node) => node.value)+ .join("") ?? ""+ );+ expect(embeddedReport).toEqual(json);+ expect(html.includes("</script><script>unsafe()")).toEqual(false);+ await Promise.all([+ readFile(path.join(reportDirectory, "report.css"), "utf8"),+ readFile(path.join(reportDirectory, "report.js"), "utf8"),+ ]);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test("writes a report from bundled code", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-report-bundle-"));+ const output = path.join(root, "screenshot-report.mjs");+ const reportDirectory = path.join(root, "report");+ try {+ await build({+ entryPoints: [+ fileURLToPath(new URL("./screenshot-report.ts", import.meta.url)),+ ],+ outfile: output,+ bundle: true,+ format: "esm",+ platform: "node",+ packages: "external",+ });+ const bundledModule = await import(pathToFileURL(output).href);+ await bundledModule.writeScreenshotComparisonReport({+ reportDirectory,+ report: {+ baselineLabel: "baseline",+ currentLabel: "current",+ durationMs: 0,+ comparisons: [],+ errors: [],+ },+ });+ await readFile(path.join(reportDirectory, "index.html"), "utf8");+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/packages/vision/src/screenshot-report.ts b/packages/vision/src/screenshot-report.tsnew file mode 100644index 0000000000..9610c39925--- /dev/null+++ b/packages/vision/src/screenshot-report.ts@@ -0,0 +1,184 @@+import { spawn } from "node:child_process";+import { copyFile, mkdir, writeFile } from "node:fs/promises";+import path from "node:path";+import type { ScreenshotDiffRegion } from "./screenshot-diff";+import {+ screenshotReportClient,+ screenshotReportCss,+ screenshotReportHtml,+} from "./screenshot-report-assets";+import type { ScreenshotTextAnalysis } from "./screenshot-text-diff";++export type ScreenshotComparisonReportItem = {+ id: string;+ title: string;+ name: string;+ status: "unchanged" | "changed" | "added" | "removed" | "error";+ baselinePath?: string;+ currentPath?: string;+ diffPath?: string;+ contextDiffPath?: string;+ differentPixels?: number;+ mismatchPercentage?: number;+ regions?: readonly ScreenshotDiffRegion[];+ textAnalysis?: ScreenshotTextAnalysis;+ warnings?: readonly string[];+ error?: string;+};++export type ScreenshotComparisonReport = {+ baselineLabel: string;+ currentLabel: string;+ durationMs: number;+ comparisons: readonly ScreenshotComparisonReportItem[];+ errors: readonly string[];+};++export type ScreenshotComparisonReportResult =+ | "passed"+ | "differences"+ | "failure";++export const classifyScreenshotComparisonReport = (+ report: ScreenshotComparisonReport+): ScreenshotComparisonReportResult => {+ if (+ report.errors.length > 0 ||+ report.comparisons.length === 0 ||+ report.comparisons.some(({ status }) => status === "error")+ ) {+ return "failure";+ }+ return report.comparisons.some(({ status }) => status !== "unchanged")+ ? "differences"+ : "passed";+};++const reportDataMarker = "__VISUAL_REPORT_DATA__";++const toPortablePath = (reportDirectory: string, file: string) =>+ path.relative(reportDirectory, file).split(path.sep).join("/");++const materializeAsset = async ({+ file,+ index,+ name,+ reportDirectory,+}: {+ file: string | undefined;+ index: number;+ name: string;+ reportDirectory: string;+}) => {+ if (file === undefined) {+ return;+ }+ const relativePath = path.relative(reportDirectory, file);+ if (+ relativePath !== "" &&+ path.isAbsolute(relativePath) === false &&+ relativePath.startsWith(`..${path.sep}`) === false+ ) {+ return toPortablePath(reportDirectory, file);+ }+ const extension = path.extname(file) || ".png";+ const destination = path.join(+ reportDirectory,+ "assets",+ "comparisons",+ String(index),+ `${name}${extension}`+ );+ await mkdir(path.dirname(destination), { recursive: true });+ await copyFile(file, destination);+ return toPortablePath(reportDirectory, destination);+};++const serializeEmbeddedJson = (value: unknown) =>+ JSON.stringify(value)+ .replaceAll("&", "\\u0026")+ .replaceAll("<", "\\u003c")+ .replaceAll(">", "\\u003e")+ .replaceAll("\u2028", "\\u2028")+ .replaceAll("\u2029", "\\u2029");++export const writeScreenshotComparisonReport = async ({+ report,+ reportDirectory,+}: {+ report: ScreenshotComparisonReport;+ reportDirectory: string;+}) => {+ await mkdir(reportDirectory, { recursive: true });+ const portableReport = {+ ...report,+ comparisons: await Promise.all(+ report.comparisons.map(async (comparison, index) => ({+ ...comparison,+ baselinePath: await materializeAsset({+ file: comparison.baselinePath,+ index,+ name: "baseline",+ reportDirectory,+ }),+ currentPath: await materializeAsset({+ file: comparison.currentPath,+ index,+ name: "current",+ reportDirectory,+ }),+ diffPath: await materializeAsset({+ file: comparison.diffPath,+ index,+ name: "diff",+ reportDirectory,+ }),+ contextDiffPath: await materializeAsset({+ file: comparison.contextDiffPath,+ index,+ name: "context-diff",+ reportDirectory,+ }),+ }))+ ),+ };+ if (screenshotReportHtml.includes(reportDataMarker) === false) {+ throw new Error(+ `Screenshot report template is missing ${reportDataMarker}.`+ );+ }+ await Promise.all([+ writeFile(path.join(reportDirectory, "report.css"), screenshotReportCss),+ writeFile(path.join(reportDirectory, "report.js"), screenshotReportClient),+ writeFile(+ path.join(reportDirectory, "report.json"),+ `${JSON.stringify(portableReport, undefined, 2)}\n`+ ),+ writeFile(+ path.join(reportDirectory, "index.html"),+ screenshotReportHtml.replace(+ reportDataMarker,+ serializeEmbeddedJson(portableReport)+ )+ ),+ ]);+};++export const openScreenshotComparisonReport = async (reportPath: string) => {+ const command =+ process.platform === "darwin"+ ? { file: "open", args: [reportPath] }+ : process.platform === "win32"+ ? { file: "cmd", args: ["/c", "start", "", reportPath] }+ : { file: "xdg-open", args: [reportPath] };++ const child = spawn(command.file, command.args, {+ detached: true,+ stdio: "ignore",+ });+ await new Promise<void>((resolve, reject) => {+ child.once("spawn", resolve);+ child.once("error", reject);+ });+ child.unref();+};diff --git a/packages/project-build/src/visual/screenshot-resize.test.ts b/packages/vision/src/screenshot-resize.test.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-resize.test.tsrename to packages/vision/src/screenshot-resize.test.tsdiff --git a/packages/project-build/src/visual/screenshot-resize.ts b/packages/vision/src/screenshot-resize.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-resize.tsrename to packages/vision/src/screenshot-resize.tsdiff --git a/packages/project-build/src/visual/screenshot-text-diff.test.ts b/packages/vision/src/screenshot-text-diff.test.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-text-diff.test.tsrename to packages/vision/src/screenshot-text-diff.test.tsdiff --git a/packages/project-build/src/visual/screenshot-text-diff.ts b/packages/vision/src/screenshot-text-diff.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot-text-diff.tsrename to packages/vision/src/screenshot-text-diff.tsdiff --git a/packages/project-build/src/visual/screenshot.test-utils.ts b/packages/vision/src/screenshot.test-utils.tssimilarity index 100%rename from packages/project-build/src/visual/screenshot.test-utils.tsrename to packages/vision/src/screenshot.test-utils.tsdiff --git a/packages/vision/src/viewports.test.ts b/packages/vision/src/viewports.test.tsnew file mode 100644index 0000000000..417118a3cb--- /dev/null+++ b/packages/vision/src/viewports.test.ts@@ -0,0 +1,53 @@+import { describe, expect, test } from "vitest";+import { selectRepresentativeViewports } from "./viewports";++describe("selectRepresentativeViewports", () => {+ test("selects common viewports nearest the middle of bounded ranges", () => {+ expect(+ selectRepresentativeViewports({+ ranges: [+ { minimumWidth: 1, maximumWidth: 479, purpose: "portrait" },+ { minimumWidth: 480, maximumWidth: 767, purpose: "landscape" },+ { minimumWidth: 768, maximumWidth: 991, purpose: "tablet" },+ { minimumWidth: 1280, purpose: "desktop" },+ ],+ })+ ).toEqual([+ { width: 390, height: 844, purposes: ["portrait"] },+ { width: 667, height: 375, purposes: ["landscape"] },+ { width: 820, height: 1180, purposes: ["tablet"] },+ { width: 1366, height: 768, purposes: ["desktop"] },+ ]);+ });++ test("uses range boundaries and merges purposes when no candidate is available", () => {+ expect(+ selectRepresentativeViewports({+ ranges: [+ { minimumWidth: 200, maximumWidth: 300, purpose: "first" },+ { minimumWidth: 250, maximumWidth: 300, purpose: "second" },+ { minimumWidth: 200, maximumWidth: 300, purpose: "first" },+ { minimumWidth: 2100, purpose: "wide" },+ ],+ })+ ).toEqual([+ { width: 300, height: 844, purposes: ["first", "second"] },+ { width: 2100, height: 900, purposes: ["wide"] },+ ]);+ });++ test("selects deterministically without mutating caller-provided candidates", () => {+ const candidates = [+ { width: 900, height: 600 },+ { width: 500, height: 700 },+ ];+ const selected = selectRepresentativeViewports({+ ranges: [{ minimumWidth: 400, maximumWidth: 1000, purpose: "bounded" }],+ candidates,+ });+ expect(selected).toEqual([+ { width: 500, height: 700, purposes: ["bounded"] },+ ]);+ expect(candidates.map(({ width }) => width)).toEqual([900, 500]);+ });+});diff --git a/packages/vision/src/viewports.ts b/packages/vision/src/viewports.tsnew file mode 100644index 0000000000..a31fd96c69--- /dev/null+++ b/packages/vision/src/viewports.ts@@ -0,0 +1,72 @@+export type Viewport = { readonly width: number; readonly height: number };++export type ViewportRange<Purpose> = {+ minimumWidth: number;+ maximumWidth?: number;+ purpose: Purpose;+};++export type RepresentativeViewport<Purpose> = Viewport & {+ purposes: Purpose[];+};++export const commonViewports: readonly Viewport[] = [+ { width: 390, height: 844 },+ { width: 667, height: 375 },+ { width: 820, height: 1180 },+ { width: 1024, height: 768 },+ { width: 1366, height: 768 },+ { width: 1440, height: 900 },+ { width: 1920, height: 1080 },+];++export const selectRepresentativeViewports = <Purpose>({+ ranges,+ candidates = commonViewports,+}: {+ ranges: readonly ViewportRange<Purpose>[];+ candidates?: readonly Viewport[];+}): RepresentativeViewport<Purpose>[] => {+ const selections = new Map<+ number,+ { height: number; purposes: Set<Purpose> }+ >();+ for (const { minimumWidth, maximumWidth, purpose } of ranges) {+ const matchingCandidates = candidates.filter(+ ({ width }) =>+ width >= minimumWidth &&+ (maximumWidth === undefined || width <= maximumWidth)+ );+ const targetWidth =+ maximumWidth === undefined+ ? minimumWidth+ : (minimumWidth + maximumWidth) / 2;+ const viewport = matchingCandidates.toSorted(+ (left, right) =>+ Math.abs(left.width - targetWidth) -+ Math.abs(right.width - targetWidth) || left.width - right.width+ )[0];+ const selected =+ viewport ??+ ({+ width: maximumWidth ?? minimumWidth,+ height: (maximumWidth ?? minimumWidth) <= 480 ? 844 : 900,+ } satisfies Viewport);+ const existing = selections.get(selected.width);+ if (existing === undefined) {+ selections.set(selected.width, {+ height: selected.height,+ purposes: new Set([purpose]),+ });+ } else {+ existing.purposes.add(purpose);+ }+ }+ return [...selections.entries()]+ .sort(([left], [right]) => left - right)+ .map(([width, { height, purposes }]) => ({+ width,+ height,+ purposes: [...purposes],+ }));+};diff --git a/packages/vision/tsconfig.json b/packages/vision/tsconfig.jsonnew file mode 100644index 0000000000..a60cadecda--- /dev/null+++ b/packages/vision/tsconfig.json@@ -0,0 +1,4 @@+{+ "extends": "@webstudio-is/tsconfig/base.json",+ "include": ["**/*.ts", "../../@types/**/*.d.ts"]+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex f38832e59c..9d80c42aff 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -62,6 +62,15 @@ importers:'@types/react':specifier: ^18.2.70version: 18.2.79+ '@types/react-dom':+ specifier: ^18.2.25+ version: 18.2.25+ '@webstudio-is/project-build':+ specifier: workspace:^+ version: link:packages/project-build+ '@webstudio-is/vision':+ specifier: workspace:^+ version: link:packages/visionesbuild:specifier: ^0.25.3version: 0.25.3@@ -74,6 +83,9 @@ importers:oxlint:specifier: ^1.73.0version: 1.73.0+ parse5:+ specifier: 7.3.0+ version: 7.3.0react:specifier: 18.3.0-canary-14898b6a9-20240318version: 18.3.0-canary-14898b6a9-20240318@@ -95,6 +107,9 @@ importers:vite:specifier: 6.3.4version: 6.3.4(@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ webstudio:+ specifier: workspace:^+ version: link:packages/cliapps/builder:dependencies:@@ -1445,6 +1460,9 @@ importers:'@webstudio-is/tsconfig':specifier: workspace:*version: link:../tsconfig+ '@webstudio-is/vision':+ specifier: workspace:*+ version: link:../vision'@webstudio-is/wsauth':specifier: workspace:*version: link:../wsauth@@ -2222,6 +2240,9 @@ importers:'@webstudio-is/trpc-interface':specifier: workspace:*version: link:../trpc-interface+ '@webstudio-is/vision':+ specifier: workspace:*+ version: link:../vision'@webstudio-is/wsauth':specifier: workspace:*version: link:../wsauth@@ -2246,9 +2267,6 @@ importers:fast-deep-equal:specifier: ^3.1.3version: 3.1.3- fast-png:- specifier: 8.0.0- version: 8.0.0fastest-levenshtein:specifier: ^1.0.16version: 1.0.16@@ -2261,9 +2279,6 @@ importers:papaparse:specifier: ^5.5.3version: 5.5.3- parse5:- specifier: 7.3.0- version: 7.3.0slugify:specifier: ^1.6.6version: 1.6.6@@ -3022,6 +3037,31 @@ importers:packages/tsconfig: {}+ packages/vision:+ dependencies:+ fast-png:+ specifier: 8.0.0+ version: 8.0.0+ fastest-levenshtein:+ specifier: ^1.0.16+ version: 1.0.16+ tsx:+ specifier: ^4.19.3+ version: 4.19.3+ devDependencies:+ '@webstudio-is/tsconfig':+ specifier: workspace:*+ version: link:../tsconfig+ esbuild:+ specifier: ^0.25.3+ version: 0.25.3+ parse5:+ specifier: 7.3.0+ version: 7.3.0+ vitest:+ specifier: ^3.1.2+ version: 3.1.2(@types/[email protected])(@types/[email protected])(@vitest/[email protected])([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected]))([email protected])([email protected])+packages/wsauth:dependencies:zod:@@ -12424,15 +12464,15 @@ snapshots:transitivePeerDependencies:- supports-color- '@babel/[email protected](@babel/[email protected])':+ '@babel/[email protected](@babel/[email protected])':dependencies:- '@babel/core': 7.26.10- '@babel/helper-plugin-utils': 7.25.9+ '@babel/core': 7.29.7+ '@babel/helper-plugin-utils': 7.29.7- '@babel/[email protected](@babel/[email protected])':+ '@babel/[email protected](@babel/[email protected])':dependencies:- '@babel/core': 7.26.10- '@babel/helper-plugin-utils': 7.25.9+ '@babel/core': 7.29.7+ '@babel/helper-plugin-utils': 7.29.7'@babel/[email protected](@babel/[email protected])':dependencies:@@ -16205,9 +16245,9 @@ snapshots:'@vitejs/[email protected]([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))':dependencies:- '@babel/core': 7.26.10- '@babel/plugin-transform-react-jsx-self': 7.25.9(@babel/[email protected])- '@babel/plugin-transform-react-jsx-source': 7.25.9(@babel/[email protected])+ '@babel/core': 7.29.7+ '@babel/plugin-transform-react-jsx-self': 7.25.9(@babel/[email protected])+ '@babel/plugin-transform-react-jsx-source': 7.25.9(@babel/[email protected])'@types/babel__core': 7.20.5react-refresh: 0.17.0vite: 6.3.4(@types/[email protected])([email protected])([email protected])([email protected])([email protected])diff --git a/scripts/visual-regression/README.md b/scripts/visual-regression/README.mdnew file mode 100644index 0000000000..2157280607--- /dev/null+++ b/scripts/visual-regression/README.md@@ -0,0 +1,54 @@+# Visual regression testing++The visual regression runner indexes the existing Storybook CSF files, builds+and serves a lightweight browser app with Vite, and renders the merge base and+current working tree in Chromium. It compares screenshots with Webstudio's+shared vision engine and does not require a hosted service.++Run the full comparison and open the visual report:++```sh+pnpm visual-regression:ui+```++Limit a local run while working on one area:++```sh+pnpm visual-regression:ui -- --grep "Design system/Button"+```++Use `--base <ref>` to compare with a branch other than `origin/main`. Reopen the+most recent report without rerunning the comparison with+`pnpm visual-regression:report`.++The HTML report shows baseline, current, and diff images for every mismatch,+plus OCR evidence for changed text. CI uploads the same self-contained report+as a `visual-regression-report` artifact.++Pixel sensitivity (`0.1`), mismatch tolerance (`0.001%`), viewport, capture,+server, and cache settings live in `config.ts`. Add+story-specific deterministic rendering options to the story's+`parameters.visualRegression` object. Intervals and Web Animations are disabled+by default after React effects settle so concurrent screenshots cannot capture+different frames. External iframe contents are hidden because their rendering+is outside the revision under test. Set `disableAnimations: false` or+`disableIntervals: false` only when a story establishes its own deterministic+state.++Story scopes are data in `.storybook/story-sources.json`. Each compared+revision loads its own copy so removing or changing a scope remains visible to+the comparison.++GitHub Actions runs the complete suite in one job and caches screenshots from+successful revisions for use as PR baselines. The runner produces the baseline+on the fly when the cache is missing, and keeps both the cache and a reusable+baseline worktree ignored locally. CI uses the Playwright Chromium version+pinned by the lockfile so those screenshots remain reusable across hosted+runners. Changes anywhere in the visual harness or shared vision source+automatically invalidate cached screenshots. Capture concurrency adapts to the+available CPUs and can be overridden with `VISUAL_CAPTURE_CONCURRENCY`.++Visual differences fail the pull request. After reviewing an intentional+change, add the `visual-change-approved` label to rerun the same comparison as+approved. A new commit removes that approval. Fork pull requests run only after+the `safe-to-deploy` label is added.diff --git a/scripts/visual-regression/capture.test.ts b/scripts/visual-regression/capture.test.tsnew file mode 100644index 0000000000..480ac7d31e--- /dev/null+++ b/scripts/visual-regression/capture.test.ts@@ -0,0 +1,63 @@+import assert from "node:assert/strict";+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { captureStoryPairs, getInitialCaptureTarget } from "./capture";+import type { VisualStoryEntry } from "./manifest";++const createEntry = (id: string): VisualStoryEntry => ({+ id,+ title: id,+ name: id,+ exportName: id,+ file: `${id}.stories.tsx`,+ titlePrefix: "Tests",+});++test("does not initialize a browser for cached removed stories", () => {+ assert.equal(+ getInitialCaptureTarget({+ baselineEntries: [createEntry("removed")],+ currentEntries: [],+ hasCachedBaseline: true,+ }),+ undefined+ );+});++test("recaptures baseline and current stories on the same browser page", async () => {+ const assetDirectory = await mkdtemp(path.join(tmpdir(), "visual-pairs-"));+ const calls: string[][] = [];+ try {+ const result = await captureStoryPairs({+ assetDirectory,+ browserPath: "/browser",+ pairs: [+ {+ baseline: createEntry("button"),+ current: createEntry("button"),+ },+ ],+ baselinePort: 6101,+ currentPort: 6102,+ session: {+ async capturePage(options, { concurrency }) {+ assert.equal(concurrency, 1);+ calls.push(options.map(({ url }) => url));+ await Promise.all(+ options.map(({ output }) => writeFile(output, "screenshot"))+ );+ },+ },+ });++ assert.deepEqual(calls, [+ ["http://127.0.0.1:6102/", "http://127.0.0.1:6101/"],+ ]);+ assert.equal(result.current.paths.has("button"), true);+ assert.equal(result.baseline.paths.has("button"), true);+ } finally {+ await rm(assetDirectory, { recursive: true, force: true });+ }+});diff --git a/scripts/visual-regression/capture.ts b/scripts/visual-regression/capture.tsnew file mode 100644index 0000000000..ef8e6d307a--- /dev/null+++ b/scripts/visual-regression/capture.ts@@ -0,0 +1,164 @@+import path from "node:path";+import type { BrowserScreenshotOptions } from "@webstudio-is/vision/browser";+import {+ captureVisualEntries,+ type VisualCaptureSession,+} from "@webstudio-is/vision/capture";+import { visualRegressionConfig } from "./config";+import type { VisualStoryEntry } from "./manifest";++const onBatchFailure = (error: unknown, count: number) => {+ console.warn(+ `Visual capture failed for ${count} stories; isolating the failure.`,+ error+ );+};++export const getInitialCaptureTarget = ({+ baselineEntries,+ currentEntries,+ hasCachedBaseline,+}: {+ baselineEntries: readonly VisualStoryEntry[];+ currentEntries: readonly VisualStoryEntry[];+ hasCachedBaseline: boolean;+}) => {+ const currentEntry = currentEntries[0];+ if (currentEntry !== undefined) {+ return { entry: currentEntry, target: "current" as const };+ }+ const baselineEntry = baselineEntries[0];+ if (hasCachedBaseline === false && baselineEntry !== undefined) {+ return { entry: baselineEntry, target: "baseline" as const };+ }+};++export const getStoryCaptureOptions = ({+ browserPath,+ entry,+ output,+ port,+}: {+ browserPath: string;+ entry: VisualStoryEntry;+ output: string;+ port: number;+}): BrowserScreenshotOptions => ({+ browserPath,+ output,+ ...visualRegressionConfig.capture.viewport,+ fullPage: true,+ includeElementGeometry: false,+ url: new URL("/", `http://127.0.0.1:${port}`).href,+ uid: process.getuid?.(),+ disableSandbox: process.env.GITHUB_ACTIONS === "true",+ waitUntil: "load",+ prepareExpression: `window.renderVisualStory(${JSON.stringify({+ file: entry.file,+ exportName: entry.exportName,+ title: entry.title,+ })}).catch(window.showVisualError)`,+ waitForSelector: "#visual-ready, #visual-error",+ failForSelector: "#visual-error",+ waitForTimeout: 0,+ finalizeExpression: "window.finishVisualStory()",+ timeout: visualRegressionConfig.capture.timeout,+ format: "png",+ scale: 1,+});++export const captureStories = async ({+ assetDirectory,+ browserPath,+ concurrency,+ entries,+ port,+ target,+ session,+}: {+ assetDirectory: string;+ browserPath: string;+ concurrency: number;+ entries: readonly VisualStoryEntry[];+ port: number;+ target: "baseline" | "current";+ session: VisualCaptureSession | undefined;+}) =>+ captureVisualEntries({+ captures: entries.map((entry) => {+ const output = path.join(assetDirectory, entry.id, `${target}.png`);+ return {+ id: entry.id,+ options: getStoryCaptureOptions({ browserPath, entry, output, port }),+ };+ }),+ concurrency,+ session,+ onBatchFailure,+ });++export const captureStoryPairs = async ({+ assetDirectory,+ browserPath,+ pairs,+ baselinePort,+ currentPort,+ session,+}: {+ assetDirectory: string;+ browserPath: string;+ pairs: readonly {+ baseline: VisualStoryEntry;+ current: VisualStoryEntry;+ }[];+ baselinePort: number;+ currentPort: number;+ session: VisualCaptureSession | undefined;+}) => {+ const targets = new Map<+ string,+ { id: string; target: "baseline" | "current" }+ >();+ const captures = pairs.flatMap(({ baseline, current }) =>+ [+ { entry: current, port: currentPort, target: "current" as const },+ { entry: baseline, port: baselinePort, target: "baseline" as const },+ ].map(({ entry, port, target }) => {+ const captureId = `${target}:${entry.id}`;+ const output = path.join(assetDirectory, entry.id, `${target}.png`);+ targets.set(captureId, { id: entry.id, target });+ return {+ id: captureId,+ options: getStoryCaptureOptions({ browserPath, entry, output, port }),+ };+ })+ );+ const result = await captureVisualEntries({+ captures,+ concurrency: 1,+ session,+ onBatchFailure,+ });+ const capturesByTarget = {+ baseline: {+ paths: new Map<string, string>(),+ errors: new Map<string, string>(),+ },+ current: {+ paths: new Map<string, string>(),+ errors: new Map<string, string>(),+ },+ };+ for (const [captureId, { id, target }] of targets) {+ const capture = capturesByTarget[target];+ const capturePath = result.paths.get(captureId);+ if (capturePath !== undefined) {+ capture.paths.set(id, capturePath);+ }+ const error = result.errors.get(captureId);+ if (error !== undefined) {+ capture.errors.set(id, error);+ }+ }+ return capturesByTarget;+};diff --git a/scripts/visual-regression/config.ts b/scripts/visual-regression/config.tsnew file mode 100644index 0000000000..b6bce6595d--- /dev/null+++ b/scripts/visual-regression/config.ts@@ -0,0 +1,34 @@+export const visualRegressionConfig = {+ servers: {+ baselinePort: 6101,+ currentPort: 6102,+ },+ capture: {+ viewport: { width: 1280, height: 800 },+ timeout: 90_000,+ concurrency: { minimum: 5, maximum: 8 },+ },+ comparison: {+ pixelThreshold: 0.1,+ maxMismatchPercentage: 0.001,+ concurrency: 4,+ },+ cache: {+ include: [+ ".storybook/preview.tsx",+ ".storybook/story-settings.ts",+ ".storybook/story-sources.json",+ "packages/vision/src/**/*",+ "scripts/visual-regression/**/*",+ "pnpm-lock.yaml",+ ],+ exclude: [+ "**/*.test.*",+ "**/*.test-utils.*",+ "**/*.d.ts",+ "**/*.md",+ "**/tsconfig.json",+ "**/__snapshots__/**",+ ],+ },+} as const;diff --git a/scripts/visual-regression/harness-modules.d.ts b/scripts/visual-regression/harness-modules.d.tsnew file mode 100644index 0000000000..59611b794b--- /dev/null+++ b/scripts/visual-regression/harness-modules.d.ts@@ -0,0 +1,10 @@+declare module "virtual:webstudio-visual-preview" {+ import type { Preview } from "@storybook/react";++ const preview: Preview;+ export default preview;+}++declare module "virtual:webstudio-visual-story-modules" {+ export const modules: Record<string, () => Promise<Record<string, unknown>>>;+}diff --git a/scripts/visual-regression/harness.tsx b/scripts/visual-regression/harness.tsxnew file mode 100644index 0000000000..9349aa28e8--- /dev/null+++ b/scripts/visual-regression/harness.tsx@@ -0,0 +1,234 @@+import React from "react";+import { flushSync } from "react-dom";+import { createRoot } from "react-dom/client";+import { composeStories, setProjectAnnotations } from "@storybook/react";+import preview from "virtual:webstudio-visual-preview";+import { modules } from "virtual:webstudio-visual-story-modules";++type VisualRegressionParameters = {+ disableAnimations?: boolean;+ disableIntervals?: boolean;+ hideSelectors?: readonly string[];+ settleTime?: number;+};++type VisualStoryInput = {+ file: string;+ exportName: string;+ title: string;+};++type ComposedStory = React.ComponentType & {+ parameters?: {+ backgrounds?: {+ default?: string;+ values?: Array<{ name: string; value: string }>;+ };+ visualRegression?: VisualRegressionParameters;+ };+};++declare global {+ interface Window {+ finishVisualStory: () => Promise<void>;+ renderVisualStory: (input: VisualStoryInput) => Promise<void>;+ showVisualError: (error: unknown) => void;+ }+}++setProjectAnnotations(preview);++const waitForFrames = () =>+ new Promise((resolve) =>+ requestAnimationFrame(() => requestAnimationFrame(resolve))+ );++const markReady = async (disableAnimations: boolean) => {+ await waitForFrames();+ if (disableAnimations) {+ for (const animation of document.getAnimations()) {+ animation.cancel();+ }+ // Animation components can react to cancellation on the next frame. Wait+ // for those updates and cancel anything they created in response.+ await waitForFrames();+ for (const animation of document.getAnimations()) {+ animation.cancel();+ }+ await waitForFrames();+ }+ if (document.activeElement instanceof HTMLElement) {+ document.activeElement.blur();+ }+ const ready = document.createElement("div");+ ready.id = "visual-ready";+ ready.hidden = true;+ document.body.append(ready);+};++const showError = (error: unknown) => {+ document.querySelector("#visual-ready")?.remove();+ document.querySelector("#visual-error")?.remove();+ const output = document.createElement("pre");+ output.id = "visual-error";+ output.textContent =+ error instanceof Error ? (error.stack ?? error.message) : String(error);+ document.body.append(output);+};+window.showVisualError = showError;++window.addEventListener("error", (event) =>+ showError(event.error ?? event.message)+);+window.addEventListener("unhandledrejection", (event) =>+ showError(event.reason)+);++const rootElement = document.querySelector("#root");+if (rootElement === null) {+ throw new Error("Visual story root is missing");+}+let root = createRoot(rootElement);+const originalSetInterval = window.setInterval;+const originalClearInterval = window.clearInterval;+const storyIntervals = new Set<number>();+let disableCurrentAnimations = true;+const OriginalDate = Date;+const fixedTime = Date.UTC(2020, 0, 1);+window.Date = new Proxy(OriginalDate, {+ construct(target, argumentsList, newTarget) {+ return Reflect.construct(+ target,+ argumentsList.length === 0 ? [fixedTime] : argumentsList,+ newTarget+ );+ },+ apply() {+ return new OriginalDate(fixedTime).toString();+ },+});+window.Date.now = () => fixedTime;++window.renderVisualStory = async ({ file, exportName, title }) => {+ flushSync(() => root.unmount());+ rootElement.replaceChildren();+ root = createRoot(rootElement);+ for (const interval of storyIntervals) {+ originalClearInterval(interval);+ }+ storyIntervals.clear();+ for (const element of document.querySelectorAll(+ "body > :not(#root), [data-visual-regression-style]"+ )) {+ element.remove();+ }+ document.body.style.background = "";++ let randomState = 0x12345678;+ Math.random = () => {+ randomState = (randomState * 1664525 + 1013904223) >>> 0;+ return randomState / 0x100000000;+ };+ const load = modules[`/${file}`];+ if (load === undefined) {+ throw new Error(`Visual story module not found: ${file}`);+ }+ const module = await load();+ const meta =+ typeof module.default === "object" && module.default !== null+ ? module.default+ : {};+ const csfModule = {+ ...module,+ default: { ...meta, title },+ } as Parameters<typeof composeStories>[0];+ const stories = composeStories(csfModule) as Record<string, ComposedStory>;+ const Story = stories[exportName];+ if (Story === undefined) {+ throw new Error(`Visual story export not found: ${exportName}`);+ }+ const options = (Story.parameters?.visualRegression ??+ {}) as VisualRegressionParameters;+ disableCurrentAnimations = options.disableAnimations !== false;+ Object.defineProperty(window, "setInterval", {+ configurable: true,+ value:+ options.disableIntervals !== false+ ? () => 0+ : (+ handler: TimerHandler,+ timeout?: number,+ ...arguments_: unknown[]+ ) => {+ const interval = originalSetInterval(+ handler,+ timeout,+ ...arguments_+ );+ storyIntervals.add(interval);+ return interval;+ },+ });+ if (+ options.disableAnimations !== false ||+ options.hideSelectors?.length !== undefined+ ) {+ const style = document.createElement("style");+ style.dataset.visualRegressionStyle = "";+ const rules =+ options.hideSelectors?.map(+ (selector) => `${selector} { visibility: hidden !important; }`+ ) ?? [];+ if (options.disableAnimations !== false) {+ rules.push(`*, *::before, *::after {+ animation-delay: 0s !important;+ animation-duration: 0s !important;+ transition: none !important;+}`);+ }+ style.textContent = rules.join("\n");+ document.head.append(style);+ }++ const backgrounds = Story.parameters?.backgrounds;+ const background = backgrounds?.values?.find(+ ({ name }: { name: string }) => name === backgrounds.default+ );+ if (background?.value !== undefined) {+ document.body.style.background = background.value;+ }++ flushSync(() => root.render(React.createElement(Story)));+ if (options.settleTime !== undefined && options.settleTime > 0) {+ await new Promise((resolve) =>+ window.setTimeout(resolve, options.settleTime)+ );+ }+ await markReady(options.disableAnimations !== false);+};++window.finishVisualStory = async () => {+ if (disableCurrentAnimations === false) {+ return;+ }+ // The browser calls this after fonts and component frames settle. Give+ // animation consumers one frame to react, then freeze the final static state+ // immediately before screenshot capture.+ for (const animation of document.getAnimations()) {+ animation.cancel();+ }+ await waitForFrames();+ for (const animation of document.getAnimations()) {+ animation.cancel();+ }+};++const params = new URLSearchParams(window.location.search);+if (params.has("file")) {+ const file = params.get("file");+ const exportName = params.get("exportName");+ const title = params.get("title");+ if (file !== null && exportName !== null && title !== null) {+ window.renderVisualStory({ file, exportName, title }).catch(showError);+ }+}diff --git a/scripts/visual-regression/index.html b/scripts/visual-regression/index.htmlnew file mode 100644index 0000000000..e9ec8f178d--- /dev/null+++ b/scripts/visual-regression/index.html@@ -0,0 +1,33 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="UTF-8" />+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />+ <style>+ html,+ body,+ #root {+ min-height: 100%;+ }++ body {+ margin: 0;+ pointer-events: none;+ }++ *,+ *::before,+ *::after {+ caret-color: transparent !important;+ }++ iframe {+ visibility: hidden !important;+ }+ </style>+ <script type="module" src="/harness.tsx"></script>+ </head>+ <body>+ <div id="root"></div>+ </body>+</html>diff --git a/scripts/visual-regression/manifest.test.ts b/scripts/visual-regression/manifest.test.tsnew file mode 100644index 0000000000..8bd8a1b698--- /dev/null+++ b/scripts/visual-regression/manifest.test.ts@@ -0,0 +1,80 @@+import assert from "node:assert/strict";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { readStoryManifest, readStorySources } from "./manifest";++test("indexes portable CSF stories with Storybook-compatible ids", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-manifest-"));+ const file = path.join(root, "apps/builder/example.stories.tsx");+ await mkdir(path.dirname(file), { recursive: true });+ await writeFile(+ file,+ `export default { title: "Controls/Button" };+export const Primary = () => null;+export const WithIcon = { render: () => null, name: "With icon" };+`+ );+ try {+ const manifest = await readStoryManifest({+ root,+ storySources: [+ {+ directory: "../apps/builder",+ files: "**/*.stories.tsx",+ titlePrefix: "Builder",+ },+ ],+ });+ assert.deepEqual(Object.keys(manifest), [+ "builder-controls-button--primary",+ "builder-controls-button--with-icon",+ ]);+ assert.equal(+ manifest["builder-controls-button--primary"]?.exportName,+ "Primary"+ );+ assert.equal(+ manifest["builder-controls-button--with-icon"]?.name,+ "With icon"+ );+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});++test("reads story source configuration from each revision", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-sources-"));+ const baseline = path.join(root, "baseline");+ const current = path.join(root, "current");+ await Promise.all(+ [+ [baseline, "Baseline"],+ [current, "Current"],+ ].map(async ([directory, titlePrefix]) => {+ const storybook = path.join(directory, ".storybook");+ await mkdir(storybook, { recursive: true });+ await writeFile(+ path.join(storybook, "story-sources.json"),+ JSON.stringify([+ {+ directory: "../stories",+ files: "**/*.stories.tsx",+ titlePrefix,+ },+ ])+ );+ })+ );+ try {+ const [baselineSources, currentSources] = await Promise.all([+ readStorySources(baseline),+ readStorySources(current),+ ]);+ assert.equal(baselineSources?.[0]?.titlePrefix, "Baseline");+ assert.equal(currentSources?.[0]?.titlePrefix, "Current");+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/scripts/visual-regression/manifest.ts b/scripts/visual-regression/manifest.tsnew file mode 100644index 0000000000..e119cd301b--- /dev/null+++ b/scripts/visual-regression/manifest.ts@@ -0,0 +1,125 @@+import { readFile } from "node:fs/promises";+import path from "node:path";+import { glob } from "node:fs/promises";+import { loadCsf } from "storybook/internal/csf-tools";+import { toId } from "storybook/internal/csf";+import type { VisualEntry } from "@webstudio-is/vision/comparison";++export type StorySource = {+ directory: string;+ files: string;+ titlePrefix: string;+};++const isStorySource = (value: unknown): value is StorySource =>+ typeof value === "object" &&+ value !== null &&+ "directory" in value &&+ typeof value.directory === "string" &&+ "files" in value &&+ typeof value.files === "string" &&+ "titlePrefix" in value &&+ typeof value.titlePrefix === "string";++export const readStorySources = async (+ root: string+): Promise<readonly StorySource[] | undefined> => {+ let source: string;+ try {+ source = await readFile(+ path.join(root, ".storybook/story-sources.json"),+ "utf8"+ );+ } catch (error) {+ if (+ typeof error === "object" &&+ error !== null &&+ "code" in error &&+ error.code === "ENOENT"+ ) {+ return;+ }+ throw error;+ }+ const value: unknown = JSON.parse(source);+ if (Array.isArray(value) === false || value.every(isStorySource) === false) {+ throw new Error(".storybook/story-sources.json is invalid.");+ }+ return value;+};++export type VisualStoryEntry = VisualEntry & {+ title: string;+ name: string;+ exportName: string;+ file: string;+ titlePrefix: string;+};++const readScopeEntries = async ({+ root,+ pattern,+ titlePrefix,+}: {+ root: string;+ pattern: string;+ titlePrefix: string;+}) => {+ const entries: VisualStoryEntry[] = [];+ for await (const file of glob(pattern, { cwd: root })) {+ const source = await readFile(path.join(root, file), "utf8");+ const csf = loadCsf(source, {+ fileName: file,+ makeTitle: (title) => `${titlePrefix}/${title}`,+ });+ csf.parse();+ for (const story of csf.indexInputs) {+ if (story.type !== "story") {+ continue;+ }+ if (+ story.title === undefined ||+ story.name === undefined ||+ story.exportName === undefined+ ) {+ throw new Error(`Story index metadata is incomplete in ${file}`);+ }+ entries.push({+ id: story.__id ?? toId(story.title, story.name),+ title: story.title,+ name: story.name,+ exportName: story.exportName,+ file,+ titlePrefix,+ });+ }+ }+ return entries;+};++export const readStoryManifest = async ({+ root,+ storySources,+}: {+ root: string;+ storySources: readonly StorySource[];+}) => {+ const entries = (+ await Promise.all(+ storySources.map(({ directory, files, titlePrefix }) => {+ const pattern = path.posix.join(".storybook", directory, files);+ return readScopeEntries({ root, pattern, titlePrefix });+ })+ )+ )+ .flat()+ .sort((left, right) => left.id.localeCompare(right.id));++ const duplicate = entries.find(+ (entry, index) => entry.id === entries[index - 1]?.id+ );+ if (duplicate !== undefined) {+ throw new Error(`Duplicate visual story id: ${duplicate.id}`);+ }+ return Object.fromEntries(entries.map((entry) => [entry.id, entry]));+};diff --git a/scripts/visual-regression/open-report.ts b/scripts/visual-regression/open-report.tsnew file mode 100644index 0000000000..5d9ee6032d--- /dev/null+++ b/scripts/visual-regression/open-report.ts@@ -0,0 +1,12 @@+import path from "node:path";+import { pathToFileURL } from "node:url";+import { openScreenshotComparisonReport } from "@webstudio-is/vision/report";++if (+ process.argv[1] !== undefined &&+ import.meta.url === pathToFileURL(process.argv[1]).href+) {+ await openScreenshotComparisonReport(+ path.resolve(".visual-regression/report/index.html")+ );+}diff --git a/scripts/visual-regression/run.ts b/scripts/visual-regression/run.tsnew file mode 100644index 0000000000..559517d16c--- /dev/null+++ b/scripts/visual-regression/run.ts@@ -0,0 +1,658 @@+import { execFile, spawn } from "node:child_process";+import { createHash } from "node:crypto";+import { access, mkdir, rm } from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { parseArgs, promisify } from "node:util";+import {+ createBrowserScreenshotSession,+ type BrowserScreenshotSession,+} from "@webstudio-is/vision/browser";+import {+ classifyScreenshotComparisonReport,+ openScreenshotComparisonReport,+ writeScreenshotComparisonReport,+ type ScreenshotComparisonReport,+} from "@webstudio-is/vision/report";+import { compareVisualEntries } from "@webstudio-is/vision/comparison";+import {+ restoreScreenshotCache,+ writeScreenshotCache,+} from "@webstudio-is/vision/screenshot-cache";+import {+ defaultScreenshotDependencies,+ resolveScreenshotBrowser,+} from "webstudio/vision";+import {+ readStoryManifest,+ readStorySources,+ type VisualStoryEntry,+} from "./manifest";+import {+ captureStoryPairs,+ captureStories,+ getStoryCaptureOptions,+ getInitialCaptureTarget,+} from "./capture";+import { visualRegressionConfig } from "./config";+import { getScreenshotRuntimeHash } from "./runtime-hash";+import {+ startVisualStoryServer,+ startVisualStoryServers,+} from "./story-server";++const repositoryRoot = process.cwd();+const outputRoot = path.join(repositoryRoot, ".visual-regression");+const reportDirectory = path.join(outputRoot, "report");+const assetDirectory = path.join(reportDirectory, "assets");+const screenshotCacheRoot = path.join(+ repositoryRoot,+ ".visual-regression-cache"+);+const defaultCaptureConcurrency = Math.min(+ visualRegressionConfig.capture.concurrency.maximum,+ Math.max(+ visualRegressionConfig.capture.concurrency.minimum,+ os.availableParallelism()+ )+);+const captureConcurrency = Number(+ process.env.VISUAL_CAPTURE_CONCURRENCY ?? defaultCaptureConcurrency+);+const execFileAsync = promisify(execFile);+if (Number.isInteger(captureConcurrency) === false || captureConcurrency < 1) {+ throw new Error("VISUAL_CAPTURE_CONCURRENCY must be a positive integer.");+}++const { values: arguments_ } = parseArgs({+ args: process.argv.slice(2).filter((argument) => argument !== "--"),+ options: {+ base: { type: "string", default: "origin/main" },+ grep: { type: "string" },+ "open-report": { type: "boolean", default: false },+ "approve-visual-changes": { type: "boolean", default: false },+ },+ strict: true,+});+const baseRef = arguments_.base;+const grep = arguments_.grep;+const openReport = arguments_["open-report"];+const approved = arguments_["approve-visual-changes"];++const run = async ({+ command,+ commandArgs,+ cwd = repositoryRoot,+}: {+ command: string;+ commandArgs: string[];+ cwd?: string;+}) => {+ const child = spawn(command, commandArgs, {+ cwd,+ stdio: ["ignore", "inherit", "inherit"],+ });+ const exitCode = await new Promise<number>((resolve, reject) => {+ child.on("error", reject);+ child.on("exit", (code, signal) => {+ if (signal !== null) {+ reject(new Error(`${command} exited with signal ${signal}`));+ return;+ }+ resolve(code ?? 1);+ });+ });+ if (exitCode !== 0) {+ throw new Error(`${command} exited with code ${exitCode}`);+ }+ return exitCode;+};++const installBaselineDependencies = (checkout: string) =>+ run({+ command: "pnpm",+ commandArgs: [+ "install",+ "--frozen-lockfile",+ "--ignore-scripts",+ "--prod=false",+ ],+ cwd: checkout,+ });++const getCommandOutput = async (command: string, commandArgs: string[]) => {+ const { stdout } = await execFileAsync(command, commandArgs, {+ cwd: repositoryRoot,+ encoding: "utf8",+ });+ return stdout.trim();+};++const getScreenshotCacheDirectory = (revision: string, runtimeHash: string) =>+ path.join(screenshotCacheRoot, `${revision}-${runtimeHash}`);++type StoryCapture = Awaited<ReturnType<typeof captureStories>>;++const replaceStoryCaptures = ({+ ids,+ capture,+ replacement,+}: {+ ids: readonly string[];+ capture: StoryCapture;+ replacement: StoryCapture;+}) => {+ for (const id of ids) {+ capture.paths.delete(id);+ capture.errors.delete(id);+ }+ for (const [id, screenshotPath] of replacement.paths) {+ capture.paths.set(id, screenshotPath);+ }+ for (const [id, error] of replacement.errors) {+ capture.errors.set(id, error);+ }+};++const compareStories = async ({+ baselineEntries,+ currentEntries,+ baselinePaths,+ currentPaths,+ baselineErrors,+ currentErrors,+ analyzeDifferences = true,+}: {+ baselineEntries: Record<string, VisualStoryEntry>;+ currentEntries: Record<string, VisualStoryEntry>;+ baselinePaths: Map<string, string>;+ currentPaths: Map<string, string>;+ baselineErrors: Map<string, string>;+ currentErrors: Map<string, string>;+ analyzeDifferences?: boolean;+}) => {+ return await compareVisualEntries({+ baselineEntries: Object.values(baselineEntries),+ currentEntries: Object.values(currentEntries),+ baselinePaths,+ currentPaths,+ baselineErrors,+ currentErrors,+ artifactDirectory: assetDirectory,+ pixelThreshold: visualRegressionConfig.comparison.pixelThreshold,+ maxMismatchPercentage:+ visualRegressionConfig.comparison.maxMismatchPercentage,+ concurrency: Math.min(+ visualRegressionConfig.comparison.concurrency,+ os.availableParallelism()+ ),+ analyzeText: analyzeDifferences ? "when-different" : false,+ writeArtifacts: analyzeDifferences ? "when-different" : false,+ });+};++const prepareBaselineCheckout = async (checkout: string) => {+ const [baselineCommit, currentCommit] = await Promise.all([+ getCommandOutput("git", ["merge-base", "HEAD", baseRef]),+ getCommandOutput("git", ["rev-parse", "HEAD"]),+ ]);+ const currentRevisionIsClean =+ (await getCommandOutput("git", ["status", "--porcelain"])) === "";+ console.info(`Comparing ${baselineCommit} with ${currentCommit}`);++ const checkoutExists = await access(path.join(checkout, ".git"))+ .then(() => true)+ .catch(() => false);+ await run(+ checkoutExists+ ? {+ command: "git",+ commandArgs: ["switch", "--detach", baselineCommit],+ cwd: checkout,+ }+ : {+ command: "git",+ commandArgs: [+ "worktree",+ "add",+ "--detach",+ checkout,+ baselineCommit,+ ],+ }+ );++ const currentSubmodules = await getCommandOutput("git", [+ "submodule",+ "status",+ "--recursive",+ ]);+ if (+ currentSubmodules+ .split("\n")+ .some((status) => status !== "" && status.startsWith("-") === false)+ ) {+ await run({+ command: "git",+ commandArgs: ["submodule", "update", "--init", "--recursive"],+ cwd: checkout,+ });+ }+ return { baselineCommit, currentCommit, currentRevisionIsClean };+};++const readComparisonEntries = async (checkout: string) => {+ const currentStorySources = await readStorySources(repositoryRoot);+ if (currentStorySources === undefined) {+ throw new Error("Current story source configuration is missing.");+ }+ const baselineStorySources =+ (await readStorySources(checkout)) ?? currentStorySources;+ const [baselineEntries, currentEntries] = await Promise.all([+ readStoryManifest({ root: checkout, storySources: baselineStorySources }),+ readStoryManifest({+ root: repositoryRoot,+ storySources: currentStorySources,+ }),+ ]);+ const matches = grep === undefined ? undefined : new RegExp(grep, "i");+ const filterEntries = (entries: Record<string, VisualStoryEntry>) =>+ Object.fromEntries(+ Object.entries(entries).filter(+ ([id, entry]) =>+ matches === undefined ||+ matches.test(`${id} ${entry.title} ${entry.name}`)+ )+ );+ return {+ baselineEntries: filterEntries(baselineEntries),+ currentEntries: filterEntries(currentEntries),+ };+};++const verifyChangedCaptures = async ({+ comparisons,+ baselineEntries,+ currentEntries,+ baselineCapture,+ currentCapture,+ cachedBaseline,+ browserPath,+ browserSession,+ checkout,+ baselineBundleDirectory,+ servers,+}: {+ comparisons: Awaited<ReturnType<typeof compareStories>>;+ baselineEntries: Record<string, VisualStoryEntry>;+ currentEntries: Record<string, VisualStoryEntry>;+ baselineCapture: StoryCapture;+ currentCapture: StoryCapture;+ cachedBaseline: boolean;+ browserPath: string;+ browserSession: BrowserScreenshotSession | undefined;+ checkout: string;+ baselineBundleDirectory: string;+ servers: Array<Awaited<ReturnType<typeof startVisualStoryServer>>>;+}) => {+ const changedIds = comparisons+ .filter(({ status }) => status === "changed")+ .map(({ id }) => id);+ if (changedIds.length === 0) {+ return { comparisons, performed: false };+ }++ console.info(+ `Verifying ${changedIds.length} visual differences with serial captures.`+ );+ const currentVerification = await captureStories({+ assetDirectory,+ browserPath,+ concurrency: 1,+ entries: changedIds.flatMap((id) => {+ const entry = currentEntries[id];+ return entry === undefined ? [] : [entry];+ }),+ port: visualRegressionConfig.servers.currentPort,+ target: "current",+ session: browserSession,+ });+ replaceStoryCaptures({+ ids: changedIds,+ capture: currentCapture,+ replacement: currentVerification,+ });+ comparisons = await compareStories({+ baselineEntries,+ currentEntries,+ baselinePaths: baselineCapture.paths,+ currentPaths: currentCapture.paths,+ baselineErrors: baselineCapture.errors,+ currentErrors: currentCapture.errors,+ analyzeDifferences: false,+ });+ const remainingChangedIds = comparisons+ .filter(({ status }) => status === "changed")+ .map(({ id }) => id);+ if (remainingChangedIds.length === 0) {+ return { comparisons, performed: true };+ }++ const changedPairs = remainingChangedIds.flatMap((id) => {+ const baseline = baselineEntries[id];+ const current = currentEntries[id];+ return baseline === undefined || current === undefined+ ? []+ : [{ baseline, current }];+ });+ if (cachedBaseline) {+ await installBaselineDependencies(checkout);+ servers.push(+ await startVisualStoryServer({+ root: checkout,+ port: visualRegressionConfig.servers.baselinePort,+ outputDirectory: baselineBundleDirectory,+ storyFiles: changedPairs.map(({ baseline }) => baseline.file),+ })+ );+ }+ const pairedVerification = await captureStoryPairs({+ assetDirectory,+ browserPath,+ pairs: changedPairs,+ baselinePort: visualRegressionConfig.servers.baselinePort,+ currentPort: visualRegressionConfig.servers.currentPort,+ session: browserSession,+ });+ replaceStoryCaptures({+ ids: remainingChangedIds,+ capture: baselineCapture,+ replacement: pairedVerification.baseline,+ });+ replaceStoryCaptures({+ ids: remainingChangedIds,+ capture: currentCapture,+ replacement: pairedVerification.current,+ });+ return {+ comparisons: await compareStories({+ baselineEntries,+ currentEntries,+ baselinePaths: baselineCapture.paths,+ currentPaths: currentCapture.paths,+ baselineErrors: baselineCapture.errors,+ currentErrors: currentCapture.errors,+ }),+ performed: true,+ };+};++const main = async () => {+ const startedAt = Date.now();+ let phaseStartedAt = startedAt;+ const logPhase = (name: string) => {+ const now = Date.now();+ console.info(`${name}: ${((now - phaseStartedAt) / 1000).toFixed(1)}s`);+ phaseStartedAt = now;+ };+ const cacheKey = createHash("sha256")+ .update(repositoryRoot)+ .digest("hex")+ .slice(0, 12);+ const temporaryRoot = path.join(+ os.tmpdir(),+ `webstudio-visual-regression-${cacheKey}`+ );+ const checkout = path.join(temporaryRoot, "checkout");+ const baselineBundleDirectory = path.join(temporaryRoot, "baseline-bundle");+ const currentBundleDirectory = path.join(temporaryRoot, "current-bundle");+ const servers: Array<Awaited<ReturnType<typeof startVisualStoryServer>>> = [];+ let browserSession: BrowserScreenshotSession | undefined;+ let report: ScreenshotComparisonReport | undefined;++ await rm(outputRoot, { recursive: true, force: true });+ await mkdir(assetDirectory, { recursive: true });+ await mkdir(temporaryRoot, { recursive: true });++ try {+ const { baselineCommit, currentCommit, currentRevisionIsClean } =+ await prepareBaselineCheckout(checkout);+ const browser = await resolveScreenshotBrowser(+ { browser: "auto" },+ defaultScreenshotDependencies+ );+ const {+ baselineEntries: filteredBaselineEntries,+ currentEntries: filteredCurrentEntries,+ } = await readComparisonEntries(checkout);+ const filteredIds = new Set([+ ...Object.keys(filteredBaselineEntries),+ ...Object.keys(filteredCurrentEntries),+ ]);+ console.info(`Running ${filteredIds.size} visual story ids.`);+ const screenshotRuntimeHash = await getScreenshotRuntimeHash({+ root: repositoryRoot,+ browserVersion: await getCommandOutput(browser.path, ["--version"]),+ });+ const screenshotCacheDirectory = getScreenshotCacheDirectory(+ baselineCommit,+ screenshotRuntimeHash+ );+ const cachedBaselinePaths = await restoreScreenshotCache({+ directory: screenshotCacheDirectory,+ ids: Object.keys(filteredBaselineEntries),+ getOutputPath: (id) => path.join(assetDirectory, id, "baseline.png"),+ });+ if (cachedBaselinePaths !== undefined) {+ console.info(`Reusing baseline screenshots for ${baselineCommit}.`);+ } else {+ console.info(+ `Baseline screenshot cache is unavailable; rendering ${+ Object.keys(filteredBaselineEntries).length+ } baseline stories.`+ );+ await installBaselineDependencies(checkout);+ }+ servers.push(+ ...(await startVisualStoryServers([+ ...(cachedBaselinePaths === undefined+ ? [+ {+ root: checkout,+ port: visualRegressionConfig.servers.baselinePort,+ outputDirectory: baselineBundleDirectory,+ storyFiles: Object.values(filteredBaselineEntries).map(+ (entry) => entry.file+ ),+ },+ ]+ : []),+ {+ root: repositoryRoot,+ port: visualRegressionConfig.servers.currentPort,+ outputDirectory: currentBundleDirectory,+ storyFiles: Object.values(filteredCurrentEntries).map(+ (entry) => entry.file+ ),+ },+ ]))+ );+ logPhase("Setup and bundles");++ const baselineEntriesToCapture = Object.values(filteredBaselineEntries);+ const currentEntriesToCapture = Object.values(filteredCurrentEntries);+ if (+ baselineEntriesToCapture.length === 0 &&+ currentEntriesToCapture.length === 0+ ) {+ throw new Error("No stories matched the visual comparison.");+ }+ const initialCapture = getInitialCaptureTarget({+ baselineEntries: baselineEntriesToCapture,+ currentEntries: currentEntriesToCapture,+ hasCachedBaseline: cachedBaselinePaths !== undefined,+ });+ if (initialCapture !== undefined) {+ const firstOutput = path.join(+ assetDirectory,+ initialCapture.entry.id,+ "browser-session.png"+ );+ await mkdir(path.dirname(firstOutput), { recursive: true });+ browserSession = await createBrowserScreenshotSession(+ getStoryCaptureOptions({+ browserPath: browser.path,+ entry: initialCapture.entry,+ output: firstOutput,+ port:+ initialCapture.target === "baseline"+ ? visualRegressionConfig.servers.baselinePort+ : visualRegressionConfig.servers.currentPort,+ })+ );+ }+ const baselineCapturePromise =+ cachedBaselinePaths === undefined+ ? captureStories({+ assetDirectory,+ browserPath: browser.path,+ concurrency: captureConcurrency,+ entries: baselineEntriesToCapture,+ port: visualRegressionConfig.servers.baselinePort,+ target: "baseline",+ session: browserSession,+ })+ : Promise.resolve({+ paths: cachedBaselinePaths,+ errors: new Map<string, string>(),+ });+ const currentCapturePromise = captureStories({+ assetDirectory,+ browserPath: browser.path,+ concurrency: captureConcurrency,+ entries: currentEntriesToCapture,+ port: visualRegressionConfig.servers.currentPort,+ target: "current",+ session: browserSession,+ });+ const [baselineCapture, currentCapture] = await Promise.all([+ baselineCapturePromise,+ currentCapturePromise,+ ]);+ logPhase("Screenshots");+ let comparisons = await compareStories({+ baselineEntries: filteredBaselineEntries,+ currentEntries: filteredCurrentEntries,+ baselinePaths: baselineCapture.paths,+ currentPaths: currentCapture.paths,+ baselineErrors: baselineCapture.errors,+ currentErrors: currentCapture.errors,+ analyzeDifferences: false,+ });+ logPhase("Image comparison");+ const verification = await verifyChangedCaptures({+ comparisons,+ baselineEntries: filteredBaselineEntries,+ currentEntries: filteredCurrentEntries,+ baselineCapture,+ currentCapture,+ cachedBaseline: cachedBaselinePaths !== undefined,+ browserPath: browser.path,+ browserSession,+ checkout,+ baselineBundleDirectory,+ servers,+ });+ comparisons = verification.comparisons;+ if (verification.performed) {+ logPhase("Serial verification");+ }+ if (+ cachedBaselinePaths === undefined &&+ baselineCapture.errors.size === 0+ ) {+ await writeScreenshotCache({+ directory: screenshotCacheDirectory,+ paths: baselineCapture.paths,+ });+ }+ if (currentRevisionIsClean && currentCapture.errors.size === 0) {+ await writeScreenshotCache({+ directory: getScreenshotCacheDirectory(+ currentCommit,+ screenshotRuntimeHash+ ),+ paths: currentCapture.paths,+ });+ }+ report = {+ baselineLabel: baselineCommit,+ currentLabel: currentCommit,+ durationMs: Date.now() - startedAt,+ comparisons,+ errors: [],+ };+ } catch (error) {+ const message =+ error instanceof Error ? (error.stack ?? error.message) : String(error);+ report = {+ baselineLabel: baseRef,+ currentLabel: "HEAD",+ durationMs: Date.now() - startedAt,+ comparisons: [],+ errors: [message],+ };+ } finally {+ await browserSession?.close().catch(() => undefined);+ await Promise.all(+ servers.map((server) => server.close().catch(() => undefined))+ );+ await Promise.all([+ rm(baselineBundleDirectory, { recursive: true, force: true }),+ rm(currentBundleDirectory, { recursive: true, force: true }),+ ]);+ }++ if (report === undefined) {+ throw new Error("Visual comparison did not produce a report.");+ }+ await writeScreenshotComparisonReport({ report, reportDirectory });+ const comparisonResult = classifyScreenshotComparisonReport(report);+ const result =+ comparisonResult === "failure"+ ? "test-failure"+ : comparisonResult === "passed"+ ? "passed"+ : approved+ ? "approved"+ : "visual-differences";+ const changed = report.comparisons.filter(+ ({ status }) => status !== "unchanged"+ ).length;+ console.info(+ `${report.comparisons.length} stories compared in ${(+ report.durationMs / 1000+ ).toFixed(1)}s; ${changed} differences.`+ );+ if (result === "approved") {+ console.info(+ "Visual differences were explicitly approved for this revision."+ );+ } else if (result === "visual-differences") {+ console.error(+ "Visual differences detected. Open .visual-regression/report/index.html."+ );+ } else if (result === "test-failure") {+ console.error(+ "Visual regression infrastructure or story rendering failed."+ );+ }+ if (openReport) {+ await openScreenshotComparisonReport(+ path.join(reportDirectory, "index.html")+ );+ }+ process.exitCode = result === "passed" || result === "approved" ? 0 : 1;+};++await main();diff --git a/scripts/visual-regression/runtime-hash.test.ts b/scripts/visual-regression/runtime-hash.test.tsnew file mode 100644index 0000000000..dc78e04e7d--- /dev/null+++ b/scripts/visual-regression/runtime-hash.test.ts@@ -0,0 +1,78 @@+import assert from "node:assert/strict";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { getScreenshotRuntimeHash } from "./runtime-hash";++const writeRuntimeFixture = async (root: string) => {+ const files = {+ ".storybook/preview.tsx": "export default {};",+ ".storybook/story-settings.ts": "export const settings = {};",+ ".storybook/story-sources.json": "[]",+ "packages/vision/src/capture.ts": "export const capture = true;",+ "scripts/visual-regression/capture.ts": "export const capture = true;",+ "pnpm-lock.yaml": "lockfileVersion: '9.0'",+ };+ await Promise.all(+ Object.entries(files).map(async ([file, contents]) => {+ const absolutePath = path.join(root, file);+ await mkdir(path.dirname(absolutePath), { recursive: true });+ await writeFile(absolutePath, contents);+ })+ );+};++test("hashes the complete screenshot runtime without a file manifest", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-runtime-hash-"));+ try {+ await writeRuntimeFixture(root);+ const initial = await getScreenshotRuntimeHash({+ root,+ browserVersion: "Chromium 1",+ });++ await writeFile(+ path.join(root, "scripts/visual-regression/new-runtime-module.ts"),+ "export const runtime = true;"+ );+ const runtimeChanged = await getScreenshotRuntimeHash({+ root,+ browserVersion: "Chromium 1",+ });+ assert.notEqual(runtimeChanged, initial);++ await writeFile(+ path.join(root, "packages/vision/src/new-capture-module.ts"),+ "export const capture = true;"+ );+ const visionChanged = await getScreenshotRuntimeHash({+ root,+ browserVersion: "Chromium 1",+ });+ assert.notEqual(visionChanged, runtimeChanged);++ await writeFile(+ path.join(root, "scripts/visual-regression/runtime.test.ts"),+ "throw new Error('test only');"+ );+ await writeFile(+ path.join(root, "scripts/visual-regression/README.md"),+ "Documentation only."+ );+ await writeFile(path.join(root, "unrelated.ts"), "export const value = 1;");+ const unrelatedChanged = await getScreenshotRuntimeHash({+ root,+ browserVersion: "Chromium 1",+ });+ assert.equal(unrelatedChanged, visionChanged);++ const browserChanged = await getScreenshotRuntimeHash({+ root,+ browserVersion: "Chromium 2",+ });+ assert.notEqual(browserChanged, unrelatedChanged);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/scripts/visual-regression/runtime-hash.ts b/scripts/visual-regression/runtime-hash.tsnew file mode 100644index 0000000000..265682bfae--- /dev/null+++ b/scripts/visual-regression/runtime-hash.ts@@ -0,0 +1,51 @@+import { createHash } from "node:crypto";+import { glob, readFile } from "node:fs/promises";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { visualRegressionConfig } from "./config";++export const getScreenshotRuntimeHash = async ({+ root,+ browserVersion,+}: {+ root: string;+ browserVersion: string;+}) => {+ const files: string[] = [];+ for await (const file of glob([...visualRegressionConfig.cache.include], {+ cwd: root,+ })) {+ if (+ visualRegressionConfig.cache.exclude.some((pattern) =>+ path.matchesGlob(file, pattern)+ ) === false+ ) {+ files.push(file);+ }+ }+ files.sort();+ const runtimeHash = createHash("sha256");+ for (const file of files) {+ const contents = await readFile(path.join(root, file));+ runtimeHash.update(file);+ runtimeHash.update("\0");+ runtimeHash.update(createHash("sha256").update(contents).digest());+ }+ runtimeHash.update("browser\0");+ runtimeHash.update(browserVersion);+ return runtimeHash.digest("hex").slice(0, 12);+};++const entrypoint = process.argv[1];+if (+ entrypoint !== undefined &&+ import.meta.url === pathToFileURL(path.resolve(entrypoint)).href+) {+ const browserVersion = process.argv[2];+ if (browserVersion === undefined) {+ throw new Error("Usage: runtime-hash.ts <browser-version>");+ }+ console.info(+ await getScreenshotRuntimeHash({ root: process.cwd(), browserVersion })+ );+}diff --git a/scripts/visual-regression/story-build.ts b/scripts/visual-regression/story-build.tsnew file mode 100644index 0000000000..8956f2204e--- /dev/null+++ b/scripts/visual-regression/story-build.ts@@ -0,0 +1,137 @@+import { realpath } from "node:fs/promises";+import path from "node:path";+import { fileURLToPath } from "node:url";+import {+ build,+ defaultClientConditions,+ type InlineConfig,+ type Plugin,+} from "vite";+import { hasPrivateStorySources } from "../../.storybook/story-settings";++const visualRoot = path.dirname(fileURLToPath(import.meta.url));+const previewModuleId = "virtual:webstudio-visual-preview";+const storyModulesId = "virtual:webstudio-visual-story-modules";++const createStoryModulesPlugin = ({+ root,+ storyFiles,+}: {+ root: string;+ storyFiles: readonly string[];+}): Plugin => {+ const modules = storyFiles+ .map(+ (file) =>+ `${JSON.stringify(`/${file}`)}: () => import(${JSON.stringify(+ path.join(root, file)+ )})`+ )+ .join(",\n");+ const sources = new Map([+ [+ previewModuleId,+ `export { default } from ${JSON.stringify(+ path.join(root, ".storybook/preview.tsx")+ )};`,+ ],+ [storyModulesId, `export const modules = {${modules}};`],+ ]);+ const resolvedPrefix = "\0webstudio-visual:";++ return {+ name: "webstudio-visual-story-modules",+ resolveId(id) {+ if (sources.has(id)) {+ return `${resolvedPrefix}${id}`;+ }+ },+ load(id) {+ if (id.startsWith(resolvedPrefix)) {+ return sources.get(id.slice(resolvedPrefix.length));+ }+ },+ };+};++const createStoryBuildConfig = ({+ root,+ outputDirectory,+ storyFiles,+}: {+ root: string;+ outputDirectory: string;+ storyFiles: readonly string[];+}): InlineConfig => ({+ root: visualRoot,+ configFile: false,+ logLevel: "warn",+ plugins: [+ createStoryModulesPlugin({+ root,+ storyFiles: [...new Set(storyFiles)].sort(),+ }),+ ],+ resolve: {+ alias: { "~": path.join(root, "apps/builder/app") },+ dedupe: ["react", "react-dom"],+ conditions: [+ ...(hasPrivateStorySources(root) ? ["webstudio-private"] : []),+ "webstudio",+ ...defaultClientConditions,+ ],+ },+ define: {+ "process.env.NODE_DEBUG": "undefined",+ "process.env.IS_STROYBOOK": "true",+ },+ build: {+ outDir: outputDirectory,+ emptyOutDir: true,+ target: "es2022",+ reportCompressedSize: false,+ chunkSizeWarningLimit: 25_000,+ rollupOptions: {+ treeshake: false,+ input: path.join(visualRoot, "index.html"),+ onwarn(warning, warn) {+ if (warning.code !== "MODULE_LEVEL_DIRECTIVE") {+ warn(warning);+ }+ },+ output: {+ assetFileNames: "assets/[name]-[hash][extname]",+ entryFileNames: "harness-[hash].js",+ inlineDynamicImports: true,+ },+ },+ },+});++export const buildVisualStoryApp = async ({+ root,+ outputDirectory,+ storyFiles,+}: {+ root: string;+ outputDirectory: string;+ storyFiles: readonly string[];+}) => {+ const nodeEnvironment = process.env.NODE_ENV;+ try {+ const resolvedRoot = await realpath(root);+ await build(+ createStoryBuildConfig({+ root: resolvedRoot,+ outputDirectory,+ storyFiles,+ })+ );+ } finally {+ if (nodeEnvironment === undefined) {+ delete process.env.NODE_ENV;+ } else {+ process.env.NODE_ENV = nodeEnvironment;+ }+ }+};diff --git a/scripts/visual-regression/story-server.test.ts b/scripts/visual-regression/story-server.test.tsnew file mode 100644index 0000000000..3f8be79e8c--- /dev/null+++ b/scripts/visual-regression/story-server.test.ts@@ -0,0 +1,114 @@+import assert from "node:assert/strict";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { parse, type DefaultTreeAdapterTypes as Html } from "parse5";+import { buildVisualStoryApp } from "./story-build";+import {+ startVisualStoryServer,+ startVisualStoryServers,+} from "./story-server";++const findElement = (+ node: Html.Node,+ predicate: (element: Html.Element) => boolean+): Html.Element | undefined => {+ if ("tagName" in node && predicate(node)) {+ return node;+ }+ if ("childNodes" in node) {+ for (const child of node.childNodes) {+ const match = findElement(child, predicate);+ if (match !== undefined) {+ return match;+ }+ }+ }+};++const readVisualStoryDocument = async () =>+ parse(await readFile(new URL("./index.html", import.meta.url), "utf8"));++test("loads the visual harness from a proper HTML entry", async () => {+ const document = await readVisualStoryDocument();+ const root = findElement(document, (element) =>+ element.attrs.some(({ name, value }) => name === "id" && value === "root")+ );+ const script = findElement(+ document,+ (element) => element.tagName === "script"+ );+ assert.equal(root?.parentNode?.nodeName, "body");+ assert.ok(+ script?.attrs.some(+ ({ name, value }) => name === "src" && value === "/harness.tsx"+ )+ );+});++test("does not disable animations before reading the story parameters", async () => {+ const document = await readVisualStoryDocument();+ const style = findElement(document, (element) => element.tagName === "style");+ const css = style?.childNodes+ .filter((node): node is Html.TextNode => node.nodeName === "#text")+ .map((node) => node.value)+ .join("");++ assert.doesNotMatch(css ?? "", /animation-(?:delay|duration)|transition/);+});++test("does not leak Vite's production environment", async () => {+ const outputDirectory = await mkdtemp(path.join(tmpdir(), "visual-build-"));+ const nodeEnvironment = process.env.NODE_ENV;+ delete process.env.NODE_ENV;+ try {+ await buildVisualStoryApp({+ root: process.cwd(),+ outputDirectory,+ storyFiles: [+ "packages/design-system/src/components/color-picker.stories.tsx",+ ],+ });+ assert.equal(process.env.NODE_ENV, undefined);+ } finally {+ if (nodeEnvironment === undefined) {+ delete process.env.NODE_ENV;+ } else {+ process.env.NODE_ENV = nodeEnvironment;+ }+ await rm(outputDirectory, { recursive: true, force: true });+ }+});++test("closes servers that started before a concurrent startup failure", async () => {+ let releaseSuccessfulServer: (() => void) | undefined;+ const successfulServerReady = new Promise<void>((resolve) => {+ releaseSuccessfulServer = resolve;+ });+ let closed = false;+ const startServer = async (+ options: Parameters<typeof startVisualStoryServer>[0]+ ) => {+ if (options.port === 6102) {+ throw new Error("Current story server failed");+ }+ await successfulServerReady;+ return {+ async close() {+ closed = true;+ },+ };+ };+ const startup = startVisualStoryServers(+ [+ { root: "/repo", port: 6101, outputDirectory: "/one", storyFiles: [] },+ { root: "/repo", port: 6102, outputDirectory: "/two", storyFiles: [] },+ ],+ startServer+ );+ releaseSuccessfulServer?.();++ await assert.rejects(startup, /Current story server failed/);+ assert.equal(closed, true);+});diff --git a/scripts/visual-regression/story-server.ts b/scripts/visual-regression/story-server.tsnew file mode 100644index 0000000000..7751c840e3--- /dev/null+++ b/scripts/visual-regression/story-server.ts@@ -0,0 +1,47 @@+import { preview as startVitePreview } from "vite";+import { buildVisualStoryApp } from "./story-build";++export const startVisualStoryServer = async ({+ root,+ port,+ outputDirectory,+ storyFiles,+}: {+ root: string;+ port: number;+ outputDirectory: string;+ storyFiles: readonly string[];+}) => {+ await buildVisualStoryApp({ root, outputDirectory, storyFiles });+ const server = await startVitePreview({+ configFile: false,+ logLevel: "error",+ build: { outDir: outputDirectory },+ preview: {+ host: "127.0.0.1",+ port,+ strictPort: true,+ },+ });+ return {+ async close() {+ await server.close();+ },+ };+};++export const startVisualStoryServers = async (+ options: readonly Parameters<typeof startVisualStoryServer>[0][],+ startServer = startVisualStoryServer+) => {+ const results = await Promise.allSettled(options.map(startServer));+ const servers = results.flatMap((result) =>+ result.status === "fulfilled" ? [result.value] : []+ );+ const failure = results.find((result) => result.status === "rejected");+ if (failure !== undefined) {+ await Promise.allSettled(servers.map((server) => server.close()));+ throw failure.reason;+ }+ return servers;+};diff --git a/scripts/visual-regression/story-settings.test.ts b/scripts/visual-regression/story-settings.test.tsnew file mode 100644index 0000000000..3990a63a42--- /dev/null+++ b/scripts/visual-regression/story-settings.test.ts@@ -0,0 +1,22 @@+import assert from "node:assert/strict";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import test from "node:test";+import { hasPrivateStorySources } from "../../.storybook/story-settings";++test("detects private story sources relative to the repository root", async () => {+ const root = await mkdtemp(path.join(tmpdir(), "visual-private-sources-"));+ const privateSources = path.join(+ root,+ "packages/sdk-components-animation/private-src"+ );+ try {+ assert.equal(hasPrivateStorySources(root), false);+ await mkdir(privateSources, { recursive: true });+ await writeFile(path.join(privateSources, "README.md"), "private");+ assert.equal(hasPrivateStorySources(root), true);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});diff --git a/scripts/visual-regression/tsconfig.json b/scripts/visual-regression/tsconfig.jsonnew file mode 100644index 0000000000..0ea33f1129--- /dev/null+++ b/scripts/visual-regression/tsconfig.json@@ -0,0 +1,15 @@+{+ "compilerOptions": {+ "customConditions": ["webstudio"],+ "jsx": "react-jsx",+ "lib": ["ES2024", "DOM", "DOM.Iterable"],+ "module": "Preserve",+ "moduleResolution": "Bundler",+ "noEmit": true,+ "skipLibCheck": true,+ "strict": true,+ "target": "ES2024",+ "types": ["node"]+ },+ "include": ["*.ts", "*.tsx", "*.d.ts"]+}