Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

feat(e2e): visual snapshots — route×theme matrix in the pinned container

Sergey Kozyrenkodc66898bc8a4
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 6ac88aee..50b53461 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -73,6 +73,53 @@ jobs:
retention-days: 7
if-no-files-found: ignore
+ # Visual snapshots: pinned container = pixel-stable baselines. Advisory —
+ # never a required check.
+ e2e-visual:
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ container:
+ image: mcr.microsoft.com/playwright:v1.61.1-noble
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v6
+
+ - name: Verify container tag matches @playwright/test
+ run: |
+ node -e "
+ const v = require('./frontend/package.json').devDependencies['@playwright/test'];
+ if (v !== '1.61.1') {
+ console.error('::error::bump the e2e-visual container tag to v' + v + '-noble (and regenerate baselines)');
+ process.exit(1);
+ }
+ "
+
+ - name: Install pnpm
+ uses: pnpm/action-setup@v4
+ with:
+ package_json_file: frontend/package.json
+
+ - name: Install dependencies
+ working-directory: frontend
+ run: pnpm install --frozen-lockfile
+
+ - name: Build
+ working-directory: frontend
+ run: pnpm run build
+
+ - name: Run visual snapshots
+ working-directory: frontend
+ run: E2E_VISUAL=1 npx playwright test -c e2e/playwright.config.ts
+
+ - name: Upload diffs
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: e2e-visual-diffs
+ path: frontend/e2e/test-results
+ retention-days: 7
+ if-no-files-found: ignore
+
# Tier 2: branch-built image + isolated compose stack + the mock-LLM agent
# loop. Not part of the PR gate — nightly and on-demand only.
e2e-local:
diff --git a/frontend/docs/e2e.md b/frontend/docs/e2e.md
index e454c6db..71c39685 100644
--- a/frontend/docs/e2e.md
+++ b/frontend/docs/e2e.md
@@ -29,6 +29,9 @@ CI=1 pnpm e2e # byte-identical reproduction of a CI run
./e2e/tools/run-local-tier.sh # Tier 2: branch image + isolated docker
# stack + mock LLM; runs specs/real/**
E2E_TIER=stand E2E_BASE_URL=https://… pnpm e2e # against a live stand
+
+pnpm e2e:visual # visual snapshots (pinned container)
+pnpm e2e:visual:update # regenerate baselines after a UI change
```
Tips for the mock tier: `vite preview` is reused between runs if you keep it
@@ -49,6 +52,18 @@ docker daemon, and removes those sandboxes on exit. The deterministic agent
transcript lives in `e2e/mock-llm/scenario.mjs`. Iterating: `E2E_SKIP_BUILD=1`
reuses the image, `E2E_KEEP_STACK=1` leaves the stack up.
+## Visual snapshots
+
+Baselines live in the repo (`e2e/specs/visual/*-snapshots/`, linux-suffixed)
+and are generated ONLY inside the pinned `mcr.microsoft.com/playwright`
+container — never run the visual project on the host: macOS pixels produce
+parallel baselines that will never match CI. `pnpm e2e:visual` derives the
+image tag from the installed `@playwright/test` version, builds `dist` on the
+host, and compares inside the container; `pnpm e2e:visual:update` regenerates
+baselines (commit them with the UI change that caused the diff). The xterm
+canvas is masked — WebGL rendering is driver-dependent. The CI `e2e-visual`
+job is advisory and never a required check.
+
## Debugging a red CI run
1. Open the failed run (link in the PR comment) and download the `e2e-report`
diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.ts
index 5a4f6641..33cc05a8 100644
--- a/frontend/e2e/playwright.config.ts
+++ b/frontend/e2e/playwright.config.ts
@@ -7,6 +7,10 @@ export type BackendTier = 'local' | 'mock' | 'stand';
const tier = (process.env.E2E_TIER ?? 'mock') as BackendTier;
const isCI = Boolean(process.env.CI);
+// Visual snapshots only ever run inside the pinned Playwright Linux container
+// (e2e/tools/run-visual.sh) — a darwin run would generate parallel baselines
+// that never match CI pixels.
+const isVisual = process.env.E2E_VISUAL === '1';
// `vite preview` listens on VITE_PORT + 100 and reuses the dev proxy config.
const PREVIEW_PORT = 8100;
@@ -38,11 +42,17 @@ export default defineConfig<BackendOptions>({
projects:
tier === 'mock'
? [
- {
- name: 'mock-chromium',
- testIgnore: '**/specs/real/**',
- use: { ...devices['Desktop Chrome'] },
- },
+ isVisual
+ ? {
+ name: 'visual',
+ testMatch: '**/specs/visual/**',
+ use: { ...devices['Desktop Chrome'] },
+ }
+ : {
+ name: 'mock-chromium',
+ testIgnore: ['**/specs/real/**', '**/specs/visual/**'],
+ use: { ...devices['Desktop Chrome'] },
+ },
]
: [
{
@@ -83,7 +93,9 @@ export default defineConfig<BackendOptions>({
webServer:
tier === 'mock'
? {
- command: 'pnpm run build && pnpm exec vite preview',
+ // The visual container cannot load host-built native vite
+ // binaries — it serves a pre-built dist with plain Node.
+ command: isVisual ? 'node e2e/tools/serve-dist.mjs' : 'pnpm run build && pnpm exec vite preview',
cwd: fileURLToPath(new URL('..', import.meta.url)),
env: { VITE_PORT: '8000', VITE_USE_HTTPS: 'false' },
reuseExistingServer: !isCI,
diff --git a/frontend/e2e/specs/visual/routes.spec.ts b/frontend/e2e/specs/visual/routes.spec.ts
new file mode 100644
index 00000000..192f615f
--- /dev/null
+++ b/frontend/e2e/specs/visual/routes.spec.ts
@@ -0,0 +1,30 @@
+import { expect, test } from '../../fixtures/test.ts';
+import { ROUTE_MANIFEST } from '../../routes.ts';
+
+const THEMES = ['light', 'dark'] as const;
+
+const slugify = (path: string) => path.replaceAll('/', '-').replace(/^-/, '') || 'root';
+
+test.describe('visual', { tag: '@visual' }, () => {
+ for (const entry of ROUTE_MANIFEST) {
+ for (const theme of THEMES) {
+ test.describe(`${entry.path} ${theme}`, () => {
+ test.use({ cassette: entry.cassette() });
+
+ test('matches the baseline', async ({ page }) => {
+ await page.addInitScript((value) => window.localStorage.setItem('theme', String(value)), theme);
+ await page.goto(entry.path);
+ await expect(entry.ready(page)).toBeVisible();
+ await page.evaluate(() => document.fonts.ready);
+
+ await expect(page).toHaveScreenshot(`${slugify(entry.path)}-${theme}.png`, {
+ fullPage: true,
+ // The terminal is a WebGL canvas: SwiftShader pixels differ
+ // between runners, so its region is masked, not compared.
+ mask: [page.locator('.xterm')],
+ });
+ });
+ });
+ }
+ }
+});
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png
new file mode 100644
index 00000000..a28c93b6
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png
new file mode 100644
index 00000000..70137b7d
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png
new file mode 100644
index 00000000..ccbc0902
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png
new file mode 100644
index 00000000..76ac600a
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png
new file mode 100644
index 00000000..2018a49e
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png
new file mode 100644
index 00000000..5bf619f9
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png
new file mode 100644
index 00000000..c6787e6f
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png
new file mode 100644
index 00000000..67aa88c6
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png
new file mode 100644
index 00000000..b6bd0d85
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png
new file mode 100644
index 00000000..a7e8810a
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png
new file mode 100644
index 00000000..6fd962a1
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png
new file mode 100644
index 00000000..9cd1edef
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png
new file mode 100644
index 00000000..9ee1ebe0
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png
new file mode 100644
index 00000000..2ee8c294
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png
new file mode 100644
index 00000000..336e0601
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png
new file mode 100644
index 00000000..b873bbf9
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png
new file mode 100644
index 00000000..28c1779f
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png
new file mode 100644
index 00000000..d93d013c
Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png differ
diff --git a/frontend/e2e/tools/run-visual.sh b/frontend/e2e/tools/run-visual.sh
new file mode 100755
index 00000000..d0903c09
--- /dev/null
+++ b/frontend/e2e/tools/run-visual.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Visual snapshots run ONLY inside the pinned Playwright container so baselines
+# are pixel-stable across every machine and CI. The image tag is derived from
+# the installed @playwright/test version — the pin cannot drift by construction.
+# Usage: run-visual.sh [--update] (E2E_SKIP_BUILD=1 reuses the existing dist)
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+FRONTEND="$REPO_ROOT/frontend"
+
+PLAYWRIGHT_VERSION="$(node -p "require('$FRONTEND/node_modules/@playwright/test/package.json').version")"
+IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble"
+
+if [[ "${E2E_SKIP_BUILD:-}" != "1" ]]; then
+ (cd "$FRONTEND" && corepack pnpm run build)
+fi
+
+UPDATE_ARGS=()
+if [[ "${1:-}" == "--update" ]]; then
+ UPDATE_ARGS=(--update-snapshots)
+ shift
+fi
+
+docker run --rm \
+ -v "$REPO_ROOT:/work" \
+ -w /work/frontend \
+ -e E2E_VISUAL=1 \
+ -e CI="${CI:-}" \
+ --ipc=host \
+ "$IMAGE" \
+ npx playwright test -c e2e/playwright.config.ts ${UPDATE_ARGS[@]+"${UPDATE_ARGS[@]}"} "$@"
diff --git a/frontend/e2e/tools/serve-dist.mjs b/frontend/e2e/tools/serve-dist.mjs
new file mode 100644
index 00000000..03e57757
--- /dev/null
+++ b/frontend/e2e/tools/serve-dist.mjs
@@ -0,0 +1,36 @@
+// Static server for the built dist with an SPA fallback. The visual project
+// runs inside the pinned Playwright container where the host-built (darwin)
+// native vite binaries cannot load — this replaces `vite preview` with
+// dependency-free Node. API calls never reach it: Playwright route mocks
+// intercept them in the browser.
+import { createReadStream, existsSync, statSync } from 'node:fs';
+import { createServer } from 'node:http';
+import { extname, join, normalize } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const DIST = fileURLToPath(new URL('../../dist', import.meta.url));
+const PORT = Number(process.env.PORT ?? 8100);
+
+const MIME = {
+ '.css': 'text/css',
+ '.html': 'text/html',
+ '.ico': 'image/x-icon',
+ '.js': 'text/javascript',
+ '.json': 'application/json',
+ '.png': 'image/png',
+ '.svg': 'image/svg+xml',
+ '.webp': 'image/webp',
+ '.woff2': 'font/woff2',
+};
+
+createServer((request, response) => {
+ const pathname = normalize(new URL(request.url ?? '/', 'http://localhost').pathname).replace(/^(\.\.[/\\])+/, '');
+ let filePath = join(DIST, pathname);
+
+ if (!existsSync(filePath) || statSync(filePath).isDirectory()) {
+ filePath = join(DIST, 'index.html');
+ }
+
+ response.writeHead(200, { 'content-type': MIME[extname(filePath)] ?? 'application/octet-stream' });
+ createReadStream(filePath).pipe(response);
+}).listen(PORT, () => console.log(`[serve-dist] ${DIST} on :${PORT}`));
diff --git a/frontend/package.json b/frontend/package.json
index c905c7d2..a14d4ca5 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -8,6 +8,8 @@
"e2e": "playwright test -c e2e/playwright.config.ts",
"e2e:setup": "playwright install chromium",
"e2e:ui": "playwright test -c e2e/playwright.config.ts --ui",
+ "e2e:visual": "./e2e/tools/run-visual.sh",
+ "e2e:visual:update": "./e2e/tools/run-visual.sh --update",
"graphql:generate": "graphql-codegen --config graphql-codegen.ts",
"lint": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\"",
"lint:fix": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\" --fix",