Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
feat(e2e): visual snapshots — route×theme matrix in the pinned container
Sergey Kozyrenkodc66898bc8a4
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex 6ac88aee..50b53461 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -73,6 +73,53 @@ jobs:retention-days: 7if-no-files-found: ignore+ # Visual snapshots: pinned container = pixel-stable baselines. Advisory —+ # never a required check.+ e2e-visual:+ runs-on: ubuntu-latest+ timeout-minutes: 20+ container:+ image: mcr.microsoft.com/playwright:v1.61.1-noble+ steps:+ - name: Checkout+ uses: actions/checkout@v6++ - name: Verify container tag matches @playwright/test+ run: |+ node -e "+ const v = require('./frontend/package.json').devDependencies['@playwright/test'];+ if (v !== '1.61.1') {+ console.error('::error::bump the e2e-visual container tag to v' + v + '-noble (and regenerate baselines)');+ process.exit(1);+ }+ "++ - name: Install pnpm+ uses: pnpm/action-setup@v4+ with:+ package_json_file: frontend/package.json++ - name: Install dependencies+ working-directory: frontend+ run: pnpm install --frozen-lockfile++ - name: Build+ working-directory: frontend+ run: pnpm run build++ - name: Run visual snapshots+ working-directory: frontend+ run: E2E_VISUAL=1 npx playwright test -c e2e/playwright.config.ts++ - name: Upload diffs+ if: failure()+ uses: actions/upload-artifact@v4+ with:+ name: e2e-visual-diffs+ path: frontend/e2e/test-results+ retention-days: 7+ if-no-files-found: ignore+# Tier 2: branch-built image + isolated compose stack + the mock-LLM agent# loop. Not part of the PR gate — nightly and on-demand only.e2e-local:diff --git a/frontend/docs/e2e.md b/frontend/docs/e2e.mdindex e454c6db..71c39685 100644--- a/frontend/docs/e2e.md+++ b/frontend/docs/e2e.md@@ -29,6 +29,9 @@ CI=1 pnpm e2e # byte-identical reproduction of a CI run./e2e/tools/run-local-tier.sh # Tier 2: branch image + isolated docker# stack + mock LLM; runs specs/real/**E2E_TIER=stand E2E_BASE_URL=https://… pnpm e2e # against a live stand++pnpm e2e:visual # visual snapshots (pinned container)+pnpm e2e:visual:update # regenerate baselines after a UI change```Tips for the mock tier: `vite preview` is reused between runs if you keep it@@ -49,6 +52,18 @@ docker daemon, and removes those sandboxes on exit. The deterministic agenttranscript lives in `e2e/mock-llm/scenario.mjs`. Iterating: `E2E_SKIP_BUILD=1`reuses the image, `E2E_KEEP_STACK=1` leaves the stack up.+## Visual snapshots++Baselines live in the repo (`e2e/specs/visual/*-snapshots/`, linux-suffixed)+and are generated ONLY inside the pinned `mcr.microsoft.com/playwright`+container — never run the visual project on the host: macOS pixels produce+parallel baselines that will never match CI. `pnpm e2e:visual` derives the+image tag from the installed `@playwright/test` version, builds `dist` on the+host, and compares inside the container; `pnpm e2e:visual:update` regenerates+baselines (commit them with the UI change that caused the diff). The xterm+canvas is masked — WebGL rendering is driver-dependent. The CI `e2e-visual`+job is advisory and never a required check.+## Debugging a red CI run1. Open the failed run (link in the PR comment) and download the `e2e-report`diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.tsindex 5a4f6641..33cc05a8 100644--- a/frontend/e2e/playwright.config.ts+++ b/frontend/e2e/playwright.config.ts@@ -7,6 +7,10 @@ export type BackendTier = 'local' | 'mock' | 'stand';const tier = (process.env.E2E_TIER ?? 'mock') as BackendTier;const isCI = Boolean(process.env.CI);+// Visual snapshots only ever run inside the pinned Playwright Linux container+// (e2e/tools/run-visual.sh) — a darwin run would generate parallel baselines+// that never match CI pixels.+const isVisual = process.env.E2E_VISUAL === '1';// `vite preview` listens on VITE_PORT + 100 and reuses the dev proxy config.const PREVIEW_PORT = 8100;@@ -38,11 +42,17 @@ export default defineConfig<BackendOptions>({projects:tier === 'mock'? [- {- name: 'mock-chromium',- testIgnore: '**/specs/real/**',- use: { ...devices['Desktop Chrome'] },- },+ isVisual+ ? {+ name: 'visual',+ testMatch: '**/specs/visual/**',+ use: { ...devices['Desktop Chrome'] },+ }+ : {+ name: 'mock-chromium',+ testIgnore: ['**/specs/real/**', '**/specs/visual/**'],+ use: { ...devices['Desktop Chrome'] },+ },]: [{@@ -83,7 +93,9 @@ export default defineConfig<BackendOptions>({webServer:tier === 'mock'? {- command: 'pnpm run build && pnpm exec vite preview',+ // The visual container cannot load host-built native vite+ // binaries — it serves a pre-built dist with plain Node.+ command: isVisual ? 'node e2e/tools/serve-dist.mjs' : 'pnpm run build && pnpm exec vite preview',cwd: fileURLToPath(new URL('..', import.meta.url)),env: { VITE_PORT: '8000', VITE_USE_HTTPS: 'false' },reuseExistingServer: !isCI,diff --git a/frontend/e2e/specs/visual/routes.spec.ts b/frontend/e2e/specs/visual/routes.spec.tsnew file mode 100644index 00000000..192f615f--- /dev/null+++ b/frontend/e2e/specs/visual/routes.spec.ts@@ -0,0 +1,30 @@+import { expect, test } from '../../fixtures/test.ts';+import { ROUTE_MANIFEST } from '../../routes.ts';++const THEMES = ['light', 'dark'] as const;++const slugify = (path: string) => path.replaceAll('/', '-').replace(/^-/, '') || 'root';++test.describe('visual', { tag: '@visual' }, () => {+ for (const entry of ROUTE_MANIFEST) {+ for (const theme of THEMES) {+ test.describe(`${entry.path} ${theme}`, () => {+ test.use({ cassette: entry.cassette() });++ test('matches the baseline', async ({ page }) => {+ await page.addInitScript((value) => window.localStorage.setItem('theme', String(value)), theme);+ await page.goto(entry.path);+ await expect(entry.ready(page)).toBeVisible();+ await page.evaluate(() => document.fonts.ready);++ await expect(page).toHaveScreenshot(`${slugify(entry.path)}-${theme}.png`, {+ fullPage: true,+ // The terminal is a WebGL canvas: SwiftShader pixels differ+ // between runners, so its region is masked, not compared.+ mask: [page.locator('.xterm')],+ });+ });+ });+ }+ }+});diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.pngnew file mode 100644index 00000000..a28c93b6Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.pngnew file mode 100644index 00000000..70137b7dBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.pngnew file mode 100644index 00000000..ccbc0902Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.pngnew file mode 100644index 00000000..76ac600aBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.pngnew file mode 100644index 00000000..2018a49eBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.pngnew file mode 100644index 00000000..5bf619f9Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.pngnew file mode 100644index 00000000..c6787e6fBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.pngnew file mode 100644index 00000000..67aa88c6Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.pngnew file mode 100644index 00000000..b6bd0d85Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.pngnew file mode 100644index 00000000..a7e8810aBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.pngnew file mode 100644index 00000000..6fd962a1Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.pngnew file mode 100644index 00000000..9cd1edefBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.pngnew file mode 100644index 00000000..9ee1ebe0Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.pngnew file mode 100644index 00000000..2ee8c294Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.pngnew file mode 100644index 00000000..336e0601Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.pngnew file mode 100644index 00000000..b873bbf9Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.pngnew file mode 100644index 00000000..28c1779fBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png differdiff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.pngnew file mode 100644index 00000000..d93d013cBinary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png differdiff --git a/frontend/e2e/tools/run-visual.sh b/frontend/e2e/tools/run-visual.shnew file mode 100755index 00000000..d0903c09--- /dev/null+++ b/frontend/e2e/tools/run-visual.sh@@ -0,0 +1,31 @@+#!/usr/bin/env bash+# Visual snapshots run ONLY inside the pinned Playwright container so baselines+# are pixel-stable across every machine and CI. The image tag is derived from+# the installed @playwright/test version — the pin cannot drift by construction.+# Usage: run-visual.sh [--update] (E2E_SKIP_BUILD=1 reuses the existing dist)+set -euo pipefail++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"+FRONTEND="$REPO_ROOT/frontend"++PLAYWRIGHT_VERSION="$(node -p "require('$FRONTEND/node_modules/@playwright/test/package.json').version")"+IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble"++if [[ "${E2E_SKIP_BUILD:-}" != "1" ]]; then+ (cd "$FRONTEND" && corepack pnpm run build)+fi++UPDATE_ARGS=()+if [[ "${1:-}" == "--update" ]]; then+ UPDATE_ARGS=(--update-snapshots)+ shift+fi++docker run --rm \+ -v "$REPO_ROOT:/work" \+ -w /work/frontend \+ -e E2E_VISUAL=1 \+ -e CI="${CI:-}" \+ --ipc=host \+ "$IMAGE" \+ npx playwright test -c e2e/playwright.config.ts ${UPDATE_ARGS[@]+"${UPDATE_ARGS[@]}"} "$@"diff --git a/frontend/e2e/tools/serve-dist.mjs b/frontend/e2e/tools/serve-dist.mjsnew file mode 100644index 00000000..03e57757--- /dev/null+++ b/frontend/e2e/tools/serve-dist.mjs@@ -0,0 +1,36 @@+// Static server for the built dist with an SPA fallback. The visual project+// runs inside the pinned Playwright container where the host-built (darwin)+// native vite binaries cannot load — this replaces `vite preview` with+// dependency-free Node. API calls never reach it: Playwright route mocks+// intercept them in the browser.+import { createReadStream, existsSync, statSync } from 'node:fs';+import { createServer } from 'node:http';+import { extname, join, normalize } from 'node:path';+import { fileURLToPath } from 'node:url';++const DIST = fileURLToPath(new URL('../../dist', import.meta.url));+const PORT = Number(process.env.PORT ?? 8100);++const MIME = {+ '.css': 'text/css',+ '.html': 'text/html',+ '.ico': 'image/x-icon',+ '.js': 'text/javascript',+ '.json': 'application/json',+ '.png': 'image/png',+ '.svg': 'image/svg+xml',+ '.webp': 'image/webp',+ '.woff2': 'font/woff2',+};++createServer((request, response) => {+ const pathname = normalize(new URL(request.url ?? '/', 'http://localhost').pathname).replace(/^(\.\.[/\\])+/, '');+ let filePath = join(DIST, pathname);++ if (!existsSync(filePath) || statSync(filePath).isDirectory()) {+ filePath = join(DIST, 'index.html');+ }++ response.writeHead(200, { 'content-type': MIME[extname(filePath)] ?? 'application/octet-stream' });+ createReadStream(filePath).pipe(response);+}).listen(PORT, () => console.log(`[serve-dist] ${DIST} on :${PORT}`));diff --git a/frontend/package.json b/frontend/package.jsonindex c905c7d2..a14d4ca5 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -8,6 +8,8 @@"e2e": "playwright test -c e2e/playwright.config.ts","e2e:setup": "playwright install chromium","e2e:ui": "playwright test -c e2e/playwright.config.ts --ui",+ "e2e:visual": "./e2e/tools/run-visual.sh",+ "e2e:visual:update": "./e2e/tools/run-visual.sh --update","graphql:generate": "graphql-codegen --config graphql-codegen.ts","lint": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\"","lint:fix": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\" --fix",