Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
ci(e2e): tighten the visual gate, CI signal integrity, and container guards
Sergey Kozyrenko3400e1e7c1dc
diff --git a/.github/workflows/e2e-report.yml b/.github/workflows/e2e-report.ymlindex 61a3a05a..24aa7e79 100644--- a/.github/workflows/e2e-report.yml+++ b/.github/workflows/e2e-report.yml@@ -36,6 +36,18 @@ jobs:run-id: ${{ github.event.workflow_run.id }}github-token: ${{ github.token }}+ # Present only when the visual snapshot step itself failed (a real pixel diff),+ # so its download outcome is the precise advisory signal — see visualDiffers below.+ - name: Download visual diffs+ id: visualdiffs+ continue-on-error: true+ uses: actions/download-artifact@v4+ with:+ name: e2e-visual-diffs+ path: visual-diffs+ run-id: ${{ github.event.workflow_run.id }}+ github-token: ${{ github.token }}+- name: Post sticky commentuses: actions/github-script@v7with:@@ -59,6 +71,7 @@ jobs:}let runJobs = [];+ let jobsFetched = false;try {const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({owner: context.repo.owner,@@ -67,15 +80,16 @@ jobs:per_page: 100,});runJobs = jobs.jobs;+ jobsFetched = true;} catch (error) {core.info(`run jobs unavailable: ${error}`);}- // Adding a label skips every gate job, so this run has nothing to say about- // the PR — commenting would overwrite the previous run's verdict with "no- // results file".+ // Only skip when we actually fetched the jobs and this run has no completed mock+ // job (e.g. it failed before e2e-mock). A transient jobs-API failure must not be+ // mistaken for that and silently leave a stale comment; fall through to results.json.const mockJob = runJobs.find((job) => /mock/.test(job.name));- if (!mockJob || mockJob.conclusion === 'skipped') {+ if (jobsFetched && (!mockJob || mockJob.conclusion === 'skipped')) {core.info('No completed mock job in this run; leaving the sticky comment as it is.');return;}@@ -94,12 +108,13 @@ jobs:// Both halves are required: results.json is written before an abort can be// observed, so a run killed by the global timeout still reports zero failures.- const mockPassed = noFailures && mockJob.conclusion === 'success';+ // When the jobs API was unreachable, fall back to results.json alone.+ const mockPassed = noFailures && (jobsFetched ? mockJob?.conclusion === 'success' : true);- // The visual job is advisory (never a required check), so the header must- // track the mock gate, not the whole-run conclusion — otherwise a visual-only- // diff reads as a mock failure over a "0 failed" stats line.- const visualDiffers = runJobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');+ // Key the advisory on the e2e-visual-diffs artifact (uploaded only when the snapshot+ // step itself failed), not on the visual job's conclusion — a tag-guard, install or+ // build failure would otherwise read as "snapshots differ".+ const visualDiffers = '${{ steps.visualdiffs.outcome }}' === 'success';const conclusion = !mockPassed ? '❌' : visualDiffers ? '⚠️' : '✅';const marker = '<!-- e2e-report -->';diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex cdabde3d..e2d0f898 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -82,9 +82,7 @@ jobs:uses: actions/upload-artifact@v4with:name: e2e-report- path: |- frontend/e2e/blob-report- frontend/e2e/test-results+ path: frontend/e2e/test-resultsretention-days: 7if-no-files-found: ignore@@ -117,15 +115,17 @@ jobs:- name: Checkoutuses: actions/checkout@v6- - name: Verify container tag matches @playwright/test+ # Compare the container's own Playwright build to @playwright/test rather than a+ # hardcoded literal, so bumping the package without re-tagging the image (or vice+ # versa) fails here instead of drifting the pixel baselines.+ - name: Verify container image matches @playwright/testrun: |- node -e "- const v = require('./frontend/package.json').devDependencies['@playwright/test'];- if (v !== '1.61.1') {- console.error('::error::bump the e2e-visual container tag to v' + v + '-noble (and regenerate baselines)');- process.exit(1);- }- "+ pkg="$(node -p "require('./frontend/package.json').devDependencies['@playwright/test'].replace(/[^0-9.]/g, '')")"+ container="$(npx playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)"+ if [ "$pkg" != "$container" ]; then+ echo "::error::e2e-visual container ships Playwright $container but @playwright/test is $pkg — bump the container tag to v$pkg-noble and regenerate baselines"+ exit 1+ fi- name: Install pnpmuses: pnpm/action-setup@v4@@ -149,11 +149,14 @@ jobs:run: pnpm run build- name: Run visual snapshots+ id: snapshotsworking-directory: frontend- run: E2E_VISUAL=1 npx playwright test -c e2e/playwright.config.ts+ run: E2E_VISUAL=1 E2E_VISUAL_CONTAINER=1 npx playwright test -c e2e/playwright.config.ts+ # Only on a real snapshot diff (not a tag-guard/install/build failure), so its+ # presence is the precise "snapshots differ" signal the report keys the advisory on.- name: Upload diffs- if: failure()+ if: failure() && steps.snapshots.outcome == 'failure'uses: actions/upload-artifact@v4with:name: e2e-visual-diffsdiff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.tsindex d656fa9e..3d117d2a 100644--- a/frontend/e2e/playwright.config.ts+++ b/frontend/e2e/playwright.config.ts@@ -7,11 +7,14 @@ const rawTier = process.env.E2E_TIER ?? 'mock';const isCI = Boolean(process.env.CI);const isVisual = process.env.E2E_VISUAL === '1';-// Baselines are linux-suffixed, so a host run writes `*-visual-darwin.png` beside-// them: new files CI never reads, leaving it red with nothing to explain why.-if (isVisual && process.platform !== 'linux') {+// Baselines are pinned-container pixels. Gate on a marker the container run sets+// (run-visual.sh and the CI job), not on the host OS: a Linux workstation passes a+// bare platform check and an --update there would overwrite the baselines with+// host-font pixels the container then fails against.+if (isVisual && process.env.E2E_VISUAL_CONTAINER !== '1') {throw new Error(- `visual snapshots must run in the pinned container — use e2e/tools/run-visual.sh (got platform "${process.platform}")`,+ 'visual snapshots must run in the pinned container — use e2e/tools/run-visual.sh ' ++ `(E2E_VISUAL_CONTAINER unset, platform "${process.platform}")`,);}@@ -39,15 +42,21 @@ if (tier === 'stand' && !TIERS.stand.baseURL) {}export default defineConfig<BackendOptions>({- // Keep Playwright's default per-pixel tolerance. Strict comparison was tried and- // reverted: the pinned container fixes the renderer but not the host, and text- // rasterisation still differs by 300-400 pixels between a local run and CI —- // consistently, not as jitter. Absorbing that needs a budget far larger than the- // ~70 pixels a palette change moves, so pixels cannot police colour here.- // cross/contrast.spec.ts measures colour numerically instead.+ // A colour change must move enough pixels to fail. The baselines and the run both+ // render in the pinned container (the host path is blocked above), so there is no+ // host-vs-CI rasterisation noise to absorb — tighten the per-pixel threshold below+ // Playwright's default 0.2 so a one-step palette shift is caught, with a small+ // maxDiffPixelRatio for anti-aliasing at glyph edges. cross/contrast.spec.ts still+ // measures colour numerically as the belt-and-suspenders check.+ expect: {+ toHaveScreenshot: { maxDiffPixelRatio: 0.01, threshold: 0.02 },+ },forbidOnly: isCI,fullyParallel: true,- globalTimeout: isCI ? 10 * 60_000 : undefined,+ // Only the mock tier: the real tiers' serial worst case (auth.setup + two flow-run+ // attempts + the stand smokes) exceeds 10 min and would abort mid-retry, reporting+ // "timed out" instead of the real failure while the job still has budget.+ globalTimeout: isCI && tier === 'mock' ? 10 * 60_000 : undefined,outputDir: './test-results',projects:tier === 'mock'@@ -59,6 +68,9 @@ export default defineConfig<BackendOptions>({use: { ...devices['Desktop Chrome'] },}: {+ // @quarantine specs are documented as excluded from the gate; wire it here+ // (there is no CLI grep) so they cannot fail the retry:0 mock run.+ grepInvert: /@quarantine/,name: 'mock-chromium',testIgnore: ['**/specs/real/**', '**/specs/visual/**'],use: { ...devices['Desktop Chrome'] },@@ -81,14 +93,12 @@ export default defineConfig<BackendOptions>({},],reporter: isCI- ? [- ['blob', { outputDir: here('./blob-report') }],- ['json', { outputFile: here('./test-results/results.json') }],- ['github'],- ['list'],- ]+ ? [['json', { outputFile: here('./test-results/results.json') }], ['github'], ['list']]: [['html', { open: 'never', outputFolder: here('./playwright-report') }], ['list']],- retries: 1,+ // No retry on the hermetic mock tier: a pinned clock/locale/network with no real+ // backend makes a retry-recovered failure a real race, not infrastructure noise, and+ // "flaky" counts as pass. The real tiers keep one retry for genuine network flake.+ retries: tier === 'mock' ? 0 : 1,testDir: './specs',use: {backend: { installMocks: TIERS[tier].installMocks },diff --git a/frontend/e2e/tools/run-visual.sh b/frontend/e2e/tools/run-visual.shindex ed1aabca..b1cd5def 100755--- a/frontend/e2e/tools/run-visual.sh+++ b/frontend/e2e/tools/run-visual.sh@@ -30,6 +30,7 @@ docker run --rm \-v "$REPO_ROOT:/work" \-w /work/frontend \-e E2E_VISUAL=1 \+ -e E2E_VISUAL_CONTAINER=1 \-e CI="${CI:-}" \-e HOME=/tmp \--user "$(id -u):$(id -g)" \