Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

ci(e2e): tighten the visual gate, CI signal integrity, and container guards

Sergey Kozyrenko3400e1e7c1dc
diff --git a/.github/workflows/e2e-report.yml b/.github/workflows/e2e-report.yml
index 61a3a05a..24aa7e79 100644
--- a/.github/workflows/e2e-report.yml
+++ b/.github/workflows/e2e-report.yml
@@ -36,6 +36,18 @@ jobs:
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ github.token }}
+ # Present only when the visual snapshot step itself failed (a real pixel diff),
+ # so its download outcome is the precise advisory signal — see visualDiffers below.
+ - name: Download visual diffs
+ id: visualdiffs
+ continue-on-error: true
+ uses: actions/download-artifact@v4
+ with:
+ name: e2e-visual-diffs
+ path: visual-diffs
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ github.token }}
+
- name: Post sticky comment
uses: actions/github-script@v7
with:
@@ -59,6 +71,7 @@ jobs:
}
let runJobs = [];
+ let jobsFetched = false;
try {
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
@@ -67,15 +80,16 @@ jobs:
per_page: 100,
});
runJobs = jobs.jobs;
+ jobsFetched = true;
} catch (error) {
core.info(`run jobs unavailable: ${error}`);
}
- // Adding a label skips every gate job, so this run has nothing to say about
- // the PR — commenting would overwrite the previous run's verdict with "no
- // results file".
+ // Only skip when we actually fetched the jobs and this run has no completed mock
+ // job (e.g. it failed before e2e-mock). A transient jobs-API failure must not be
+ // mistaken for that and silently leave a stale comment; fall through to results.json.
const mockJob = runJobs.find((job) => /mock/.test(job.name));
- if (!mockJob || mockJob.conclusion === 'skipped') {
+ if (jobsFetched && (!mockJob || mockJob.conclusion === 'skipped')) {
core.info('No completed mock job in this run; leaving the sticky comment as it is.');
return;
}
@@ -94,12 +108,13 @@ jobs:
// Both halves are required: results.json is written before an abort can be
// observed, so a run killed by the global timeout still reports zero failures.
- const mockPassed = noFailures && mockJob.conclusion === 'success';
+ // When the jobs API was unreachable, fall back to results.json alone.
+ const mockPassed = noFailures && (jobsFetched ? mockJob?.conclusion === 'success' : true);
- // The visual job is advisory (never a required check), so the header must
- // track the mock gate, not the whole-run conclusion — otherwise a visual-only
- // diff reads as a mock failure over a "0 failed" stats line.
- const visualDiffers = runJobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');
+ // Key the advisory on the e2e-visual-diffs artifact (uploaded only when the snapshot
+ // step itself failed), not on the visual job's conclusion — a tag-guard, install or
+ // build failure would otherwise read as "snapshots differ".
+ const visualDiffers = '${{ steps.visualdiffs.outcome }}' === 'success';
const conclusion = !mockPassed ? '❌' : visualDiffers ? '⚠️' : '✅';
const marker = '<!-- e2e-report -->';
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index cdabde3d..e2d0f898 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -82,9 +82,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-report
- path: |
- frontend/e2e/blob-report
- frontend/e2e/test-results
+ path: frontend/e2e/test-results
retention-days: 7
if-no-files-found: ignore
@@ -117,15 +115,17 @@ jobs:
- name: Checkout
uses: actions/checkout@v6
- - name: Verify container tag matches @playwright/test
+ # Compare the container's own Playwright build to @playwright/test rather than a
+ # hardcoded literal, so bumping the package without re-tagging the image (or vice
+ # versa) fails here instead of drifting the pixel baselines.
+ - name: Verify container image matches @playwright/test
run: |
- node -e "
- const v = require('./frontend/package.json').devDependencies['@playwright/test'];
- if (v !== '1.61.1') {
- console.error('::error::bump the e2e-visual container tag to v' + v + '-noble (and regenerate baselines)');
- process.exit(1);
- }
- "
+ pkg="$(node -p "require('./frontend/package.json').devDependencies['@playwright/test'].replace(/[^0-9.]/g, '')")"
+ container="$(npx playwright --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)"
+ if [ "$pkg" != "$container" ]; then
+ echo "::error::e2e-visual container ships Playwright $container but @playwright/test is $pkg — bump the container tag to v$pkg-noble and regenerate baselines"
+ exit 1
+ fi
- name: Install pnpm
uses: pnpm/action-setup@v4
@@ -149,11 +149,14 @@ jobs:
run: pnpm run build
- name: Run visual snapshots
+ id: snapshots
working-directory: frontend
- run: E2E_VISUAL=1 npx playwright test -c e2e/playwright.config.ts
+ run: E2E_VISUAL=1 E2E_VISUAL_CONTAINER=1 npx playwright test -c e2e/playwright.config.ts
+ # Only on a real snapshot diff (not a tag-guard/install/build failure), so its
+ # presence is the precise "snapshots differ" signal the report keys the advisory on.
- name: Upload diffs
- if: failure()
+ if: failure() && steps.snapshots.outcome == 'failure'
uses: actions/upload-artifact@v4
with:
name: e2e-visual-diffs
diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.ts
index d656fa9e..3d117d2a 100644
--- a/frontend/e2e/playwright.config.ts
+++ b/frontend/e2e/playwright.config.ts
@@ -7,11 +7,14 @@ const rawTier = process.env.E2E_TIER ?? 'mock';
const isCI = Boolean(process.env.CI);
const isVisual = process.env.E2E_VISUAL === '1';
-// Baselines are linux-suffixed, so a host run writes `*-visual-darwin.png` beside
-// them: new files CI never reads, leaving it red with nothing to explain why.
-if (isVisual && process.platform !== 'linux') {
+// Baselines are pinned-container pixels. Gate on a marker the container run sets
+// (run-visual.sh and the CI job), not on the host OS: a Linux workstation passes a
+// bare platform check and an --update there would overwrite the baselines with
+// host-font pixels the container then fails against.
+if (isVisual && process.env.E2E_VISUAL_CONTAINER !== '1') {
throw new Error(
- `visual snapshots must run in the pinned container — use e2e/tools/run-visual.sh (got platform "${process.platform}")`,
+ 'visual snapshots must run in the pinned container — use e2e/tools/run-visual.sh ' +
+ `(E2E_VISUAL_CONTAINER unset, platform "${process.platform}")`,
);
}
@@ -39,15 +42,21 @@ if (tier === 'stand' && !TIERS.stand.baseURL) {
}
export default defineConfig<BackendOptions>({
- // Keep Playwright's default per-pixel tolerance. Strict comparison was tried and
- // reverted: the pinned container fixes the renderer but not the host, and text
- // rasterisation still differs by 300-400 pixels between a local run and CI —
- // consistently, not as jitter. Absorbing that needs a budget far larger than the
- // ~70 pixels a palette change moves, so pixels cannot police colour here.
- // cross/contrast.spec.ts measures colour numerically instead.
+ // A colour change must move enough pixels to fail. The baselines and the run both
+ // render in the pinned container (the host path is blocked above), so there is no
+ // host-vs-CI rasterisation noise to absorb — tighten the per-pixel threshold below
+ // Playwright's default 0.2 so a one-step palette shift is caught, with a small
+ // maxDiffPixelRatio for anti-aliasing at glyph edges. cross/contrast.spec.ts still
+ // measures colour numerically as the belt-and-suspenders check.
+ expect: {
+ toHaveScreenshot: { maxDiffPixelRatio: 0.01, threshold: 0.02 },
+ },
forbidOnly: isCI,
fullyParallel: true,
- globalTimeout: isCI ? 10 * 60_000 : undefined,
+ // Only the mock tier: the real tiers' serial worst case (auth.setup + two flow-run
+ // attempts + the stand smokes) exceeds 10 min and would abort mid-retry, reporting
+ // "timed out" instead of the real failure while the job still has budget.
+ globalTimeout: isCI && tier === 'mock' ? 10 * 60_000 : undefined,
outputDir: './test-results',
projects:
tier === 'mock'
@@ -59,6 +68,9 @@ export default defineConfig<BackendOptions>({
use: { ...devices['Desktop Chrome'] },
}
: {
+ // @quarantine specs are documented as excluded from the gate; wire it here
+ // (there is no CLI grep) so they cannot fail the retry:0 mock run.
+ grepInvert: /@quarantine/,
name: 'mock-chromium',
testIgnore: ['**/specs/real/**', '**/specs/visual/**'],
use: { ...devices['Desktop Chrome'] },
@@ -81,14 +93,12 @@ export default defineConfig<BackendOptions>({
},
],
reporter: isCI
- ? [
- ['blob', { outputDir: here('./blob-report') }],
- ['json', { outputFile: here('./test-results/results.json') }],
- ['github'],
- ['list'],
- ]
+ ? [['json', { outputFile: here('./test-results/results.json') }], ['github'], ['list']]
: [['html', { open: 'never', outputFolder: here('./playwright-report') }], ['list']],
- retries: 1,
+ // No retry on the hermetic mock tier: a pinned clock/locale/network with no real
+ // backend makes a retry-recovered failure a real race, not infrastructure noise, and
+ // "flaky" counts as pass. The real tiers keep one retry for genuine network flake.
+ retries: tier === 'mock' ? 0 : 1,
testDir: './specs',
use: {
backend: { installMocks: TIERS[tier].installMocks },
diff --git a/frontend/e2e/tools/run-visual.sh b/frontend/e2e/tools/run-visual.sh
index ed1aabca..b1cd5def 100755
--- a/frontend/e2e/tools/run-visual.sh
+++ b/frontend/e2e/tools/run-visual.sh
@@ -30,6 +30,7 @@ docker run --rm \
-v "$REPO_ROOT:/work" \
-w /work/frontend \
-e E2E_VISUAL=1 \
+ -e E2E_VISUAL_CONTAINER=1 \
-e CI="${CI:-}" \
-e HOME=/tmp \
--user "$(id -u):$(id -g)" \