Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

ci(e2e): stop the PR comment lying, and keep stand artifacts free of the stand

Sergey Kozyrenko291a0af0dc71
diff --git a/.github/workflows/e2e-report.yml b/.github/workflows/e2e-report.yml
index 23c3973a..61a3a05a 100644
--- a/.github/workflows/e2e-report.yml
+++ b/.github/workflows/e2e-report.yml
@@ -16,7 +16,7 @@ permissions:
# A cancelled superseded run must not race the report of the run that
# replaced it and become the sticky comment's last writer.
concurrency:
- group: e2e-report-${{ github.event.workflow_run.head_branch }}
+ group: e2e-report-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}
cancel-in-progress: true
jobs:
@@ -58,33 +58,48 @@ jobs:
return;
}
+ let runJobs = [];
+ try {
+ const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ run_id: run.id,
+ per_page: 100,
+ });
+ runJobs = jobs.jobs;
+ } catch (error) {
+ core.info(`run jobs unavailable: ${error}`);
+ }
+
+ // Adding a label skips every gate job, so this run has nothing to say about
+ // the PR — commenting would overwrite the previous run's verdict with "no
+ // results file".
+ const mockJob = runJobs.find((job) => /mock/.test(job.name));
+ if (!mockJob || mockJob.conclusion === 'skipped') {
+ core.info('No completed mock job in this run; leaving the sticky comment as it is.');
+ return;
+ }
+
let statsLine = 'The run produced no results file (it likely failed before the tests started).';
- let mockPassed = false;
+ let noFailures = false;
try {
const report = JSON.parse(fs.readFileSync('results/results.json', 'utf8'));
const n = (value) => Number(value) || 0;
const { expected, unexpected, flaky, skipped } = report.stats ?? {};
statsLine = `**${n(expected)} passed**, ${n(unexpected)} failed, ${n(flaky)} flaky, ${n(skipped)} skipped.`;
- mockPassed = n(unexpected) === 0;
+ noFailures = n(unexpected) === 0;
} catch (error) {
core.info(`results.json unavailable: ${error}`);
}
+ // Both halves are required: results.json is written before an abort can be
+ // observed, so a run killed by the global timeout still reports zero failures.
+ const mockPassed = noFailures && mockJob.conclusion === 'success';
+
// The visual job is advisory (never a required check), so the header must
// track the mock gate, not the whole-run conclusion — otherwise a visual-only
// diff reads as a mock failure over a "0 failed" stats line.
- let visualDiffers = false;
- try {
- const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
- owner: context.repo.owner,
- repo: context.repo.repo,
- run_id: run.id,
- per_page: 100,
- });
- visualDiffers = jobs.jobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');
- } catch (error) {
- core.info(`run jobs unavailable: ${error}`);
- }
+ const visualDiffers = runJobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');
const conclusion = !mockPassed ? '❌' : visualDiffers ? '⚠️' : '✅';
const marker = '<!-- e2e-report -->';
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index d55174cc..fff06ddb 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -31,10 +31,11 @@ permissions:
contents: read
# Keyed on the PR number (not the branch name — two fork PRs named identically
-# would cancel each other) and the event action (a `labeled` run must not
-# cancel an in-flight `synchronize` run of the same PR).
+# would cancel each other) and a labeled/not-labeled bucket: a `labeled` run must
+# not cancel an in-flight `synchronize` run, while successive pushes to the same
+# PR must still supersede each other.
concurrency:
- group: e2e-${{ github.event.action || 'run' }}-${{ github.event.pull_request.number || github.run_id }}
+ group: e2e-${{ github.event.action == 'labeled' && 'labeled' || 'run' }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
@@ -265,9 +266,9 @@ jobs:
E2E_USER: ${{ secrets.E2E_STAND_USER }}
E2E_PASSWORD: ${{ secrets.E2E_STAND_PASSWORD }}
- # Traces/videos are disabled on this tier (they would embed the live
- # session cookie in a public-repo artifact) — this uploads results.json
- # and failure screenshots only; never storageState.
+ # Traces, videos and screenshots are all disabled on this tier — they would
+ # embed the live session cookie, the stand URL or the login into a
+ # public-repo artifact. This uploads results.json only; never storageState.
- name: Upload report
if: always()
uses: actions/upload-artifact@v4
diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.ts
index 619049ae..9fd0eb25 100644
--- a/frontend/e2e/playwright.config.ts
+++ b/frontend/e2e/playwright.config.ts
@@ -105,7 +105,7 @@ export default defineConfig<BackendOptions>({
baseURL: TIERS[tier].baseURL,
ignoreHTTPSErrors: tier === 'local',
locale: 'en-US',
- screenshot: 'only-on-failure',
+ screenshot: tier === 'stand' ? 'off' : 'only-on-failure',
timezoneId: 'UTC',
// Stand traces/videos embed the live session cookie and the setup
// project's password fill; CI uploads test-results as a public-repo
diff --git a/frontend/e2e/tools/run-local-tier.sh b/frontend/e2e/tools/run-local-tier.sh
index a9724d05..bbda709e 100755
--- a/frontend/e2e/tools/run-local-tier.sh
+++ b/frontend/e2e/tools/run-local-tier.sh
@@ -35,8 +35,10 @@ cleanup() {
mkdir -p "$REPO_ROOT/frontend/e2e/test-results"
compose logs --no-color pentagi mock-llm \
> "$REPO_ROOT/frontend/e2e/test-results/compose.log" 2>&1 || true
- remove_e2e_sandboxes
- compose down -v --remove-orphans
+ # The trap is the script's last statement, so an unguarded teardown failure
+ # would become the exit status and turn a passing run red.
+ remove_e2e_sandboxes || true
+ compose down -v --remove-orphans || true
}
trap cleanup EXIT