Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
ci(e2e): stop the PR comment lying, and keep stand artifacts free of the stand
Sergey Kozyrenko291a0af0dc71
diff --git a/.github/workflows/e2e-report.yml b/.github/workflows/e2e-report.ymlindex 23c3973a..61a3a05a 100644--- a/.github/workflows/e2e-report.yml+++ b/.github/workflows/e2e-report.yml@@ -16,7 +16,7 @@ permissions:# A cancelled superseded run must not race the report of the run that# replaced it and become the sticky comment's last writer.concurrency:- group: e2e-report-${{ github.event.workflow_run.head_branch }}+ group: e2e-report-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}cancel-in-progress: truejobs:@@ -58,33 +58,48 @@ jobs:return;}+ let runJobs = [];+ try {+ const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({+ owner: context.repo.owner,+ repo: context.repo.repo,+ run_id: run.id,+ per_page: 100,+ });+ runJobs = jobs.jobs;+ } catch (error) {+ core.info(`run jobs unavailable: ${error}`);+ }++ // Adding a label skips every gate job, so this run has nothing to say about+ // the PR — commenting would overwrite the previous run's verdict with "no+ // results file".+ const mockJob = runJobs.find((job) => /mock/.test(job.name));+ if (!mockJob || mockJob.conclusion === 'skipped') {+ core.info('No completed mock job in this run; leaving the sticky comment as it is.');+ return;+ }+let statsLine = 'The run produced no results file (it likely failed before the tests started).';- let mockPassed = false;+ let noFailures = false;try {const report = JSON.parse(fs.readFileSync('results/results.json', 'utf8'));const n = (value) => Number(value) || 0;const { expected, unexpected, flaky, skipped } = report.stats ?? {};statsLine = `**${n(expected)} passed**, ${n(unexpected)} failed, ${n(flaky)} flaky, ${n(skipped)} skipped.`;- mockPassed = n(unexpected) === 0;+ noFailures = n(unexpected) === 0;} catch (error) {core.info(`results.json unavailable: ${error}`);}+ // Both halves are required: results.json is written before an abort can be+ // observed, so a run killed by the global timeout still reports zero failures.+ const mockPassed = noFailures && mockJob.conclusion === 'success';+// The visual job is advisory (never a required check), so the header must// track the mock gate, not the whole-run conclusion — otherwise a visual-only// diff reads as a mock failure over a "0 failed" stats line.- let visualDiffers = false;- try {- const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({- owner: context.repo.owner,- repo: context.repo.repo,- run_id: run.id,- per_page: 100,- });- visualDiffers = jobs.jobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');- } catch (error) {- core.info(`run jobs unavailable: ${error}`);- }+ const visualDiffers = runJobs.some((job) => /visual/.test(job.name) && job.conclusion === 'failure');const conclusion = !mockPassed ? '❌' : visualDiffers ? '⚠️' : '✅';const marker = '<!-- e2e-report -->';diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex d55174cc..fff06ddb 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -31,10 +31,11 @@ permissions:contents: read# Keyed on the PR number (not the branch name — two fork PRs named identically-# would cancel each other) and the event action (a `labeled` run must not-# cancel an in-flight `synchronize` run of the same PR).+# would cancel each other) and a labeled/not-labeled bucket: a `labeled` run must+# not cancel an in-flight `synchronize` run, while successive pushes to the same+# PR must still supersede each other.concurrency:- group: e2e-${{ github.event.action || 'run' }}-${{ github.event.pull_request.number || github.run_id }}+ group: e2e-${{ github.event.action == 'labeled' && 'labeled' || 'run' }}-${{ github.event.pull_request.number || github.run_id }}cancel-in-progress: ${{ github.event_name == 'pull_request' }}jobs:@@ -265,9 +266,9 @@ jobs:E2E_USER: ${{ secrets.E2E_STAND_USER }}E2E_PASSWORD: ${{ secrets.E2E_STAND_PASSWORD }}- # Traces/videos are disabled on this tier (they would embed the live- # session cookie in a public-repo artifact) — this uploads results.json- # and failure screenshots only; never storageState.+ # Traces, videos and screenshots are all disabled on this tier — they would+ # embed the live session cookie, the stand URL or the login into a+ # public-repo artifact. This uploads results.json only; never storageState.- name: Upload reportif: always()uses: actions/upload-artifact@v4diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.tsindex 619049ae..9fd0eb25 100644--- a/frontend/e2e/playwright.config.ts+++ b/frontend/e2e/playwright.config.ts@@ -105,7 +105,7 @@ export default defineConfig<BackendOptions>({baseURL: TIERS[tier].baseURL,ignoreHTTPSErrors: tier === 'local',locale: 'en-US',- screenshot: 'only-on-failure',+ screenshot: tier === 'stand' ? 'off' : 'only-on-failure',timezoneId: 'UTC',// Stand traces/videos embed the live session cookie and the setup// project's password fill; CI uploads test-results as a public-repodiff --git a/frontend/e2e/tools/run-local-tier.sh b/frontend/e2e/tools/run-local-tier.shindex a9724d05..bbda709e 100755--- a/frontend/e2e/tools/run-local-tier.sh+++ b/frontend/e2e/tools/run-local-tier.sh@@ -35,8 +35,10 @@ cleanup() {mkdir -p "$REPO_ROOT/frontend/e2e/test-results"compose logs --no-color pentagi mock-llm \> "$REPO_ROOT/frontend/e2e/test-results/compose.log" 2>&1 || true- remove_e2e_sandboxes- compose down -v --remove-orphans+ # The trap is the script's last statement, so an unguarded teardown failure+ # would become the exit status and turn a passing run red.+ remove_e2e_sandboxes || true+ compose down -v --remove-orphans || true}trap cleanup EXIT