High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat(ci): run the extended suites on every merge to main and smoke nightly
Luigi Agosti2519c4e659cb
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.ymlindex 1b746794..a41dfffc 100644--- a/.github/workflows/ci-e2e.yml+++ b/.github/workflows/ci-e2e.yml@@ -13,6 +13,15 @@ on:- '.gitignore'- 'skills/**'- 'plugin/**'+ push:+ branches: [main]+ paths-ignore:+ - '**.md'+ - 'docs/**'+ - 'LICENSE'+ - '.gitignore'+ - 'skills/**'+ - 'plugin/**'workflow_dispatch:inputs:suite:@@ -49,11 +58,9 @@ jobs:./scripts/ci/detect-e2e-suites.sh >> "$GITHUB_OUTPUT"api:- needs: [detect-changes]if: >-- always() &&- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'api')+ github.event_name == 'pull_request' ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api'))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -61,22 +68,10 @@ jobs:summary_title: E2E API Teststimeout_minutes: 15- api-extended-pr:- needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true'- uses: ./.github/workflows/reusable-e2e.yml- with:- ref: ${{ github.sha }}- suite: api-extended- summary_title: E2E API Extended Tests (PR)- timeout_minutes: 20-cli:- needs: [detect-changes]if: >-- always() &&- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'cli')+ github.event_name == 'pull_request' ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli'))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -84,22 +79,10 @@ jobs:summary_title: E2E CLI Teststimeout_minutes: 15- cli-extended-pr:- needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true'- uses: ./.github/workflows/reusable-e2e.yml- with:- ref: ${{ github.sha }}- suite: cli-extended- summary_title: E2E CLI Extended Tests (PR)- timeout_minutes: 20-infra:- needs: [detect-changes]if: >-- always() &&- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'infra')+ github.event_name == 'pull_request' ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra'))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -107,26 +90,14 @@ jobs:summary_title: E2E Infra Teststimeout_minutes: 15- infra-extended-pr:- needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true'- uses: ./.github/workflows/reusable-e2e.yml- with:- ref: ${{ github.sha }}- suite: infra-extended- summary_title: E2E Infra Extended Tests (PR)- timeout_minutes: 25-- smoke-pr:- needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true'- uses: ./.github/workflows/reusable-smoke.yml- with:- ref: ${{ github.sha }}-- # Extended suites for manual dispatchapi-extended:- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api')+ needs: [detect-changes]+ if: >-+ always() &&+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&+ (github.event_name == 'push' ||+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true') ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api')))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -135,7 +106,13 @@ jobs:timeout_minutes: 20cli-extended:- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli')+ needs: [detect-changes]+ if: >-+ always() &&+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&+ (github.event_name == 'push' ||+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true') ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli')))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -144,7 +121,13 @@ jobs:timeout_minutes: 20infra-extended:- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra')+ needs: [detect-changes]+ if: >-+ always() &&+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&+ (github.event_name == 'push' ||+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true') ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra')))uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -153,7 +136,12 @@ jobs:timeout_minutes: 25smoke:- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'smoke')+ needs: [detect-changes]+ if: >-+ always() &&+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&+ ((github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true') ||+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'smoke')))uses: ./.github/workflows/reusable-smoke.ymlwith:ref: ${{ github.sha }}diff --git a/.github/workflows/ci-smoke.yml b/.github/workflows/ci-smoke.ymlindex ca64e3b8..781ecdcf 100644--- a/.github/workflows/ci-smoke.yml+++ b/.github/workflows/ci-smoke.yml@@ -1,6 +1,8 @@name: CI / Smokeon:+ schedule:+ - cron: '0 3 * * *'workflow_dispatch:permissions:diff --git a/TESTING.md b/TESTING.mdindex 8e823834..3f3eb90c 100644--- a/TESTING.md+++ b/TESTING.md@@ -79,12 +79,27 @@ The map reads product paths as well as test paths:| audit implementation — `internal/audit/`, `pkg/pinchtabaudit/`, the audit handlers, `internal/cli/actions/actions_audit*`, `cmd/pinchtab/cmd_audit*` | `api-extended`, `cli-extended` || compare implementation — `internal/cli/actions/actions_compare*`, `cmd/pinchtab/cmd_compare*` (`internal/audit/compare.go` is already audit) | `cli-extended` |-Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim still runs on `workflow_dispatch` only.+Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim is not lost — it runs after merge, on the lane below.Audit and compare are also declared there as *enrolled areas*: every tracked source file matching an area must escalate that area's suites, with a short exclusion list for files that only share the word (`internal/authn/audit.go` is security audit logging). So a new file in an enrolled family cannot quietly miss the map — which is how the CLI audit implementation went unenrolled while its cobra wrapper was covered. Enrolling another area is one entry in `enrolledAreas`.The multi-page audit runs — `api/audit-extended.sh`, `cli/audit-cli-extended.sh`, `cli/audit-seaportal-extended.sh`, `cli/compare-extended.sh` — sit in the extended tier rather than the basic one, so a pull request pays for them only when it touches audit or compare. The PR path keeps the single-page audit signal: `api/audit-page-basic.sh`, `api/a11y-audit-basic.sh`, `api/audit-fixtures-basic.sh`, `cli/audit-auth-basic.sh` and `cli/audit-report-basic.sh`.+### What a merge and the nightly run++Every push to `main` runs the three extended suites without consulting the map, so coverage a pull request skipped is exercised at merge rather than never. Extended suites include the matching `*-basic.sh` scenarios, so the merge lane does not repeat the basic suites.++Smoke is the expensive lane — it builds images — so it runs nightly on the `schedule` trigger in `.github/workflows/ci-smoke.yml` rather than per merge, on top of the pull requests whose diff the map escalates to `smoke`.++| Lane | Trigger | Suites |+| --- | --- | --- |+| pull request | `pull_request` on `main` | the three basic suites, plus whatever the escalation map claims |+| merge | `push` on `main` | `api-extended`, `cli-extended`, `infra-extended` |+| nightly | `schedule` in `ci-smoke.yml` | `smoke` |+| manual | `workflow_dispatch` | whichever suite is chosen |++`go test ./internal/devtools/` reads both workflows and fails if a suite the map can escalate has no automatic trigger, so a suite cannot quietly fall back to `workflow_dispatch` only.+### Basic Suites```bashdiff --git a/internal/devtools/e2e_escalation_test.go b/internal/devtools/e2e_escalation_test.goindex 92b0eb84..df7dac8e 100644--- a/internal/devtools/e2e_escalation_test.go+++ b/internal/devtools/e2e_escalation_test.go@@ -9,12 +9,15 @@ import ("sort""strings""testing"++ "gopkg.in/yaml.v3")const (escalationScript = "scripts/ci/detect-e2e-suites.sh"escalationMap = "scripts/ci/e2e-escalation.map"escalationWorkflow = ".github/workflows/ci-e2e.yml"+ smokeWorkflow = ".github/workflows/ci-smoke.yml")func TestEscalationFollowsProductPathsAndTestPaths(t *testing.T) {@@ -144,6 +147,84 @@ func TestTheWorkflowConsumesExactlyTheEmittedOutputs(t *testing.T) {}}+func TestEveryEscalatableSuiteAlsoRunsWithoutADiffToClaimIt(t *testing.T) {+ root := repoRoot(t)+ automatic := suitesOnAutomaticTriggers(t, root)++ for _, name := range detectSuiteNames(t, root) {+ suite := strings.ReplaceAll(strings.TrimPrefix(name, "run_"), "_", "-")+ if !automatic[suite] {+ t.Errorf("the %s suite runs only when a diff escalates it or a human dispatches it, so a merge the map does not claim never exercises the coverage it holds; give it a job gated on github.event_name == '<event>' under an automatic trigger in %s or %s", suite, escalationWorkflow, smokeWorkflow)+ }+ }+}++type ciWorkflow struct {+ On map[string]any `yaml:"on"`+ Jobs map[string]ciJob `yaml:"jobs"`+}++type ciJob struct {+ If string `yaml:"if"`+ Uses string `yaml:"uses"`+ With struct {+ Suite string `yaml:"suite"`+ } `yaml:"with"`+}++func suitesOnAutomaticTriggers(t *testing.T, root string) map[string]bool {+ t.Helper()+ automatic := map[string]bool{}+ for _, rel := range []string{escalationWorkflow, smokeWorkflow} {+ workflow := parseWorkflow(t, root, rel)+ for _, event := range automaticEvents(workflow) {+ for _, job := range workflow.Jobs {+ if suite := jobSuite(job); suite != "" && jobRunsOnEvent(job, event) {+ automatic[suite] = true+ }+ }+ }+ }+ return automatic+}++func automaticEvents(workflow ciWorkflow) []string {+ var events []string+ for event := range workflow.On {+ if event != "pull_request" && event != "workflow_dispatch" {+ events = append(events, event)+ }+ }+ sort.Strings(events)+ return events+}++func jobSuite(job ciJob) string {+ if job.With.Suite != "" {+ return job.With.Suite+ }+ if strings.Contains(job.Uses, "reusable-smoke") {+ return "smoke"+ }+ return ""+}++func jobRunsOnEvent(job ciJob, event string) bool {+ return job.If == "" || strings.Contains(job.If, "github.event_name == '"+event+"'")+}++func parseWorkflow(t *testing.T, root, rel string) ciWorkflow {+ t.Helper()+ var workflow ciWorkflow+ if err := yaml.Unmarshal([]byte(readRepoFile(t, root, rel)), &workflow); err != nil {+ t.Fatalf("cannot parse %s: %v", rel, err)+ }+ if len(workflow.On) == 0 || len(workflow.Jobs) == 0 {+ t.Fatalf("%s declares %d triggers and %d jobs, so every trigger guard over it would pass vacuously", rel, len(workflow.On), len(workflow.Jobs))+ }+ return workflow+}+func detectSuites(t *testing.T, root string, changed []string) []string {t.Helper()var escalated []stringdiff --git a/tests/e2e/README.md b/tests/e2e/README.mdindex cebd5b25..1ffd3303 100644--- a/tests/e2e/README.md+++ b/tests/e2e/README.md@@ -114,8 +114,8 @@ CI uses the Go runner directly:- `.github/workflows/reusable-e2e.yml` runs `go run ./tests/tools/runner e2e --suite <suite> --logs hide`- `.github/workflows/reusable-smoke.yml` runs `go run ./tests/tools/runner e2e --suite smoke --logs hide`-- `.github/workflows/ci-e2e.yml` chooses PR, extended-on-touch, manual, and smoke jobs, then delegates execution to the reusable workflows-- `.github/workflows/ci-smoke.yml` is the manual smoke entrypoint+- `.github/workflows/ci-e2e.yml` chooses the PR, extended-on-touch, extended-on-merge, manual, and smoke jobs, then delegates execution to the reusable workflows+- `.github/workflows/ci-smoke.yml` is the nightly smoke entrypoint, and the manual oneWorkflow YAML should stay thin: it decides when to run suites, while scenario selection, services, logs, reports, failure summaries, and GitHub Actions outputs stay in the Go layer.