Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat(ci): run the extended suites on every merge to main and smoke nightly

Luigi Agosti2519c4e659cb
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml
index 1b746794..a41dfffc 100644
--- a/.github/workflows/ci-e2e.yml
+++ b/.github/workflows/ci-e2e.yml
@@ -13,6 +13,15 @@ on:
- '.gitignore'
- 'skills/**'
- 'plugin/**'
+ push:
+ branches: [main]
+ paths-ignore:
+ - '**.md'
+ - 'docs/**'
+ - 'LICENSE'
+ - '.gitignore'
+ - 'skills/**'
+ - 'plugin/**'
workflow_dispatch:
inputs:
suite:
@@ -49,11 +58,9 @@ jobs:
./scripts/ci/detect-e2e-suites.sh >> "$GITHUB_OUTPUT"
api:
- needs: [detect-changes]
if: >-
- always() &&
- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'api')
+ github.event_name == 'pull_request' ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api'))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -61,22 +68,10 @@ jobs:
summary_title: E2E API Tests
timeout_minutes: 15
- api-extended-pr:
- needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true'
- uses: ./.github/workflows/reusable-e2e.yml
- with:
- ref: ${{ github.sha }}
- suite: api-extended
- summary_title: E2E API Extended Tests (PR)
- timeout_minutes: 20
-
cli:
- needs: [detect-changes]
if: >-
- always() &&
- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'cli')
+ github.event_name == 'pull_request' ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli'))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -84,22 +79,10 @@ jobs:
summary_title: E2E CLI Tests
timeout_minutes: 15
- cli-extended-pr:
- needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true'
- uses: ./.github/workflows/reusable-e2e.yml
- with:
- ref: ${{ github.sha }}
- suite: cli-extended
- summary_title: E2E CLI Extended Tests (PR)
- timeout_minutes: 20
-
infra:
- needs: [detect-changes]
if: >-
- always() &&
- (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
- (github.event_name != 'workflow_dispatch' || inputs.suite == 'all' || inputs.suite == 'infra')
+ github.event_name == 'pull_request' ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra'))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -107,26 +90,14 @@ jobs:
summary_title: E2E Infra Tests
timeout_minutes: 15
- infra-extended-pr:
- needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true'
- uses: ./.github/workflows/reusable-e2e.yml
- with:
- ref: ${{ github.sha }}
- suite: infra-extended
- summary_title: E2E Infra Extended Tests (PR)
- timeout_minutes: 25
-
- smoke-pr:
- needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true'
- uses: ./.github/workflows/reusable-smoke.yml
- with:
- ref: ${{ github.sha }}
-
- # Extended suites for manual dispatch
api-extended:
- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api')
+ needs: [detect-changes]
+ if: >-
+ always() &&
+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
+ (github.event_name == 'push' ||
+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true') ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'api')))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -135,7 +106,13 @@ jobs:
timeout_minutes: 20
cli-extended:
- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli')
+ needs: [detect-changes]
+ if: >-
+ always() &&
+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
+ (github.event_name == 'push' ||
+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true') ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'cli')))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -144,7 +121,13 @@ jobs:
timeout_minutes: 20
infra-extended:
- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra')
+ needs: [detect-changes]
+ if: >-
+ always() &&
+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
+ (github.event_name == 'push' ||
+ (github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true') ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'infra')))
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -153,7 +136,12 @@ jobs:
timeout_minutes: 25
smoke:
- if: github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'smoke')
+ needs: [detect-changes]
+ if: >-
+ always() &&
+ (needs.detect-changes.result == 'success' || needs.detect-changes.result == 'skipped') &&
+ ((github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true') ||
+ (github.event_name == 'workflow_dispatch' && (inputs.suite == 'all' || inputs.suite == 'smoke')))
uses: ./.github/workflows/reusable-smoke.yml
with:
ref: ${{ github.sha }}
diff --git a/.github/workflows/ci-smoke.yml b/.github/workflows/ci-smoke.yml
index ca64e3b8..781ecdcf 100644
--- a/.github/workflows/ci-smoke.yml
+++ b/.github/workflows/ci-smoke.yml
@@ -1,6 +1,8 @@
name: CI / Smoke
on:
+ schedule:
+ - cron: '0 3 * * *'
workflow_dispatch:
permissions:
diff --git a/TESTING.md b/TESTING.md
index 8e823834..3f3eb90c 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -79,12 +79,27 @@ The map reads product paths as well as test paths:
| audit implementation — `internal/audit/`, `pkg/pinchtabaudit/`, the audit handlers, `internal/cli/actions/actions_audit*`, `cmd/pinchtab/cmd_audit*` | `api-extended`, `cli-extended` |
| compare implementation — `internal/cli/actions/actions_compare*`, `cmd/pinchtab/cmd_compare*` (`internal/audit/compare.go` is already audit) | `cli-extended` |
-Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim still runs on `workflow_dispatch` only.
+Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim is not lost — it runs after merge, on the lane below.
Audit and compare are also declared there as *enrolled areas*: every tracked source file matching an area must escalate that area's suites, with a short exclusion list for files that only share the word (`internal/authn/audit.go` is security audit logging). So a new file in an enrolled family cannot quietly miss the map — which is how the CLI audit implementation went unenrolled while its cobra wrapper was covered. Enrolling another area is one entry in `enrolledAreas`.
The multi-page audit runs — `api/audit-extended.sh`, `cli/audit-cli-extended.sh`, `cli/audit-seaportal-extended.sh`, `cli/compare-extended.sh` — sit in the extended tier rather than the basic one, so a pull request pays for them only when it touches audit or compare. The PR path keeps the single-page audit signal: `api/audit-page-basic.sh`, `api/a11y-audit-basic.sh`, `api/audit-fixtures-basic.sh`, `cli/audit-auth-basic.sh` and `cli/audit-report-basic.sh`.
+### What a merge and the nightly run
+
+Every push to `main` runs the three extended suites without consulting the map, so coverage a pull request skipped is exercised at merge rather than never. Extended suites include the matching `*-basic.sh` scenarios, so the merge lane does not repeat the basic suites.
+
+Smoke is the expensive lane — it builds images — so it runs nightly on the `schedule` trigger in `.github/workflows/ci-smoke.yml` rather than per merge, on top of the pull requests whose diff the map escalates to `smoke`.
+
+| Lane | Trigger | Suites |
+| --- | --- | --- |
+| pull request | `pull_request` on `main` | the three basic suites, plus whatever the escalation map claims |
+| merge | `push` on `main` | `api-extended`, `cli-extended`, `infra-extended` |
+| nightly | `schedule` in `ci-smoke.yml` | `smoke` |
+| manual | `workflow_dispatch` | whichever suite is chosen |
+
+`go test ./internal/devtools/` reads both workflows and fails if a suite the map can escalate has no automatic trigger, so a suite cannot quietly fall back to `workflow_dispatch` only.
+
### Basic Suites
```bash
diff --git a/internal/devtools/e2e_escalation_test.go b/internal/devtools/e2e_escalation_test.go
index 92b0eb84..df7dac8e 100644
--- a/internal/devtools/e2e_escalation_test.go
+++ b/internal/devtools/e2e_escalation_test.go
@@ -9,12 +9,15 @@ import (
"sort"
"strings"
"testing"
+
+ "gopkg.in/yaml.v3"
)
const (
escalationScript = "scripts/ci/detect-e2e-suites.sh"
escalationMap = "scripts/ci/e2e-escalation.map"
escalationWorkflow = ".github/workflows/ci-e2e.yml"
+ smokeWorkflow = ".github/workflows/ci-smoke.yml"
)
func TestEscalationFollowsProductPathsAndTestPaths(t *testing.T) {
@@ -144,6 +147,84 @@ func TestTheWorkflowConsumesExactlyTheEmittedOutputs(t *testing.T) {
}
}
+func TestEveryEscalatableSuiteAlsoRunsWithoutADiffToClaimIt(t *testing.T) {
+ root := repoRoot(t)
+ automatic := suitesOnAutomaticTriggers(t, root)
+
+ for _, name := range detectSuiteNames(t, root) {
+ suite := strings.ReplaceAll(strings.TrimPrefix(name, "run_"), "_", "-")
+ if !automatic[suite] {
+ t.Errorf("the %s suite runs only when a diff escalates it or a human dispatches it, so a merge the map does not claim never exercises the coverage it holds; give it a job gated on github.event_name == '<event>' under an automatic trigger in %s or %s", suite, escalationWorkflow, smokeWorkflow)
+ }
+ }
+}
+
+type ciWorkflow struct {
+ On map[string]any `yaml:"on"`
+ Jobs map[string]ciJob `yaml:"jobs"`
+}
+
+type ciJob struct {
+ If string `yaml:"if"`
+ Uses string `yaml:"uses"`
+ With struct {
+ Suite string `yaml:"suite"`
+ } `yaml:"with"`
+}
+
+func suitesOnAutomaticTriggers(t *testing.T, root string) map[string]bool {
+ t.Helper()
+ automatic := map[string]bool{}
+ for _, rel := range []string{escalationWorkflow, smokeWorkflow} {
+ workflow := parseWorkflow(t, root, rel)
+ for _, event := range automaticEvents(workflow) {
+ for _, job := range workflow.Jobs {
+ if suite := jobSuite(job); suite != "" && jobRunsOnEvent(job, event) {
+ automatic[suite] = true
+ }
+ }
+ }
+ }
+ return automatic
+}
+
+func automaticEvents(workflow ciWorkflow) []string {
+ var events []string
+ for event := range workflow.On {
+ if event != "pull_request" && event != "workflow_dispatch" {
+ events = append(events, event)
+ }
+ }
+ sort.Strings(events)
+ return events
+}
+
+func jobSuite(job ciJob) string {
+ if job.With.Suite != "" {
+ return job.With.Suite
+ }
+ if strings.Contains(job.Uses, "reusable-smoke") {
+ return "smoke"
+ }
+ return ""
+}
+
+func jobRunsOnEvent(job ciJob, event string) bool {
+ return job.If == "" || strings.Contains(job.If, "github.event_name == '"+event+"'")
+}
+
+func parseWorkflow(t *testing.T, root, rel string) ciWorkflow {
+ t.Helper()
+ var workflow ciWorkflow
+ if err := yaml.Unmarshal([]byte(readRepoFile(t, root, rel)), &workflow); err != nil {
+ t.Fatalf("cannot parse %s: %v", rel, err)
+ }
+ if len(workflow.On) == 0 || len(workflow.Jobs) == 0 {
+ t.Fatalf("%s declares %d triggers and %d jobs, so every trigger guard over it would pass vacuously", rel, len(workflow.On), len(workflow.Jobs))
+ }
+ return workflow
+}
+
func detectSuites(t *testing.T, root string, changed []string) []string {
t.Helper()
var escalated []string
diff --git a/tests/e2e/README.md b/tests/e2e/README.md
index cebd5b25..1ffd3303 100644
--- a/tests/e2e/README.md
+++ b/tests/e2e/README.md
@@ -114,8 +114,8 @@ CI uses the Go runner directly:
- `.github/workflows/reusable-e2e.yml` runs `go run ./tests/tools/runner e2e --suite <suite> --logs hide`
- `.github/workflows/reusable-smoke.yml` runs `go run ./tests/tools/runner e2e --suite smoke --logs hide`
-- `.github/workflows/ci-e2e.yml` chooses PR, extended-on-touch, manual, and smoke jobs, then delegates execution to the reusable workflows
-- `.github/workflows/ci-smoke.yml` is the manual smoke entrypoint
+- `.github/workflows/ci-e2e.yml` chooses the PR, extended-on-touch, extended-on-merge, manual, and smoke jobs, then delegates execution to the reusable workflows
+- `.github/workflows/ci-smoke.yml` is the nightly smoke entrypoint, and the manual one
Workflow YAML should stay thin: it decides when to run suites, while scenario selection, services, logs, reports, failure summaries, and GitHub Actions outputs stay in the Go layer.