Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(ci): escalate extended suites from product paths, not only test churn

Luigi Agostib32843706ebf
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml
index 1e35de1e..1b746794 100644
--- a/.github/workflows/ci-e2e.yml
+++ b/.github/workflows/ci-e2e.yml
@@ -33,45 +33,20 @@ jobs:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
outputs:
- api_extended_touched: ${{ steps.changes.outputs.api_extended_touched }}
- cli_extended_touched: ${{ steps.changes.outputs.cli_extended_touched }}
- infra_extended_touched: ${{ steps.changes.outputs.infra_extended_touched }}
- smoke_touched: ${{ steps.changes.outputs.smoke_touched }}
+ run_api_extended: ${{ steps.changes.outputs.run_api_extended }}
+ run_cli_extended: ${{ steps.changes.outputs.run_cli_extended }}
+ run_infra_extended: ${{ steps.changes.outputs.run_infra_extended }}
+ run_smoke: ${{ steps.changes.outputs.run_smoke }}
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- - name: Detect touched E2E files
+ - name: Decide which suites this diff needs
id: changes
+ shell: bash
run: |
- CHANGED=$(git diff --name-only origin/main...HEAD)
-
- # Touching any non-basic, non-smoke scenario should trigger the matching
- # extended suite in its native environment rather than trying to
- # inject that file into the basic-stack PR jobs.
- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/api/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then
- echo "api_extended_touched=true" >> "$GITHUB_OUTPUT"
- else
- echo "api_extended_touched=false" >> "$GITHUB_OUTPUT"
- fi
-
- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/cli/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then
- echo "cli_extended_touched=true" >> "$GITHUB_OUTPUT"
- else
- echo "cli_extended_touched=false" >> "$GITHUB_OUTPUT"
- fi
-
- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/infra/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then
- echo "infra_extended_touched=true" >> "$GITHUB_OUTPUT"
- else
- echo "infra_extended_touched=false" >> "$GITHUB_OUTPUT"
- fi
-
- if echo "$CHANGED" | grep -E '^(tests/e2e/scenarios/[^/]+/[^/]+-smoke\.sh|scripts/docker-.*smoke\.sh|tests/tools/docker/chrome-cft-smoke\.Dockerfile|Dockerfile|\.dockerignore|\.github/workflows/(ci-smoke|reusable-smoke)\.yml)$' >/dev/null; then
- echo "smoke_touched=true" >> "$GITHUB_OUTPUT"
- else
- echo "smoke_touched=false" >> "$GITHUB_OUTPUT"
- fi
+ git diff --name-only origin/main...HEAD |
+ ./scripts/ci/detect-e2e-suites.sh >> "$GITHUB_OUTPUT"
api:
needs: [detect-changes]
@@ -88,7 +63,7 @@ jobs:
api-extended-pr:
needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.api_extended_touched == 'true'
+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true'
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -111,7 +86,7 @@ jobs:
cli-extended-pr:
needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.cli_extended_touched == 'true'
+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true'
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -134,7 +109,7 @@ jobs:
infra-extended-pr:
needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.infra_extended_touched == 'true'
+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true'
uses: ./.github/workflows/reusable-e2e.yml
with:
ref: ${{ github.sha }}
@@ -144,7 +119,7 @@ jobs:
smoke-pr:
needs: [detect-changes]
- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.smoke_touched == 'true'
+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true'
uses: ./.github/workflows/reusable-smoke.yml
with:
ref: ${{ github.sha }}
diff --git a/TESTING.md b/TESTING.md
index 7b2e5859..c05a95ff 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -64,6 +64,22 @@ Add new scenarios under `tests/e2e/scenarios/<group>/`, choose the tier by filen
CI uses `.github/workflows/reusable-e2e.yml` and `.github/workflows/reusable-smoke.yml`, both calling the Go runner directly. The workflow layer decides when to run; the Go layer decides what to run and how to report it.
+### What a pull request runs
+
+Every pull request runs the three basic suites. Which extended and smoke suites join them comes from `scripts/ci/e2e-escalation.map`, evaluated against the diff by `scripts/ci/detect-e2e-suites.sh` in the `detect-changes` job of `.github/workflows/ci-e2e.yml`. One rule per line maps a path expression to the suites that cover that path; the first rule matching a changed path decides, and a path matching no rule adds nothing.
+
+The map reads product paths as well as test paths:
+
+| Changed path | Suites added |
+| --- | --- |
+| `tests/e2e/scenarios/<group>/*-basic.sh` | none — the basic suites already run |
+| `tests/e2e/scenarios/<group>/*-smoke.sh` | `smoke` |
+| any other `*.sh` under `tests/e2e/scenarios/api/`, `cli/` or `infra/` | that group's extended suite |
+| `Dockerfile`, `.dockerignore`, `scripts/docker-*smoke.sh`, the smoke Dockerfile and workflows | `smoke` |
+| audit implementation — `internal/audit/`, `pkg/pinchtabaudit/`, the audit handlers, `cmd/pinchtab/cmd_audit*` | `api-extended`, `cli-extended` |
+
+Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim still runs on `workflow_dispatch` only.
+
### Basic Suites
```bash
diff --git a/internal/devtools/e2e_escalation_test.go b/internal/devtools/e2e_escalation_test.go
new file mode 100644
index 00000000..795e07d9
--- /dev/null
+++ b/internal/devtools/e2e_escalation_test.go
@@ -0,0 +1,221 @@
+package devtools
+
+import (
+ "bytes"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+)
+
+const (
+ escalationScript = "scripts/ci/detect-e2e-suites.sh"
+ escalationMap = "scripts/ci/e2e-escalation.map"
+ escalationWorkflow = ".github/workflows/ci-e2e.yml"
+)
+
+func TestEscalationFollowsProductPathsAndTestPaths(t *testing.T) {
+ root := repoRoot(t)
+
+ cases := []struct {
+ name string
+ changed []string
+ want []string
+ }{
+ {
+ name: "audit implementation with no scenario file",
+ changed: []string{"internal/audit/run.go", "pkg/pinchtabaudit/client.go", "internal/handlers/audit.go", "cmd/pinchtab/cmd_audit.go"},
+ want: []string{"run_api_extended", "run_cli_extended"},
+ },
+ {
+ name: "audit implementation alongside a basic scenario",
+ changed: []string{"internal/audit/enrich.go", "tests/e2e/scenarios/api/audit-basic.sh"},
+ want: []string{"run_api_extended", "run_cli_extended"},
+ },
+ {
+ name: "api extended scenario churn",
+ changed: []string{"tests/e2e/scenarios/api/tabs-extended.sh"},
+ want: []string{"run_api_extended"},
+ },
+ {
+ name: "cli extended scenario churn",
+ changed: []string{"tests/e2e/scenarios/cli/audit-pdf-extended.sh"},
+ want: []string{"run_cli_extended"},
+ },
+ {
+ name: "infra extended scenario churn",
+ changed: []string{"tests/e2e/scenarios/infra/auth-extended.sh"},
+ want: []string{"run_infra_extended"},
+ },
+ {
+ name: "standalone scenario churn",
+ changed: []string{"tests/e2e/scenarios/api/network-retain-body.sh"},
+ want: []string{"run_api_extended"},
+ },
+ {
+ name: "smoke inputs",
+ changed: []string{"tests/e2e/scenarios/cli/tabs-smoke.sh", "Dockerfile"},
+ want: []string{"run_smoke"},
+ },
+ {
+ name: "basic scenario churn only",
+ changed: []string{"tests/e2e/scenarios/api/tabs-basic.sh", "tests/e2e/scenarios/infra/network-basic.sh"},
+ want: nil,
+ },
+ {
+ name: "product paths no extended suite covers",
+ changed: []string{"internal/dashboard/dashboard.go", "README.md", "internal/scroll/scroll.go"},
+ want: nil,
+ },
+ {
+ name: "empty diff",
+ changed: nil,
+ want: nil,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := detectSuites(t, root, tc.changed)
+ if strings.Join(got, " ") != strings.Join(tc.want, " ") {
+ t.Errorf("%s escalated %v, want %v", strings.Join(tc.changed, ", "), got, tc.want)
+ }
+ })
+ }
+}
+
+func TestEveryEscalationRuleMatchesATrackedPath(t *testing.T) {
+ root := repoRoot(t)
+ tracked := trackedPaths(t, root)
+
+ for _, rule := range escalationRules(t, root) {
+ pattern, err := regexp.Compile(rule)
+ if err != nil {
+ t.Errorf("rule %q is not a valid expression: %v", rule, err)
+ continue
+ }
+ matched := false
+ for _, path := range tracked {
+ if pattern.MatchString(path) {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ t.Errorf("rule %q matches no tracked file, so the coverage it claims is not wired to anything; re-point it at the path that moved rather than leaving it", rule)
+ }
+ }
+}
+
+func TestTheWorkflowConsumesExactlyTheEmittedOutputs(t *testing.T) {
+ root := repoRoot(t)
+ body := readRepoFile(t, root, escalationWorkflow)
+
+ if !strings.Contains(body, escalationScript) {
+ t.Fatalf("%s does not invoke %s, so the mapping decides nothing", escalationWorkflow, escalationScript)
+ }
+
+ emitted := map[string]bool{}
+ for _, name := range detectSuiteNames(t, root) {
+ emitted[name] = true
+ if !strings.Contains(body, "steps.changes.outputs."+name) {
+ t.Errorf("%s emits %s but %s never exports it, so the suite it gates can never run", escalationScript, name, escalationWorkflow)
+ }
+ }
+
+ consumed := regexp.MustCompile(`detect-changes\.outputs\.([a-z_]+)`)
+ for _, match := range consumed.FindAllStringSubmatch(body, -1) {
+ if !emitted[match[1]] {
+ t.Errorf("%s gates a job on %s, which %s never emits, so the job never runs", escalationWorkflow, match[1], escalationScript)
+ }
+ }
+}
+
+func detectSuites(t *testing.T, root string, changed []string) []string {
+ t.Helper()
+ var escalated []string
+ for _, line := range runDetect(t, root, changed) {
+ name, value, ok := strings.Cut(line, "=")
+ if !ok {
+ t.Fatalf("%s emitted %q, which is not a name=value pair", escalationScript, line)
+ }
+ if value == "true" {
+ escalated = append(escalated, name)
+ }
+ }
+ sort.Strings(escalated)
+ return escalated
+}
+
+func detectSuiteNames(t *testing.T, root string) []string {
+ t.Helper()
+ var names []string
+ for _, line := range runDetect(t, root, nil) {
+ name, _, _ := strings.Cut(line, "=")
+ names = append(names, name)
+ }
+ if len(names) == 0 {
+ t.Fatalf("%s emitted no output, so every guard over it is vacuous", escalationScript)
+ }
+ return names
+}
+
+func runDetect(t *testing.T, root string, changed []string) []string {
+ t.Helper()
+ if readRepoFile(t, root, escalationScript) == "" || readRepoFile(t, root, escalationMap) == "" {
+ t.Fatalf("%s and %s must both hold content, and reading them here is also what keeps a cached green from surviving an edit to either", escalationScript, escalationMap)
+ }
+ cmd := exec.Command("bash", filepath.Join(root, escalationScript))
+ cmd.Stdin = strings.NewReader(strings.Join(changed, "\n"))
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout = &stdout
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ t.Fatalf("%s failed on %v: %v\n%s", escalationScript, changed, err, stderr.String())
+ }
+ var lines []string
+ for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") {
+ if line != "" {
+ lines = append(lines, line)
+ }
+ }
+ return lines
+}
+
+func escalationRules(t *testing.T, root string) []string {
+ t.Helper()
+ var rules []string
+ for _, line := range strings.Split(readRepoFile(t, root, escalationMap), "\n") {
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ rules = append(rules, strings.Fields(line)[0])
+ }
+ if len(rules) == 0 {
+ t.Fatalf("%s holds no rule", escalationMap)
+ }
+ return rules
+}
+
+func trackedPaths(t *testing.T, root string) []string {
+ t.Helper()
+ cmd := exec.Command("git", "ls-files")
+ cmd.Dir = root
+ out, err := cmd.Output()
+ if err != nil {
+ t.Fatalf("cannot list tracked files, so staleness is unprovable: %v", err)
+ }
+ return strings.Split(strings.TrimSpace(string(out)), "\n")
+}
+
+func readRepoFile(t *testing.T, root, rel string) string {
+ t.Helper()
+ body, err := os.ReadFile(filepath.Join(root, rel))
+ if err != nil {
+ t.Fatalf("cannot read %s: %v", rel, err)
+ }
+ return string(body)
+}
diff --git a/scripts/ci/detect-e2e-suites.sh b/scripts/ci/detect-e2e-suites.sh
new file mode 100755
index 00000000..c473df79
--- /dev/null
+++ b/scripts/ci/detect-e2e-suites.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+map_file="$script_dir/e2e-escalation.map"
+
+suites=(api-extended cli-extended infra-extended smoke)
+
+known_suite() {
+ local candidate
+ for candidate in "${suites[@]}"; do
+ [ "$1" = "$candidate" ] && return 0
+ done
+ return 1
+}
+
+patterns=()
+targets=()
+while IFS= read -r line || [ -n "$line" ]; do
+ case "$line" in '' | '#'*) continue ;; esac
+ read -r pattern rule_suites <<<"$line"
+ if [ -z "${rule_suites:-}" ]; then
+ echo "detect-e2e-suites: rule names no suite: $line" >&2
+ exit 1
+ fi
+ for suite in $rule_suites; do
+ if [ "$suite" != "-" ] && ! known_suite "$suite"; then
+ echo "detect-e2e-suites: unknown suite \"$suite\" in rule: $line" >&2
+ exit 1
+ fi
+ done
+ patterns+=("$pattern")
+ targets+=("$rule_suites")
+done <"$map_file"
+
+if [ ${#patterns[@]} -eq 0 ]; then
+ echo "detect-e2e-suites: $map_file holds no rule" >&2
+ exit 1
+fi
+
+escalated=""
+while IFS= read -r path || [ -n "$path" ]; do
+ [ -n "$path" ] || continue
+ for i in "${!patterns[@]}"; do
+ if [[ $path =~ ${patterns[$i]} ]]; then
+ escalated+=" ${targets[$i]} "
+ break
+ fi
+ done
+done
+
+for suite in "${suites[@]}"; do
+ if [[ $escalated == *" $suite "* ]]; then
+ echo "run_${suite//-/_}=true"
+ else
+ echo "run_${suite//-/_}=false"
+ fi
+done
diff --git a/scripts/ci/e2e-escalation.map b/scripts/ci/e2e-escalation.map
new file mode 100644
index 00000000..9ff58958
--- /dev/null
+++ b/scripts/ci/e2e-escalation.map
@@ -0,0 +1,21 @@
+# Which E2E suites a changed path pulls into a pull request run.
+#
+# One rule per line: <ERE matched against the repo-relative path> <suites, space
+# separated, or "-" for none>. The first matching rule decides; a path matching no
+# rule escalates nothing. Adding coverage for a product area is one line.
+
+^tests/e2e/scenarios/[^/]+/[^/]+-smoke\.sh$ smoke
+^tests/e2e/scenarios/[^/]+/[^/]+-basic\.sh$ -
+^tests/e2e/scenarios/api/[^/]+\.sh$ api-extended
+^tests/e2e/scenarios/cli/[^/]+\.sh$ cli-extended
+^tests/e2e/scenarios/infra/[^/]+\.sh$ infra-extended
+
+^(Dockerfile|\.dockerignore)$ smoke
+^scripts/docker-.*smoke\.sh$ smoke
+^tests/tools/docker/chrome-cft-smoke\.Dockerfile$ smoke
+^\.github/workflows/(ci-smoke|reusable-smoke)\.yml$ smoke
+
+^internal/audit/ api-extended cli-extended
+^pkg/pinchtabaudit/ api-extended cli-extended
+^internal/handlers/(a11y_)?audit api-extended cli-extended
+^cmd/pinchtab/cmd_audit api-extended cli-extended