High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(ci): escalate extended suites from product paths, not only test churn
Luigi Agostib32843706ebf
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.ymlindex 1e35de1e..1b746794 100644--- a/.github/workflows/ci-e2e.yml+++ b/.github/workflows/ci-e2e.yml@@ -33,45 +33,20 @@ jobs:if: github.event_name == 'pull_request'runs-on: ubuntu-latestoutputs:- api_extended_touched: ${{ steps.changes.outputs.api_extended_touched }}- cli_extended_touched: ${{ steps.changes.outputs.cli_extended_touched }}- infra_extended_touched: ${{ steps.changes.outputs.infra_extended_touched }}- smoke_touched: ${{ steps.changes.outputs.smoke_touched }}+ run_api_extended: ${{ steps.changes.outputs.run_api_extended }}+ run_cli_extended: ${{ steps.changes.outputs.run_cli_extended }}+ run_infra_extended: ${{ steps.changes.outputs.run_infra_extended }}+ run_smoke: ${{ steps.changes.outputs.run_smoke }}steps:- uses: actions/checkout@v6with:fetch-depth: 0- - name: Detect touched E2E files+ - name: Decide which suites this diff needsid: changes+ shell: bashrun: |- CHANGED=$(git diff --name-only origin/main...HEAD)-- # Touching any non-basic, non-smoke scenario should trigger the matching- # extended suite in its native environment rather than trying to- # inject that file into the basic-stack PR jobs.- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/api/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then- echo "api_extended_touched=true" >> "$GITHUB_OUTPUT"- else- echo "api_extended_touched=false" >> "$GITHUB_OUTPUT"- fi-- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/cli/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then- echo "cli_extended_touched=true" >> "$GITHUB_OUTPUT"- else- echo "cli_extended_touched=false" >> "$GITHUB_OUTPUT"- fi-- if echo "$CHANGED" | grep -E '^tests/e2e/scenarios/infra/[^/]+\.sh$' | grep -Ev '(^|/)[^/]+-(basic|smoke)\.sh$' >/dev/null; then- echo "infra_extended_touched=true" >> "$GITHUB_OUTPUT"- else- echo "infra_extended_touched=false" >> "$GITHUB_OUTPUT"- fi-- if echo "$CHANGED" | grep -E '^(tests/e2e/scenarios/[^/]+/[^/]+-smoke\.sh|scripts/docker-.*smoke\.sh|tests/tools/docker/chrome-cft-smoke\.Dockerfile|Dockerfile|\.dockerignore|\.github/workflows/(ci-smoke|reusable-smoke)\.yml)$' >/dev/null; then- echo "smoke_touched=true" >> "$GITHUB_OUTPUT"- else- echo "smoke_touched=false" >> "$GITHUB_OUTPUT"- fi+ git diff --name-only origin/main...HEAD |+ ./scripts/ci/detect-e2e-suites.sh >> "$GITHUB_OUTPUT"api:needs: [detect-changes]@@ -88,7 +63,7 @@ jobs:api-extended-pr:needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.api_extended_touched == 'true'+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_api_extended == 'true'uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -111,7 +86,7 @@ jobs:cli-extended-pr:needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.cli_extended_touched == 'true'+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_cli_extended == 'true'uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -134,7 +109,7 @@ jobs:infra-extended-pr:needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.infra_extended_touched == 'true'+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_infra_extended == 'true'uses: ./.github/workflows/reusable-e2e.ymlwith:ref: ${{ github.sha }}@@ -144,7 +119,7 @@ jobs:smoke-pr:needs: [detect-changes]- if: github.event_name == 'pull_request' && needs.detect-changes.outputs.smoke_touched == 'true'+ if: github.event_name == 'pull_request' && needs.detect-changes.outputs.run_smoke == 'true'uses: ./.github/workflows/reusable-smoke.ymlwith:ref: ${{ github.sha }}diff --git a/TESTING.md b/TESTING.mdindex 7b2e5859..c05a95ff 100644--- a/TESTING.md+++ b/TESTING.md@@ -64,6 +64,22 @@ Add new scenarios under `tests/e2e/scenarios/<group>/`, choose the tier by filenCI uses `.github/workflows/reusable-e2e.yml` and `.github/workflows/reusable-smoke.yml`, both calling the Go runner directly. The workflow layer decides when to run; the Go layer decides what to run and how to report it.+### What a pull request runs++Every pull request runs the three basic suites. Which extended and smoke suites join them comes from `scripts/ci/e2e-escalation.map`, evaluated against the diff by `scripts/ci/detect-e2e-suites.sh` in the `detect-changes` job of `.github/workflows/ci-e2e.yml`. One rule per line maps a path expression to the suites that cover that path; the first rule matching a changed path decides, and a path matching no rule adds nothing.++The map reads product paths as well as test paths:++| Changed path | Suites added |+| --- | --- |+| `tests/e2e/scenarios/<group>/*-basic.sh` | none — the basic suites already run |+| `tests/e2e/scenarios/<group>/*-smoke.sh` | `smoke` |+| any other `*.sh` under `tests/e2e/scenarios/api/`, `cli/` or `infra/` | that group's extended suite |+| `Dockerfile`, `.dockerignore`, `scripts/docker-*smoke.sh`, the smoke Dockerfile and workflows | `smoke` |+| audit implementation — `internal/audit/`, `pkg/pinchtabaudit/`, the audit handlers, `cmd/pinchtab/cmd_audit*` | `api-extended`, `cli-extended` |++Covering a new product area is one line in the map. `go test ./internal/devtools/` drives the script over synthetic file lists, so a mapping is verified by running it rather than by reading the YAML. Extended coverage the map does not claim still runs on `workflow_dispatch` only.+### Basic Suites```bashdiff --git a/internal/devtools/e2e_escalation_test.go b/internal/devtools/e2e_escalation_test.gonew file mode 100644index 00000000..795e07d9--- /dev/null+++ b/internal/devtools/e2e_escalation_test.go@@ -0,0 +1,221 @@+package devtools++import (+ "bytes"+ "os"+ "os/exec"+ "path/filepath"+ "regexp"+ "sort"+ "strings"+ "testing"+)++const (+ escalationScript = "scripts/ci/detect-e2e-suites.sh"+ escalationMap = "scripts/ci/e2e-escalation.map"+ escalationWorkflow = ".github/workflows/ci-e2e.yml"+)++func TestEscalationFollowsProductPathsAndTestPaths(t *testing.T) {+ root := repoRoot(t)++ cases := []struct {+ name string+ changed []string+ want []string+ }{+ {+ name: "audit implementation with no scenario file",+ changed: []string{"internal/audit/run.go", "pkg/pinchtabaudit/client.go", "internal/handlers/audit.go", "cmd/pinchtab/cmd_audit.go"},+ want: []string{"run_api_extended", "run_cli_extended"},+ },+ {+ name: "audit implementation alongside a basic scenario",+ changed: []string{"internal/audit/enrich.go", "tests/e2e/scenarios/api/audit-basic.sh"},+ want: []string{"run_api_extended", "run_cli_extended"},+ },+ {+ name: "api extended scenario churn",+ changed: []string{"tests/e2e/scenarios/api/tabs-extended.sh"},+ want: []string{"run_api_extended"},+ },+ {+ name: "cli extended scenario churn",+ changed: []string{"tests/e2e/scenarios/cli/audit-pdf-extended.sh"},+ want: []string{"run_cli_extended"},+ },+ {+ name: "infra extended scenario churn",+ changed: []string{"tests/e2e/scenarios/infra/auth-extended.sh"},+ want: []string{"run_infra_extended"},+ },+ {+ name: "standalone scenario churn",+ changed: []string{"tests/e2e/scenarios/api/network-retain-body.sh"},+ want: []string{"run_api_extended"},+ },+ {+ name: "smoke inputs",+ changed: []string{"tests/e2e/scenarios/cli/tabs-smoke.sh", "Dockerfile"},+ want: []string{"run_smoke"},+ },+ {+ name: "basic scenario churn only",+ changed: []string{"tests/e2e/scenarios/api/tabs-basic.sh", "tests/e2e/scenarios/infra/network-basic.sh"},+ want: nil,+ },+ {+ name: "product paths no extended suite covers",+ changed: []string{"internal/dashboard/dashboard.go", "README.md", "internal/scroll/scroll.go"},+ want: nil,+ },+ {+ name: "empty diff",+ changed: nil,+ want: nil,+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := detectSuites(t, root, tc.changed)+ if strings.Join(got, " ") != strings.Join(tc.want, " ") {+ t.Errorf("%s escalated %v, want %v", strings.Join(tc.changed, ", "), got, tc.want)+ }+ })+ }+}++func TestEveryEscalationRuleMatchesATrackedPath(t *testing.T) {+ root := repoRoot(t)+ tracked := trackedPaths(t, root)++ for _, rule := range escalationRules(t, root) {+ pattern, err := regexp.Compile(rule)+ if err != nil {+ t.Errorf("rule %q is not a valid expression: %v", rule, err)+ continue+ }+ matched := false+ for _, path := range tracked {+ if pattern.MatchString(path) {+ matched = true+ break+ }+ }+ if !matched {+ t.Errorf("rule %q matches no tracked file, so the coverage it claims is not wired to anything; re-point it at the path that moved rather than leaving it", rule)+ }+ }+}++func TestTheWorkflowConsumesExactlyTheEmittedOutputs(t *testing.T) {+ root := repoRoot(t)+ body := readRepoFile(t, root, escalationWorkflow)++ if !strings.Contains(body, escalationScript) {+ t.Fatalf("%s does not invoke %s, so the mapping decides nothing", escalationWorkflow, escalationScript)+ }++ emitted := map[string]bool{}+ for _, name := range detectSuiteNames(t, root) {+ emitted[name] = true+ if !strings.Contains(body, "steps.changes.outputs."+name) {+ t.Errorf("%s emits %s but %s never exports it, so the suite it gates can never run", escalationScript, name, escalationWorkflow)+ }+ }++ consumed := regexp.MustCompile(`detect-changes\.outputs\.([a-z_]+)`)+ for _, match := range consumed.FindAllStringSubmatch(body, -1) {+ if !emitted[match[1]] {+ t.Errorf("%s gates a job on %s, which %s never emits, so the job never runs", escalationWorkflow, match[1], escalationScript)+ }+ }+}++func detectSuites(t *testing.T, root string, changed []string) []string {+ t.Helper()+ var escalated []string+ for _, line := range runDetect(t, root, changed) {+ name, value, ok := strings.Cut(line, "=")+ if !ok {+ t.Fatalf("%s emitted %q, which is not a name=value pair", escalationScript, line)+ }+ if value == "true" {+ escalated = append(escalated, name)+ }+ }+ sort.Strings(escalated)+ return escalated+}++func detectSuiteNames(t *testing.T, root string) []string {+ t.Helper()+ var names []string+ for _, line := range runDetect(t, root, nil) {+ name, _, _ := strings.Cut(line, "=")+ names = append(names, name)+ }+ if len(names) == 0 {+ t.Fatalf("%s emitted no output, so every guard over it is vacuous", escalationScript)+ }+ return names+}++func runDetect(t *testing.T, root string, changed []string) []string {+ t.Helper()+ if readRepoFile(t, root, escalationScript) == "" || readRepoFile(t, root, escalationMap) == "" {+ t.Fatalf("%s and %s must both hold content, and reading them here is also what keeps a cached green from surviving an edit to either", escalationScript, escalationMap)+ }+ cmd := exec.Command("bash", filepath.Join(root, escalationScript))+ cmd.Stdin = strings.NewReader(strings.Join(changed, "\n"))+ var stdout, stderr bytes.Buffer+ cmd.Stdout = &stdout+ cmd.Stderr = &stderr+ if err := cmd.Run(); err != nil {+ t.Fatalf("%s failed on %v: %v\n%s", escalationScript, changed, err, stderr.String())+ }+ var lines []string+ for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") {+ if line != "" {+ lines = append(lines, line)+ }+ }+ return lines+}++func escalationRules(t *testing.T, root string) []string {+ t.Helper()+ var rules []string+ for _, line := range strings.Split(readRepoFile(t, root, escalationMap), "\n") {+ if line == "" || strings.HasPrefix(line, "#") {+ continue+ }+ rules = append(rules, strings.Fields(line)[0])+ }+ if len(rules) == 0 {+ t.Fatalf("%s holds no rule", escalationMap)+ }+ return rules+}++func trackedPaths(t *testing.T, root string) []string {+ t.Helper()+ cmd := exec.Command("git", "ls-files")+ cmd.Dir = root+ out, err := cmd.Output()+ if err != nil {+ t.Fatalf("cannot list tracked files, so staleness is unprovable: %v", err)+ }+ return strings.Split(strings.TrimSpace(string(out)), "\n")+}++func readRepoFile(t *testing.T, root, rel string) string {+ t.Helper()+ body, err := os.ReadFile(filepath.Join(root, rel))+ if err != nil {+ t.Fatalf("cannot read %s: %v", rel, err)+ }+ return string(body)+}diff --git a/scripts/ci/detect-e2e-suites.sh b/scripts/ci/detect-e2e-suites.shnew file mode 100755index 00000000..c473df79--- /dev/null+++ b/scripts/ci/detect-e2e-suites.sh@@ -0,0 +1,58 @@+#!/usr/bin/env bash+set -euo pipefail++script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)+map_file="$script_dir/e2e-escalation.map"++suites=(api-extended cli-extended infra-extended smoke)++known_suite() {+ local candidate+ for candidate in "${suites[@]}"; do+ [ "$1" = "$candidate" ] && return 0+ done+ return 1+}++patterns=()+targets=()+while IFS= read -r line || [ -n "$line" ]; do+ case "$line" in '' | '#'*) continue ;; esac+ read -r pattern rule_suites <<<"$line"+ if [ -z "${rule_suites:-}" ]; then+ echo "detect-e2e-suites: rule names no suite: $line" >&2+ exit 1+ fi+ for suite in $rule_suites; do+ if [ "$suite" != "-" ] && ! known_suite "$suite"; then+ echo "detect-e2e-suites: unknown suite \"$suite\" in rule: $line" >&2+ exit 1+ fi+ done+ patterns+=("$pattern")+ targets+=("$rule_suites")+done <"$map_file"++if [ ${#patterns[@]} -eq 0 ]; then+ echo "detect-e2e-suites: $map_file holds no rule" >&2+ exit 1+fi++escalated=""+while IFS= read -r path || [ -n "$path" ]; do+ [ -n "$path" ] || continue+ for i in "${!patterns[@]}"; do+ if [[ $path =~ ${patterns[$i]} ]]; then+ escalated+=" ${targets[$i]} "+ break+ fi+ done+done++for suite in "${suites[@]}"; do+ if [[ $escalated == *" $suite "* ]]; then+ echo "run_${suite//-/_}=true"+ else+ echo "run_${suite//-/_}=false"+ fi+donediff --git a/scripts/ci/e2e-escalation.map b/scripts/ci/e2e-escalation.mapnew file mode 100644index 00000000..9ff58958--- /dev/null+++ b/scripts/ci/e2e-escalation.map@@ -0,0 +1,21 @@+# Which E2E suites a changed path pulls into a pull request run.+#+# One rule per line: <ERE matched against the repo-relative path> <suites, space+# separated, or "-" for none>. The first matching rule decides; a path matching no+# rule escalates nothing. Adding coverage for a product area is one line.++^tests/e2e/scenarios/[^/]+/[^/]+-smoke\.sh$ smoke+^tests/e2e/scenarios/[^/]+/[^/]+-basic\.sh$ -+^tests/e2e/scenarios/api/[^/]+\.sh$ api-extended+^tests/e2e/scenarios/cli/[^/]+\.sh$ cli-extended+^tests/e2e/scenarios/infra/[^/]+\.sh$ infra-extended++^(Dockerfile|\.dockerignore)$ smoke+^scripts/docker-.*smoke\.sh$ smoke+^tests/tools/docker/chrome-cft-smoke\.Dockerfile$ smoke+^\.github/workflows/(ci-smoke|reusable-smoke)\.yml$ smoke++^internal/audit/ api-extended cli-extended+^pkg/pinchtabaudit/ api-extended cli-extended+^internal/handlers/(a11y_)?audit api-extended cli-extended+^cmd/pinchtab/cmd_audit api-extended cli-extended