Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(e2e): decide a scenario's verdict from its EXIT trap so an explicit exit cannot skip it

Luigi Agostib92458762f50
diff --git a/TESTING.md b/TESTING.md
index 049341b4..31588530 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -225,8 +225,9 @@ itself.
Failure accounting crosses the subshell boundary through exit status: a scenario
that ends with any failed test exits non-zero and the executor fails the suite.
-The `E2E_RESULT` lines the Go runner parses are unaffected — they are printed,
-not accumulated in a variable.
+That holds however the file ends — a guard that records a failure and then calls
+`exit 0` to stop the file still fails it. The `E2E_RESULT` lines the Go runner
+parses are unaffected — they are printed, not accumulated in a variable.
To clean up after a scenario, define `scenario_cleanup`; the executor calls it
when the file finishes, including when the file exits early:
diff --git a/tests/e2e/executor_test.go b/tests/e2e/executor_test.go
new file mode 100644
index 00000000..0604317f
--- /dev/null
+++ b/tests/e2e/executor_test.go
@@ -0,0 +1,181 @@
+package e2e
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+)
+
+// runProbeSuite drives the real tests/e2e/run.sh over throwaway scenario files. It needs
+// no Docker and no server: the readiness loop skips a target whose variable is unset, so
+// the executor's own accounting is exercised in isolation from everything it usually
+// coordinates.
+func runProbeSuite(t *testing.T, scenarios map[string]string, order ...string) (string, int) {
+ t.Helper()
+
+ dir := t.TempDir()
+ for name, body := range scenarios {
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ args := []string{"run.sh"}
+ for _, name := range order {
+ args = append(args, "scenario="+name)
+ }
+ cmd := exec.Command("bash", args...)
+ cmd.Dir = "."
+ cmd.Env = append(os.Environ(),
+ "E2E_HELPER=api",
+ "E2E_SCENARIO_DIR="+dir,
+ "E2E_REQUIRED_COMMANDS=echo",
+ "E2E_READY_TARGETS=PROBE_TARGET_DELIBERATELY_UNSET",
+ "E2E_SUMMARY_TITLE=probe",
+ "E2E_TEST_FILTER=",
+ "E2E_LOGS=hide",
+ )
+ out, err := cmd.CombinedOutput()
+ code := 0
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ code = exitErr.ExitCode()
+ } else if err != nil {
+ t.Fatalf("running run.sh: %v\n%s", err, out)
+ }
+ return string(out), code
+}
+
+const passingScenario = `
+start_test "probe passes"
+pass_assert "ok"
+end_test
+`
+
+// The executor decides a scenario's verdict from TESTS_FAILED, which the subshell cannot
+// write back to the parent. Every way a scenario file can end has to reach that verdict —
+// including the ways that skip the last statement.
+func TestScenarioVerdictSurvivesEveryWayAFileCanEnd(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("the executor under test is a bash script")
+ }
+
+ for _, tc := range []struct {
+ name string
+ body string
+ wantFail bool
+ }{
+ {
+ name: "passing file",
+ body: passingScenario,
+ wantFail: false,
+ },
+ {
+ name: "failed assertion, file ends normally",
+ body: "\nstart_test \"probe fails\"\nfail_assert \"deliberate\"\nend_test\n",
+ wantFail: true,
+ },
+ {
+ // The shape that shipped broken: a guard that records a failure and then
+ // exits 0 to stop the file. cli/library-mode-smoke.sh does exactly this when
+ // its binary is missing from the runner image, so a lane could print a failed
+ // assertion and still go green.
+ name: "failed assertion then an explicit exit 0",
+ body: "\nstart_test \"probe fails then bails\"\nfail_assert \"deliberate\"\nend_test\nexit 0\n",
+ wantFail: true,
+ },
+ {
+ name: "explicit non-zero exit with nothing recorded",
+ body: "\nstart_test \"probe bails hard\"\npass_assert \"ok\"\nend_test\nexit 1\n",
+ wantFail: true,
+ },
+ {
+ // A scenario whose last statement happens to return non-zero has not failed
+ // a test; only recorded failures decide the verdict.
+ name: "last statement returns non-zero but nothing failed",
+ body: passingScenario + "\ngrep -q nothing-here /dev/null\n",
+ wantFail: false,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ out, code := runProbeSuite(t, map[string]string{"probe.sh": tc.body}, "probe.sh")
+ if failed := code != 0; failed != tc.wantFail {
+ t.Errorf("suite exit = %d (failed=%v), want failed=%v; a scenario's recorded failures must decide its verdict\n%s",
+ code, failed, tc.wantFail, out)
+ }
+ })
+ }
+}
+
+// A file that dies mid-way must take nothing with it: not the files after it, not its
+// cleanup, and not its variables.
+func TestOneScenarioDyingLeavesTheRestOfTheSuiteIntact(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("the executor under test is a bash script")
+ }
+
+ dying := `
+leaked_var="from-first"
+leaked_fn() { echo leaked; }
+scenario_cleanup() { echo PROBE-CLEANUP-RAN; }
+
+start_test "first scenario fails then dies"
+fail_assert "deliberate"
+end_test
+
+exit 1
+
+start_test "never reached"
+pass_assert "unreachable"
+end_test
+`
+ following := `
+start_test "second scenario runs anyway"
+if [ -n "${leaked_var:-}" ]; then
+ fail_assert "leaked_var crossed the file boundary: ${leaked_var}"
+else
+ pass_assert "no variable crossed the file boundary"
+fi
+if declare -F leaked_fn >/dev/null 2>&1; then
+ fail_assert "leaked_fn crossed the file boundary"
+else
+ pass_assert "no function crossed the file boundary"
+fi
+end_test
+`
+ out, code := runProbeSuite(t,
+ map[string]string{"first.sh": dying, "second.sh": following},
+ "first.sh", "second.sh")
+
+ if code == 0 {
+ t.Errorf("suite passed despite a failed scenario:\n%s", out)
+ }
+ if !strings.Contains(out, "PROBE-CLEANUP-RAN") {
+ t.Errorf("scenario_cleanup did not run for a file that exited early:\n%s", out)
+ }
+ if !strings.Contains(out, "second scenario runs anyway") {
+ t.Errorf("the scenario after the dying one never ran, so one file still amputates the suite:\n%s", out)
+ }
+ if strings.Contains(out, "never reached") {
+ t.Errorf("execution continued past the exit inside the dying file:\n%s", out)
+ }
+ // The second scenario asserts the isolation itself, so its own verdict is the
+ // signal: a leak makes it record a failure rather than a pass.
+ if !hasResultLine(out, "passed", "[second] second scenario runs anyway") {
+ t.Errorf("the second scenario did not pass, so state crossed the subshell boundary:\n%s", out)
+ }
+}
+
+// hasResultLine looks for one machine-readable result line by status and test name. The
+// duration between them varies per run, so the two halves are matched separately rather
+// than as one literal.
+func hasResultLine(out, status, name string) bool {
+ for _, line := range strings.Split(out, "\n") {
+ if strings.HasPrefix(line, "E2E_RESULT\t"+status+"\t") && strings.HasSuffix(strings.TrimRight(line, "\r"), name) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh
index 21bb5c90..1d573bcd 100644
--- a/tests/e2e/run.sh
+++ b/tests/e2e/run.sh
@@ -205,6 +205,17 @@ source_filtered_scenario() {
return 0
}
+# finish_scenario is the scenario's verdict, and it runs from the subshell's EXIT trap
+# so that a scenario which calls `exit` cannot skip it. A file ending in
+# `fail_assert; end_test; exit 0` — a live shape in the smoke tier — otherwise reported a
+# counted failure as a passing file, which is the green-lane-that-stopped-checking this
+# card exists to prevent. Leaving the status alone when nothing failed keeps an explicit
+# non-zero exit intact.
+finish_scenario() {
+ run_scenario_cleanup
+ [ "${TESTS_FAILED:-0}" -eq 0 ] || exit 1
+}
+
# Each scenario file runs in its own subshell: file-scope variables, functions
# and traps die with the file, so no scenario can be reached by state another
# one left behind. The subshell cannot write TESTS_FAILED back, so it reports
@@ -214,7 +225,7 @@ run_scenario_file() {
(
CURRENT_SCENARIO_FILE="${script_name%.sh}"
TESTS_FAILED=0
- trap run_scenario_cleanup EXIT
+ trap finish_scenario EXIT
if [ -n "${TEST_FILTER}" ]; then
if ! source_filtered_scenario "${script_path}" "${TEST_FILTER}"; then
echo -e "${MUTED} no matching test in ${script_name}${NC}"
diff --git a/tests/e2e/scenarios/infra/stealth-extended.sh b/tests/e2e/scenarios/infra/stealth-extended.sh
index f020c8a5..d535dc41 100644
--- a/tests/e2e/scenarios/infra/stealth-extended.sh
+++ b/tests/e2e/scenarios/infra/stealth-extended.sh
@@ -490,70 +490,69 @@ if [ -n "$FULL_URL" ]; then
echo ""
echo -e "${BLUE}Testing FULL stealth mode (permissive instance)${NC}"
- ORIG_URL="$E2E_SERVER"
- E2E_SERVER="$FULL_URL"
+ permissive_full_stealth_checks() {
+ start_test "bot-detect-full: can navigate with permissive full stealth"
- start_test "bot-detect-full: can navigate with permissive full stealth"
+ pt_post /navigate "{\"url\":\"${FIXTURES_URL}/bot-detect.html\"}"
+ assert_ok "navigate to bot-detect fixture (permissive full stealth)"
- pt_post /navigate "{\"url\":\"${FIXTURES_URL}/bot-detect.html\"}"
- assert_ok "navigate to bot-detect fixture (permissive full stealth)"
+ end_test
- end_test
+ start_test "bot-detect-full: WebGL not SwiftShader (full stealth spoofs GPU)"
- start_test "bot-detect-full: WebGL not SwiftShader (full stealth spoofs GPU)"
+ pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl.getExtension(\"WEBGL_debug_renderer_info\"); return gl.getParameter(d.UNMASKED_RENDERER_WEBGL).toLowerCase().includes(\"swiftshader\"); } catch(e) { return false; } })()"}'
+ assert_json_eq "$RESULT" '.result' 'false' "WebGL renderer not SwiftShader"
- pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl.getExtension(\"WEBGL_debug_renderer_info\"); return gl.getParameter(d.UNMASKED_RENDERER_WEBGL).toLowerCase().includes(\"swiftshader\"); } catch(e) { return false; } })()"}'
- assert_json_eq "$RESULT" '.result' 'false' "WebGL renderer not SwiftShader"
-
- end_test
+ end_test
- start_test "bot-detect-full: WebGL renderer matches platform when observable"
+ start_test "bot-detect-full: WebGL renderer matches platform when observable"
- pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl && gl.getExtension(\"WEBGL_debug_renderer_info\"); const r = d ? gl.getParameter(d.UNMASKED_RENDERER_WEBGL) : \"\"; const p = navigator.platform || \"\"; const lower = r.toLowerCase(); const ok = !d || (!lower.includes(\"swiftshader\") && ((p === \"MacIntel\" && !lower.includes(\"direct3d\")) || (p.includes(\"Linux\") && !lower.includes(\"direct3d\")) || (p.includes(\"Win\") && lower.includes(\"direct3d\")))); return JSON.stringify({available: !!d, renderer: r, platform: p, ok}); } catch(e) { return JSON.stringify({available:false, renderer:\"\", platform:navigator.platform || \"\", ok:false}); } })()"}'
- webgl_json=$(echo "$RESULT" | jq -r '.result // "{}"')
- webgl_available=$(echo "$webgl_json" | jq -r '.available // false')
- webgl_ok=$(echo "$webgl_json" | jq -r '.ok // false')
- webgl_renderer=$(echo "$webgl_json" | jq -r '.renderer // ""')
- webgl_platform=$(echo "$webgl_json" | jq -r '.platform // ""')
- if [ "$webgl_available" = "true" ]; then
- if [ "$webgl_ok" = "true" ]; then
- echo -e " ${GREEN}✓${NC} WebGL renderer matches platform (${webgl_platform})"
- ((ASSERTIONS_PASSED++)) || true
+ pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl && gl.getExtension(\"WEBGL_debug_renderer_info\"); const r = d ? gl.getParameter(d.UNMASKED_RENDERER_WEBGL) : \"\"; const p = navigator.platform || \"\"; const lower = r.toLowerCase(); const ok = !d || (!lower.includes(\"swiftshader\") && ((p === \"MacIntel\" && !lower.includes(\"direct3d\")) || (p.includes(\"Linux\") && !lower.includes(\"direct3d\")) || (p.includes(\"Win\") && lower.includes(\"direct3d\")))); return JSON.stringify({available: !!d, renderer: r, platform: p, ok}); } catch(e) { return JSON.stringify({available:false, renderer:\"\", platform:navigator.platform || \"\", ok:false}); } })()"}'
+ webgl_json=$(echo "$RESULT" | jq -r '.result // "{}"')
+ webgl_available=$(echo "$webgl_json" | jq -r '.available // false')
+ webgl_ok=$(echo "$webgl_json" | jq -r '.ok // false')
+ webgl_renderer=$(echo "$webgl_json" | jq -r '.renderer // ""')
+ webgl_platform=$(echo "$webgl_json" | jq -r '.platform // ""')
+ if [ "$webgl_available" = "true" ]; then
+ if [ "$webgl_ok" = "true" ]; then
+ echo -e " ${GREEN}✓${NC} WebGL renderer matches platform (${webgl_platform})"
+ ((ASSERTIONS_PASSED++)) || true
+ else
+ echo -e " ${RED}✗${NC} WebGL renderer mismatches platform ${webgl_platform}: ${webgl_renderer}"
+ ((ASSERTIONS_FAILED++)) || true
+ fi
else
- echo -e " ${RED}✗${NC} WebGL renderer mismatches platform ${webgl_platform}: ${webgl_renderer}"
- ((ASSERTIONS_FAILED++)) || true
+ echo -e " ${MUTED}(skipped - WEBGL_debug_renderer_info unavailable)${NC}"
+ ((ASSERTIONS_PASSED++)) || true
fi
- else
- echo -e " ${MUTED}(skipped - WEBGL_debug_renderer_info unavailable)${NC}"
- ((ASSERTIONS_PASSED++)) || true
- fi
- end_test
+ end_test
- start_test "bot-detect-full: permissive full score check"
+ start_test "bot-detect-full: permissive full score check"
- if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then
- skip_test "cloak removes surfaces this composite score rewards (chrome.runtime/battery/outer dims)"
- else
- pt_post /evaluate '{"expression":"JSON.stringify(window.__botDetectScore || {})"}'
- score_json=$(echo "$RESULT" | jq -r '.result // "{}"')
- critical_passed=$(echo "$score_json" | jq -r '.critical // 0')
- critical_total=$(echo "$score_json" | jq -r '.criticalTotal // 0')
+ if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then
+ skip_test "cloak removes surfaces this composite score rewards (chrome.runtime/battery/outer dims)"
+ else
+ pt_post /evaluate '{"expression":"JSON.stringify(window.__botDetectScore || {})"}'
+ score_json=$(echo "$RESULT" | jq -r '.result // "{}"')
+ critical_passed=$(echo "$score_json" | jq -r '.critical // 0')
+ critical_total=$(echo "$score_json" | jq -r '.criticalTotal // 0')
- echo -e " ${MUTED}Score: critical ${critical_passed}/${critical_total} (full)${NC}"
+ echo -e " ${MUTED}Score: critical ${critical_passed}/${critical_total} (full)${NC}"
- if [ "$critical_passed" -ge "$critical_total" ]; then
- echo -e " ${GREEN}✓${NC} score meets full expectations"
- ((ASSERTIONS_PASSED++)) || true
- else
- echo -e " ${RED}✗${NC} score below full expectations (${critical_passed}/${critical_total})"
- ((ASSERTIONS_FAILED++)) || true
+ if [ "$critical_passed" -ge "$critical_total" ]; then
+ echo -e " ${GREEN}✓${NC} score meets full expectations"
+ ((ASSERTIONS_PASSED++)) || true
+ else
+ echo -e " ${RED}✗${NC} score below full expectations (${critical_passed}/${critical_total})"
+ ((ASSERTIONS_FAILED++)) || true
+ fi
fi
- fi
- end_test
+ end_test
+ }
- E2E_SERVER="$ORIG_URL"
+ with_server "$FULL_URL" permissive_full_stealth_checks
fi
echo ""