High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(e2e): decide a scenario's verdict from its EXIT trap so an explicit exit cannot skip it
Luigi Agostib92458762f50
diff --git a/TESTING.md b/TESTING.mdindex 049341b4..31588530 100644--- a/TESTING.md+++ b/TESTING.md@@ -225,8 +225,9 @@ itself.Failure accounting crosses the subshell boundary through exit status: a scenariothat ends with any failed test exits non-zero and the executor fails the suite.-The `E2E_RESULT` lines the Go runner parses are unaffected — they are printed,-not accumulated in a variable.+That holds however the file ends — a guard that records a failure and then calls+`exit 0` to stop the file still fails it. The `E2E_RESULT` lines the Go runner+parses are unaffected — they are printed, not accumulated in a variable.To clean up after a scenario, define `scenario_cleanup`; the executor calls itwhen the file finishes, including when the file exits early:diff --git a/tests/e2e/executor_test.go b/tests/e2e/executor_test.gonew file mode 100644index 00000000..0604317f--- /dev/null+++ b/tests/e2e/executor_test.go@@ -0,0 +1,181 @@+package e2e++import (+ "os"+ "os/exec"+ "path/filepath"+ "runtime"+ "strings"+ "testing"+)++// runProbeSuite drives the real tests/e2e/run.sh over throwaway scenario files. It needs+// no Docker and no server: the readiness loop skips a target whose variable is unset, so+// the executor's own accounting is exercised in isolation from everything it usually+// coordinates.+func runProbeSuite(t *testing.T, scenarios map[string]string, order ...string) (string, int) {+ t.Helper()++ dir := t.TempDir()+ for name, body := range scenarios {+ if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {+ t.Fatal(err)+ }+ }++ args := []string{"run.sh"}+ for _, name := range order {+ args = append(args, "scenario="+name)+ }+ cmd := exec.Command("bash", args...)+ cmd.Dir = "."+ cmd.Env = append(os.Environ(),+ "E2E_HELPER=api",+ "E2E_SCENARIO_DIR="+dir,+ "E2E_REQUIRED_COMMANDS=echo",+ "E2E_READY_TARGETS=PROBE_TARGET_DELIBERATELY_UNSET",+ "E2E_SUMMARY_TITLE=probe",+ "E2E_TEST_FILTER=",+ "E2E_LOGS=hide",+ )+ out, err := cmd.CombinedOutput()+ code := 0+ if exitErr, ok := err.(*exec.ExitError); ok {+ code = exitErr.ExitCode()+ } else if err != nil {+ t.Fatalf("running run.sh: %v\n%s", err, out)+ }+ return string(out), code+}++const passingScenario = `+start_test "probe passes"+pass_assert "ok"+end_test+`++// The executor decides a scenario's verdict from TESTS_FAILED, which the subshell cannot+// write back to the parent. Every way a scenario file can end has to reach that verdict —+// including the ways that skip the last statement.+func TestScenarioVerdictSurvivesEveryWayAFileCanEnd(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("the executor under test is a bash script")+ }++ for _, tc := range []struct {+ name string+ body string+ wantFail bool+ }{+ {+ name: "passing file",+ body: passingScenario,+ wantFail: false,+ },+ {+ name: "failed assertion, file ends normally",+ body: "\nstart_test \"probe fails\"\nfail_assert \"deliberate\"\nend_test\n",+ wantFail: true,+ },+ {+ // The shape that shipped broken: a guard that records a failure and then+ // exits 0 to stop the file. cli/library-mode-smoke.sh does exactly this when+ // its binary is missing from the runner image, so a lane could print a failed+ // assertion and still go green.+ name: "failed assertion then an explicit exit 0",+ body: "\nstart_test \"probe fails then bails\"\nfail_assert \"deliberate\"\nend_test\nexit 0\n",+ wantFail: true,+ },+ {+ name: "explicit non-zero exit with nothing recorded",+ body: "\nstart_test \"probe bails hard\"\npass_assert \"ok\"\nend_test\nexit 1\n",+ wantFail: true,+ },+ {+ // A scenario whose last statement happens to return non-zero has not failed+ // a test; only recorded failures decide the verdict.+ name: "last statement returns non-zero but nothing failed",+ body: passingScenario + "\ngrep -q nothing-here /dev/null\n",+ wantFail: false,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ out, code := runProbeSuite(t, map[string]string{"probe.sh": tc.body}, "probe.sh")+ if failed := code != 0; failed != tc.wantFail {+ t.Errorf("suite exit = %d (failed=%v), want failed=%v; a scenario's recorded failures must decide its verdict\n%s",+ code, failed, tc.wantFail, out)+ }+ })+ }+}++// A file that dies mid-way must take nothing with it: not the files after it, not its+// cleanup, and not its variables.+func TestOneScenarioDyingLeavesTheRestOfTheSuiteIntact(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("the executor under test is a bash script")+ }++ dying := `+leaked_var="from-first"+leaked_fn() { echo leaked; }+scenario_cleanup() { echo PROBE-CLEANUP-RAN; }++start_test "first scenario fails then dies"+fail_assert "deliberate"+end_test++exit 1++start_test "never reached"+pass_assert "unreachable"+end_test+`+ following := `+start_test "second scenario runs anyway"+if [ -n "${leaked_var:-}" ]; then+ fail_assert "leaked_var crossed the file boundary: ${leaked_var}"+else+ pass_assert "no variable crossed the file boundary"+fi+if declare -F leaked_fn >/dev/null 2>&1; then+ fail_assert "leaked_fn crossed the file boundary"+else+ pass_assert "no function crossed the file boundary"+fi+end_test+`+ out, code := runProbeSuite(t,+ map[string]string{"first.sh": dying, "second.sh": following},+ "first.sh", "second.sh")++ if code == 0 {+ t.Errorf("suite passed despite a failed scenario:\n%s", out)+ }+ if !strings.Contains(out, "PROBE-CLEANUP-RAN") {+ t.Errorf("scenario_cleanup did not run for a file that exited early:\n%s", out)+ }+ if !strings.Contains(out, "second scenario runs anyway") {+ t.Errorf("the scenario after the dying one never ran, so one file still amputates the suite:\n%s", out)+ }+ if strings.Contains(out, "never reached") {+ t.Errorf("execution continued past the exit inside the dying file:\n%s", out)+ }+ // The second scenario asserts the isolation itself, so its own verdict is the+ // signal: a leak makes it record a failure rather than a pass.+ if !hasResultLine(out, "passed", "[second] second scenario runs anyway") {+ t.Errorf("the second scenario did not pass, so state crossed the subshell boundary:\n%s", out)+ }+}++// hasResultLine looks for one machine-readable result line by status and test name. The+// duration between them varies per run, so the two halves are matched separately rather+// than as one literal.+func hasResultLine(out, status, name string) bool {+ for _, line := range strings.Split(out, "\n") {+ if strings.HasPrefix(line, "E2E_RESULT\t"+status+"\t") && strings.HasSuffix(strings.TrimRight(line, "\r"), name) {+ return true+ }+ }+ return false+}diff --git a/tests/e2e/run.sh b/tests/e2e/run.shindex 21bb5c90..1d573bcd 100644--- a/tests/e2e/run.sh+++ b/tests/e2e/run.sh@@ -205,6 +205,17 @@ source_filtered_scenario() {return 0}+# finish_scenario is the scenario's verdict, and it runs from the subshell's EXIT trap+# so that a scenario which calls `exit` cannot skip it. A file ending in+# `fail_assert; end_test; exit 0` — a live shape in the smoke tier — otherwise reported a+# counted failure as a passing file, which is the green-lane-that-stopped-checking this+# card exists to prevent. Leaving the status alone when nothing failed keeps an explicit+# non-zero exit intact.+finish_scenario() {+ run_scenario_cleanup+ [ "${TESTS_FAILED:-0}" -eq 0 ] || exit 1+}+# Each scenario file runs in its own subshell: file-scope variables, functions# and traps die with the file, so no scenario can be reached by state another# one left behind. The subshell cannot write TESTS_FAILED back, so it reports@@ -214,7 +225,7 @@ run_scenario_file() {(CURRENT_SCENARIO_FILE="${script_name%.sh}"TESTS_FAILED=0- trap run_scenario_cleanup EXIT+ trap finish_scenario EXITif [ -n "${TEST_FILTER}" ]; thenif ! source_filtered_scenario "${script_path}" "${TEST_FILTER}"; thenecho -e "${MUTED} no matching test in ${script_name}${NC}"diff --git a/tests/e2e/scenarios/infra/stealth-extended.sh b/tests/e2e/scenarios/infra/stealth-extended.shindex f020c8a5..d535dc41 100644--- a/tests/e2e/scenarios/infra/stealth-extended.sh+++ b/tests/e2e/scenarios/infra/stealth-extended.sh@@ -490,70 +490,69 @@ if [ -n "$FULL_URL" ]; thenecho ""echo -e "${BLUE}Testing FULL stealth mode (permissive instance)${NC}"- ORIG_URL="$E2E_SERVER"- E2E_SERVER="$FULL_URL"+ permissive_full_stealth_checks() {+ start_test "bot-detect-full: can navigate with permissive full stealth"- start_test "bot-detect-full: can navigate with permissive full stealth"+ pt_post /navigate "{\"url\":\"${FIXTURES_URL}/bot-detect.html\"}"+ assert_ok "navigate to bot-detect fixture (permissive full stealth)"- pt_post /navigate "{\"url\":\"${FIXTURES_URL}/bot-detect.html\"}"- assert_ok "navigate to bot-detect fixture (permissive full stealth)"+ end_test- end_test+ start_test "bot-detect-full: WebGL not SwiftShader (full stealth spoofs GPU)"- start_test "bot-detect-full: WebGL not SwiftShader (full stealth spoofs GPU)"+ pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl.getExtension(\"WEBGL_debug_renderer_info\"); return gl.getParameter(d.UNMASKED_RENDERER_WEBGL).toLowerCase().includes(\"swiftshader\"); } catch(e) { return false; } })()"}'+ assert_json_eq "$RESULT" '.result' 'false' "WebGL renderer not SwiftShader"- pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl.getExtension(\"WEBGL_debug_renderer_info\"); return gl.getParameter(d.UNMASKED_RENDERER_WEBGL).toLowerCase().includes(\"swiftshader\"); } catch(e) { return false; } })()"}'- assert_json_eq "$RESULT" '.result' 'false' "WebGL renderer not SwiftShader"-- end_test+ end_test- start_test "bot-detect-full: WebGL renderer matches platform when observable"+ start_test "bot-detect-full: WebGL renderer matches platform when observable"- pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl && gl.getExtension(\"WEBGL_debug_renderer_info\"); const r = d ? gl.getParameter(d.UNMASKED_RENDERER_WEBGL) : \"\"; const p = navigator.platform || \"\"; const lower = r.toLowerCase(); const ok = !d || (!lower.includes(\"swiftshader\") && ((p === \"MacIntel\" && !lower.includes(\"direct3d\")) || (p.includes(\"Linux\") && !lower.includes(\"direct3d\")) || (p.includes(\"Win\") && lower.includes(\"direct3d\")))); return JSON.stringify({available: !!d, renderer: r, platform: p, ok}); } catch(e) { return JSON.stringify({available:false, renderer:\"\", platform:navigator.platform || \"\", ok:false}); } })()"}'- webgl_json=$(echo "$RESULT" | jq -r '.result // "{}"')- webgl_available=$(echo "$webgl_json" | jq -r '.available // false')- webgl_ok=$(echo "$webgl_json" | jq -r '.ok // false')- webgl_renderer=$(echo "$webgl_json" | jq -r '.renderer // ""')- webgl_platform=$(echo "$webgl_json" | jq -r '.platform // ""')- if [ "$webgl_available" = "true" ]; then- if [ "$webgl_ok" = "true" ]; then- echo -e " ${GREEN}✓${NC} WebGL renderer matches platform (${webgl_platform})"- ((ASSERTIONS_PASSED++)) || true+ pt_post /evaluate '{"expression":"(() => { try { const c = document.createElement(\"canvas\"); const gl = c.getContext(\"webgl\"); const d = gl && gl.getExtension(\"WEBGL_debug_renderer_info\"); const r = d ? gl.getParameter(d.UNMASKED_RENDERER_WEBGL) : \"\"; const p = navigator.platform || \"\"; const lower = r.toLowerCase(); const ok = !d || (!lower.includes(\"swiftshader\") && ((p === \"MacIntel\" && !lower.includes(\"direct3d\")) || (p.includes(\"Linux\") && !lower.includes(\"direct3d\")) || (p.includes(\"Win\") && lower.includes(\"direct3d\")))); return JSON.stringify({available: !!d, renderer: r, platform: p, ok}); } catch(e) { return JSON.stringify({available:false, renderer:\"\", platform:navigator.platform || \"\", ok:false}); } })()"}'+ webgl_json=$(echo "$RESULT" | jq -r '.result // "{}"')+ webgl_available=$(echo "$webgl_json" | jq -r '.available // false')+ webgl_ok=$(echo "$webgl_json" | jq -r '.ok // false')+ webgl_renderer=$(echo "$webgl_json" | jq -r '.renderer // ""')+ webgl_platform=$(echo "$webgl_json" | jq -r '.platform // ""')+ if [ "$webgl_available" = "true" ]; then+ if [ "$webgl_ok" = "true" ]; then+ echo -e " ${GREEN}✓${NC} WebGL renderer matches platform (${webgl_platform})"+ ((ASSERTIONS_PASSED++)) || true+ else+ echo -e " ${RED}✗${NC} WebGL renderer mismatches platform ${webgl_platform}: ${webgl_renderer}"+ ((ASSERTIONS_FAILED++)) || true+ fielse- echo -e " ${RED}✗${NC} WebGL renderer mismatches platform ${webgl_platform}: ${webgl_renderer}"- ((ASSERTIONS_FAILED++)) || true+ echo -e " ${MUTED}(skipped - WEBGL_debug_renderer_info unavailable)${NC}"+ ((ASSERTIONS_PASSED++)) || truefi- else- echo -e " ${MUTED}(skipped - WEBGL_debug_renderer_info unavailable)${NC}"- ((ASSERTIONS_PASSED++)) || true- fi- end_test+ end_test- start_test "bot-detect-full: permissive full score check"+ start_test "bot-detect-full: permissive full score check"- if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then- skip_test "cloak removes surfaces this composite score rewards (chrome.runtime/battery/outer dims)"- else- pt_post /evaluate '{"expression":"JSON.stringify(window.__botDetectScore || {})"}'- score_json=$(echo "$RESULT" | jq -r '.result // "{}"')- critical_passed=$(echo "$score_json" | jq -r '.critical // 0')- critical_total=$(echo "$score_json" | jq -r '.criticalTotal // 0')+ if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then+ skip_test "cloak removes surfaces this composite score rewards (chrome.runtime/battery/outer dims)"+ else+ pt_post /evaluate '{"expression":"JSON.stringify(window.__botDetectScore || {})"}'+ score_json=$(echo "$RESULT" | jq -r '.result // "{}"')+ critical_passed=$(echo "$score_json" | jq -r '.critical // 0')+ critical_total=$(echo "$score_json" | jq -r '.criticalTotal // 0')- echo -e " ${MUTED}Score: critical ${critical_passed}/${critical_total} (full)${NC}"+ echo -e " ${MUTED}Score: critical ${critical_passed}/${critical_total} (full)${NC}"- if [ "$critical_passed" -ge "$critical_total" ]; then- echo -e " ${GREEN}✓${NC} score meets full expectations"- ((ASSERTIONS_PASSED++)) || true- else- echo -e " ${RED}✗${NC} score below full expectations (${critical_passed}/${critical_total})"- ((ASSERTIONS_FAILED++)) || true+ if [ "$critical_passed" -ge "$critical_total" ]; then+ echo -e " ${GREEN}✓${NC} score meets full expectations"+ ((ASSERTIONS_PASSED++)) || true+ else+ echo -e " ${RED}✗${NC} score below full expectations (${critical_passed}/${critical_total})"+ ((ASSERTIONS_FAILED++)) || true+ fifi- fi- end_test+ end_test+ }- E2E_SERVER="$ORIG_URL"+ with_server "$FULL_URL" permissive_full_stealth_checksfiecho ""