Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

5.6 KB
Raw Blame
package e2e

import (
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

// runProbeSuite drives the real tests/e2e/run.sh over throwaway scenario files. It needs
// no Docker and no server: the readiness loop skips a target whose variable is unset, so
// the executor's own accounting is exercised in isolation from everything it usually
// coordinates.
func runProbeSuite(t *testing.T, scenarios map[string]string, order ...string) (string, int) {
	t.Helper()

	dir := t.TempDir()
	for name, body := range scenarios {
		if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
			t.Fatal(err)
		}
	}

	args := []string{"run.sh"}
	for _, name := range order {
		args = append(args, "scenario="+name)
	}
	cmd := exec.Command("bash", args...)
	cmd.Dir = "."
	cmd.Env = append(os.Environ(),
		"E2E_HELPER=api",
		"E2E_SCENARIO_DIR="+dir,
		"E2E_REQUIRED_COMMANDS=echo",
		"E2E_READY_TARGETS=PROBE_TARGET_DELIBERATELY_UNSET",
		"E2E_SUMMARY_TITLE=probe",
		"E2E_TEST_FILTER=",
		"E2E_LOGS=hide",
	)
	out, err := cmd.CombinedOutput()
	code := 0
	if exitErr, ok := err.(*exec.ExitError); ok {
		code = exitErr.ExitCode()
	} else if err != nil {
		t.Fatalf("running run.sh: %v\n%s", err, out)
	}
	return string(out), code
}

const passingScenario = `
start_test "probe passes"
pass_assert "ok"
end_test
`

// The executor decides a scenario's verdict from TESTS_FAILED, which the subshell cannot
// write back to the parent. Every way a scenario file can end has to reach that verdict —
// including the ways that skip the last statement.
func TestScenarioVerdictSurvivesEveryWayAFileCanEnd(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("the executor under test is a bash script")
	}

	for _, tc := range []struct {
		name     string
		body     string
		wantFail bool
	}{
		{
			name:     "passing file",
			body:     passingScenario,
			wantFail: false,
		},
		{
			name:     "failed assertion, file ends normally",
			body:     "\nstart_test \"probe fails\"\nfail_assert \"deliberate\"\nend_test\n",
			wantFail: true,
		},
		{
			// The shape that shipped broken: a guard that records a failure and then
			// exits 0 to stop the file. cli/library-mode-smoke.sh does exactly this when
			// its binary is missing from the runner image, so a lane could print a failed
			// assertion and still go green.
			name:     "failed assertion then an explicit exit 0",
			body:     "\nstart_test \"probe fails then bails\"\nfail_assert \"deliberate\"\nend_test\nexit 0\n",
			wantFail: true,
		},
		{
			name:     "explicit non-zero exit with nothing recorded",
			body:     "\nstart_test \"probe bails hard\"\npass_assert \"ok\"\nend_test\nexit 1\n",
			wantFail: true,
		},
		{
			// A scenario whose last statement happens to return non-zero has not failed
			// a test; only recorded failures decide the verdict.
			name:     "last statement returns non-zero but nothing failed",
			body:     passingScenario + "\ngrep -q nothing-here /dev/null\n",
			wantFail: false,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			out, code := runProbeSuite(t, map[string]string{"probe.sh": tc.body}, "probe.sh")
			if failed := code != 0; failed != tc.wantFail {
				t.Errorf("suite exit = %d (failed=%v), want failed=%v; a scenario's recorded failures must decide its verdict\n%s",
					code, failed, tc.wantFail, out)
			}
		})
	}
}

// A file that dies mid-way must take nothing with it: not the files after it, not its
// cleanup, and not its variables.
func TestOneScenarioDyingLeavesTheRestOfTheSuiteIntact(t *testing.T) {
	if runtime.GOOS == "windows" {
		t.Skip("the executor under test is a bash script")
	}

	dying := `
leaked_var="from-first"
leaked_fn() { echo leaked; }
scenario_cleanup() { echo PROBE-CLEANUP-RAN; }

start_test "first scenario fails then dies"
fail_assert "deliberate"
end_test

exit 1

start_test "never reached"
pass_assert "unreachable"
end_test
`
	following := `
start_test "second scenario runs anyway"
if [ -n "${leaked_var:-}" ]; then
  fail_assert "leaked_var crossed the file boundary: ${leaked_var}"
else
  pass_assert "no variable crossed the file boundary"
fi
if declare -F leaked_fn >/dev/null 2>&1; then
  fail_assert "leaked_fn crossed the file boundary"
else
  pass_assert "no function crossed the file boundary"
fi
end_test
`
	out, code := runProbeSuite(t,
		map[string]string{"first.sh": dying, "second.sh": following},
		"first.sh", "second.sh")

	if code == 0 {
		t.Errorf("suite passed despite a failed scenario:\n%s", out)
	}
	if !strings.Contains(out, "PROBE-CLEANUP-RAN") {
		t.Errorf("scenario_cleanup did not run for a file that exited early:\n%s", out)
	}
	if !strings.Contains(out, "second scenario runs anyway") {
		t.Errorf("the scenario after the dying one never ran, so one file still amputates the suite:\n%s", out)
	}
	if strings.Contains(out, "never reached") {
		t.Errorf("execution continued past the exit inside the dying file:\n%s", out)
	}
	// The second scenario asserts the isolation itself, so its own verdict is the
	// signal: a leak makes it record a failure rather than a pass.
	if !hasResultLine(out, "passed", "[second] second scenario runs anyway") {
		t.Errorf("the second scenario did not pass, so state crossed the subshell boundary:\n%s", out)
	}
}

// hasResultLine looks for one machine-readable result line by status and test name. The
// duration between them varies per run, so the two halves are matched separately rather
// than as one literal.
func hasResultLine(out, status, name string) bool {
	for _, line := range strings.Split(out, "\n") {
		if strings.HasPrefix(line, "E2E_RESULT\t"+status+"\t") && strings.HasSuffix(strings.TrimRight(line, "\r"), name) {
			return true
		}
	}
	return false
}