High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: verify cloak browser behavior in doctor
Luigi Agosti886be290f17d
diff --git a/cmd/pinchtab/cmd_doctor_test.go b/cmd/pinchtab/cmd_doctor_test.goindex 7796b52e..7a11d70a 100644--- a/cmd/pinchtab/cmd_doctor_test.go+++ b/cmd/pinchtab/cmd_doctor_test.go@@ -89,6 +89,15 @@ func TestBrowserOverviewCloakHintsMatchPresenceState(t *testing.T) {t.Fatalf("broken binary hint = %q, must not claim browser was found", browserOverviewHint(brokenPath))}+ unverified := base+ unverified.Checks = []doctor.CheckResult{+ {Name: "cloakbrowser_present", Status: doctor.StatusWarn, Detail: `/tmp/google-chrome: could not parse version from "not-a-version-string"`},+ {Name: "cdp_reachable", Status: doctor.StatusFail},+ }+ if hint := browserOverviewHint(unverified); strings.Contains(hint, "was found") {+ t.Fatalf("unverified binary hint = %q, must not claim CloakBrowser was found", hint)+ }+cdpFailure := basecdpFailure.Checks = []doctor.CheckResult{{Name: "cloakbrowser_present", Status: doctor.StatusPass, Detail: "/opt/cloak/chrome -> 130.0.0 (>= 120.0.0)"},diff --git a/internal/browsers/chrome/cdp_probe.go b/internal/browsers/chrome/cdp_probe.goindex 62d175f8..f06be209 100644--- a/internal/browsers/chrome/cdp_probe.go+++ b/internal/browsers/chrome/cdp_probe.go@@ -3,6 +3,7 @@ package chromeimport ("bufio""context"+ "encoding/json""errors""fmt""io"@@ -11,11 +12,14 @@ import ("os/exec""regexp""time"++ "github.com/chromedp/chromedp")type CDPProbeResult struct {- Port int- VersionURL string+ Port int+ VersionURL string+ WebSocketURL string}var devtoolsRe = regexp.MustCompile(`DevTools listening on ws://[^:]+:(\d+)/`)@@ -23,6 +27,26 @@ var devtoolsRe = regexp.MustCompile(`DevTools listening on ws://[^:]+:(\d+)/`)// LaunchAndProbe starts binary headless, waits for the DevTools banner,// confirms /json/version responds, and tears the browser down before return.func LaunchAndProbe(ctx context.Context, binary string, extraArgs []string, timeout time.Duration) (CDPProbeResult, error) {+ return launchAndProbe(ctx, binary, extraArgs, timeout, nil)+}++// LaunchAndEvaluate performs the launch probe and evaluates one expression+// before tearing the browser down. It lets provider diagnostics verify behavior+// that a version string or a successfully opened CDP port cannot prove.+func LaunchAndEvaluate(ctx context.Context, binary string, extraArgs []string, timeout time.Duration, expression string, value any) (CDPProbeResult, error) {+ return launchAndProbe(ctx, binary, extraArgs, timeout, func(ctx context.Context, result CDPProbeResult) error {+ allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, result.WebSocketURL, chromedp.NoModifyURL)+ defer allocCancel()+ browserCtx, browserCancel := chromedp.NewContext(allocCtx)+ defer browserCancel()+ if err := chromedp.Run(browserCtx, chromedp.Evaluate(expression, value)); err != nil {+ return fmt.Errorf("evaluate %q: %w", expression, err)+ }+ return nil+ })+}++func launchAndProbe(ctx context.Context, binary string, extraArgs []string, timeout time.Duration, inspect func(context.Context, CDPProbeResult) error) (CDPProbeResult, error) {cctx, cancel := context.WithTimeout(ctx, timeout)defer cancel()@@ -90,10 +114,17 @@ func LaunchAndProbe(ctx context.Context, binary string, extraArgs []string, time}versionURL := fmt.Sprintf("http://127.0.0.1:%d/json/version", port)- if err := probeCDPVersionWithRetry(cctx, versionURL); err != nil {- return CDPProbeResult{Port: port, VersionURL: versionURL}, err+ webSocketURL, err := probeCDPVersionWithRetry(cctx, versionURL)+ result := CDPProbeResult{Port: port, VersionURL: versionURL, WebSocketURL: webSocketURL}+ if err != nil {+ return result, err}- return CDPProbeResult{Port: port, VersionURL: versionURL}, nil+ if inspect != nil {+ if err := inspect(cctx, result); err != nil {+ return result, fmt.Errorf("inspect launched browser: %w", err)+ }+ }+ return result, nil}func scrapeDevtoolsPort(r io.ReadCloser, portCh chan<- int, errCh chan<- error) {@@ -124,7 +155,7 @@ func scrapeDevtoolsPort(r io.ReadCloser, portCh chan<- int, errCh chan<- error)errCh <- errors.New("stderr closed before DevTools banner appeared")}-func probeCDPVersionWithRetry(ctx context.Context, url string) error {+func probeCDPVersionWithRetry(ctx context.Context, url string) (string, error) {deadline, ok := ctx.Deadline()if !ok {deadline = time.Now().Add(5 * time.Second)@@ -136,11 +167,11 @@ func probeCDPVersionWithRetry(ctx context.Context, url string) error {if lastErr == nil {lastErr = errors.New("deadline exceeded")}- return fmt.Errorf("probe %s: %w", url, lastErr)+ return "", fmt.Errorf("probe %s: %w", url, lastErr)}req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)if err != nil {- return fmt.Errorf("probe %s: %w", url, err)+ return "", fmt.Errorf("probe %s: %w", url, err)}resp, err := client.Do(req)if err != nil {@@ -148,11 +179,25 @@ func probeCDPVersionWithRetry(ctx context.Context, url string) error {time.Sleep(100 * time.Millisecond)continue}+ if resp.StatusCode != http.StatusOK {+ _ = resp.Body.Close()+ lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)+ time.Sleep(100 * time.Millisecond)+ continue+ }+ var version struct {+ WebSocketURL string `json:"webSocketDebuggerUrl"`+ }+ err = json.NewDecoder(resp.Body).Decode(&version)_ = resp.Body.Close()- if resp.StatusCode == http.StatusOK {- return nil+ if err == nil && version.WebSocketURL != "" {+ return version.WebSocketURL, nil+ }+ if err != nil {+ lastErr = fmt.Errorf("decode response: %w", err)+ } else {+ lastErr = errors.New("response missing webSocketDebuggerUrl")}- lastErr = fmt.Errorf("HTTP %d", resp.StatusCode)time.Sleep(100 * time.Millisecond)}}diff --git a/internal/browsers/cloak/cloak_test.go b/internal/browsers/cloak/cloak_test.goindex bf4f56f6..9047cdfd 100644--- a/internal/browsers/cloak/cloak_test.go+++ b/internal/browsers/cloak/cloak_test.go@@ -2,9 +2,6 @@ package cloak_testimport ("context"- "os"- "path/filepath"- "runtime""slices""strings""testing"@@ -768,43 +765,6 @@ func TestCloak_CanHandle_StateChanging(t *testing.T) {}}-// TestCloakPresentHonorsBrowserBinaryOverride is the original report: the-// CloakBrowser installer drops its Chromium in a versioned dir that isn't on the-// static probe list, so a configured browser.binary must PASS cloakbrowser_present-// rather than report a false "cloakbrowser not found" FAIL.-func TestCloakPresentHonorsBrowserBinaryOverride(t *testing.T) {- if runtime.GOOS == "windows" {- t.Skip("cloakbrowser_present is skipped on windows")- }- dir := t.TempDir()- fake := filepath.Join(dir, "chromium-145.0.0.0", "chrome")- if err := os.MkdirAll(filepath.Dir(fake), 0o755); err != nil {- t.Fatal(err)- }- if err := os.WriteFile(fake, []byte("#!/bin/sh\necho 'Chromium 145.0.0.0'\n"), 0o755); err != nil {- t.Fatal(err)- }-- checks := (&cloak.Browser{}).DoctorChecks(browsers.TargetConfig{})- var present *browsers.DoctorCheck- for i := range checks {- if checks[i].ID == "cloakbrowser_present" {- present = &checks[i]- break- }- }- if present == nil {- t.Fatal("cloakbrowser_present check not found")- }- res := present.Fn(context.Background(), &browsers.DoctorEnv{Binary: fake})- if res.Status != browsers.DoctorPass {- t.Fatalf("cloakbrowser_present with override status = %v, want DoctorPass; detail: %s", res.Status, res.Detail)- }- if !strings.Contains(res.Detail, fake) {- t.Errorf("expected detail to reference the override path %q; got %q", fake, res.Detail)- }-}-func TestCloakHandleDecisionsCheck(t *testing.T) {b := &cloak.Browser{}checks := b.DoctorChecks(browsers.TargetConfig{})diff --git a/internal/browsers/cloak/doctor.go b/internal/browsers/cloak/doctor.goindex ccf5f856..e16e66bc 100644--- a/internal/browsers/cloak/doctor.go+++ b/internal/browsers/cloak/doctor.go@@ -13,6 +13,9 @@ import ()const cloakMinVersion = "120.0.0"+const cloakIdentityPlatform = "windows"++var launchAndEvaluate = chrome.LaunchAndEvaluate// DoctorChecks overrides the inherited Chrome DoctorChecks method.func (Browser) DoctorChecks(_ browsers.TargetConfig) []browsers.DoctorCheck {@@ -121,6 +124,27 @@ func cloakPresenceCheck(ctx context.Context, cfg interface{}) browsers.DoctorCheDetail: fmt.Sprintf("%s: could not parse version from %q", found, line),}}+ var platform string+ identityArgs := []string{"--fingerprint-platform=" + cloakIdentityPlatform}+ if env, ok := cfg.(*browsers.DoctorEnv); ok && env != nil && env.NoSandbox {+ identityArgs = append(identityArgs, "--no-sandbox")+ }+ _, err = launchAndEvaluate(ctx, found, identityArgs, 10*time.Second, "navigator.platform", &platform)+ if err != nil || platform != "Win32" {+ reason := fmt.Sprintf("fingerprint platform probe returned %q, want Win32", platform)+ if err != nil {+ reason = err.Error()+ }+ prefix := fmt.Sprintf("binary %q", found)+ if override != "" {+ prefix = fmt.Sprintf("configured browser.binary %q", found)+ }+ return browsers.DoctorCheckResult{+ Status: browsers.DoctorWarn,+ Detail: fmt.Sprintf("%s reports version %s but did not exhibit CloakBrowser fingerprint behavior: %s", prefix, token, reason),+ Err: err,+ }+ }if browserprobe.CompareSemver(token, cloakMinVersion) < 0 {return browsers.DoctorCheckResult{Status: browsers.DoctorWarn,@@ -133,13 +157,9 @@ func cloakPresenceCheck(ctx context.Context, cfg interface{}) browsers.DoctorCheDetail: fmt.Sprintf("CloakBrowser found at %s -> %s, but browser.binary is unset", found, token),}}- // A configured browser.binary is trusted on version alone: CloakBrowser's- // forked Chromium reports "Chromium <ver>" from --version, indistinguishable- // from vanilla Chromium, so there is no product string to gate identity on.- // Say so plainly rather than implying the binary was verified to be CloakBrowser.return browsers.DoctorCheckResult{Status: browsers.DoctorPass,- Detail: fmt.Sprintf("%s -> %s (>= %s); trusted from browser.binary without CloakBrowser identity check", found, token, cloakMinVersion),+ Detail: fmt.Sprintf("%s -> %s (>= %s)", found, token, cloakMinVersion),}}@@ -152,7 +172,11 @@ func cdpReachableCheck(ctx context.Context, cfg interface{}) browsers.DoctorChecif bin == "" {return browsers.DoctorCheckResult{Status: browsers.DoctorSkip, Detail: "skipped — no browser.binary set (see cloakbrowser_present)"}}- res, err := chrome.LaunchAndProbe(ctx, bin, nil, 10*time.Second)+ var args []string+ if env.NoSandbox {+ args = append(args, "--no-sandbox")+ }+ res, err := chrome.LaunchAndProbe(ctx, bin, args, 10*time.Second)if err != nil {return browsers.DoctorCheckResult{Status: browsers.DoctorFail, Detail: err.Error(), Err: err}}@@ -192,6 +216,9 @@ func fingerprintFlagsCheck(ctx context.Context, cfg interface{}) browsers.Doctorif len(fpFlags) == 0 {return browsers.DoctorCheckResult{Status: browsers.DoctorSkip, Detail: "no cloak fingerprint flags configured"}}+ if env.NoSandbox {+ fpFlags = append(fpFlags, "--no-sandbox")+ }res, err := chrome.LaunchAndProbe(ctx, bin, fpFlags, 10*time.Second)if err != nil {return browsers.DoctorCheckResult{diff --git a/internal/browsers/cloak/doctor_test.go b/internal/browsers/cloak/doctor_test.gonew file mode 100644index 00000000..7e1bfab1--- /dev/null+++ b/internal/browsers/cloak/doctor_test.go@@ -0,0 +1,98 @@+package cloak++import (+ "context"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"+ "time"++ "github.com/pinchtab/pinchtab/internal/browsers"+ "github.com/pinchtab/pinchtab/internal/browsers/chrome"+)++func configuredPresenceCheck(t *testing.T, binary, versionOutput, platform string) browsers.DoctorCheckResult {+ t.Helper()+ if runtime.GOOS == "windows" {+ t.Skip("cloakbrowser_present is skipped on windows")+ }+ if err := os.MkdirAll(filepath.Dir(binary), 0o755); err != nil {+ t.Fatal(err)+ }+ contents := []byte("#!/bin/sh\necho '" + versionOutput + "'\n")+ if err := os.WriteFile(binary, contents, 0o755); err != nil {+ t.Fatal(err)+ }++ original := launchAndEvaluate+ launchAndEvaluate = func(_ context.Context, gotBinary string, args []string, _ time.Duration, expression string, value any) (chrome.CDPProbeResult, error) {+ if gotBinary != binary {+ t.Fatalf("binary = %q, want %q", gotBinary, binary)+ }+ if len(args) != 1 || args[0] != "--fingerprint-platform=windows" {+ t.Fatalf("identity args = %v, want controlled Windows fingerprint probe", args)+ }+ if expression != "navigator.platform" {+ t.Fatalf("expression = %q, want navigator.platform", expression)+ }+ result, ok := value.(*string)+ if !ok {+ t.Fatalf("probe result type = %T, want *string", value)+ }+ *result = platform+ return chrome.CDPProbeResult{Port: 9222}, nil+ }+ t.Cleanup(func() { launchAndEvaluate = original })++ return cloakPresenceCheck(context.Background(), &browsers.DoctorEnv{Binary: binary})+}++func TestCloakPresenceAcceptsBehaviorInsteadOfPathNaming(t *testing.T) {+ binary := filepath.Join(t.TempDir(), "chromium-145.0.0.0", "chrome")+ result := configuredPresenceCheck(t, binary, "Chromium 145.0.0.0", "Win32")+ if result.Status != browsers.DoctorPass {+ t.Fatalf("status = %v, want pass: %s", result.Status, result.Detail)+ }+}++func TestCloakPresenceWarnsWhenBehavioralCloakIsDiscoveredButUnconfigured(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("cloakbrowser_present is skipped on windows")+ }+ dir := t.TempDir()+ binary := filepath.Join(dir, "cloakbrowser")+ if err := os.WriteFile(binary, []byte("#!/bin/sh\necho 'Chromium 145.0.0.0'\n"), 0o755); err != nil {+ t.Fatal(err)+ }+ t.Setenv("PATH", dir)+ t.Setenv("HOME", t.TempDir())++ original := launchAndEvaluate+ launchAndEvaluate = func(_ context.Context, _ string, _ []string, _ time.Duration, _ string, value any) (chrome.CDPProbeResult, error) {+ *value.(*string) = "Win32"+ return chrome.CDPProbeResult{Port: 9222}, nil+ }+ t.Cleanup(func() { launchAndEvaluate = original })++ result := cloakPresenceCheck(context.Background(), &browsers.DoctorEnv{})+ if result.Status != browsers.DoctorWarn || !strings.Contains(result.Detail, "browser.binary is unset") {+ t.Fatalf("result = %v %q, want verified discovery setup warning", result.Status, result.Detail)+ }+}++func TestCloakPresenceRejectsChromeEvenUnderCloakNamedDirectory(t *testing.T) {+ for _, version := range []string{"145.0.7632.109", "118.0.0"} {+ t.Run(version, func(t *testing.T) {+ binary := filepath.Join(t.TempDir(), "cloakbrowser", "google-chrome")+ result := configuredPresenceCheck(t, binary, "Google Chrome "+version, "Linux x86_64")+ if result.Status == browsers.DoctorPass {+ t.Fatalf("ordinary Chrome passed cloakbrowser_present: %s", result.Detail)+ }+ if !strings.Contains(result.Detail, "did not exhibit CloakBrowser fingerprint behavior") {+ t.Fatalf("detail = %q, want behavioral identity warning", result.Detail)+ }+ })+ }+}diff --git a/internal/browsers/config.go b/internal/browsers/config.goindex a922c729..b7b24157 100644--- a/internal/browsers/config.go+++ b/internal/browsers/config.go@@ -114,6 +114,7 @@ type DoctorCheck struct {// DoctorEnv avoids browser sub-packages importing the config package.type DoctorEnv struct {- Binary string- Cloak CloakFingerprint+ Binary string+ Cloak CloakFingerprint+ NoSandbox bool}diff --git a/internal/doctor/checks_system_test.go b/internal/doctor/checks_system_test.goindex f4acaebb..8e4d793e 100644--- a/internal/doctor/checks_system_test.go+++ b/internal/doctor/checks_system_test.go@@ -253,15 +253,15 @@ func TestCloakBrowserPresent_FailWhenAbsentAndConfigured(t *testing.T) {}}-func TestCloakBrowserPresent_WarnsWhenDiscoveredButUnconfigured(t *testing.T) {+func TestCloakBrowserPresent_DoesNotTrustDiscoveredNameAlone(t *testing.T) {withStubBinary(t, "cloakbrowser", "CloakBrowser 130.0.6723.91")t.Setenv("HOME", t.TempDir())r := runCloakPresent(t)if r.Status != StatusWarn {t.Fatalf("status = %v want warn; detail=%q err=%v", r.Status, r.Detail, r.Err)}- if !strings.Contains(r.Detail, "browser.binary is unset") {- t.Fatalf("detail = %q, want browser.binary setup guidance", r.Detail)+ if !strings.Contains(r.Detail, "did not exhibit CloakBrowser fingerprint behavior") {+ t.Fatalf("detail = %q, want behavioral identity warning", r.Detail)}}diff --git a/internal/doctor/runner.go b/internal/doctor/runner.goindex f2844321..f08be99e 100644--- a/internal/doctor/runner.go+++ b/internal/doctor/runner.go@@ -3,10 +3,13 @@ package doctorimport ("context"+ "os"+ "runtime""strings""time""github.com/pinchtab/pinchtab/internal/browsers"+ "github.com/pinchtab/pinchtab/internal/browsers/runtimekit""github.com/pinchtab/pinchtab/internal/config")@@ -194,8 +197,10 @@ func buildDoctorEnv(cfg *config.RuntimeConfig) *browsers.DoctorEnv {if cfg == nil {return nil}+ _, containerErr := os.Stat("/.dockerenv")return &browsers.DoctorEnv{- Binary: strings.TrimSpace(cfg.BrowserBinary),+ Binary: strings.TrimSpace(cfg.BrowserBinary),+ NoSandbox: runtimekit.ChromeNeedsNoSandbox(runtime.GOOS, os.Geteuid(), containerErr == nil),Cloak: browsers.CloakFingerprint{FingerprintSeed: cfg.Cloak.FingerprintSeed,Platform: cfg.Cloak.Platform,diff --git a/tests/e2e/fixtures/test-extension/content.js b/tests/e2e/fixtures/test-extension/content.jsnew file mode 100644index 00000000..c6298059--- /dev/null+++ b/tests/e2e/fixtures/test-extension/content.js@@ -0,0 +1 @@+document.documentElement.setAttribute("data-pinchtab-test-extension", "loaded");diff --git a/tests/e2e/scenarios/infra/system-basic.sh b/tests/e2e/scenarios/infra/system-basic.shindex 0a8df4b4..d330ac99 100644--- a/tests/e2e/scenarios/infra/system-basic.sh+++ b/tests/e2e/scenarios/infra/system-basic.sh@@ -142,77 +142,6 @@ end_testORCH_URL=$E2E_SERVERORIG_URL=$E2E_SERVER-assert_instance_logs_poll() {- local inst_id="$1"- local needle="$2"- local desc="$3"- local attempts="${4:-15}"- local delay="${5:-1}"-- local i- for i in $(seq 1 "$attempts"); do- E2E_SERVER=$ORCH_URL pt_get "/instances/${inst_id}/logs" >/dev/null- # Avoid a false negative under `set -o pipefail`: `grep -q` exits early- # after a match, which can SIGPIPE the writer side of a pipeline.- if [[ "$HTTP_STATUS" =~ ^2 ]] && grep -Fq -- "$needle" <<<"$RESULT"; then- echo -e " ${GREEN}✓${NC} $desc"- ((ASSERTIONS_PASSED++)) || true- return 0- fi- sleep "$delay"- done-- echo -e " ${RED}✗${NC} $desc (missing: $needle)"- ((ASSERTIONS_FAILED++)) || true- return 1-}--assert_instance_logs_poll_all() {- local inst_id="$1"- local desc="$2"- shift 2-- local attempts=15- local delay=1- local i needle ok- for i in $(seq 1 "$attempts"); do- E2E_SERVER=$ORCH_URL pt_get "/instances/${inst_id}/logs" >/dev/null- if [[ "$HTTP_STATUS" =~ ^2 ]]; then- ok=1- for needle in "$@"; do- if ! grep -Fq -- "$needle" <<<"$RESULT"; then- ok=0- break- fi- done- if [ "$ok" -eq 1 ]; then- echo -e " ${GREEN}✓${NC} $desc"- ((ASSERTIONS_PASSED++)) || true- return 0- fi- fi- sleep "$delay"- done-- echo -e " ${RED}✗${NC} $desc (missing fragments: $*)"- ((ASSERTIONS_FAILED++)) || true- return 1-}--print_extension_hints() {- local inst_id="${1:-}"- echo ""- echo " ${YELLOW}${BOLD}🔍 Troubleshooting Extension Failure:${NC}"- echo " - Check if /extensions/test-extension exists and is readable in the pinchtab container."- echo " - Check Manifest V3 host_permissions matches: [\"*://*/*\"]"- if [ -n "$inst_id" ]; then- E2E_SERVER=$ORCH_URL pt_get "/instances/${inst_id}/logs" >/dev/null- echo " - Recent instance log tail:"- printf '%s\n' "$RESULT" | tail -n 12 | sed 's/^/ /'- fi- echo ""-}-# --- T1: Default instance loads configured extension path ---start_test "Extension config: default instance loads configured extension path"@@ -229,29 +158,10 @@ fipt_post /navigate "{\"url\":\"${FIXTURES_URL}/index.html\"}"assert_ok "navigate"--DEFAULT_LOG_PASS=1-if [ -n "$DEFAULT_INST_ID" ] && [ "$DEFAULT_INST_ID" != "null" ]; then- assert_instance_logs_poll_all \- "$DEFAULT_INST_ID" \- "default instance logs configured extension path" \- "loading extensions" \- "paths=/extensions/test-extension"- DEFAULT_LOG_PASS=$?-- assert_instance_logs_poll \- "$DEFAULT_INST_ID" \- "browser initialized successfully" \- "default instance browser initialized"-fi--if [ $DEFAULT_LOG_PASS -ne 0 ]; then- print_extension_hints "$DEFAULT_INST_ID"- # Debug: dump full instance logs for analysis- echo " ${YELLOW}📋 Full instance logs:${NC}"- E2E_SERVER=$ORCH_URL pt_get "/instances/${DEFAULT_INST_ID}/logs" >/dev/null- printf '%s\n' "$RESULT" | head -30 | sed 's/^/ /'-fi+EXTENSION_TAB_ID=$(echo "$RESULT" | jq -r '.tabId // empty')+pt_post "/tabs/${EXTENSION_TAB_ID}/evaluate" '{"expression":"document.documentElement.getAttribute(\"data-pinchtab-test-extension\")"}'+assert_ok "evaluate extension marker"+assert_result_eq ".result" "loaded" "configured extension executed on the default instance"end_test