High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(orchestrator): harden startup tab readiness
Marioa2aa41f58300
diff --git a/internal/orchestrator/handlers_tabs.go b/internal/orchestrator/handlers_tabs.goindex 944419de..29f62131 100644--- a/internal/orchestrator/handlers_tabs.go+++ b/internal/orchestrator/handlers_tabs.go@@ -2,10 +2,13 @@ package orchestratorimport ("bytes"+ "context""encoding/json""fmt""io""net/http"+ "net/url"+ "strings""github.com/pinchtab/pinchtab/internal/httpx")@@ -57,5 +60,85 @@ func (o *Orchestrator) handleInstanceTabOpen(w http.ResponseWriter, r *http.Requhttpx.Error(w, 502, err)return}- o.proxyToURL(w, proxyReq, targetURL)+ resp, body, err := o.doInstanceRequest(r.Context(), proxyReq.Method, proxyReq.Header, payload, targetURL, inst)+ if err != nil {+ httpx.Error(w, 502, err)+ return+ }+ if shouldRetryInstanceTabOpen(resp.StatusCode, body) {+ ensureURL, ensureErr := o.instancePathURL(inst, "/ensure-chrome", "")+ if ensureErr == nil {+ if err := o.ensureInstanceChrome(inst, ensureURL); err == nil {+ if retryResp, retryBody, retryErr := o.doInstanceRequest(r.Context(), proxyReq.Method, proxyReq.Header, payload, targetURL, inst); retryErr == nil {+ resp = retryResp+ body = retryBody+ }+ }+ }+ }+ o.handleProxyResponseHeaders(r, resp, inst.ID)+ enrichActivityFromResponse(r, body)+ copyProxyResponse(w, resp, body)+}++func (o *Orchestrator) ensureInstanceChrome(inst *InstanceInternal, targetURL *url.URL) error {+ if inst == nil || targetURL == nil {+ return fmt.Errorf("instance ensure-chrome target missing")+ }+ req, err := http.NewRequest(http.MethodPost, targetURL.String(), nil)+ if err != nil {+ return err+ }+ tagOrchestratorMonitoringRequest(req)+ o.applyInstanceAuth(req, inst)+ resp, err := o.client.Do(req)+ if err != nil {+ return err+ }+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ if resp.StatusCode != http.StatusOK {+ return fmt.Errorf("ensure-chrome HTTP %d: %s", resp.StatusCode, compactBody(body))+ }+ return nil+}++func (o *Orchestrator) doInstanceRequest(ctx context.Context, method string, header http.Header, body []byte, targetURL *url.URL, inst *InstanceInternal) (*http.Response, []byte, error) {+ proxyReq, err := http.NewRequestWithContext(ctx, method, targetURL.String(), bytes.NewReader(body))+ if err != nil {+ return nil, nil, err+ }+ proxyReq.ContentLength = int64(len(body))+ proxyReq.Header = header.Clone()+ tagOrchestratorMonitoringRequest(proxyReq)+ o.applyInstanceAuth(proxyReq, inst)++ resp, err := o.client.Do(proxyReq)+ if err != nil {+ return nil, nil, err+ }+ body, readErr := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if readErr != nil {+ return nil, nil, readErr+ }+ return resp, body, nil+}++func copyProxyResponse(w http.ResponseWriter, resp *http.Response, body []byte) {+ for key, values := range resp.Header {+ for _, v := range values {+ w.Header().Add(key, v)+ }+ }+ w.WriteHeader(resp.StatusCode)+ _, _ = w.Write(body)+}++func shouldRetryInstanceTabOpen(statusCode int, body []byte) bool {+ if statusCode != http.StatusInternalServerError {+ return false+ }+ msg := strings.ToLower(string(body))+ return strings.Contains(msg, "context canceled") || strings.Contains(msg, "context cancelled")}diff --git a/internal/orchestrator/health.go b/internal/orchestrator/health.goindex 7699b538..7656e755 100644--- a/internal/orchestrator/health.go+++ b/internal/orchestrator/health.go@@ -1,8 +1,10 @@package orchestratorimport (+ "bytes""encoding/json""fmt"+ "io""log/slog""net""net/http"@@ -66,24 +68,12 @@ func (o *Orchestrator) monitor(inst *InstanceInternal) {lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())continue}- req, reqErr := http.NewRequest(http.MethodGet, healthProbeURL(targetBaseURL), nil)- if reqErr != nil {- lastProbe = fmt.Sprintf("%s -> %s", baseURL, reqErr.Error())- continue- }- tagOrchestratorMonitoringRequest(req)- o.applyInstanceAuth(req, inst)- resp, err := o.client.Do(req)- if err == nil {- _ = resp.Body.Close()- lastProbe = fmt.Sprintf("%s -> HTTP %d", baseURL, resp.StatusCode)- if isInstanceHealthyStatus(resp.StatusCode) {- healthy = true- resolvedURL = baseURL- break- }- } else {- lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())+ ready, probe := o.probeChildInstanceReady(inst, targetBaseURL)+ lastProbe = fmt.Sprintf("%s -> %s", baseURL, probe)+ if ready {+ healthy = true+ resolvedURL = baseURL+ break}}if healthy {@@ -234,6 +224,109 @@ func (o *Orchestrator) probeInstanceHealth(inst *InstanceInternal) (bool, stringreturn false, "", lastProbe}+func (o *Orchestrator) probeChildInstanceReady(inst *InstanceInternal, baseURL *url.URL) (bool, string) {+ req, reqErr := http.NewRequest(http.MethodGet, healthProbeURL(baseURL), nil)+ if reqErr != nil {+ return false, reqErr.Error()+ }+ tagOrchestratorMonitoringRequest(req)+ o.applyInstanceAuth(req, inst)+ resp, err := o.client.Do(req)+ if err != nil {+ return false, err.Error()+ }+ _ = resp.Body.Close()+ if !isInstanceHealthyStatus(resp.StatusCode) {+ return false, fmt.Sprintf("health HTTP %d", resp.StatusCode)+ }++ tabID, err := o.warmInstanceTabLifecycle(inst, baseURL)+ if err != nil {+ if tabID != "" {+ slog.Debug("instance startup warmup left tab open", "id", inst.ID, "tabId", tabID, "err", err)+ }+ return false, fmt.Sprintf("warmup failed: %v", err)+ }+ return true, "ready"+}++func (o *Orchestrator) warmInstanceTabLifecycle(inst *InstanceInternal, baseURL *url.URL) (string, error) {+ payload := []byte(`{"action":"new","url":"about:blank"}`)+ tabURL := *baseURL+ tabURL.Path = "/tab"+ createReq, err := http.NewRequest(http.MethodPost, tabURL.String(), bytes.NewReader(payload))+ if err != nil {+ return "", err+ }+ createReq.Header.Set("Content-Type", "application/json")+ tagOrchestratorMonitoringRequest(createReq)+ o.applyInstanceAuth(createReq, inst)++ createResp, err := o.client.Do(createReq)+ if err != nil {+ return "", err+ }+ defer func() { _ = createResp.Body.Close() }()++ body, readErr := io.ReadAll(createResp.Body)+ if readErr != nil {+ return "", fmt.Errorf("read create-tab response: %w", readErr)+ }+ if createResp.StatusCode != http.StatusOK {+ return "", fmt.Errorf("create tab HTTP %d: %s", createResp.StatusCode, compactBody(body))+ }++ var result struct {+ TabID string `json:"tabId"`+ }+ if err := json.Unmarshal(body, &result); err != nil {+ return "", fmt.Errorf("decode create-tab response: %w", err)+ }+ if strings.TrimSpace(result.TabID) == "" {+ return "", fmt.Errorf("create-tab response missing tabId")+ }++ closePayload, err := json.Marshal(map[string]string{"tabId": result.TabID})+ if err != nil {+ return result.TabID, err+ }+ closeURL := *baseURL+ closeURL.Path = "/close"+ closeReq, err := http.NewRequest(http.MethodPost, closeURL.String(), bytes.NewReader(closePayload))+ if err != nil {+ return result.TabID, err+ }+ closeReq.Header.Set("Content-Type", "application/json")+ tagOrchestratorMonitoringRequest(closeReq)+ o.applyInstanceAuth(closeReq, inst)++ closeResp, err := o.client.Do(closeReq)+ if err != nil {+ return result.TabID, fmt.Errorf("close warmup tab: %w", err)+ }+ defer func() { _ = closeResp.Body.Close() }()+ closeBody, readErr := io.ReadAll(closeResp.Body)+ if readErr != nil {+ return result.TabID, fmt.Errorf("read close-tab response: %w", readErr)+ }+ if closeResp.StatusCode != http.StatusOK {+ return result.TabID, fmt.Errorf("close warmup tab HTTP %d: %s", closeResp.StatusCode, compactBody(closeBody))+ }+ return result.TabID, nil+}++func compactBody(body []byte) string {+ trimmed := strings.TrimSpace(string(body))+ if trimmed == "" {+ return "<empty>"+ }+ const max = 220+ if len(trimmed) > max {+ return trimmed[:max]+ }+ return trimmed+}+type remoteTab struct {ID string `json:"id"`URL string `json:"url"`diff --git a/internal/orchestrator/health_test.go b/internal/orchestrator/health_test.goindex 34baed98..ad1460d4 100644--- a/internal/orchestrator/health_test.go+++ b/internal/orchestrator/health_test.go@@ -3,8 +3,13 @@ package orchestratorimport ("io""net/http"+ "net/http/httptest"+ "net/url"+ "strconv""strings"+ "sync/atomic""testing"+ "time""github.com/pinchtab/pinchtab/internal/bridge""github.com/pinchtab/pinchtab/internal/config"@@ -259,3 +264,172 @@ func TestFetchMetrics_TagsBridgeRequestAsOrchestrator(t *testing.T) {t.Fatalf("X-PinchTab-Source = %q, want orchestrator", source)}}++type blockingWaitCmd struct {+ pid int+ waitCh chan error+}++func (c *blockingWaitCmd) Wait() error { return <-c.waitCh }+func (c *blockingWaitCmd) PID() int { return c.pid }+func (c *blockingWaitCmd) Cancel() {}++func TestProbeChildInstanceReady_RequiresWarmTabSuccess(t *testing.T) {+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/health":+ w.WriteHeader(http.StatusOK)+ case "/tab":+ w.WriteHeader(http.StatusInternalServerError)+ _, _ = w.Write([]byte(`{"code":"error","error":"create target: context canceled"}`))+ default:+ t.Fatalf("unexpected path %s", r.URL.Path)+ }+ }))+ defer srv.Close()++ baseURL, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)+ }++ ready, probe := o.probeChildInstanceReady(&InstanceInternal{Instance: bridge.Instance{ID: "inst_1"}}, baseURL)+ if ready {+ t.Fatal("expected warmup failure to keep instance unready")+ }+ if !strings.Contains(probe, "create target: context canceled") {+ t.Fatalf("probe = %q, want create-target failure", probe)+ }+}++func TestProbeChildInstanceReady_WarmsTabLifecycle(t *testing.T) {+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})++ var closeCalls int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/health":+ w.WriteHeader(http.StatusOK)+ case "/tab":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"tabId":"tab-1","url":"about:blank","title":""}`))+ case "/close":+ atomic.AddInt32(&closeCalls, 1)+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"closed":true,"tabId":"tab-1"}`))+ default:+ t.Fatalf("unexpected path %s", r.URL.Path)+ }+ }))+ defer srv.Close()++ baseURL, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)+ }++ ready, probe := o.probeChildInstanceReady(&InstanceInternal{Instance: bridge.Instance{ID: "inst_1"}}, baseURL)+ if !ready {+ t.Fatalf("expected readiness success, got probe %q", probe)+ }+ if probe != "ready" {+ t.Fatalf("probe = %q, want ready", probe)+ }+ if atomic.LoadInt32(&closeCalls) != 1 {+ t.Fatalf("close calls = %d, want 1", closeCalls)+ }+}++func TestMonitor_DelaysRunningUntilWarmTabSucceeds(t *testing.T) {+ old := processAliveFunc+ processAliveFunc = func(pid int) bool { return pid > 0 }+ defer func() { processAliveFunc = old }()++ var tabAttempts int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/health":+ w.WriteHeader(http.StatusOK)+ case "/tab":+ attempt := atomic.AddInt32(&tabAttempts, 1)+ if attempt == 1 {+ w.WriteHeader(http.StatusInternalServerError)+ _, _ = w.Write([]byte(`{"code":"error","error":"create target: context canceled"}`))+ return+ }+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"tabId":"tab-2","url":"about:blank","title":""}`))+ case "/close":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte(`{"closed":true,"tabId":"tab-2"}`))+ default:+ t.Fatalf("unexpected path %s", r.URL.Path)+ }+ }))+ defer srv.Close()++ parsed, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)+ }+ host, portStr, ok := strings.Cut(parsed.Host, ":")+ if !ok {+ t.Fatalf("expected host:port in %q", parsed.Host)+ }+ port, err := strconv.Atoi(portStr)+ if err != nil {+ t.Fatalf("Atoi(%q) error = %v", portStr, err)+ }++ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})+ o.ApplyRuntimeConfig(&config.RuntimeConfig{Bind: host})+ o.client = srv.Client()++ cmd := &blockingWaitCmd{pid: 4321, waitCh: make(chan error, 1)}+ inst := &InstanceInternal{+ Instance: bridge.Instance{+ ID: "inst_1",+ Port: strconv.Itoa(port),+ Status: "starting",+ StartTime: time.Now(),+ ProfileName: "profile1",+ },+ cmd: cmd,+ }++ done := make(chan struct{})+ go func() {+ o.monitor(inst)+ close(done)+ }()++ deadline := time.Now().Add(4 * time.Second)+ for time.Now().Before(deadline) {+ o.mu.RLock()+ status := inst.Status+ o.mu.RUnlock()+ if status == "running" {+ break+ }+ time.Sleep(50 * time.Millisecond)+ }++ o.mu.RLock()+ status := inst.Status+ o.mu.RUnlock()+ if status != "running" {+ cmd.waitCh <- nil+ <-done+ t.Fatalf("status = %q, want running", status)+ }+ if atomic.LoadInt32(&tabAttempts) < 2 {+ cmd.waitCh <- nil+ <-done+ t.Fatalf("tab attempts = %d, want at least 2", tabAttempts)+ }++ cmd.waitCh <- nil+ <-done+}diff --git a/tests/e2e/scenarios/infra/orchestrator-smoke.sh b/tests/e2e/scenarios/infra/orchestrator-smoke.shindex 3380da86..45a2a5be 100644--- a/tests/e2e/scenarios/infra/orchestrator-smoke.sh+++ b/tests/e2e/scenarios/infra/orchestrator-smoke.sh@@ -62,6 +62,12 @@ ACTIVE_INST_IDS+=("$INST1" "$INST2")wait_for_orchestrator_instance_status "${E2E_SERVER}" "${INST1}" "running" 30+# Issue 545 contract: once an instance reports running, the first immediate+# tab open must succeed without a startup-race 500.+pt_post "/instances/${INST1}/tabs/open" '{"url":"about:blank"}'+assert_ok "running instance accepts immediate tabs/open"+assert_json_exists "$RESULT" '.tabId' "immediate tabs/open returns tab id"+pt_get /instancesassert_ok "list after launch"assert_instance_list_contains "$INST1" "instance $INST1 in list" "instance $INST1 not in list"