Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(orchestrator): harden startup tab readiness

Marioa2aa41f58300
diff --git a/internal/orchestrator/handlers_tabs.go b/internal/orchestrator/handlers_tabs.go
index 944419de..29f62131 100644
--- a/internal/orchestrator/handlers_tabs.go
+++ b/internal/orchestrator/handlers_tabs.go
@@ -2,10 +2,13 @@ package orchestrator
import (
"bytes"
+ "context"
"encoding/json"
"fmt"
"io"
"net/http"
+ "net/url"
+ "strings"
"github.com/pinchtab/pinchtab/internal/httpx"
)
@@ -57,5 +60,85 @@ func (o *Orchestrator) handleInstanceTabOpen(w http.ResponseWriter, r *http.Requ
httpx.Error(w, 502, err)
return
}
- o.proxyToURL(w, proxyReq, targetURL)
+ resp, body, err := o.doInstanceRequest(r.Context(), proxyReq.Method, proxyReq.Header, payload, targetURL, inst)
+ if err != nil {
+ httpx.Error(w, 502, err)
+ return
+ }
+ if shouldRetryInstanceTabOpen(resp.StatusCode, body) {
+ ensureURL, ensureErr := o.instancePathURL(inst, "/ensure-chrome", "")
+ if ensureErr == nil {
+ if err := o.ensureInstanceChrome(inst, ensureURL); err == nil {
+ if retryResp, retryBody, retryErr := o.doInstanceRequest(r.Context(), proxyReq.Method, proxyReq.Header, payload, targetURL, inst); retryErr == nil {
+ resp = retryResp
+ body = retryBody
+ }
+ }
+ }
+ }
+ o.handleProxyResponseHeaders(r, resp, inst.ID)
+ enrichActivityFromResponse(r, body)
+ copyProxyResponse(w, resp, body)
+}
+
+func (o *Orchestrator) ensureInstanceChrome(inst *InstanceInternal, targetURL *url.URL) error {
+ if inst == nil || targetURL == nil {
+ return fmt.Errorf("instance ensure-chrome target missing")
+ }
+ req, err := http.NewRequest(http.MethodPost, targetURL.String(), nil)
+ if err != nil {
+ return err
+ }
+ tagOrchestratorMonitoringRequest(req)
+ o.applyInstanceAuth(req, inst)
+ resp, err := o.client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("ensure-chrome HTTP %d: %s", resp.StatusCode, compactBody(body))
+ }
+ return nil
+}
+
+func (o *Orchestrator) doInstanceRequest(ctx context.Context, method string, header http.Header, body []byte, targetURL *url.URL, inst *InstanceInternal) (*http.Response, []byte, error) {
+ proxyReq, err := http.NewRequestWithContext(ctx, method, targetURL.String(), bytes.NewReader(body))
+ if err != nil {
+ return nil, nil, err
+ }
+ proxyReq.ContentLength = int64(len(body))
+ proxyReq.Header = header.Clone()
+ tagOrchestratorMonitoringRequest(proxyReq)
+ o.applyInstanceAuth(proxyReq, inst)
+
+ resp, err := o.client.Do(proxyReq)
+ if err != nil {
+ return nil, nil, err
+ }
+ body, readErr := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if readErr != nil {
+ return nil, nil, readErr
+ }
+ return resp, body, nil
+}
+
+func copyProxyResponse(w http.ResponseWriter, resp *http.Response, body []byte) {
+ for key, values := range resp.Header {
+ for _, v := range values {
+ w.Header().Add(key, v)
+ }
+ }
+ w.WriteHeader(resp.StatusCode)
+ _, _ = w.Write(body)
+}
+
+func shouldRetryInstanceTabOpen(statusCode int, body []byte) bool {
+ if statusCode != http.StatusInternalServerError {
+ return false
+ }
+ msg := strings.ToLower(string(body))
+ return strings.Contains(msg, "context canceled") || strings.Contains(msg, "context cancelled")
}
diff --git a/internal/orchestrator/health.go b/internal/orchestrator/health.go
index 7699b538..7656e755 100644
--- a/internal/orchestrator/health.go
+++ b/internal/orchestrator/health.go
@@ -1,8 +1,10 @@
package orchestrator
import (
+ "bytes"
"encoding/json"
"fmt"
+ "io"
"log/slog"
"net"
"net/http"
@@ -66,24 +68,12 @@ func (o *Orchestrator) monitor(inst *InstanceInternal) {
lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())
continue
}
- req, reqErr := http.NewRequest(http.MethodGet, healthProbeURL(targetBaseURL), nil)
- if reqErr != nil {
- lastProbe = fmt.Sprintf("%s -> %s", baseURL, reqErr.Error())
- continue
- }
- tagOrchestratorMonitoringRequest(req)
- o.applyInstanceAuth(req, inst)
- resp, err := o.client.Do(req)
- if err == nil {
- _ = resp.Body.Close()
- lastProbe = fmt.Sprintf("%s -> HTTP %d", baseURL, resp.StatusCode)
- if isInstanceHealthyStatus(resp.StatusCode) {
- healthy = true
- resolvedURL = baseURL
- break
- }
- } else {
- lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())
+ ready, probe := o.probeChildInstanceReady(inst, targetBaseURL)
+ lastProbe = fmt.Sprintf("%s -> %s", baseURL, probe)
+ if ready {
+ healthy = true
+ resolvedURL = baseURL
+ break
}
}
if healthy {
@@ -234,6 +224,109 @@ func (o *Orchestrator) probeInstanceHealth(inst *InstanceInternal) (bool, string
return false, "", lastProbe
}
+func (o *Orchestrator) probeChildInstanceReady(inst *InstanceInternal, baseURL *url.URL) (bool, string) {
+ req, reqErr := http.NewRequest(http.MethodGet, healthProbeURL(baseURL), nil)
+ if reqErr != nil {
+ return false, reqErr.Error()
+ }
+ tagOrchestratorMonitoringRequest(req)
+ o.applyInstanceAuth(req, inst)
+ resp, err := o.client.Do(req)
+ if err != nil {
+ return false, err.Error()
+ }
+ _ = resp.Body.Close()
+ if !isInstanceHealthyStatus(resp.StatusCode) {
+ return false, fmt.Sprintf("health HTTP %d", resp.StatusCode)
+ }
+
+ tabID, err := o.warmInstanceTabLifecycle(inst, baseURL)
+ if err != nil {
+ if tabID != "" {
+ slog.Debug("instance startup warmup left tab open", "id", inst.ID, "tabId", tabID, "err", err)
+ }
+ return false, fmt.Sprintf("warmup failed: %v", err)
+ }
+ return true, "ready"
+}
+
+func (o *Orchestrator) warmInstanceTabLifecycle(inst *InstanceInternal, baseURL *url.URL) (string, error) {
+ payload := []byte(`{"action":"new","url":"about:blank"}`)
+ tabURL := *baseURL
+ tabURL.Path = "/tab"
+ createReq, err := http.NewRequest(http.MethodPost, tabURL.String(), bytes.NewReader(payload))
+ if err != nil {
+ return "", err
+ }
+ createReq.Header.Set("Content-Type", "application/json")
+ tagOrchestratorMonitoringRequest(createReq)
+ o.applyInstanceAuth(createReq, inst)
+
+ createResp, err := o.client.Do(createReq)
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = createResp.Body.Close() }()
+
+ body, readErr := io.ReadAll(createResp.Body)
+ if readErr != nil {
+ return "", fmt.Errorf("read create-tab response: %w", readErr)
+ }
+ if createResp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("create tab HTTP %d: %s", createResp.StatusCode, compactBody(body))
+ }
+
+ var result struct {
+ TabID string `json:"tabId"`
+ }
+ if err := json.Unmarshal(body, &result); err != nil {
+ return "", fmt.Errorf("decode create-tab response: %w", err)
+ }
+ if strings.TrimSpace(result.TabID) == "" {
+ return "", fmt.Errorf("create-tab response missing tabId")
+ }
+
+ closePayload, err := json.Marshal(map[string]string{"tabId": result.TabID})
+ if err != nil {
+ return result.TabID, err
+ }
+ closeURL := *baseURL
+ closeURL.Path = "/close"
+ closeReq, err := http.NewRequest(http.MethodPost, closeURL.String(), bytes.NewReader(closePayload))
+ if err != nil {
+ return result.TabID, err
+ }
+ closeReq.Header.Set("Content-Type", "application/json")
+ tagOrchestratorMonitoringRequest(closeReq)
+ o.applyInstanceAuth(closeReq, inst)
+
+ closeResp, err := o.client.Do(closeReq)
+ if err != nil {
+ return result.TabID, fmt.Errorf("close warmup tab: %w", err)
+ }
+ defer func() { _ = closeResp.Body.Close() }()
+ closeBody, readErr := io.ReadAll(closeResp.Body)
+ if readErr != nil {
+ return result.TabID, fmt.Errorf("read close-tab response: %w", readErr)
+ }
+ if closeResp.StatusCode != http.StatusOK {
+ return result.TabID, fmt.Errorf("close warmup tab HTTP %d: %s", closeResp.StatusCode, compactBody(closeBody))
+ }
+ return result.TabID, nil
+}
+
+func compactBody(body []byte) string {
+ trimmed := strings.TrimSpace(string(body))
+ if trimmed == "" {
+ return "<empty>"
+ }
+ const max = 220
+ if len(trimmed) > max {
+ return trimmed[:max]
+ }
+ return trimmed
+}
+
type remoteTab struct {
ID string `json:"id"`
URL string `json:"url"`
diff --git a/internal/orchestrator/health_test.go b/internal/orchestrator/health_test.go
index 34baed98..ad1460d4 100644
--- a/internal/orchestrator/health_test.go
+++ b/internal/orchestrator/health_test.go
@@ -3,8 +3,13 @@ package orchestrator
import (
"io"
"net/http"
+ "net/http/httptest"
+ "net/url"
+ "strconv"
"strings"
+ "sync/atomic"
"testing"
+ "time"
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/config"
@@ -259,3 +264,172 @@ func TestFetchMetrics_TagsBridgeRequestAsOrchestrator(t *testing.T) {
t.Fatalf("X-PinchTab-Source = %q, want orchestrator", source)
}
}
+
+type blockingWaitCmd struct {
+ pid int
+ waitCh chan error
+}
+
+func (c *blockingWaitCmd) Wait() error { return <-c.waitCh }
+func (c *blockingWaitCmd) PID() int { return c.pid }
+func (c *blockingWaitCmd) Cancel() {}
+
+func TestProbeChildInstanceReady_RequiresWarmTabSuccess(t *testing.T) {
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/health":
+ w.WriteHeader(http.StatusOK)
+ case "/tab":
+ w.WriteHeader(http.StatusInternalServerError)
+ _, _ = w.Write([]byte(`{"code":"error","error":"create target: context canceled"}`))
+ default:
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ }))
+ defer srv.Close()
+
+ baseURL, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)
+ }
+
+ ready, probe := o.probeChildInstanceReady(&InstanceInternal{Instance: bridge.Instance{ID: "inst_1"}}, baseURL)
+ if ready {
+ t.Fatal("expected warmup failure to keep instance unready")
+ }
+ if !strings.Contains(probe, "create target: context canceled") {
+ t.Fatalf("probe = %q, want create-target failure", probe)
+ }
+}
+
+func TestProbeChildInstanceReady_WarmsTabLifecycle(t *testing.T) {
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+
+ var closeCalls int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/health":
+ w.WriteHeader(http.StatusOK)
+ case "/tab":
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"tabId":"tab-1","url":"about:blank","title":""}`))
+ case "/close":
+ atomic.AddInt32(&closeCalls, 1)
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"closed":true,"tabId":"tab-1"}`))
+ default:
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ }))
+ defer srv.Close()
+
+ baseURL, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)
+ }
+
+ ready, probe := o.probeChildInstanceReady(&InstanceInternal{Instance: bridge.Instance{ID: "inst_1"}}, baseURL)
+ if !ready {
+ t.Fatalf("expected readiness success, got probe %q", probe)
+ }
+ if probe != "ready" {
+ t.Fatalf("probe = %q, want ready", probe)
+ }
+ if atomic.LoadInt32(&closeCalls) != 1 {
+ t.Fatalf("close calls = %d, want 1", closeCalls)
+ }
+}
+
+func TestMonitor_DelaysRunningUntilWarmTabSucceeds(t *testing.T) {
+ old := processAliveFunc
+ processAliveFunc = func(pid int) bool { return pid > 0 }
+ defer func() { processAliveFunc = old }()
+
+ var tabAttempts int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/health":
+ w.WriteHeader(http.StatusOK)
+ case "/tab":
+ attempt := atomic.AddInt32(&tabAttempts, 1)
+ if attempt == 1 {
+ w.WriteHeader(http.StatusInternalServerError)
+ _, _ = w.Write([]byte(`{"code":"error","error":"create target: context canceled"}`))
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"tabId":"tab-2","url":"about:blank","title":""}`))
+ case "/close":
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(`{"closed":true,"tabId":"tab-2"}`))
+ default:
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ }))
+ defer srv.Close()
+
+ parsed, err := url.Parse(srv.URL)
+ if err != nil {
+ t.Fatalf("Parse(%q) error = %v", srv.URL, err)
+ }
+ host, portStr, ok := strings.Cut(parsed.Host, ":")
+ if !ok {
+ t.Fatalf("expected host:port in %q", parsed.Host)
+ }
+ port, err := strconv.Atoi(portStr)
+ if err != nil {
+ t.Fatalf("Atoi(%q) error = %v", portStr, err)
+ }
+
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+ o.ApplyRuntimeConfig(&config.RuntimeConfig{Bind: host})
+ o.client = srv.Client()
+
+ cmd := &blockingWaitCmd{pid: 4321, waitCh: make(chan error, 1)}
+ inst := &InstanceInternal{
+ Instance: bridge.Instance{
+ ID: "inst_1",
+ Port: strconv.Itoa(port),
+ Status: "starting",
+ StartTime: time.Now(),
+ ProfileName: "profile1",
+ },
+ cmd: cmd,
+ }
+
+ done := make(chan struct{})
+ go func() {
+ o.monitor(inst)
+ close(done)
+ }()
+
+ deadline := time.Now().Add(4 * time.Second)
+ for time.Now().Before(deadline) {
+ o.mu.RLock()
+ status := inst.Status
+ o.mu.RUnlock()
+ if status == "running" {
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+
+ o.mu.RLock()
+ status := inst.Status
+ o.mu.RUnlock()
+ if status != "running" {
+ cmd.waitCh <- nil
+ <-done
+ t.Fatalf("status = %q, want running", status)
+ }
+ if atomic.LoadInt32(&tabAttempts) < 2 {
+ cmd.waitCh <- nil
+ <-done
+ t.Fatalf("tab attempts = %d, want at least 2", tabAttempts)
+ }
+
+ cmd.waitCh <- nil
+ <-done
+}
diff --git a/tests/e2e/scenarios/infra/orchestrator-smoke.sh b/tests/e2e/scenarios/infra/orchestrator-smoke.sh
index 3380da86..45a2a5be 100644
--- a/tests/e2e/scenarios/infra/orchestrator-smoke.sh
+++ b/tests/e2e/scenarios/infra/orchestrator-smoke.sh
@@ -62,6 +62,12 @@ ACTIVE_INST_IDS+=("$INST1" "$INST2")
wait_for_orchestrator_instance_status "${E2E_SERVER}" "${INST1}" "running" 30
+# Issue 545 contract: once an instance reports running, the first immediate
+# tab open must succeed without a startup-race 500.
+pt_post "/instances/${INST1}/tabs/open" '{"url":"about:blank"}'
+assert_ok "running instance accepts immediate tabs/open"
+assert_json_exists "$RESULT" '.tabId' "immediate tabs/open returns tab id"
+
pt_get /instances
assert_ok "list after launch"
assert_instance_list_contains "$INST1" "instance $INST1 in list" "instance $INST1 not in list"