Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: monitor attached bridge health (#362)

Jamil Zakirov4b9c78185cf9
diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.go
index 418833ba..7422bdf5 100644
--- a/internal/bridge/bridge.go
+++ b/internal/bridge/bridge.go
@@ -181,11 +181,26 @@ func (b *Bridge) EnsureChrome(cfg *config.RuntimeConfig) error {
defer b.initMu.Unlock()
if b.initialized && b.BrowserCtx != nil {
- return nil // Already initialized
+ // Check if browser context is still alive
+ if b.BrowserCtx.Err() == nil {
+ return nil
+ }
+ // Chrome died — reset state for re-initialization
+ slog.Warn("chrome context cancelled, re-initializing")
+ b.initialized = false
+ b.BrowserCtx = nil
+ b.BrowserCancel = nil
+ b.AllocCtx = nil
+ b.AllocCancel = nil
+ b.TabManager = nil
}
if b.BrowserCtx != nil {
- return nil // Already has browser context
+ if b.BrowserCtx.Err() == nil {
+ return nil
+ }
+ b.BrowserCtx = nil
+ b.BrowserCancel = nil
}
slog.Debug("ensure chrome called", "headless", cfg.Headless, "profile", cfg.ProfileDir)
diff --git a/internal/orchestrator/health.go b/internal/orchestrator/health.go
index 4a538db2..2b99ef67 100644
--- a/internal/orchestrator/health.go
+++ b/internal/orchestrator/health.go
@@ -11,8 +11,9 @@ import (
)
const (
- instanceHealthPollInterval = 500 * time.Millisecond
- instanceStartupTimeout = 45 * time.Second
+ instanceHealthPollInterval = 500 * time.Millisecond
+ instanceStartupTimeout = 45 * time.Second
+ attachedBridgeHealthPollInterval = 60 * time.Second
)
func (o *Orchestrator) monitor(inst *InstanceInternal) {
@@ -37,32 +38,7 @@ func (o *Orchestrator) monitor(inst *InstanceInternal) {
}
time.Sleep(instanceHealthPollInterval)
- for _, baseURL := range instanceBaseURLs(inst.URL, inst.Port) {
- baseParsed, parseErr := url.Parse(baseURL)
- if parseErr != nil {
- lastProbe = fmt.Sprintf("%s -> %s", baseURL, parseErr.Error())
- continue
- }
- target := &url.URL{Scheme: baseParsed.Scheme, Host: baseParsed.Host, Path: "/health"}
- req, reqErr := http.NewRequest(http.MethodGet, target.String(), nil)
- if reqErr != nil {
- lastProbe = fmt.Sprintf("%s -> %s", baseURL, reqErr.Error())
- continue
- }
- o.applyInstanceAuth(req, inst)
- resp, err := o.client.Do(req)
- if err == nil {
- _ = resp.Body.Close()
- lastProbe = fmt.Sprintf("%s -> HTTP %d", baseURL, resp.StatusCode)
- if isInstanceHealthyStatus(resp.StatusCode) {
- healthy = true
- resolvedURL = baseURL
- break
- }
- } else {
- lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())
- }
- }
+ healthy, resolvedURL, lastProbe = o.probeInstanceHealth(inst)
if healthy {
break
}
@@ -127,6 +103,74 @@ func (o *Orchestrator) monitor(inst *InstanceInternal) {
slog.Info("instance exited", "id", inst.ID)
}
+func (o *Orchestrator) monitorAttachedBridge(inst *InstanceInternal) {
+ ticker := time.NewTicker(attachedBridgeHealthPollInterval)
+ defer ticker.Stop()
+
+ for range ticker.C {
+ if !o.checkAttachedBridgeHealth(inst) {
+ return
+ }
+ }
+}
+
+func (o *Orchestrator) checkAttachedBridgeHealth(inst *InstanceInternal) bool {
+ o.mu.RLock()
+ current, ok := o.instances[inst.ID]
+ shouldStop := !ok || current != inst || inst.Status != "running" || !inst.Attached || inst.AttachType != "bridge"
+ o.mu.RUnlock()
+ if shouldStop {
+ return false
+ }
+
+ healthy, resolvedURL, lastProbe := o.probeInstanceHealth(inst)
+ if healthy {
+ if resolvedURL != "" && resolvedURL != inst.URL {
+ o.mu.Lock()
+ if current, ok := o.instances[inst.ID]; ok && current == inst {
+ inst.URL = resolvedURL
+ inst.Instance.URL = resolvedURL
+ o.syncInstanceToManager(&inst.Instance)
+ }
+ o.mu.Unlock()
+ }
+ return true
+ }
+
+ slog.Warn("attached bridge unreachable, removing", "id", inst.ID, "probe", lastProbe)
+ o.markStopped(inst.ID)
+ return false
+}
+
+func (o *Orchestrator) probeInstanceHealth(inst *InstanceInternal) (bool, string, string) {
+ lastProbe := "no response"
+ for _, baseURL := range instanceBaseURLs(inst.URL, inst.Port) {
+ baseParsed, parseErr := url.Parse(baseURL)
+ if parseErr != nil {
+ lastProbe = fmt.Sprintf("%s -> %s", baseURL, parseErr.Error())
+ continue
+ }
+ target := &url.URL{Scheme: baseParsed.Scheme, Host: baseParsed.Host, Path: "/health"}
+ req, reqErr := http.NewRequest(http.MethodGet, target.String(), nil)
+ if reqErr != nil {
+ lastProbe = fmt.Sprintf("%s -> %s", baseURL, reqErr.Error())
+ continue
+ }
+ o.applyInstanceAuth(req, inst)
+ resp, err := o.client.Do(req)
+ if err != nil {
+ lastProbe = fmt.Sprintf("%s -> %s", baseURL, err.Error())
+ continue
+ }
+ _ = resp.Body.Close()
+ lastProbe = fmt.Sprintf("%s -> HTTP %d", baseURL, resp.StatusCode)
+ if isInstanceHealthyStatus(resp.StatusCode) {
+ return true, baseURL, lastProbe
+ }
+ }
+ return false, "", lastProbe
+}
+
type remoteTab struct {
ID string `json:"id"`
URL string `json:"url"`
diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go
index faab05f6..252ae047 100644
--- a/internal/orchestrator/orchestrator.go
+++ b/internal/orchestrator/orchestrator.go
@@ -488,6 +488,12 @@ func (o *Orchestrator) AttachBridge(name, baseURL, token string) (*bridge.Instan
slog.Info("attached to external bridge", "id", inst.ID, "name", name, "url", internalurls.RedactForLog(inst.URL))
o.emitEvent("instance.attached", inst)
+ o.mu.RLock()
+ internal := o.instances[inst.ID]
+ o.mu.RUnlock()
+ if internal != nil {
+ go o.monitorAttachedBridge(internal)
+ }
return inst, nil
}
diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go
index 85ffaf3d..503dfa56 100644
--- a/internal/orchestrator/orchestrator_test.go
+++ b/internal/orchestrator/orchestrator_test.go
@@ -409,6 +409,45 @@ func TestOrchestrator_AttachBridge(t *testing.T) {
}
}
+func TestOrchestrator_AttachBridge_RemovesUnhealthyBridge(t *testing.T) {
+ unhealthy := false
+ backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/health" {
+ t.Fatalf("path = %q, want /health", r.URL.Path)
+ }
+ if unhealthy {
+ http.Error(w, "unhealthy", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer backend.Close()
+
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+ o.client = backend.Client()
+
+ inst, err := o.AttachBridge("bridge1", backend.URL, "bridge-token")
+ if err != nil {
+ t.Fatalf("AttachBridge failed: %v", err)
+ }
+
+ unhealthy = true
+
+ o.mu.RLock()
+ internal := o.instances[inst.ID]
+ o.mu.RUnlock()
+ if internal == nil {
+ t.Fatalf("attached instance %q missing from orchestrator", inst.ID)
+ }
+
+ if o.checkAttachedBridgeHealth(internal) {
+ t.Fatal("expected unhealthy attached bridge to stop monitoring")
+ }
+ if len(o.List()) != 0 {
+ t.Fatalf("expected attached bridge to be removed, got %d instances", len(o.List()))
+ }
+}
+
func TestValidateAttachURL_AllowsBridgeHTTP(t *testing.T) {
o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
o.ApplyRuntimeConfig(&config.RuntimeConfig{