Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(orchestrator): coalesce the responsiveness fan-out and release /health at a short join wait so a wedged instance cannot time out the CLI presence checks (PIN-340)

Luigi Agosti9d8bec9a258b
diff --git a/docs/reference/health.md b/docs/reference/health.md
index 51fefaa0..8751cda7 100644
--- a/docs/reference/health.md
+++ b/docs/reference/health.md
@@ -76,7 +76,7 @@ Notes:
- use `defaultInstance.status == "running"` when you want to confirm Chrome is ready
- strategies such as `always-on` can create an instance automatically at startup
- `status` does not degrade on a browser crash: the instance is relaunched and is serving again. The crash is history, so it rides beside `status` as `crashes`; a crashed-then-relaunched instance differs from one that never crashed by that key alone. Every tab the dead browser held is gone, and a call to one answers `404` with code `browser_crashed` and a `hint` saying so
-- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: on every `/health` the front door probes each running instance's `/tabs` under a short budget beside its crash fetch, and an instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`
+- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: every `/health` starts a probe of each running instance's `/tabs` under a short budget beside its crash fetch, or joins the one already in flight, and waits for it at most 250ms; a probe that takes longer finishes in the background, so that `/health` reports the last recorded result and a later one reports the new result. An instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`
## Related Pages
diff --git a/docs/reference/instances.md b/docs/reference/instances.md
index 6c706b75..eefd1458 100644
--- a/docs/reference/instances.md
+++ b/docs/reference/instances.md
@@ -118,7 +118,7 @@ Instance responses include:
- `mode`: `"headless"` or `"headed"`
- `headless`: boolean kept for compatibility
-- `responsiveness`: whether the instance's browser routes answer, measured on the last front-door `/health` or monitoring snapshot by probing the instance's `/tabs` under a short budget. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`
+- `responsiveness`: whether the instance's browser routes answer, measured by the latest completed probe of the instance's `/tabs` under a short budget, which a front-door `/health` or monitoring snapshot starts and waits for at most 250ms; a slower probe is reported by the next read. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`
## Get Instance Logs
diff --git a/internal/dashboard/monitoring.go b/internal/dashboard/monitoring.go
index 27b5eb80..402c5f51 100644
--- a/internal/dashboard/monitoring.go
+++ b/internal/dashboard/monitoring.go
@@ -79,7 +79,7 @@ func (d *Dashboard) monitoringSnapshot(includeMemory bool) MonitoringSnapshot {
if d.monitoring != nil {
if refresher, ok := d.monitoring.(interface {
- RefreshCrashes() map[string]bridge.CrashSummary
+ RefreshCrashes()
}); ok {
refresher.RefreshCrashes()
}
diff --git a/internal/orchestrator/instance_query.go b/internal/orchestrator/instance_query.go
index c835c30f..9074666a 100644
--- a/internal/orchestrator/instance_query.go
+++ b/internal/orchestrator/instance_query.go
@@ -71,11 +71,40 @@ func (o *Orchestrator) probeInstance(inst *InstanceInternal) instanceProbe {
return instanceProbe{crashes: crashes, responsiveness: classifyResponsiveness(healthErr, tabsErr)}
}
+var refreshJoinWait = 250 * time.Millisecond
+
// RefreshCrashes asks every live instance for its crash record and probes its
// tabs route, so List can carry both and CrashSummary can merge the crashes.
// Browser crashes are recorded by the process that owns the browser, which in
// server mode is never this one.
-func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {
+func (o *Orchestrator) RefreshCrashes() {
+ timer := time.NewTimer(refreshJoinWait)
+ defer timer.Stop()
+ select {
+ case <-o.startRefresh():
+ case <-timer.C:
+ }
+}
+
+func (o *Orchestrator) startRefresh() <-chan struct{} {
+ o.refreshMu.Lock()
+ defer o.refreshMu.Unlock()
+ if o.refreshDone != nil {
+ return o.refreshDone
+ }
+ done := make(chan struct{})
+ o.refreshDone = done
+ go func() {
+ o.refreshInstances()
+ o.refreshMu.Lock()
+ o.refreshDone = nil
+ o.refreshMu.Unlock()
+ close(done)
+ }()
+ return done
+}
+
+func (o *Orchestrator) refreshInstances() {
o.mu.RLock()
instances := make([]*InstanceInternal, 0, len(o.instances))
for _, inst := range o.instances {
@@ -96,7 +125,6 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {
}
wg.Wait()
- fresh := make(map[string]bridge.CrashSummary, len(instances))
o.mu.Lock()
if o.crashes == nil {
o.crashes = map[string]bridge.CrashSummary{}
@@ -109,11 +137,9 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {
if probes[i].crashes == nil {
continue
}
- fresh[inst.ID] = *probes[i].crashes
o.crashes[inst.ID] = *probes[i].crashes
}
o.mu.Unlock()
- return fresh
}
// CrashSummary merges the instances' crash records into the shape bridge /health
diff --git a/internal/orchestrator/instance_responsiveness_test.go b/internal/orchestrator/instance_responsiveness_test.go
index c26ad6d0..da9209c9 100644
--- a/internal/orchestrator/instance_responsiveness_test.go
+++ b/internal/orchestrator/instance_responsiveness_test.go
@@ -173,3 +173,110 @@ func TestTheTabsProbeReachesTheChildAsOrchestratorActivityNotClientActivity(t *t
}
}
}
+
+func awaitRefresh(t *testing.T, o *Orchestrator) time.Time {
+ t.Helper()
+ o.refreshMu.Lock()
+ done := o.refreshDone
+ o.refreshMu.Unlock()
+ if done != nil {
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("the crash fan-out never finished")
+ }
+ }
+ return time.Now()
+}
+
+func addStubInstance(o *Orchestrator, id, url string) {
+ o.mu.Lock()
+ defer o.mu.Unlock()
+ o.instances[id] = &InstanceInternal{
+ Instance: bridge.Instance{ID: id, ProfileName: id, Status: "running"},
+ URL: url,
+ }
+}
+
+func TestAHangingTabsProbeReleasesTheRequestAtTheJoinWaitAndRecordsItsVerdictLater(t *testing.T) {
+ previous := responsivenessProbeBudget
+ responsivenessProbeBudget = 800 * time.Millisecond
+ t.Cleanup(func() { responsivenessProbeBudget = previous })
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
+
+ started := time.Now()
+ o.CrashSummary()
+ elapsed := time.Since(started)
+ if elapsed < refreshJoinWait || elapsed > refreshJoinWait+150*time.Millisecond || elapsed >= 500*time.Millisecond {
+ t.Fatalf("CrashSummary took %v with one hanging child, want the %v join wait, under the CLI's 500ms presence check", elapsed, refreshJoinWait)
+ }
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
+ t.Fatalf("responsiveness = %q before the probe finished, want unknown", got)
+ }
+ awaitRefresh(t, o)
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnresponsive {
+ t.Fatalf("responsiveness = %q after the probe finished, want unresponsive", got)
+ }
+}
+
+func TestConcurrentRefreshesShareOneFanOut(t *testing.T) {
+ previous := responsivenessProbeBudget
+ responsivenessProbeBudget = 800 * time.Millisecond
+ t.Cleanup(func() { responsivenessProbeBudget = previous })
+ var mu sync.Mutex
+ tabsHits := 0
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ tabsHits++
+ mu.Unlock()
+ hangUntilClientGivesUp(w, r)
+ }))
+
+ o.RefreshCrashes()
+ o.RefreshCrashes()
+ awaitRefresh(t, o)
+ mu.Lock()
+ defer mu.Unlock()
+ if tabsHits != 1 {
+ t.Fatalf("two refreshes during one in-flight fan-out probed /tabs %d times, want 1", tabsHits)
+ }
+}
+
+func TestTheFanOutProbesInstancesInParallel(t *testing.T) {
+ budget := 300 * time.Millisecond
+ previous := responsivenessProbeBudget
+ responsivenessProbeBudget = budget
+ t.Cleanup(func() { responsivenessProbeBudget = previous })
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
+ o.mu.RLock()
+ url := o.instances[stubInstanceID].URL
+ o.mu.RUnlock()
+ addStubInstance(o, "inst_probe_2", url)
+
+ started := time.Now()
+ o.RefreshCrashes()
+ elapsed := awaitRefresh(t, o).Sub(started)
+ if elapsed > budget+150*time.Millisecond {
+ t.Fatalf("fan-out over two hanging instances took %v, want one %v budget: the probes ran serially", elapsed, budget)
+ }
+ for _, inst := range o.List() {
+ if inst.Responsiveness != bridge.ResponsivenessUnresponsive {
+ t.Fatalf("%s responsiveness = %q, want unresponsive", inst.ID, inst.Responsiveness)
+ }
+ }
+}
+
+func TestATabsProbeCutOffWithoutAnAnswerIsUnknownNotUnresponsive(t *testing.T) {
+ shortProbeBudget(t)
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, _ *http.Request) {
+ conn, _, err := w.(http.Hijacker).Hijack()
+ if err == nil {
+ _ = conn.Close()
+ }
+ }))
+ o.RefreshCrashes()
+ awaitRefresh(t, o)
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
+ t.Fatalf("responsiveness = %q when /tabs dropped the connection, want unknown: only a probe that outlasts the budget is unresponsive", got)
+ }
+}
diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go
index 520e9943..b8122dba 100644
--- a/internal/orchestrator/orchestrator.go
+++ b/internal/orchestrator/orchestrator.go
@@ -27,6 +27,8 @@ type EventHandler func(InstanceEvent)
type Orchestrator struct {
instances map[string]*InstanceInternal
crashes map[string]bridge.CrashSummary
+ refreshMu sync.Mutex
+ refreshDone chan struct{}
baseDir string
binary string
profiles *profiles.ProfileManager