High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(orchestrator): coalesce the responsiveness fan-out and release /health at a short join wait so a wedged instance cannot time out the CLI presence checks (PIN-340)
Luigi Agosti9d8bec9a258b
diff --git a/docs/reference/health.md b/docs/reference/health.mdindex 51fefaa0..8751cda7 100644--- a/docs/reference/health.md+++ b/docs/reference/health.md@@ -76,7 +76,7 @@ Notes:- use `defaultInstance.status == "running"` when you want to confirm Chrome is ready- strategies such as `always-on` can create an instance automatically at startup- `status` does not degrade on a browser crash: the instance is relaunched and is serving again. The crash is history, so it rides beside `status` as `crashes`; a crashed-then-relaunched instance differs from one that never crashed by that key alone. Every tab the dead browser held is gone, and a call to one answers `404` with code `browser_crashed` and a `hint` saying so-- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: on every `/health` the front door probes each running instance's `/tabs` under a short budget beside its crash fetch, and an instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`+- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: every `/health` starts a probe of each running instance's `/tabs` under a short budget beside its crash fetch, or joins the one already in flight, and waits for it at most 250ms; a probe that takes longer finishes in the background, so that `/health` reports the last recorded result and a later one reports the new result. An instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`## Related Pagesdiff --git a/docs/reference/instances.md b/docs/reference/instances.mdindex 6c706b75..eefd1458 100644--- a/docs/reference/instances.md+++ b/docs/reference/instances.md@@ -118,7 +118,7 @@ Instance responses include:- `mode`: `"headless"` or `"headed"`- `headless`: boolean kept for compatibility-- `responsiveness`: whether the instance's browser routes answer, measured on the last front-door `/health` or monitoring snapshot by probing the instance's `/tabs` under a short budget. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`+- `responsiveness`: whether the instance's browser routes answer, measured by the latest completed probe of the instance's `/tabs` under a short budget, which a front-door `/health` or monitoring snapshot starts and waits for at most 250ms; a slower probe is reported by the next read. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`## Get Instance Logsdiff --git a/internal/dashboard/monitoring.go b/internal/dashboard/monitoring.goindex 27b5eb80..402c5f51 100644--- a/internal/dashboard/monitoring.go+++ b/internal/dashboard/monitoring.go@@ -79,7 +79,7 @@ func (d *Dashboard) monitoringSnapshot(includeMemory bool) MonitoringSnapshot {if d.monitoring != nil {if refresher, ok := d.monitoring.(interface {- RefreshCrashes() map[string]bridge.CrashSummary+ RefreshCrashes()}); ok {refresher.RefreshCrashes()}diff --git a/internal/orchestrator/instance_query.go b/internal/orchestrator/instance_query.goindex c835c30f..9074666a 100644--- a/internal/orchestrator/instance_query.go+++ b/internal/orchestrator/instance_query.go@@ -71,11 +71,40 @@ func (o *Orchestrator) probeInstance(inst *InstanceInternal) instanceProbe {return instanceProbe{crashes: crashes, responsiveness: classifyResponsiveness(healthErr, tabsErr)}}+var refreshJoinWait = 250 * time.Millisecond+// RefreshCrashes asks every live instance for its crash record and probes its// tabs route, so List can carry both and CrashSummary can merge the crashes.// Browser crashes are recorded by the process that owns the browser, which in// server mode is never this one.-func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {+func (o *Orchestrator) RefreshCrashes() {+ timer := time.NewTimer(refreshJoinWait)+ defer timer.Stop()+ select {+ case <-o.startRefresh():+ case <-timer.C:+ }+}++func (o *Orchestrator) startRefresh() <-chan struct{} {+ o.refreshMu.Lock()+ defer o.refreshMu.Unlock()+ if o.refreshDone != nil {+ return o.refreshDone+ }+ done := make(chan struct{})+ o.refreshDone = done+ go func() {+ o.refreshInstances()+ o.refreshMu.Lock()+ o.refreshDone = nil+ o.refreshMu.Unlock()+ close(done)+ }()+ return done+}++func (o *Orchestrator) refreshInstances() {o.mu.RLock()instances := make([]*InstanceInternal, 0, len(o.instances))for _, inst := range o.instances {@@ -96,7 +125,6 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {}wg.Wait()- fresh := make(map[string]bridge.CrashSummary, len(instances))o.mu.Lock()if o.crashes == nil {o.crashes = map[string]bridge.CrashSummary{}@@ -109,11 +137,9 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {if probes[i].crashes == nil {continue}- fresh[inst.ID] = *probes[i].crasheso.crashes[inst.ID] = *probes[i].crashes}o.mu.Unlock()- return fresh}// CrashSummary merges the instances' crash records into the shape bridge /healthdiff --git a/internal/orchestrator/instance_responsiveness_test.go b/internal/orchestrator/instance_responsiveness_test.goindex c26ad6d0..da9209c9 100644--- a/internal/orchestrator/instance_responsiveness_test.go+++ b/internal/orchestrator/instance_responsiveness_test.go@@ -173,3 +173,110 @@ func TestTheTabsProbeReachesTheChildAsOrchestratorActivityNotClientActivity(t *t}}}++func awaitRefresh(t *testing.T, o *Orchestrator) time.Time {+ t.Helper()+ o.refreshMu.Lock()+ done := o.refreshDone+ o.refreshMu.Unlock()+ if done != nil {+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("the crash fan-out never finished")+ }+ }+ return time.Now()+}++func addStubInstance(o *Orchestrator, id, url string) {+ o.mu.Lock()+ defer o.mu.Unlock()+ o.instances[id] = &InstanceInternal{+ Instance: bridge.Instance{ID: id, ProfileName: id, Status: "running"},+ URL: url,+ }+}++func TestAHangingTabsProbeReleasesTheRequestAtTheJoinWaitAndRecordsItsVerdictLater(t *testing.T) {+ previous := responsivenessProbeBudget+ responsivenessProbeBudget = 800 * time.Millisecond+ t.Cleanup(func() { responsivenessProbeBudget = previous })+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))++ started := time.Now()+ o.CrashSummary()+ elapsed := time.Since(started)+ if elapsed < refreshJoinWait || elapsed > refreshJoinWait+150*time.Millisecond || elapsed >= 500*time.Millisecond {+ t.Fatalf("CrashSummary took %v with one hanging child, want the %v join wait, under the CLI's 500ms presence check", elapsed, refreshJoinWait)+ }+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {+ t.Fatalf("responsiveness = %q before the probe finished, want unknown", got)+ }+ awaitRefresh(t, o)+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnresponsive {+ t.Fatalf("responsiveness = %q after the probe finished, want unresponsive", got)+ }+}++func TestConcurrentRefreshesShareOneFanOut(t *testing.T) {+ previous := responsivenessProbeBudget+ responsivenessProbeBudget = 800 * time.Millisecond+ t.Cleanup(func() { responsivenessProbeBudget = previous })+ var mu sync.Mutex+ tabsHits := 0+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, r *http.Request) {+ mu.Lock()+ tabsHits+++ mu.Unlock()+ hangUntilClientGivesUp(w, r)+ }))++ o.RefreshCrashes()+ o.RefreshCrashes()+ awaitRefresh(t, o)+ mu.Lock()+ defer mu.Unlock()+ if tabsHits != 1 {+ t.Fatalf("two refreshes during one in-flight fan-out probed /tabs %d times, want 1", tabsHits)+ }+}++func TestTheFanOutProbesInstancesInParallel(t *testing.T) {+ budget := 300 * time.Millisecond+ previous := responsivenessProbeBudget+ responsivenessProbeBudget = budget+ t.Cleanup(func() { responsivenessProbeBudget = previous })+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))+ o.mu.RLock()+ url := o.instances[stubInstanceID].URL+ o.mu.RUnlock()+ addStubInstance(o, "inst_probe_2", url)++ started := time.Now()+ o.RefreshCrashes()+ elapsed := awaitRefresh(t, o).Sub(started)+ if elapsed > budget+150*time.Millisecond {+ t.Fatalf("fan-out over two hanging instances took %v, want one %v budget: the probes ran serially", elapsed, budget)+ }+ for _, inst := range o.List() {+ if inst.Responsiveness != bridge.ResponsivenessUnresponsive {+ t.Fatalf("%s responsiveness = %q, want unresponsive", inst.ID, inst.Responsiveness)+ }+ }+}++func TestATabsProbeCutOffWithoutAnAnswerIsUnknownNotUnresponsive(t *testing.T) {+ shortProbeBudget(t)+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, func(w http.ResponseWriter, _ *http.Request) {+ conn, _, err := w.(http.Hijacker).Hijack()+ if err == nil {+ _ = conn.Close()+ }+ }))+ o.RefreshCrashes()+ awaitRefresh(t, o)+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {+ t.Fatalf("responsiveness = %q when /tabs dropped the connection, want unknown: only a probe that outlasts the budget is unresponsive", got)+ }+}diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.goindex 520e9943..b8122dba 100644--- a/internal/orchestrator/orchestrator.go+++ b/internal/orchestrator/orchestrator.go@@ -27,6 +27,8 @@ type EventHandler func(InstanceEvent)type Orchestrator struct {instances map[string]*InstanceInternalcrashes map[string]bridge.CrashSummary+ refreshMu sync.Mutex+ refreshDone chan struct{}baseDir stringbinary stringprofiles *profiles.ProfileManager