High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(orchestrator): report an instance that answers /health but not its browser routes as unresponsive (PIN-340)
Luigi Agosti34f69ab367d0
diff --git a/dashboard/src/generated/types.ts b/dashboard/src/generated/types.tsindex 37cce480..274177a3 100644--- a/dashboard/src/generated/types.ts+++ b/dashboard/src/generated/types.ts@@ -59,6 +59,7 @@ export interface Instance {fallbackFrom?: string;fallbackReason?: string;crashes?: CrashSummary;+ responsiveness: string; // responsive/unresponsive/unknown}export interface CrashEvent {time: string;diff --git a/docs/reference/health.md b/docs/reference/health.mdindex 49ca2305..51fefaa0 100644--- a/docs/reference/health.md+++ b/docs/reference/health.md@@ -56,18 +56,19 @@ curl http://localhost:9867/health| Field | Description || --- | --- |-| `status` | `ok` when server is healthy |+| `status` | `ok` when server is healthy; `degraded` while at least one instance is unresponsive || `mode` | `dashboard` in server mode || `version` | PinchTab version || `uptime` | Milliseconds since server start || `authRequired` | `true` when a server token is configured || `profiles` | Number of configured profiles || `instances` | Number of managed instances |-| `defaultInstance` | First managed instance info, when present |+| `defaultInstance` | First managed instance info, when present: `id`, `status` and `responsiveness` || `agents` | Connected agent count || `restartRequired` | `true` when file-based config changes need restart || `restartReasons` | Restart reason list when required || `crashes` | Present once any instance's browser has crashed: `total` and `recent`, the same block bridge `/health` carries, each event naming its `instanceId` |+| `unresponsiveInstances` | Present while any instance answers its own `/health` but not its browser routes: the ids of those instances |Notes:@@ -75,6 +76,7 @@ Notes:- use `defaultInstance.status == "running"` when you want to confirm Chrome is ready- strategies such as `always-on` can create an instance automatically at startup- `status` does not degrade on a browser crash: the instance is relaunched and is serving again. The crash is history, so it rides beside `status` as `crashes`; a crashed-then-relaunched instance differs from one that never crashed by that key alone. Every tab the dead browser held is gone, and a call to one answers `404` with code `browser_crashed` and a `hint` saying so+- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: on every `/health` the front door probes each running instance's `/tabs` under a short budget beside its crash fetch, and an instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`## Related Pagesdiff --git a/docs/reference/instances.md b/docs/reference/instances.mdindex 08844cf4..6c706b75 100644--- a/docs/reference/instances.md+++ b/docs/reference/instances.md@@ -37,6 +37,7 @@ Response shape:"mode": "headed","headless": false,"status": "running",+ "responsiveness": "responsive","securityPolicy": {"allowedDomains": ["127.0.0.1", "localhost", "::1", "wikipedia.org"]}@@ -117,6 +118,7 @@ Instance responses include:- `mode`: `"headless"` or `"headed"`- `headless`: boolean kept for compatibility+- `responsiveness`: whether the instance's browser routes answer, measured on the last front-door `/health` or monitoring snapshot by probing the instance's `/tabs` under a short budget. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`## Get Instance Logsdiff --git a/internal/api/types/types.go b/internal/api/types/types.goindex 65df6f55..85faae04 100644--- a/internal/api/types/types.go+++ b/internal/api/types/types.go@@ -57,6 +57,8 @@ type Instance struct {FallbackReason string `json:"fallbackReason,omitempty"`Crashes *CrashSummary `json:"crashes,omitempty"`++ Responsiveness string `json:"responsiveness"` // responsive/unresponsive/unknown}type CrashEvent struct {diff --git a/internal/bridge/api.go b/internal/bridge/api.goindex 066d19df..7fcb77b6 100644--- a/internal/bridge/api.go+++ b/internal/bridge/api.go@@ -329,12 +329,30 @@ type Instance struct {FallbackReason string `json:"fallbackReason,omitempty"`Crashes *CrashSummary `json:"crashes,omitempty"`++ Responsiveness string `json:"responsiveness"`+}++const (+ ResponsivenessResponsive = "responsive"+ ResponsivenessUnresponsive = "unresponsive"+ ResponsivenessUnknown = "unknown"+)++func NormalizeResponsiveness(value string) string {+ switch value {+ case ResponsivenessResponsive, ResponsivenessUnresponsive:+ return value+ default:+ return ResponsivenessUnknown+ }}func (i Instance) MarshalJSON() ([]byte, error) {type alias Instancecopy := alias(i)copy.Mode = normalizeInstanceMode(copy.Mode, copy.Headless)+ copy.Responsiveness = NormalizeResponsiveness(copy.Responsiveness)return json.Marshal(copy)}diff --git a/internal/dashboard/config_health.go b/internal/dashboard/config_health.goindex 7f40a4af..54797df2 100644--- a/internal/dashboard/config_health.go+++ b/internal/dashboard/config_health.go@@ -12,8 +12,9 @@ import ()type healthInstanceInfo struct {- ID string `json:"id"`- Status string `json:"status"`+ ID string `json:"id"`+ Status string `json:"status"`+ Responsiveness string `json:"responsiveness"`}type healthSecurityInfo struct {@@ -26,21 +27,37 @@ type healthSecurityInfo struct {}type healthEnvelope struct {- Status string `json:"status"`- Mode string `json:"mode"`- Version string `json:"version"`- Uptime int64 `json:"uptime"`- AuthRequired bool `json:"authRequired"`- Profiles int `json:"profiles"`- TemporaryProfiles int `json:"temporaryProfiles"`- QuarantinedProfiles int `json:"quarantinedProfiles"`- Instances int `json:"instances"`- DefaultInstance *healthInstanceInfo `json:"defaultInstance,omitempty"`- Agents int `json:"agents"`- RestartRequired bool `json:"restartRequired"`- RestartReasons []string `json:"restartReasons,omitempty"`- Security *healthSecurityInfo `json:"security,omitempty"`- Crashes *bridge.CrashSummary `json:"crashes,omitempty"`+ Status string `json:"status"`+ Mode string `json:"mode"`+ Version string `json:"version"`+ Uptime int64 `json:"uptime"`+ AuthRequired bool `json:"authRequired"`+ Profiles int `json:"profiles"`+ TemporaryProfiles int `json:"temporaryProfiles"`+ QuarantinedProfiles int `json:"quarantinedProfiles"`+ Instances int `json:"instances"`+ DefaultInstance *healthInstanceInfo `json:"defaultInstance,omitempty"`+ Agents int `json:"agents"`+ RestartRequired bool `json:"restartRequired"`+ RestartReasons []string `json:"restartReasons,omitempty"`+ Security *healthSecurityInfo `json:"security,omitempty"`+ Crashes *bridge.CrashSummary `json:"crashes,omitempty"`+ UnresponsiveInstances []string `json:"unresponsiveInstances,omitempty"`+}++const (+ healthStatusOK = "ok"+ healthStatusDegraded = "degraded"+)++func unresponsiveInstanceIDs(instances []bridge.Instance) []string {+ var ids []string+ for _, inst := range instances {+ if inst.Responsiveness == bridge.ResponsivenessUnresponsive {+ ids = append(ids, inst.ID)+ }+ }+ return ids}type crashReporter interface {@@ -75,47 +92,57 @@ func (c *ConfigAPI) healthInfo(includeSecurity bool) (healthEnvelope, error) {}}+ var crashes *bridge.CrashSummary+ if reporter, ok := c.instances.(crashReporter); ok {+ if summary := reporter.CrashSummary(); summary.Total > 0 {+ crashes = &summary+ }+ }instanceCount := 0var defaultInst *healthInstanceInfo+ var unresponsive []stringif c.instances != nil {instances := c.instances.List()instanceCount = len(instances)+ unresponsive = unresponsiveInstanceIDs(instances)if len(instances) > 0 {defaultInst = &healthInstanceInfo{- ID: instances[0].ID,- Status: instances[0].Status,+ ID: instances[0].ID,+ Status: instances[0].Status,+ Responsiveness: instances[0].Responsiveness,}}}+ status := healthStatusOK+ if len(unresponsive) > 0 {+ status = healthStatusDegraded+ }agentCount := 0if c.agents != nil {agentCount = c.agents.AgentCount()}cfg := c.cfg()out := healthEnvelope{- Status: "ok",- Mode: "dashboard",- Version: c.version,- Uptime: int64(time.Since(c.startedAt).Milliseconds()),- AuthRequired: cfg != nil && strings.TrimSpace(cfg.Token) != "",- Profiles: profileCount,- TemporaryProfiles: temporaryCount,- QuarantinedProfiles: quarantinedCount,- Instances: instanceCount,- DefaultInstance: defaultInst,- Agents: agentCount,- RestartRequired: len(restartReasons) > 0,- RestartReasons: restartReasons,+ Status: status,+ Mode: "dashboard",+ Version: c.version,+ Uptime: int64(time.Since(c.startedAt).Milliseconds()),+ AuthRequired: cfg != nil && strings.TrimSpace(cfg.Token) != "",+ Profiles: profileCount,+ TemporaryProfiles: temporaryCount,+ QuarantinedProfiles: quarantinedCount,+ Instances: instanceCount,+ DefaultInstance: defaultInst,+ Agents: agentCount,+ RestartRequired: len(restartReasons) > 0,+ RestartReasons: restartReasons,+ Crashes: crashes,+ UnresponsiveInstances: unresponsive,}if includeSecurity {security := runtimeSecurityInfo(cfg)out.Security = &security}- if reporter, ok := c.instances.(crashReporter); ok {- if crashes := reporter.CrashSummary(); crashes.Total > 0 {- out.Crashes = &crashes- }- }return out, nil}diff --git a/internal/dashboard/health_responsiveness_test.go b/internal/dashboard/health_responsiveness_test.gonew file mode 100644index 00000000..476a8f5f--- /dev/null+++ b/internal/dashboard/health_responsiveness_test.go@@ -0,0 +1,62 @@+package dashboard++import (+ "testing"++ "github.com/pinchtab/pinchtab/internal/bridge"+)++type probingInstances struct {+ instances []bridge.Instance+ probed string+}++func (s *probingInstances) List() []bridge.Instance { return s.instances }++func (s *probingInstances) CrashSummary() bridge.CrashSummary {+ for i := range s.instances {+ s.instances[i].Responsiveness = s.probed+ }+ return bridge.CrashSummary{}+}++func TestServerModeHealthDegradesWhileAnInstanceIsUnresponsive(t *testing.T) {+ body := healthBody(t, plainInstances{instances: []bridge.Instance{+ {ID: "inst_ok", Status: "running", Responsiveness: bridge.ResponsivenessResponsive},+ {ID: "inst_wedged", Status: "running", Responsiveness: bridge.ResponsivenessUnresponsive},+ }})+ if body["status"] != "degraded" {+ t.Errorf("status = %v, want degraded while an instance is unresponsive", body["status"])+ }+ named, _ := body["unresponsiveInstances"].([]any)+ if len(named) != 1 || named[0] != "inst_wedged" {+ t.Errorf("unresponsiveInstances = %v, want [inst_wedged]", body["unresponsiveInstances"])+ }+ def, _ := body["defaultInstance"].(map[string]any)+ if def["responsiveness"] != bridge.ResponsivenessResponsive || def["status"] != "running" {+ t.Errorf("defaultInstance = %v, want its responsiveness beside an unchanged status", def)+ }+}++func TestServerModeHealthStaysOkForResponsiveAndUnknownInstances(t *testing.T) {+ for _, value := range []string{bridge.ResponsivenessResponsive, bridge.ResponsivenessUnknown, ""} {+ body := healthBody(t, plainInstances{instances: []bridge.Instance{{ID: "inst_1", Status: "running", Responsiveness: value}}})+ if body["status"] != "ok" {+ t.Errorf("responsiveness %q: status = %v, want ok", value, body["status"])+ }+ if _, present := body["unresponsiveInstances"]; present {+ t.Errorf("responsiveness %q: unresponsiveInstances present: %v", value, body["unresponsiveInstances"])+ }+ }+}++func TestServerModeHealthReportsTheResponsivenessItJustProbed(t *testing.T) {+ source := &probingInstances{+ instances: []bridge.Instance{{ID: "inst_1", Status: "running", Responsiveness: bridge.ResponsivenessResponsive}},+ probed: bridge.ResponsivenessUnresponsive,+ }+ body := healthBody(t, source)+ if body["status"] != "degraded" {+ t.Errorf("status = %v, want degraded: the list must be read after the probe, not before", body["status"])+ }+}diff --git a/internal/orchestrator/health_transport.go b/internal/orchestrator/health_transport.goindex 5fb1a027..f357294d 100644--- a/internal/orchestrator/health_transport.go+++ b/internal/orchestrator/health_transport.go@@ -1,10 +1,13 @@package orchestratorimport (+ "context""encoding/json"+ "errors""fmt""net/http""strings"+ "time""github.com/pinchtab/pinchtab/internal/activity""github.com/pinchtab/pinchtab/internal/bridge"@@ -34,19 +37,22 @@ type memoryMetrics struct {UnreadableTargets int `json:"unreadableTargets"`}-func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {- target, err := o.instancePathURL(inst, "/tabs", "")+func (o *Orchestrator) instanceGet(ctx context.Context, inst *InstanceInternal, path string) (*http.Response, error) {+ target, err := o.instancePathURL(inst, path, "")if err != nil {return nil, err}- req, err := http.NewRequest(http.MethodGet, target.String(), nil)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil)if err != nil {return nil, err}tagOrchestratorMonitoringRequest(req)o.applyInstanceAuth(req, inst)+ return o.client.Do(req)+}- resp, err := o.client.Do(req)+func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {+ resp, err := o.instanceGet(context.Background(), inst, "/tabs")if err != nil {return nil, err}@@ -66,18 +72,7 @@ func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {}func (o *Orchestrator) fetchMetrics(inst *InstanceInternal) (*memoryMetrics, error) {- target, err := o.instancePathURL(inst, "/metrics", "")- if err != nil {- return nil, err- }- req, err := http.NewRequest(http.MethodGet, target.String(), nil)- if err != nil {- return nil, err- }- tagOrchestratorMonitoringRequest(req)- o.applyInstanceAuth(req, inst)-- resp, err := o.client.Do(req)+ resp, err := o.instanceGet(context.Background(), inst, "/metrics")if err != nil {return nil, err}@@ -94,19 +89,8 @@ func (o *Orchestrator) fetchMetrics(inst *InstanceInternal) (*memoryMetrics, errreturn result.Memory, nil}-func (o *Orchestrator) fetchCrashes(inst *InstanceInternal) (*bridge.CrashSummary, error) {- target, err := o.instancePathURL(inst, "/health", "")- if err != nil {- return nil, err- }- req, err := http.NewRequest(http.MethodGet, target.String(), nil)- if err != nil {- return nil, err- }- tagOrchestratorMonitoringRequest(req)- o.applyInstanceAuth(req, inst)-- resp, err := o.client.Do(req)+func (o *Orchestrator) fetchCrashes(ctx context.Context, inst *InstanceInternal) (*bridge.CrashSummary, error) {+ resp, err := o.instanceGet(ctx, inst, "/health")if err != nil {return nil, err}@@ -122,6 +106,30 @@ func (o *Orchestrator) fetchCrashes(inst *InstanceInternal) (*bridge.CrashSummarreturn result.Crashes, nil}+var responsivenessProbeBudget = 3 * time.Second++func (o *Orchestrator) probeTabs(ctx context.Context, inst *InstanceInternal) error {+ resp, err := o.instanceGet(ctx, inst, "/tabs")+ if err != nil {+ return err+ }+ _ = resp.Body.Close()+ return nil+}++func classifyResponsiveness(healthErr, tabsErr error) string {+ switch {+ case healthErr != nil:+ return bridge.ResponsivenessUnknown+ case tabsErr == nil:+ return bridge.ResponsivenessResponsive+ case errors.Is(tabsErr, context.DeadlineExceeded):+ return bridge.ResponsivenessUnresponsive+ default:+ return bridge.ResponsivenessUnknown+ }+}+func tagOrchestratorMonitoringRequest(req *http.Request) {if req == nil {returndiff --git a/internal/orchestrator/instance_query.go b/internal/orchestrator/instance_query.goindex 55225fd5..c835c30f 100644--- a/internal/orchestrator/instance_query.go+++ b/internal/orchestrator/instance_query.go@@ -1,11 +1,13 @@package orchestratorimport (+ "context""fmt""net/http""net/url""sort""strings"+ "sync""time""github.com/pinchtab/pinchtab/internal/api/types"@@ -35,6 +37,7 @@ func (o *Orchestrator) List() []bridge.Instance {for _, inst := range o.instances {copyInst := inst.InstancecopyInst.Status = effectiveInstanceStatus(copyInst.Status, instanceIsActive(inst))+ copyInst.Responsiveness = bridge.NormalizeResponsiveness(copyInst.Responsiveness)if crashes, ok := o.crashes[inst.ID]; ok && crashes.Total > 0 {summary := crashescopyInst.Crashes = &summary@@ -44,10 +47,34 @@ func (o *Orchestrator) List() []bridge.Instance {return result}-// RefreshCrashes asks every live instance for its crash record and keeps the-// answers, so List can carry them and CrashSummary can merge them. Browser-// crashes are recorded by the process that owns the browser, which in server-// mode is never this one.+type instanceProbe struct {+ crashes *bridge.CrashSummary+ responsiveness string+}++func (o *Orchestrator) probeInstance(inst *InstanceInternal) instanceProbe {+ ctx, cancel := context.WithTimeout(context.Background(), responsivenessProbeBudget)+ defer cancel()+ var tabsErr error+ done := make(chan struct{})+ go func() {+ tabsErr = o.probeTabs(ctx, inst)+ close(done)+ }()+ crashes, healthErr := o.fetchCrashes(ctx, inst)+ <-done+ if crashes != nil {+ for i := range crashes.Recent {+ crashes.Recent[i].InstanceID = inst.ID+ }+ }+ return instanceProbe{crashes: crashes, responsiveness: classifyResponsiveness(healthErr, tabsErr)}+}++// RefreshCrashes asks every live instance for its crash record and probes its+// tabs route, so List can carry both and CrashSummary can merge the crashes.+// Browser crashes are recorded by the process that owns the browser, which in+// server mode is never this one.func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {o.mu.RLock()instances := make([]*InstanceInternal, 0, len(o.instances))@@ -58,24 +85,32 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {}o.mu.RUnlock()- fresh := make(map[string]bridge.CrashSummary, len(instances))- for _, inst := range instances {- crashes, err := o.fetchCrashes(inst)- if err != nil || crashes == nil {- continue- }- for i := range crashes.Recent {- crashes.Recent[i].InstanceID = inst.ID- }- fresh[inst.ID] = *crashes+ probes := make([]instanceProbe, len(instances))+ var wg sync.WaitGroup+ for i, inst := range instances {+ wg.Add(1)+ go func(i int, inst *InstanceInternal) {+ defer wg.Done()+ probes[i] = o.probeInstance(inst)+ }(i, inst)}+ wg.Wait()+ fresh := make(map[string]bridge.CrashSummary, len(instances))o.mu.Lock()if o.crashes == nil {o.crashes = map[string]bridge.CrashSummary{}}- for id, crashes := range fresh {- o.crashes[id] = crashes+ for _, inst := range o.instances {+ inst.Responsiveness = bridge.ResponsivenessUnknown+ }+ for i, inst := range instances {+ inst.Responsiveness = probes[i].responsiveness+ if probes[i].crashes == nil {+ continue+ }+ fresh[inst.ID] = *probes[i].crashes+ o.crashes[inst.ID] = *probes[i].crashes}o.mu.Unlock()return freshdiff --git a/internal/orchestrator/instance_responsiveness_test.go b/internal/orchestrator/instance_responsiveness_test.gonew file mode 100644index 00000000..c26ad6d0--- /dev/null+++ b/internal/orchestrator/instance_responsiveness_test.go@@ -0,0 +1,175 @@+package orchestrator++import (+ "net/http"+ "net/http/httptest"+ "sync"+ "testing"+ "time"++ "github.com/pinchtab/pinchtab/internal/activity"+ "github.com/pinchtab/pinchtab/internal/bridge"+)++const stubInstanceID = "inst_probe"++func hangUntilClientGivesUp(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() }++func answerHealth(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"status":"ok","tabs":0}`))+}++func answerTabs(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"tabs":[]}`))+}++func shortProbeBudget(t *testing.T) time.Duration {+ t.Helper()+ previous := responsivenessProbeBudget+ responsivenessProbeBudget = 150 * time.Millisecond+ t.Cleanup(func() { responsivenessProbeBudget = previous })+ return responsivenessProbeBudget+}++func orchestratorOverStubChild(t *testing.T, handler http.Handler) (*Orchestrator, *mockRunner) {+ t.Helper()+ backend := httptest.NewServer(handler)+ t.Cleanup(backend.Close)+ runner := &mockRunner{portAvail: true}+ o := NewOrchestratorWithRunner(t.TempDir(), runner)+ o.client = backend.Client()+ o.mu.Lock()+ o.instances[stubInstanceID] = &InstanceInternal{+ Instance: bridge.Instance{ID: stubInstanceID, ProfileName: "p1", Status: "running"},+ URL: backend.URL,+ }+ o.mu.Unlock()+ return o, runner+}++func stubChild(health, tabs http.HandlerFunc) http.Handler {+ mux := http.NewServeMux()+ mux.HandleFunc("/health", health)+ mux.HandleFunc("/tabs", tabs)+ return mux+}++func TestResponsivenessIsThreeDistinctValuesNeverSharingARepresentation(t *testing.T) {+ values := map[string]bool{+ bridge.ResponsivenessResponsive: true,+ bridge.ResponsivenessUnresponsive: true,+ bridge.ResponsivenessUnknown: true,+ }+ if len(values) != 3 {+ t.Fatalf("responsiveness values collide: %v", values)+ }+ for _, v := range []string{"", "garbage"} {+ if got := bridge.NormalizeResponsiveness(v); got != bridge.ResponsivenessUnknown {+ t.Fatalf("NormalizeResponsiveness(%q) = %q, want unknown", v, got)+ }+ }+}++func TestAnInstanceAnsweringHealthButNotTabsIsUnresponsiveAndStillRunning(t *testing.T) {+ shortProbeBudget(t)+ o, runner := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))+ o.mu.RLock()+ before := o.instances[stubInstanceID]+ o.mu.RUnlock()++ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {+ t.Fatalf("never-probed instance reads %q, want unknown", got)+ }+ o.RefreshCrashes()++ inst := o.List()[0]+ if inst.Responsiveness != bridge.ResponsivenessUnresponsive {+ t.Fatalf("responsiveness = %q, want unresponsive", inst.Responsiveness)+ }+ if inst.Status != "running" {+ t.Fatalf("status = %q, want running: responsiveness never changes routing", inst.Status)+ }+ o.mu.RLock()+ after := o.instances[stubInstanceID]+ o.mu.RUnlock()+ if runner.runCalled || after != before {+ t.Fatalf("an unresponsive result restarted the instance (runCalled=%v, replaced=%v)", runner.runCalled, after != before)+ }+}++func TestAnInstanceBlockingEveryRouteIsUnknownNotUnresponsive(t *testing.T) {+ shortProbeBudget(t)+ o, _ := orchestratorOverStubChild(t, stubChild(hangUntilClientGivesUp, hangUntilClientGivesUp))+ o.RefreshCrashes()+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {+ t.Fatalf("responsiveness = %q, want unknown: its /health never answered either", got)+ }+}++func TestAnInstanceAnsweringBothRoutesIsResponsive(t *testing.T) {+ shortProbeBudget(t)+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))+ o.RefreshCrashes()+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessResponsive {+ t.Fatalf("responsiveness = %q, want responsive", got)+ }+}++func TestAStoppedInstanceReadsUnknownAgainAfterTheNextRefresh(t *testing.T) {+ shortProbeBudget(t)+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))+ o.RefreshCrashes()+ o.mu.Lock()+ o.instances[stubInstanceID].Status = "stopped"+ o.mu.Unlock()+ o.RefreshCrashes()+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {+ t.Fatalf("responsiveness = %q after the instance stopped, want unknown", got)+ }+}++func TestTheProbeBudgetBoundsTheCrashSummaryLatency(t *testing.T) {+ budget := shortProbeBudget(t)+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))+ started := time.Now()+ o.CrashSummary()+ elapsed := time.Since(started)+ if elapsed < budget || elapsed > budget+500*time.Millisecond {+ t.Fatalf("CrashSummary took %v with one hanging child, want within the %v probe budget plus a small margin", elapsed, budget)+ }+}++type capturingRecorder struct {+ mu sync.Mutex+ events []activity.Event+}++func (c *capturingRecorder) Enabled() bool { return true }+func (c *capturingRecorder) Record(evt activity.Event) error {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.events = append(c.events, evt)+ return nil+}+func (c *capturingRecorder) Query(activity.Filter) ([]activity.Event, error) { return nil, nil }++func TestTheTabsProbeReachesTheChildAsOrchestratorActivityNotClientActivity(t *testing.T) {+ shortProbeBudget(t)+ rec := &capturingRecorder{}+ o, _ := orchestratorOverStubChild(t, activity.Middleware(rec, "client", stubChild(answerHealth, answerTabs)))+ o.RefreshCrashes()++ rec.mu.Lock()+ defer rec.mu.Unlock()+ paths := map[string]string{}+ for _, evt := range rec.events {+ paths[evt.Path] = evt.Source+ }+ for _, path := range []string{"/tabs", "/health"} {+ if source, ok := paths[path]; !ok || source != orchestratorActivitySource {+ t.Fatalf("probe of %s recorded with source %q (seen=%v), want %q so it counts as neither activity nor an idle reset", path, source, ok, orchestratorActivitySource)+ }+ }+}