Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(orchestrator): report an instance that answers /health but not its browser routes as unresponsive (PIN-340)

Luigi Agosti34f69ab367d0
diff --git a/dashboard/src/generated/types.ts b/dashboard/src/generated/types.ts
index 37cce480..274177a3 100644
--- a/dashboard/src/generated/types.ts
+++ b/dashboard/src/generated/types.ts
@@ -59,6 +59,7 @@ export interface Instance {
fallbackFrom?: string;
fallbackReason?: string;
crashes?: CrashSummary;
+ responsiveness: string; // responsive/unresponsive/unknown
}
export interface CrashEvent {
time: string;
diff --git a/docs/reference/health.md b/docs/reference/health.md
index 49ca2305..51fefaa0 100644
--- a/docs/reference/health.md
+++ b/docs/reference/health.md
@@ -56,18 +56,19 @@ curl http://localhost:9867/health
| Field | Description |
| --- | --- |
-| `status` | `ok` when server is healthy |
+| `status` | `ok` when server is healthy; `degraded` while at least one instance is unresponsive |
| `mode` | `dashboard` in server mode |
| `version` | PinchTab version |
| `uptime` | Milliseconds since server start |
| `authRequired` | `true` when a server token is configured |
| `profiles` | Number of configured profiles |
| `instances` | Number of managed instances |
-| `defaultInstance` | First managed instance info, when present |
+| `defaultInstance` | First managed instance info, when present: `id`, `status` and `responsiveness` |
| `agents` | Connected agent count |
| `restartRequired` | `true` when file-based config changes need restart |
| `restartReasons` | Restart reason list when required |
| `crashes` | Present once any instance's browser has crashed: `total` and `recent`, the same block bridge `/health` carries, each event naming its `instanceId` |
+| `unresponsiveInstances` | Present while any instance answers its own `/health` but not its browser routes: the ids of those instances |
Notes:
@@ -75,6 +76,7 @@ Notes:
- use `defaultInstance.status == "running"` when you want to confirm Chrome is ready
- strategies such as `always-on` can create an instance automatically at startup
- `status` does not degrade on a browser crash: the instance is relaunched and is serving again. The crash is history, so it rides beside `status` as `crashes`; a crashed-then-relaunched instance differs from one that never crashed by that key alone. Every tab the dead browser held is gone, and a call to one answers `404` with code `browser_crashed` and a `hint` saying so
+- `status` does degrade to `degraded` while an instance is unresponsive, because that condition is current: on every `/health` the front door probes each running instance's `/tabs` under a short budget beside its crash fetch, and an instance whose `/health` answers while `/tabs` exceeds the budget is named in `unresponsiveInstances`. Its own `status` stays `running` and nothing restarts it; the field is a report, not a remedy. See `responsiveness` on `GET /instances`
## Related Pages
diff --git a/docs/reference/instances.md b/docs/reference/instances.md
index 08844cf4..6c706b75 100644
--- a/docs/reference/instances.md
+++ b/docs/reference/instances.md
@@ -37,6 +37,7 @@ Response shape:
"mode": "headed",
"headless": false,
"status": "running",
+ "responsiveness": "responsive",
"securityPolicy": {
"allowedDomains": ["127.0.0.1", "localhost", "::1", "wikipedia.org"]
}
@@ -117,6 +118,7 @@ Instance responses include:
- `mode`: `"headless"` or `"headed"`
- `headless`: boolean kept for compatibility
+- `responsiveness`: whether the instance's browser routes answer, measured on the last front-door `/health` or monitoring snapshot by probing the instance's `/tabs` under a short budget. `responsive` when it answered, `unresponsive` when its `/health` answered but `/tabs` did not within the budget, `unknown` when it has not been probed yet or the probe could not connect. It never changes `status`, which stays `running` for an unresponsive instance, and it triggers no restart. The front door's `/health` degrades while any instance is `unresponsive`
## Get Instance Logs
diff --git a/internal/api/types/types.go b/internal/api/types/types.go
index 65df6f55..85faae04 100644
--- a/internal/api/types/types.go
+++ b/internal/api/types/types.go
@@ -57,6 +57,8 @@ type Instance struct {
FallbackReason string `json:"fallbackReason,omitempty"`
Crashes *CrashSummary `json:"crashes,omitempty"`
+
+ Responsiveness string `json:"responsiveness"` // responsive/unresponsive/unknown
}
type CrashEvent struct {
diff --git a/internal/bridge/api.go b/internal/bridge/api.go
index 066d19df..7fcb77b6 100644
--- a/internal/bridge/api.go
+++ b/internal/bridge/api.go
@@ -329,12 +329,30 @@ type Instance struct {
FallbackReason string `json:"fallbackReason,omitempty"`
Crashes *CrashSummary `json:"crashes,omitempty"`
+
+ Responsiveness string `json:"responsiveness"`
+}
+
+const (
+ ResponsivenessResponsive = "responsive"
+ ResponsivenessUnresponsive = "unresponsive"
+ ResponsivenessUnknown = "unknown"
+)
+
+func NormalizeResponsiveness(value string) string {
+ switch value {
+ case ResponsivenessResponsive, ResponsivenessUnresponsive:
+ return value
+ default:
+ return ResponsivenessUnknown
+ }
}
func (i Instance) MarshalJSON() ([]byte, error) {
type alias Instance
copy := alias(i)
copy.Mode = normalizeInstanceMode(copy.Mode, copy.Headless)
+ copy.Responsiveness = NormalizeResponsiveness(copy.Responsiveness)
return json.Marshal(copy)
}
diff --git a/internal/dashboard/config_health.go b/internal/dashboard/config_health.go
index 7f40a4af..54797df2 100644
--- a/internal/dashboard/config_health.go
+++ b/internal/dashboard/config_health.go
@@ -12,8 +12,9 @@ import (
)
type healthInstanceInfo struct {
- ID string `json:"id"`
- Status string `json:"status"`
+ ID string `json:"id"`
+ Status string `json:"status"`
+ Responsiveness string `json:"responsiveness"`
}
type healthSecurityInfo struct {
@@ -26,21 +27,37 @@ type healthSecurityInfo struct {
}
type healthEnvelope struct {
- Status string `json:"status"`
- Mode string `json:"mode"`
- Version string `json:"version"`
- Uptime int64 `json:"uptime"`
- AuthRequired bool `json:"authRequired"`
- Profiles int `json:"profiles"`
- TemporaryProfiles int `json:"temporaryProfiles"`
- QuarantinedProfiles int `json:"quarantinedProfiles"`
- Instances int `json:"instances"`
- DefaultInstance *healthInstanceInfo `json:"defaultInstance,omitempty"`
- Agents int `json:"agents"`
- RestartRequired bool `json:"restartRequired"`
- RestartReasons []string `json:"restartReasons,omitempty"`
- Security *healthSecurityInfo `json:"security,omitempty"`
- Crashes *bridge.CrashSummary `json:"crashes,omitempty"`
+ Status string `json:"status"`
+ Mode string `json:"mode"`
+ Version string `json:"version"`
+ Uptime int64 `json:"uptime"`
+ AuthRequired bool `json:"authRequired"`
+ Profiles int `json:"profiles"`
+ TemporaryProfiles int `json:"temporaryProfiles"`
+ QuarantinedProfiles int `json:"quarantinedProfiles"`
+ Instances int `json:"instances"`
+ DefaultInstance *healthInstanceInfo `json:"defaultInstance,omitempty"`
+ Agents int `json:"agents"`
+ RestartRequired bool `json:"restartRequired"`
+ RestartReasons []string `json:"restartReasons,omitempty"`
+ Security *healthSecurityInfo `json:"security,omitempty"`
+ Crashes *bridge.CrashSummary `json:"crashes,omitempty"`
+ UnresponsiveInstances []string `json:"unresponsiveInstances,omitempty"`
+}
+
+const (
+ healthStatusOK = "ok"
+ healthStatusDegraded = "degraded"
+)
+
+func unresponsiveInstanceIDs(instances []bridge.Instance) []string {
+ var ids []string
+ for _, inst := range instances {
+ if inst.Responsiveness == bridge.ResponsivenessUnresponsive {
+ ids = append(ids, inst.ID)
+ }
+ }
+ return ids
}
type crashReporter interface {
@@ -75,47 +92,57 @@ func (c *ConfigAPI) healthInfo(includeSecurity bool) (healthEnvelope, error) {
}
}
+ var crashes *bridge.CrashSummary
+ if reporter, ok := c.instances.(crashReporter); ok {
+ if summary := reporter.CrashSummary(); summary.Total > 0 {
+ crashes = &summary
+ }
+ }
instanceCount := 0
var defaultInst *healthInstanceInfo
+ var unresponsive []string
if c.instances != nil {
instances := c.instances.List()
instanceCount = len(instances)
+ unresponsive = unresponsiveInstanceIDs(instances)
if len(instances) > 0 {
defaultInst = &healthInstanceInfo{
- ID: instances[0].ID,
- Status: instances[0].Status,
+ ID: instances[0].ID,
+ Status: instances[0].Status,
+ Responsiveness: instances[0].Responsiveness,
}
}
}
+ status := healthStatusOK
+ if len(unresponsive) > 0 {
+ status = healthStatusDegraded
+ }
agentCount := 0
if c.agents != nil {
agentCount = c.agents.AgentCount()
}
cfg := c.cfg()
out := healthEnvelope{
- Status: "ok",
- Mode: "dashboard",
- Version: c.version,
- Uptime: int64(time.Since(c.startedAt).Milliseconds()),
- AuthRequired: cfg != nil && strings.TrimSpace(cfg.Token) != "",
- Profiles: profileCount,
- TemporaryProfiles: temporaryCount,
- QuarantinedProfiles: quarantinedCount,
- Instances: instanceCount,
- DefaultInstance: defaultInst,
- Agents: agentCount,
- RestartRequired: len(restartReasons) > 0,
- RestartReasons: restartReasons,
+ Status: status,
+ Mode: "dashboard",
+ Version: c.version,
+ Uptime: int64(time.Since(c.startedAt).Milliseconds()),
+ AuthRequired: cfg != nil && strings.TrimSpace(cfg.Token) != "",
+ Profiles: profileCount,
+ TemporaryProfiles: temporaryCount,
+ QuarantinedProfiles: quarantinedCount,
+ Instances: instanceCount,
+ DefaultInstance: defaultInst,
+ Agents: agentCount,
+ RestartRequired: len(restartReasons) > 0,
+ RestartReasons: restartReasons,
+ Crashes: crashes,
+ UnresponsiveInstances: unresponsive,
}
if includeSecurity {
security := runtimeSecurityInfo(cfg)
out.Security = &security
}
- if reporter, ok := c.instances.(crashReporter); ok {
- if crashes := reporter.CrashSummary(); crashes.Total > 0 {
- out.Crashes = &crashes
- }
- }
return out, nil
}
diff --git a/internal/dashboard/health_responsiveness_test.go b/internal/dashboard/health_responsiveness_test.go
new file mode 100644
index 00000000..476a8f5f
--- /dev/null
+++ b/internal/dashboard/health_responsiveness_test.go
@@ -0,0 +1,62 @@
+package dashboard
+
+import (
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/bridge"
+)
+
+type probingInstances struct {
+ instances []bridge.Instance
+ probed string
+}
+
+func (s *probingInstances) List() []bridge.Instance { return s.instances }
+
+func (s *probingInstances) CrashSummary() bridge.CrashSummary {
+ for i := range s.instances {
+ s.instances[i].Responsiveness = s.probed
+ }
+ return bridge.CrashSummary{}
+}
+
+func TestServerModeHealthDegradesWhileAnInstanceIsUnresponsive(t *testing.T) {
+ body := healthBody(t, plainInstances{instances: []bridge.Instance{
+ {ID: "inst_ok", Status: "running", Responsiveness: bridge.ResponsivenessResponsive},
+ {ID: "inst_wedged", Status: "running", Responsiveness: bridge.ResponsivenessUnresponsive},
+ }})
+ if body["status"] != "degraded" {
+ t.Errorf("status = %v, want degraded while an instance is unresponsive", body["status"])
+ }
+ named, _ := body["unresponsiveInstances"].([]any)
+ if len(named) != 1 || named[0] != "inst_wedged" {
+ t.Errorf("unresponsiveInstances = %v, want [inst_wedged]", body["unresponsiveInstances"])
+ }
+ def, _ := body["defaultInstance"].(map[string]any)
+ if def["responsiveness"] != bridge.ResponsivenessResponsive || def["status"] != "running" {
+ t.Errorf("defaultInstance = %v, want its responsiveness beside an unchanged status", def)
+ }
+}
+
+func TestServerModeHealthStaysOkForResponsiveAndUnknownInstances(t *testing.T) {
+ for _, value := range []string{bridge.ResponsivenessResponsive, bridge.ResponsivenessUnknown, ""} {
+ body := healthBody(t, plainInstances{instances: []bridge.Instance{{ID: "inst_1", Status: "running", Responsiveness: value}}})
+ if body["status"] != "ok" {
+ t.Errorf("responsiveness %q: status = %v, want ok", value, body["status"])
+ }
+ if _, present := body["unresponsiveInstances"]; present {
+ t.Errorf("responsiveness %q: unresponsiveInstances present: %v", value, body["unresponsiveInstances"])
+ }
+ }
+}
+
+func TestServerModeHealthReportsTheResponsivenessItJustProbed(t *testing.T) {
+ source := &probingInstances{
+ instances: []bridge.Instance{{ID: "inst_1", Status: "running", Responsiveness: bridge.ResponsivenessResponsive}},
+ probed: bridge.ResponsivenessUnresponsive,
+ }
+ body := healthBody(t, source)
+ if body["status"] != "degraded" {
+ t.Errorf("status = %v, want degraded: the list must be read after the probe, not before", body["status"])
+ }
+}
diff --git a/internal/orchestrator/health_transport.go b/internal/orchestrator/health_transport.go
index 5fb1a027..f357294d 100644
--- a/internal/orchestrator/health_transport.go
+++ b/internal/orchestrator/health_transport.go
@@ -1,10 +1,13 @@
package orchestrator
import (
+ "context"
"encoding/json"
+ "errors"
"fmt"
"net/http"
"strings"
+ "time"
"github.com/pinchtab/pinchtab/internal/activity"
"github.com/pinchtab/pinchtab/internal/bridge"
@@ -34,19 +37,22 @@ type memoryMetrics struct {
UnreadableTargets int `json:"unreadableTargets"`
}
-func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {
- target, err := o.instancePathURL(inst, "/tabs", "")
+func (o *Orchestrator) instanceGet(ctx context.Context, inst *InstanceInternal, path string) (*http.Response, error) {
+ target, err := o.instancePathURL(inst, path, "")
if err != nil {
return nil, err
}
- req, err := http.NewRequest(http.MethodGet, target.String(), nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil)
if err != nil {
return nil, err
}
tagOrchestratorMonitoringRequest(req)
o.applyInstanceAuth(req, inst)
+ return o.client.Do(req)
+}
- resp, err := o.client.Do(req)
+func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {
+ resp, err := o.instanceGet(context.Background(), inst, "/tabs")
if err != nil {
return nil, err
}
@@ -66,18 +72,7 @@ func (o *Orchestrator) fetchTabs(inst *InstanceInternal) ([]remoteTab, error) {
}
func (o *Orchestrator) fetchMetrics(inst *InstanceInternal) (*memoryMetrics, error) {
- target, err := o.instancePathURL(inst, "/metrics", "")
- if err != nil {
- return nil, err
- }
- req, err := http.NewRequest(http.MethodGet, target.String(), nil)
- if err != nil {
- return nil, err
- }
- tagOrchestratorMonitoringRequest(req)
- o.applyInstanceAuth(req, inst)
-
- resp, err := o.client.Do(req)
+ resp, err := o.instanceGet(context.Background(), inst, "/metrics")
if err != nil {
return nil, err
}
@@ -94,19 +89,8 @@ func (o *Orchestrator) fetchMetrics(inst *InstanceInternal) (*memoryMetrics, err
return result.Memory, nil
}
-func (o *Orchestrator) fetchCrashes(inst *InstanceInternal) (*bridge.CrashSummary, error) {
- target, err := o.instancePathURL(inst, "/health", "")
- if err != nil {
- return nil, err
- }
- req, err := http.NewRequest(http.MethodGet, target.String(), nil)
- if err != nil {
- return nil, err
- }
- tagOrchestratorMonitoringRequest(req)
- o.applyInstanceAuth(req, inst)
-
- resp, err := o.client.Do(req)
+func (o *Orchestrator) fetchCrashes(ctx context.Context, inst *InstanceInternal) (*bridge.CrashSummary, error) {
+ resp, err := o.instanceGet(ctx, inst, "/health")
if err != nil {
return nil, err
}
@@ -122,6 +106,30 @@ func (o *Orchestrator) fetchCrashes(inst *InstanceInternal) (*bridge.CrashSummar
return result.Crashes, nil
}
+var responsivenessProbeBudget = 3 * time.Second
+
+func (o *Orchestrator) probeTabs(ctx context.Context, inst *InstanceInternal) error {
+ resp, err := o.instanceGet(ctx, inst, "/tabs")
+ if err != nil {
+ return err
+ }
+ _ = resp.Body.Close()
+ return nil
+}
+
+func classifyResponsiveness(healthErr, tabsErr error) string {
+ switch {
+ case healthErr != nil:
+ return bridge.ResponsivenessUnknown
+ case tabsErr == nil:
+ return bridge.ResponsivenessResponsive
+ case errors.Is(tabsErr, context.DeadlineExceeded):
+ return bridge.ResponsivenessUnresponsive
+ default:
+ return bridge.ResponsivenessUnknown
+ }
+}
+
func tagOrchestratorMonitoringRequest(req *http.Request) {
if req == nil {
return
diff --git a/internal/orchestrator/instance_query.go b/internal/orchestrator/instance_query.go
index 55225fd5..c835c30f 100644
--- a/internal/orchestrator/instance_query.go
+++ b/internal/orchestrator/instance_query.go
@@ -1,11 +1,13 @@
package orchestrator
import (
+ "context"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
+ "sync"
"time"
"github.com/pinchtab/pinchtab/internal/api/types"
@@ -35,6 +37,7 @@ func (o *Orchestrator) List() []bridge.Instance {
for _, inst := range o.instances {
copyInst := inst.Instance
copyInst.Status = effectiveInstanceStatus(copyInst.Status, instanceIsActive(inst))
+ copyInst.Responsiveness = bridge.NormalizeResponsiveness(copyInst.Responsiveness)
if crashes, ok := o.crashes[inst.ID]; ok && crashes.Total > 0 {
summary := crashes
copyInst.Crashes = &summary
@@ -44,10 +47,34 @@ func (o *Orchestrator) List() []bridge.Instance {
return result
}
-// RefreshCrashes asks every live instance for its crash record and keeps the
-// answers, so List can carry them and CrashSummary can merge them. Browser
-// crashes are recorded by the process that owns the browser, which in server
-// mode is never this one.
+type instanceProbe struct {
+ crashes *bridge.CrashSummary
+ responsiveness string
+}
+
+func (o *Orchestrator) probeInstance(inst *InstanceInternal) instanceProbe {
+ ctx, cancel := context.WithTimeout(context.Background(), responsivenessProbeBudget)
+ defer cancel()
+ var tabsErr error
+ done := make(chan struct{})
+ go func() {
+ tabsErr = o.probeTabs(ctx, inst)
+ close(done)
+ }()
+ crashes, healthErr := o.fetchCrashes(ctx, inst)
+ <-done
+ if crashes != nil {
+ for i := range crashes.Recent {
+ crashes.Recent[i].InstanceID = inst.ID
+ }
+ }
+ return instanceProbe{crashes: crashes, responsiveness: classifyResponsiveness(healthErr, tabsErr)}
+}
+
+// RefreshCrashes asks every live instance for its crash record and probes its
+// tabs route, so List can carry both and CrashSummary can merge the crashes.
+// Browser crashes are recorded by the process that owns the browser, which in
+// server mode is never this one.
func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {
o.mu.RLock()
instances := make([]*InstanceInternal, 0, len(o.instances))
@@ -58,24 +85,32 @@ func (o *Orchestrator) RefreshCrashes() map[string]bridge.CrashSummary {
}
o.mu.RUnlock()
- fresh := make(map[string]bridge.CrashSummary, len(instances))
- for _, inst := range instances {
- crashes, err := o.fetchCrashes(inst)
- if err != nil || crashes == nil {
- continue
- }
- for i := range crashes.Recent {
- crashes.Recent[i].InstanceID = inst.ID
- }
- fresh[inst.ID] = *crashes
+ probes := make([]instanceProbe, len(instances))
+ var wg sync.WaitGroup
+ for i, inst := range instances {
+ wg.Add(1)
+ go func(i int, inst *InstanceInternal) {
+ defer wg.Done()
+ probes[i] = o.probeInstance(inst)
+ }(i, inst)
}
+ wg.Wait()
+ fresh := make(map[string]bridge.CrashSummary, len(instances))
o.mu.Lock()
if o.crashes == nil {
o.crashes = map[string]bridge.CrashSummary{}
}
- for id, crashes := range fresh {
- o.crashes[id] = crashes
+ for _, inst := range o.instances {
+ inst.Responsiveness = bridge.ResponsivenessUnknown
+ }
+ for i, inst := range instances {
+ inst.Responsiveness = probes[i].responsiveness
+ if probes[i].crashes == nil {
+ continue
+ }
+ fresh[inst.ID] = *probes[i].crashes
+ o.crashes[inst.ID] = *probes[i].crashes
}
o.mu.Unlock()
return fresh
diff --git a/internal/orchestrator/instance_responsiveness_test.go b/internal/orchestrator/instance_responsiveness_test.go
new file mode 100644
index 00000000..c26ad6d0
--- /dev/null
+++ b/internal/orchestrator/instance_responsiveness_test.go
@@ -0,0 +1,175 @@
+package orchestrator
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/pinchtab/pinchtab/internal/activity"
+ "github.com/pinchtab/pinchtab/internal/bridge"
+)
+
+const stubInstanceID = "inst_probe"
+
+func hangUntilClientGivesUp(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() }
+
+func answerHealth(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"status":"ok","tabs":0}`))
+}
+
+func answerTabs(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"tabs":[]}`))
+}
+
+func shortProbeBudget(t *testing.T) time.Duration {
+ t.Helper()
+ previous := responsivenessProbeBudget
+ responsivenessProbeBudget = 150 * time.Millisecond
+ t.Cleanup(func() { responsivenessProbeBudget = previous })
+ return responsivenessProbeBudget
+}
+
+func orchestratorOverStubChild(t *testing.T, handler http.Handler) (*Orchestrator, *mockRunner) {
+ t.Helper()
+ backend := httptest.NewServer(handler)
+ t.Cleanup(backend.Close)
+ runner := &mockRunner{portAvail: true}
+ o := NewOrchestratorWithRunner(t.TempDir(), runner)
+ o.client = backend.Client()
+ o.mu.Lock()
+ o.instances[stubInstanceID] = &InstanceInternal{
+ Instance: bridge.Instance{ID: stubInstanceID, ProfileName: "p1", Status: "running"},
+ URL: backend.URL,
+ }
+ o.mu.Unlock()
+ return o, runner
+}
+
+func stubChild(health, tabs http.HandlerFunc) http.Handler {
+ mux := http.NewServeMux()
+ mux.HandleFunc("/health", health)
+ mux.HandleFunc("/tabs", tabs)
+ return mux
+}
+
+func TestResponsivenessIsThreeDistinctValuesNeverSharingARepresentation(t *testing.T) {
+ values := map[string]bool{
+ bridge.ResponsivenessResponsive: true,
+ bridge.ResponsivenessUnresponsive: true,
+ bridge.ResponsivenessUnknown: true,
+ }
+ if len(values) != 3 {
+ t.Fatalf("responsiveness values collide: %v", values)
+ }
+ for _, v := range []string{"", "garbage"} {
+ if got := bridge.NormalizeResponsiveness(v); got != bridge.ResponsivenessUnknown {
+ t.Fatalf("NormalizeResponsiveness(%q) = %q, want unknown", v, got)
+ }
+ }
+}
+
+func TestAnInstanceAnsweringHealthButNotTabsIsUnresponsiveAndStillRunning(t *testing.T) {
+ shortProbeBudget(t)
+ o, runner := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
+ o.mu.RLock()
+ before := o.instances[stubInstanceID]
+ o.mu.RUnlock()
+
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
+ t.Fatalf("never-probed instance reads %q, want unknown", got)
+ }
+ o.RefreshCrashes()
+
+ inst := o.List()[0]
+ if inst.Responsiveness != bridge.ResponsivenessUnresponsive {
+ t.Fatalf("responsiveness = %q, want unresponsive", inst.Responsiveness)
+ }
+ if inst.Status != "running" {
+ t.Fatalf("status = %q, want running: responsiveness never changes routing", inst.Status)
+ }
+ o.mu.RLock()
+ after := o.instances[stubInstanceID]
+ o.mu.RUnlock()
+ if runner.runCalled || after != before {
+ t.Fatalf("an unresponsive result restarted the instance (runCalled=%v, replaced=%v)", runner.runCalled, after != before)
+ }
+}
+
+func TestAnInstanceBlockingEveryRouteIsUnknownNotUnresponsive(t *testing.T) {
+ shortProbeBudget(t)
+ o, _ := orchestratorOverStubChild(t, stubChild(hangUntilClientGivesUp, hangUntilClientGivesUp))
+ o.RefreshCrashes()
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
+ t.Fatalf("responsiveness = %q, want unknown: its /health never answered either", got)
+ }
+}
+
+func TestAnInstanceAnsweringBothRoutesIsResponsive(t *testing.T) {
+ shortProbeBudget(t)
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))
+ o.RefreshCrashes()
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessResponsive {
+ t.Fatalf("responsiveness = %q, want responsive", got)
+ }
+}
+
+func TestAStoppedInstanceReadsUnknownAgainAfterTheNextRefresh(t *testing.T) {
+ shortProbeBudget(t)
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, answerTabs))
+ o.RefreshCrashes()
+ o.mu.Lock()
+ o.instances[stubInstanceID].Status = "stopped"
+ o.mu.Unlock()
+ o.RefreshCrashes()
+ if got := o.List()[0].Responsiveness; got != bridge.ResponsivenessUnknown {
+ t.Fatalf("responsiveness = %q after the instance stopped, want unknown", got)
+ }
+}
+
+func TestTheProbeBudgetBoundsTheCrashSummaryLatency(t *testing.T) {
+ budget := shortProbeBudget(t)
+ o, _ := orchestratorOverStubChild(t, stubChild(answerHealth, hangUntilClientGivesUp))
+ started := time.Now()
+ o.CrashSummary()
+ elapsed := time.Since(started)
+ if elapsed < budget || elapsed > budget+500*time.Millisecond {
+ t.Fatalf("CrashSummary took %v with one hanging child, want within the %v probe budget plus a small margin", elapsed, budget)
+ }
+}
+
+type capturingRecorder struct {
+ mu sync.Mutex
+ events []activity.Event
+}
+
+func (c *capturingRecorder) Enabled() bool { return true }
+func (c *capturingRecorder) Record(evt activity.Event) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.events = append(c.events, evt)
+ return nil
+}
+func (c *capturingRecorder) Query(activity.Filter) ([]activity.Event, error) { return nil, nil }
+
+func TestTheTabsProbeReachesTheChildAsOrchestratorActivityNotClientActivity(t *testing.T) {
+ shortProbeBudget(t)
+ rec := &capturingRecorder{}
+ o, _ := orchestratorOverStubChild(t, activity.Middleware(rec, "client", stubChild(answerHealth, answerTabs)))
+ o.RefreshCrashes()
+
+ rec.mu.Lock()
+ defer rec.mu.Unlock()
+ paths := map[string]string{}
+ for _, evt := range rec.events {
+ paths[evt.Path] = evt.Source
+ }
+ for _, path := range []string{"/tabs", "/health"} {
+ if source, ok := paths[path]; !ok || source != orchestratorActivitySource {
+ t.Fatalf("probe of %s recorded with source %q (seen=%v), want %q so it counts as neither activity nor an idle reset", path, source, ok, orchestratorActivitySource)
+ }
+ }
+}