Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(orchestrator): startup monitors outlive the orchestrator that started them

Darshan Jain2d519e490aa5
diff --git a/internal/orchestrator/health.go b/internal/orchestrator/health.go
index c10ecbc6..c14455a5 100644
--- a/internal/orchestrator/health.go
+++ b/internal/orchestrator/health.go
@@ -27,6 +27,40 @@ type startupProbe struct {
waitCh chan error
}
+// startMonitor runs monitor for inst and registers it with o.monitors, so
+// Shutdown can wait for it. Every monitor must start this way; a bare
+// `go o.monitor(inst)` is a goroutine that outlives its orchestrator.
+func (o *Orchestrator) startMonitor(inst *InstanceInternal) {
+ o.monitors.Add(1)
+ go func() {
+ defer o.monitors.Done()
+ o.monitor(inst)
+ }()
+}
+
+// shuttingDown reports whether Shutdown has been called. Safe on an
+// Orchestrator built without the constructor: a nil channel is never ready.
+func (o *Orchestrator) shuttingDown() bool {
+ select {
+ case <-o.shutdownCh:
+ return true
+ default:
+ return false
+ }
+}
+
+// sleepOrShutdown waits for d, or returns early once Shutdown is called. It
+// replaces a bare time.Sleep in the probe loop so a shutdown does not have to
+// wait out a poll interval per instance.
+func (o *Orchestrator) sleepOrShutdown(d time.Duration) {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ case <-o.shutdownCh:
+ }
+}
+
func (o *Orchestrator) monitor(inst *InstanceInternal) {
p := o.probeStartupHealth(inst)
o.applyStartupOutcome(inst, p)
@@ -56,7 +90,18 @@ func (o *Orchestrator) probeStartupHealth(inst *InstanceInternal) startupProbe {
if portErr != nil {
break
}
- time.Sleep(instanceHealthPollInterval)
+ // Stop probing a starting instance once the orchestrator is going down;
+ // applyStartupOutcome below leaves a stopping/stopped instance alone, so
+ // bailing here loses nothing.
+ if o.shuttingDown() {
+ p.lastProbe = "orchestrator shutting down"
+ break
+ }
+ o.sleepOrShutdown(instanceHealthPollInterval)
+ if o.shuttingDown() {
+ p.lastProbe = "orchestrator shutting down"
+ break
+ }
for _, baseURL := range instanceBaseURLs(configuredChildBind(o.runtimeCfg), probePort) {
targetBaseURL, err := o.validatedHealthProbeBaseURL(baseURL, "", healthProbePolicyLoopback)
@@ -129,7 +174,15 @@ func (o *Orchestrator) applyStartupOutcome(inst *InstanceInternal, p startupProb
func (o *Orchestrator) finalizeInstanceExit(inst *InstanceInternal, p startupProbe) {
if !p.exitedEarly {
- <-p.waitCh
+ // Interruptible: this waits for the child process to report exit, which
+ // for a process that never exits is forever. Shutdown waits on the
+ // monitors, so a bare receive here makes shutdown unbounded. Stop has
+ // already signalled the process by the time shutdownCh closes; the state
+ // transition below still runs.
+ select {
+ case <-p.waitCh:
+ case <-o.shutdownCh:
+ }
}
o.mu.Lock()
wasStopped := false
diff --git a/internal/orchestrator/instance_attach.go b/internal/orchestrator/instance_attach.go
index 402e4cc6..ed6ed59c 100644
--- a/internal/orchestrator/instance_attach.go
+++ b/internal/orchestrator/instance_attach.go
@@ -209,7 +209,7 @@ func (o *Orchestrator) AttachWithOptions(name, cdpURL string, opts AttachOptions
reserved = false
released = true
- go o.monitor(internal)
+ o.startMonitor(internal)
healthTimeout := o.attachHealthCheckTimeout
if healthTimeout <= 0 {
diff --git a/internal/orchestrator/instance_launch.go b/internal/orchestrator/instance_launch.go
index c655985b..9179d9d5 100644
--- a/internal/orchestrator/instance_launch.go
+++ b/internal/orchestrator/instance_launch.go
@@ -228,7 +228,7 @@ func (o *Orchestrator) LaunchWithOptions(name, port string, headless bool, opts
o.mu.Unlock()
reservedPorts = nil
- go o.monitor(inst)
+ o.startMonitor(inst)
return &inst.Instance, nil
}
diff --git a/internal/orchestrator/instance_stop.go b/internal/orchestrator/instance_stop.go
index eeed553a..551961dc 100644
--- a/internal/orchestrator/instance_stop.go
+++ b/internal/orchestrator/instance_stop.go
@@ -288,6 +288,49 @@ func (o *Orchestrator) Shutdown() {
}(id)
}
wg.Wait()
+
+ // Signal after the instances are stopped, so a monitor still probing a
+ // healthy instance gets to record its outcome, then wait for every monitor
+ // to return. Bounded by one poll interval rather than
+ // instanceStartupTimeout, because the probe loop watches shutdownCh.
+ o.signalShutdown()
+ if !waitGroupWithin(&o.monitors, monitorShutdownGrace) {
+ // Belt and braces. finalizeInstanceExit is interruptible, so this should
+ // not fire; if a monitor ever blocks somewhere else, shutdown must still
+ // return rather than hang the process.
+ slog.Warn("startup monitors did not finish within the shutdown grace period",
+ "grace", monitorShutdownGrace)
+ }
+}
+
+// monitorShutdownGrace bounds how long Shutdown waits for startup monitors.
+// Generous next to the poll interval they watch shutdownCh on, and far below
+// instanceStartupTimeout.
+const monitorShutdownGrace = 5 * time.Second
+
+// waitGroupWithin reports whether wg finished inside d.
+func waitGroupWithin(wg *sync.WaitGroup, d time.Duration) bool {
+ done := make(chan struct{})
+ go func() {
+ wg.Wait()
+ close(done)
+ }()
+ select {
+ case <-done:
+ return true
+ case <-time.After(d):
+ return false
+ }
+}
+
+// signalShutdown closes shutdownCh exactly once. Safe to call on an
+// Orchestrator built without the constructor, where the channel is nil.
+func (o *Orchestrator) signalShutdown() {
+ o.shutdownOnce.Do(func() {
+ if o.shutdownCh != nil {
+ close(o.shutdownCh)
+ }
+ })
}
func (o *Orchestrator) ForceShutdown() {
diff --git a/internal/orchestrator/monitor_lifecycle_test.go b/internal/orchestrator/monitor_lifecycle_test.go
new file mode 100644
index 00000000..22159c2c
--- /dev/null
+++ b/internal/orchestrator/monitor_lifecycle_test.go
@@ -0,0 +1,124 @@
+package orchestrator
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+ "time"
+)
+
+// Shutdown must not leave a startup monitor running. A monitor writes instance
+// state for up to instanceStartupTimeout after launch, so one that outlives its
+// orchestrator keeps touching memory the owner has finished with — and, under
+// test, package vars the test has already restored.
+//
+// It must also not wait that timeout out. With a runner whose instance never
+// becomes healthy, the probe loop would otherwise poll for the full 45s; the
+// point of shutdownCh is that the loop notices and returns.
+func TestShutdownStopsAndJoinsStartupMonitors(t *testing.T) {
+ old := processAliveFunc
+ processAliveFunc = func(pid int) bool { return pid > 0 }
+ t.Cleanup(func() { processAliveFunc = old })
+ stubPortAvailability(t, func(int) bool { return true })
+
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+ if _, err := o.LaunchWithOptions("monitor-lifecycle", "9071", true, LaunchOptions{}); err != nil {
+ t.Fatalf("LaunchWithOptions: %v", err)
+ }
+
+ // The instance is still starting, so the monitor is inside its probe loop.
+ done := make(chan time.Duration, 1)
+ go func() {
+ start := time.Now()
+ o.Shutdown()
+ done <- time.Since(start)
+ }()
+
+ select {
+ case elapsed := <-done:
+ // The bound is half instanceStartupTimeout, which is the property under
+ // test: Shutdown must not wait the startup probe out. It is not a
+ // performance assertion — Stop already spends up to
+ // gracefulProcessStopTimeout per instance waiting for the process to
+ // exit, and this same test measures ~10s against an unmodified
+ // orchestrator too. Joining the monitors adds nothing to that.
+ if elapsed > instanceStartupTimeout/2 {
+ t.Fatalf("Shutdown took %s; it waited out the startup probe instead of interrupting it", elapsed)
+ }
+ case <-time.After(instanceStartupTimeout):
+ t.Fatal("Shutdown did not return; it is waiting out the full startup timeout")
+ }
+
+ // Shutdown returning means monitors.Wait() returned, so every monitor has
+ // finished. Calling it again must stay safe — closing shutdownCh twice
+ // would panic without the sync.Once.
+ o.Shutdown()
+}
+
+// Shutdown is reachable more than once (an explicit call plus a signal handler),
+// and on an Orchestrator built as a bare struct literal, which several tests in
+// this package do. Neither may panic on the nil channel.
+func TestSignalShutdownIsIdempotentAndNilSafe(t *testing.T) {
+ bare := &Orchestrator{}
+ if bare.shuttingDown() {
+ t.Fatal("a fresh orchestrator reports itself shutting down")
+ }
+ bare.signalShutdown()
+ bare.signalShutdown()
+
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+ if o.shuttingDown() {
+ t.Fatal("a fresh orchestrator reports itself shutting down")
+ }
+ o.signalShutdown()
+ if !o.shuttingDown() {
+ t.Fatal("shuttingDown() is false after signalShutdown()")
+ }
+ o.signalShutdown()
+}
+
+// startMonitor is what registers a monitor with the WaitGroup Shutdown waits on.
+// A bare `go o.monitor(...)` compiles, runs, and silently reintroduces the leak,
+// so pin the call shape rather than trusting review to catch it.
+func TestEveryMonitorGoesThroughStartMonitor(t *testing.T) {
+ bareGoMonitor := regexp.MustCompile(`go\s+o\.monitor\(`)
+
+ entries, err := os.ReadDir(".")
+ if err != nil {
+ t.Fatalf("cannot read package dir, so this rule would check nothing: %v", err)
+ }
+
+ scanned := 0
+ var offenders []string
+ for _, e := range entries {
+ name := e.Name()
+ if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
+ continue
+ }
+ body, readErr := os.ReadFile(filepath.Join(".", name))
+ if readErr != nil {
+ t.Fatalf("read %s: %v", name, readErr)
+ }
+ scanned++
+ for _, line := range strings.Split(string(body), "\n") {
+ // Comments describe the shape; only code reintroduces it.
+ if strings.HasPrefix(strings.TrimSpace(line), "//") {
+ continue
+ }
+ if bareGoMonitor.MatchString(line) {
+ offenders = append(offenders, name)
+ break
+ }
+ }
+ }
+
+ if scanned < 20 {
+ t.Fatalf("scanned %d non-test sources, want at least 20; the walk matched almost nothing and this rule would pass vacuously", scanned)
+ }
+ if len(offenders) > 0 {
+ t.Fatalf("monitor started outside startMonitor in %s; use o.startMonitor(inst) so Shutdown can wait for it",
+ strings.Join(offenders, ", "))
+ }
+}
diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go
index 155132db..9181911f 100644
--- a/internal/orchestrator/orchestrator.go
+++ b/internal/orchestrator/orchestrator.go
@@ -41,6 +41,23 @@ type Orchestrator struct {
// race with stubbed package vars.
detachedStops sync.WaitGroup
+ // monitors tracks the per-instance startup monitors for the same reason.
+ // A monitor writes instance state for up to instanceStartupTimeout after
+ // launch, so one that outlives its orchestrator keeps touching memory the
+ // owner has finished with — and, in tests, package vars the test has already
+ // restored. Shutdown waits on this.
+ monitors sync.WaitGroup
+
+ // shutdownCh is closed once, by Shutdown, to cut short in-flight startup
+ // probes. Without it, waiting on monitors would mean waiting out the full
+ // instanceStartupTimeout for any instance still starting.
+ //
+ // Nil when an Orchestrator is built as a bare struct literal rather than
+ // through the constructor, which some tests do. A nil channel is never
+ // ready, so every select below simply falls through to its other case.
+ shutdownCh chan struct{}
+ shutdownOnce sync.Once
+
// strictCrossInstanceTab toggles the cross-instance explicit-tab rule.
// When false (default), a request that targets a tab on a different
// instance than the caller's existing identity binding rebinds the
@@ -155,6 +172,7 @@ func NewOrchestratorWithRunner(baseDir string, runner HostRunner) *Orchestrator
tabsCache: NewTabsCache(0, nil),
portAllocator: NewPortAllocator(9868, 9968),
idMgr: ids.NewManager(),
+ shutdownCh: make(chan struct{}),
}
orch.registerInstanceCleanupHook()
diff --git a/internal/strategy/simple/simple_test.go b/internal/strategy/simple/simple_test.go
index bcc2e2ea..39eb2e20 100644
--- a/internal/strategy/simple/simple_test.go
+++ b/internal/strategy/simple/simple_test.go
@@ -151,11 +151,10 @@ func TestStrategy_EnsureRunning_BrowserTargetAutoLaunchesRequestedTarget(t *test
"cloak": {Provider: config.BrowserCloak},
},
})
- t.Cleanup(func() {
- for _, inst := range orch.List() {
- _ = orch.Stop(inst.ID)
- }
- })
+ // Shutdown, not a Stop loop: it also waits for the startup monitors, which
+ // otherwise outlive the test and keep reading http.DefaultTransport while
+ // the cleanup registered above restores it.
+ t.Cleanup(orch.Shutdown)
req := httptest.NewRequest(http.MethodPost, "/navigate?browser=cloak", nil)