High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(orchestrator): startup monitors outlive the orchestrator that started them
Darshan Jain2d519e490aa5
diff --git a/internal/orchestrator/health.go b/internal/orchestrator/health.goindex c10ecbc6..c14455a5 100644--- a/internal/orchestrator/health.go+++ b/internal/orchestrator/health.go@@ -27,6 +27,40 @@ type startupProbe struct {waitCh chan error}+// startMonitor runs monitor for inst and registers it with o.monitors, so+// Shutdown can wait for it. Every monitor must start this way; a bare+// `go o.monitor(inst)` is a goroutine that outlives its orchestrator.+func (o *Orchestrator) startMonitor(inst *InstanceInternal) {+ o.monitors.Add(1)+ go func() {+ defer o.monitors.Done()+ o.monitor(inst)+ }()+}++// shuttingDown reports whether Shutdown has been called. Safe on an+// Orchestrator built without the constructor: a nil channel is never ready.+func (o *Orchestrator) shuttingDown() bool {+ select {+ case <-o.shutdownCh:+ return true+ default:+ return false+ }+}++// sleepOrShutdown waits for d, or returns early once Shutdown is called. It+// replaces a bare time.Sleep in the probe loop so a shutdown does not have to+// wait out a poll interval per instance.+func (o *Orchestrator) sleepOrShutdown(d time.Duration) {+ timer := time.NewTimer(d)+ defer timer.Stop()+ select {+ case <-timer.C:+ case <-o.shutdownCh:+ }+}+func (o *Orchestrator) monitor(inst *InstanceInternal) {p := o.probeStartupHealth(inst)o.applyStartupOutcome(inst, p)@@ -56,7 +90,18 @@ func (o *Orchestrator) probeStartupHealth(inst *InstanceInternal) startupProbe {if portErr != nil {break}- time.Sleep(instanceHealthPollInterval)+ // Stop probing a starting instance once the orchestrator is going down;+ // applyStartupOutcome below leaves a stopping/stopped instance alone, so+ // bailing here loses nothing.+ if o.shuttingDown() {+ p.lastProbe = "orchestrator shutting down"+ break+ }+ o.sleepOrShutdown(instanceHealthPollInterval)+ if o.shuttingDown() {+ p.lastProbe = "orchestrator shutting down"+ break+ }for _, baseURL := range instanceBaseURLs(configuredChildBind(o.runtimeCfg), probePort) {targetBaseURL, err := o.validatedHealthProbeBaseURL(baseURL, "", healthProbePolicyLoopback)@@ -129,7 +174,15 @@ func (o *Orchestrator) applyStartupOutcome(inst *InstanceInternal, p startupProbfunc (o *Orchestrator) finalizeInstanceExit(inst *InstanceInternal, p startupProbe) {if !p.exitedEarly {- <-p.waitCh+ // Interruptible: this waits for the child process to report exit, which+ // for a process that never exits is forever. Shutdown waits on the+ // monitors, so a bare receive here makes shutdown unbounded. Stop has+ // already signalled the process by the time shutdownCh closes; the state+ // transition below still runs.+ select {+ case <-p.waitCh:+ case <-o.shutdownCh:+ }}o.mu.Lock()wasStopped := falsediff --git a/internal/orchestrator/instance_attach.go b/internal/orchestrator/instance_attach.goindex 402e4cc6..ed6ed59c 100644--- a/internal/orchestrator/instance_attach.go+++ b/internal/orchestrator/instance_attach.go@@ -209,7 +209,7 @@ func (o *Orchestrator) AttachWithOptions(name, cdpURL string, opts AttachOptionsreserved = falsereleased = true- go o.monitor(internal)+ o.startMonitor(internal)healthTimeout := o.attachHealthCheckTimeoutif healthTimeout <= 0 {diff --git a/internal/orchestrator/instance_launch.go b/internal/orchestrator/instance_launch.goindex c655985b..9179d9d5 100644--- a/internal/orchestrator/instance_launch.go+++ b/internal/orchestrator/instance_launch.go@@ -228,7 +228,7 @@ func (o *Orchestrator) LaunchWithOptions(name, port string, headless bool, optso.mu.Unlock()reservedPorts = nil- go o.monitor(inst)+ o.startMonitor(inst)return &inst.Instance, nil}diff --git a/internal/orchestrator/instance_stop.go b/internal/orchestrator/instance_stop.goindex eeed553a..551961dc 100644--- a/internal/orchestrator/instance_stop.go+++ b/internal/orchestrator/instance_stop.go@@ -288,6 +288,49 @@ func (o *Orchestrator) Shutdown() {}(id)}wg.Wait()++ // Signal after the instances are stopped, so a monitor still probing a+ // healthy instance gets to record its outcome, then wait for every monitor+ // to return. Bounded by one poll interval rather than+ // instanceStartupTimeout, because the probe loop watches shutdownCh.+ o.signalShutdown()+ if !waitGroupWithin(&o.monitors, monitorShutdownGrace) {+ // Belt and braces. finalizeInstanceExit is interruptible, so this should+ // not fire; if a monitor ever blocks somewhere else, shutdown must still+ // return rather than hang the process.+ slog.Warn("startup monitors did not finish within the shutdown grace period",+ "grace", monitorShutdownGrace)+ }+}++// monitorShutdownGrace bounds how long Shutdown waits for startup monitors.+// Generous next to the poll interval they watch shutdownCh on, and far below+// instanceStartupTimeout.+const monitorShutdownGrace = 5 * time.Second++// waitGroupWithin reports whether wg finished inside d.+func waitGroupWithin(wg *sync.WaitGroup, d time.Duration) bool {+ done := make(chan struct{})+ go func() {+ wg.Wait()+ close(done)+ }()+ select {+ case <-done:+ return true+ case <-time.After(d):+ return false+ }+}++// signalShutdown closes shutdownCh exactly once. Safe to call on an+// Orchestrator built without the constructor, where the channel is nil.+func (o *Orchestrator) signalShutdown() {+ o.shutdownOnce.Do(func() {+ if o.shutdownCh != nil {+ close(o.shutdownCh)+ }+ })}func (o *Orchestrator) ForceShutdown() {diff --git a/internal/orchestrator/monitor_lifecycle_test.go b/internal/orchestrator/monitor_lifecycle_test.gonew file mode 100644index 00000000..22159c2c--- /dev/null+++ b/internal/orchestrator/monitor_lifecycle_test.go@@ -0,0 +1,124 @@+package orchestrator++import (+ "os"+ "path/filepath"+ "regexp"+ "strings"+ "testing"+ "time"+)++// Shutdown must not leave a startup monitor running. A monitor writes instance+// state for up to instanceStartupTimeout after launch, so one that outlives its+// orchestrator keeps touching memory the owner has finished with — and, under+// test, package vars the test has already restored.+//+// It must also not wait that timeout out. With a runner whose instance never+// becomes healthy, the probe loop would otherwise poll for the full 45s; the+// point of shutdownCh is that the loop notices and returns.+func TestShutdownStopsAndJoinsStartupMonitors(t *testing.T) {+ old := processAliveFunc+ processAliveFunc = func(pid int) bool { return pid > 0 }+ t.Cleanup(func() { processAliveFunc = old })+ stubPortAvailability(t, func(int) bool { return true })++ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})+ if _, err := o.LaunchWithOptions("monitor-lifecycle", "9071", true, LaunchOptions{}); err != nil {+ t.Fatalf("LaunchWithOptions: %v", err)+ }++ // The instance is still starting, so the monitor is inside its probe loop.+ done := make(chan time.Duration, 1)+ go func() {+ start := time.Now()+ o.Shutdown()+ done <- time.Since(start)+ }()++ select {+ case elapsed := <-done:+ // The bound is half instanceStartupTimeout, which is the property under+ // test: Shutdown must not wait the startup probe out. It is not a+ // performance assertion — Stop already spends up to+ // gracefulProcessStopTimeout per instance waiting for the process to+ // exit, and this same test measures ~10s against an unmodified+ // orchestrator too. Joining the monitors adds nothing to that.+ if elapsed > instanceStartupTimeout/2 {+ t.Fatalf("Shutdown took %s; it waited out the startup probe instead of interrupting it", elapsed)+ }+ case <-time.After(instanceStartupTimeout):+ t.Fatal("Shutdown did not return; it is waiting out the full startup timeout")+ }++ // Shutdown returning means monitors.Wait() returned, so every monitor has+ // finished. Calling it again must stay safe — closing shutdownCh twice+ // would panic without the sync.Once.+ o.Shutdown()+}++// Shutdown is reachable more than once (an explicit call plus a signal handler),+// and on an Orchestrator built as a bare struct literal, which several tests in+// this package do. Neither may panic on the nil channel.+func TestSignalShutdownIsIdempotentAndNilSafe(t *testing.T) {+ bare := &Orchestrator{}+ if bare.shuttingDown() {+ t.Fatal("a fresh orchestrator reports itself shutting down")+ }+ bare.signalShutdown()+ bare.signalShutdown()++ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})+ if o.shuttingDown() {+ t.Fatal("a fresh orchestrator reports itself shutting down")+ }+ o.signalShutdown()+ if !o.shuttingDown() {+ t.Fatal("shuttingDown() is false after signalShutdown()")+ }+ o.signalShutdown()+}++// startMonitor is what registers a monitor with the WaitGroup Shutdown waits on.+// A bare `go o.monitor(...)` compiles, runs, and silently reintroduces the leak,+// so pin the call shape rather than trusting review to catch it.+func TestEveryMonitorGoesThroughStartMonitor(t *testing.T) {+ bareGoMonitor := regexp.MustCompile(`go\s+o\.monitor\(`)++ entries, err := os.ReadDir(".")+ if err != nil {+ t.Fatalf("cannot read package dir, so this rule would check nothing: %v", err)+ }++ scanned := 0+ var offenders []string+ for _, e := range entries {+ name := e.Name()+ if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {+ continue+ }+ body, readErr := os.ReadFile(filepath.Join(".", name))+ if readErr != nil {+ t.Fatalf("read %s: %v", name, readErr)+ }+ scanned+++ for _, line := range strings.Split(string(body), "\n") {+ // Comments describe the shape; only code reintroduces it.+ if strings.HasPrefix(strings.TrimSpace(line), "//") {+ continue+ }+ if bareGoMonitor.MatchString(line) {+ offenders = append(offenders, name)+ break+ }+ }+ }++ if scanned < 20 {+ t.Fatalf("scanned %d non-test sources, want at least 20; the walk matched almost nothing and this rule would pass vacuously", scanned)+ }+ if len(offenders) > 0 {+ t.Fatalf("monitor started outside startMonitor in %s; use o.startMonitor(inst) so Shutdown can wait for it",+ strings.Join(offenders, ", "))+ }+}diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.goindex 155132db..9181911f 100644--- a/internal/orchestrator/orchestrator.go+++ b/internal/orchestrator/orchestrator.go@@ -41,6 +41,23 @@ type Orchestrator struct {// race with stubbed package vars.detachedStops sync.WaitGroup+ // monitors tracks the per-instance startup monitors for the same reason.+ // A monitor writes instance state for up to instanceStartupTimeout after+ // launch, so one that outlives its orchestrator keeps touching memory the+ // owner has finished with — and, in tests, package vars the test has already+ // restored. Shutdown waits on this.+ monitors sync.WaitGroup++ // shutdownCh is closed once, by Shutdown, to cut short in-flight startup+ // probes. Without it, waiting on monitors would mean waiting out the full+ // instanceStartupTimeout for any instance still starting.+ //+ // Nil when an Orchestrator is built as a bare struct literal rather than+ // through the constructor, which some tests do. A nil channel is never+ // ready, so every select below simply falls through to its other case.+ shutdownCh chan struct{}+ shutdownOnce sync.Once+// strictCrossInstanceTab toggles the cross-instance explicit-tab rule.// When false (default), a request that targets a tab on a different// instance than the caller's existing identity binding rebinds the@@ -155,6 +172,7 @@ func NewOrchestratorWithRunner(baseDir string, runner HostRunner) *OrchestratortabsCache: NewTabsCache(0, nil),portAllocator: NewPortAllocator(9868, 9968),idMgr: ids.NewManager(),+ shutdownCh: make(chan struct{}),}orch.registerInstanceCleanupHook()diff --git a/internal/strategy/simple/simple_test.go b/internal/strategy/simple/simple_test.goindex bcc2e2ea..39eb2e20 100644--- a/internal/strategy/simple/simple_test.go+++ b/internal/strategy/simple/simple_test.go@@ -151,11 +151,10 @@ func TestStrategy_EnsureRunning_BrowserTargetAutoLaunchesRequestedTarget(t *test"cloak": {Provider: config.BrowserCloak},},})- t.Cleanup(func() {- for _, inst := range orch.List() {- _ = orch.Stop(inst.ID)- }- })+ // Shutdown, not a Stop loop: it also waits for the startup monitors, which+ // otherwise outlive the test and keep reading http.DefaultTransport while+ // the cleanup registered above restores it.+ t.Cleanup(orch.Shutdown)req := httptest.NewRequest(http.MethodPost, "/navigate?browser=cloak", nil)