Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(orchestrator): join detached failed-attempt teardowns on shutdown

Luigi Agosti5cec7f15db13
diff --git a/internal/orchestrator/instance_stop.go b/internal/orchestrator/instance_stop.go
index 551961dc..ea59f987 100644
--- a/internal/orchestrator/instance_stop.go
+++ b/internal/orchestrator/instance_stop.go
@@ -51,30 +51,33 @@ func (o *Orchestrator) Stop(id string) error {
}
pid := inst.cmd.PID()
-
- reqCtx, cancel := context.WithTimeout(context.Background(), shutdownRequestTimeout)
- defer cancel()
- if targetURL, targetErr := o.instancePathURL(inst, "/shutdown", ""); targetErr == nil {
- req, _ := http.NewRequestWithContext(reqCtx, http.MethodPost, targetURL.String(), nil)
- o.applyInstanceAuth(req, inst)
- resp, err := o.client.Do(req)
- if err == nil {
- _ = resp.Body.Close()
- }
+ if !o.stopChildProcess(id, inst) {
+ o.setStopError(id, fmt.Sprintf("failed to stop process %d; still running", pid))
+ return fmt.Errorf("failed to stop instance %q gracefully", id)
}
+ o.markStopped(id)
+ return nil
+}
+
+// stopChildProcess runs the child shutdown escalation: ask the bridge to shut
+// down, then SIGTERM, SIGKILL, and finally cancel the command context. It
+// reports whether the process is gone. Stop and the detached failed-attempt
+// teardown share it; they differ only in the bookkeeping around it.
+func (o *Orchestrator) stopChildProcess(id string, inst *InstanceInternal) bool {
+ o.requestChildShutdown(inst)
+
+ pid := inst.cmd.PID()
if pid > 0 {
if waitForProcessExit(pid, gracefulProcessStopTimeout) {
- o.markStopped(id)
- return nil
+ return true
}
if err := killProcessGroup(pid, sigTERM); err != nil {
slog.Warn("failed to send SIGTERM to instance", "id", id, "pid", pid, "err", err)
}
if waitForProcessExit(pid, termProcessStopTimeout) {
- o.markStopped(id)
- return nil
+ return true
}
if err := killProcessGroup(pid, sigKILL); err != nil {
@@ -83,18 +86,23 @@ func (o *Orchestrator) Stop(id string) error {
}
inst.cmd.Cancel()
+ return pid <= 0 || waitForProcessExit(pid, killProcessStopTimeout)
+}
- if pid > 0 {
- if waitForProcessExit(pid, killProcessStopTimeout) {
- o.markStopped(id)
- return nil
- }
- o.setStopError(id, fmt.Sprintf("failed to stop process %d; still running", pid))
- return fmt.Errorf("failed to stop instance %q gracefully", id)
+// requestChildShutdown asks a child bridge to shut itself down. Failures are
+// the caller's cue to escalate, not an error to report.
+func (o *Orchestrator) requestChildShutdown(inst *InstanceInternal) {
+ reqCtx, cancel := context.WithTimeout(context.Background(), shutdownRequestTimeout)
+ defer cancel()
+ targetURL, targetErr := o.instancePathURL(inst, "/shutdown", "")
+ if targetErr != nil {
+ return
+ }
+ req, _ := http.NewRequestWithContext(reqCtx, http.MethodPost, targetURL.String(), nil)
+ o.applyInstanceAuth(req, inst)
+ if resp, err := o.client.Do(req); err == nil {
+ _ = resp.Body.Close()
}
-
- o.markStopped(id)
- return nil
}
func (o *Orchestrator) stopRegisteredBridge(inst *InstanceInternal) error {
@@ -242,7 +250,6 @@ func (o *Orchestrator) cleanupStoppedProfile(profileName, browser string) {
providerhooks.CleanupProfile(browser, profilePath)
if strings.HasPrefix(profileName, "instance-") {
- profilePath := filepath.Join(o.baseDir, profileName)
if err := os.RemoveAll(profilePath); err != nil {
slog.Warn("failed to delete temporary profile directory", "name", profileName, "err", err)
} else {
@@ -290,17 +297,25 @@ func (o *Orchestrator) Shutdown() {
wg.Wait()
// Signal after the instances are stopped, so a monitor still probing a
- // healthy instance gets to record its outcome, then wait for every monitor
- // to return. Bounded by one poll interval rather than
- // instanceStartupTimeout, because the probe loop watches shutdownCh.
+ // healthy instance gets to record its outcome, then join every goroutine
+ // the orchestrator detached. The monitors are bounded by one poll interval
+ // rather than instanceStartupTimeout because the probe loop watches
+ // shutdownCh; the teardowns run a failed attempt's own process escalation,
+ // which nothing else stops for them.
o.signalShutdown()
- if !waitGroupWithin(&o.monitors, monitorShutdownGrace) {
- // Belt and braces. finalizeInstanceExit is interruptible, so this should
- // not fire; if a monitor ever blocks somewhere else, shutdown must still
- // return rather than hang the process.
- slog.Warn("startup monitors did not finish within the shutdown grace period",
- "grace", monitorShutdownGrace)
+ joinDetached("startup monitors", &o.monitors, monitorShutdownGrace)
+ joinDetached("failed-attempt teardowns", &o.detachedStops, detachedStopShutdownGrace)
+}
+
+// joinDetached waits for wg, then reports rather than hangs. Every wait here is
+// bounded so a goroutine blocked somewhere unexpected can never hold the
+// process open.
+func joinDetached(work string, wg *sync.WaitGroup, grace time.Duration) {
+ if waitGroupWithin(wg, grace) {
+ return
}
+ slog.Warn("detached goroutines did not finish within the shutdown grace period",
+ "work", work, "grace", grace)
}
// monitorShutdownGrace bounds how long Shutdown waits for startup monitors.
@@ -308,6 +323,13 @@ func (o *Orchestrator) Shutdown() {
// instanceStartupTimeout.
const monitorShutdownGrace = 5 * time.Second
+// detachedStopShutdownGrace bounds how long Shutdown waits for failed-attempt
+// teardowns. One runs stopChildProcess, whose own escalation is bounded by the
+// shutdown request plus the three process waits, so a teardown that is making
+// progress finishes inside this.
+var detachedStopShutdownGrace = shutdownRequestTimeout + gracefulProcessStopTimeout +
+ termProcessStopTimeout + killProcessStopTimeout + time.Second
+
// waitGroupWithin reports whether wg finished inside d.
func waitGroupWithin(wg *sync.WaitGroup, d time.Duration) bool {
done := make(chan struct{})
diff --git a/internal/orchestrator/launch_with_fallback.go b/internal/orchestrator/launch_with_fallback.go
index f83d6c67..d0d7a52a 100644
--- a/internal/orchestrator/launch_with_fallback.go
+++ b/internal/orchestrator/launch_with_fallback.go
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"log/slog"
- "net/http"
"strings"
"time"
@@ -381,47 +380,13 @@ func (o *Orchestrator) stopDetachedFailedAttempt(instanceID string, inst *Instan
if inst.cmd == nil {
if inst.AttachType == "bridge" {
- o.requestDetachedShutdown(inst)
+ o.requestChildShutdown(inst)
}
return
}
- o.requestDetachedShutdown(inst)
- pid := inst.cmd.PID()
- if pid > 0 {
- if waitForProcessExit(pid, 5*time.Second) {
- return
- }
-
- if err := killProcessGroup(pid, sigTERM); err != nil {
- slog.Warn("failed to send SIGTERM to failed fallback attempt", "id", instanceID, "pid", pid, "err", err)
- }
- if waitForProcessExit(pid, 3*time.Second) {
- return
- }
-
- if err := killProcessGroup(pid, sigKILL); err != nil {
- slog.Warn("failed to send SIGKILL to failed fallback attempt", "id", instanceID, "pid", pid, "err", err)
- }
- }
-
- inst.cmd.Cancel()
- if pid > 0 && !waitForProcessExit(pid, 2*time.Second) {
- slog.Warn("failed fallback attempt still running after teardown", "id", instanceID, "pid", pid)
- }
-}
-
-func (o *Orchestrator) requestDetachedShutdown(inst *InstanceInternal) {
- reqCtx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
- defer cancel()
- targetURL, targetErr := o.instancePathURL(inst, "/shutdown", "")
- if targetErr != nil {
- return
- }
- req, _ := http.NewRequestWithContext(reqCtx, http.MethodPost, targetURL.String(), nil)
- o.applyInstanceAuth(req, inst)
- if resp, err := o.client.Do(req); err == nil {
- _ = resp.Body.Close()
+ if !o.stopChildProcess(instanceID, inst) {
+ slog.Warn("failed fallback attempt still running after teardown", "id", instanceID, "pid", inst.cmd.PID())
}
}
diff --git a/internal/orchestrator/monitor_lifecycle_test.go b/internal/orchestrator/monitor_lifecycle_test.go
index 22159c2c..f3ab51c8 100644
--- a/internal/orchestrator/monitor_lifecycle_test.go
+++ b/internal/orchestrator/monitor_lifecycle_test.go
@@ -5,6 +5,7 @@ import (
"path/filepath"
"regexp"
"strings"
+ "sync/atomic"
"testing"
"time"
)
@@ -122,3 +123,28 @@ func TestEveryMonitorGoesThroughStartMonitor(t *testing.T) {
strings.Join(offenders, ", "))
}
}
+
+// Shutdown must also join the detached failed-attempt teardowns. A teardown
+// kills a browser process and deletes its profile for up to its own escalation
+// bound after Shutdown is called, so one that outlives the orchestrator is the
+// same leak the monitors had — and its field comment already promised that
+// shutdown paths wait for it. That tearDownFailedAttempt registers with the
+// group is pinned by TestTearDownFailedAttempt_DetachesBeforeAsyncStop; this pins
+// the other half.
+func TestShutdownJoinsDetachedFailedAttemptTeardowns(t *testing.T) {
+ o := NewOrchestratorWithRunner(t.TempDir(), &mockRunner{portAvail: true})
+
+ var finished atomic.Bool
+ o.detachedStops.Add(1)
+ go func() {
+ defer o.detachedStops.Done()
+ time.Sleep(300 * time.Millisecond)
+ finished.Store(true)
+ }()
+
+ o.Shutdown()
+
+ if !finished.Load() {
+ t.Fatal("Shutdown returned while a detached failed-attempt teardown was still running")
+ }
+}