Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Forwarder reload waits for the old launchd job to exit before bootstrapping

shukieshah8e9f2b7906c8
diff --git a/cli/beacon/internal/endpoint/service/forwarder.go b/cli/beacon/internal/endpoint/service/forwarder.go
index 429a4cee..f64ec5c3 100644
--- a/cli/beacon/internal/endpoint/service/forwarder.go
+++ b/cli/beacon/internal/endpoint/service/forwarder.go
@@ -149,9 +149,22 @@ func (m ForwarderManager) Load() error {
return err
}
domain := serviceDomain(m.UserMode)
- // bootout first so a config rewrite (re-enrollment rotates the key) restarts Vector.
- _ = runLaunchctlWithContext(domain, ForwarderLabel, "", "bootout", domain+"/"+ForwarderLabel)
- return loadLaunchdJob(domain, ForwarderLabel, path)
+ target := domain + "/" + ForwarderLabel
+ // bootout first so a config rewrite (re-enrollment rotates the key) restarts Vector, then
+ // wait for the old instance to be gone: it drains in-flight requests for up to a minute,
+ // and a bootstrap issued while it is still registered is silently lost with it. Finally
+ // confirm the new instance is actually running rather than trusting the old pid.
+ _ = runLaunchctlWithContext(domain, ForwarderLabel, "", "bootout", target)
+ if !waitForLaunchdJobGone(domain, ForwarderLabel) {
+ return fmt.Errorf("the previous forwarder did not stop within %s; check `launchctl print %s`", launchdStopTimeout, target)
+ }
+ if err := loadLaunchdJob(domain, ForwarderLabel, path); err != nil {
+ return err
+ }
+ if !waitForLaunchdJobRunning(domain, ForwarderLabel) {
+ return fmt.Errorf("the forwarder was loaded but has not started within %s; check `launchctl print %s` and Vector's log", launchdStartTimeout, target)
+ }
+ return nil
}
// Unload stops and deregisters the forwarder. A missing unit is not an error.
diff --git a/cli/beacon/internal/endpoint/service/forwarder_test.go b/cli/beacon/internal/endpoint/service/forwarder_test.go
index b9ebc4db..865aad1f 100644
--- a/cli/beacon/internal/endpoint/service/forwarder_test.go
+++ b/cli/beacon/internal/endpoint/service/forwarder_test.go
@@ -1,6 +1,8 @@
package service
import (
+ "errors"
+ "fmt"
"os"
"path/filepath"
"runtime"
@@ -106,3 +108,84 @@ func TestForwarderRefusesSupervisedAndForeignBackends(t *testing.T) {
}
}
}
+
+// Re-enrollment reloads a running forwarder. launchd's bootout returns before the old Vector
+// has exited (it drains in-flight requests for up to a minute), and a bootstrap issued while
+// the old job is still registered is lost when it finally goes; the old pid also satisfies a
+// naive "running" check. Load must wait for the old job to disappear, then prove the new one
+// started. Seen live on 2026-09-04: connect reported running, no forwarder was left.
+func TestForwarderLoadWaitsForTheOldJobAndVerifiesTheNewOne(t *testing.T) {
+ if runtime.GOOS != "darwin" {
+ t.Skip("launchd only")
+ }
+ shrinkLaunchdWaits(t)
+ t.Setenv("HOME", t.TempDir())
+ var calls []string
+ printsAfterBootout := 0
+ bootstrapped := false
+ oldRun := runLaunchctlCommand
+ runLaunchctlCommand = func(args ...string) (string, error) {
+ calls = append(calls, strings.Join(args, " "))
+ switch args[0] {
+ case "bootout":
+ return "", nil
+ case "print":
+ if bootstrapped {
+ return "state = running\npid = 200\n", nil
+ }
+ printsAfterBootout++
+ if printsAfterBootout <= 2 {
+ return "state = running\npid = 100\n", nil // the old instance, still draining
+ }
+ return "Could not find service", errors.New("exit status 113")
+ case "bootstrap":
+ if printsAfterBootout <= 2 {
+ t.Fatalf("bootstrap issued while the old job was still registered: %#v", calls)
+ }
+ bootstrapped = true
+ return "", nil
+ }
+ return "", fmt.Errorf("unexpected launchctl call: %s", strings.Join(args, " "))
+ }
+ t.Cleanup(func() { runLaunchctlCommand = oldRun })
+
+ m := ForwarderManager{UserMode: true, Kind: KindLaunchd}
+ if err := m.Load(); err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ if !bootstrapped || !strings.HasPrefix(calls[0], "bootout ") {
+ t.Fatalf("expected bootout, wait, bootstrap, verify: %#v", calls)
+ }
+}
+
+func TestForwarderLoadFailsWhenTheNewInstanceNeverStarts(t *testing.T) {
+ if runtime.GOOS != "darwin" {
+ t.Skip("launchd only")
+ }
+ shrinkLaunchdWaits(t)
+ t.Setenv("HOME", t.TempDir())
+ bootstrapped := false
+ oldRun := runLaunchctlCommand
+ runLaunchctlCommand = func(args ...string) (string, error) {
+ switch args[0] {
+ case "bootout":
+ return "", nil
+ case "bootstrap":
+ bootstrapped = true
+ return "", nil
+ case "print":
+ if !bootstrapped {
+ return "Could not find service", errors.New("exit status 113") // old job gone
+ }
+ return "state = waiting\n", nil // loaded, never gets a pid
+ }
+ return "", fmt.Errorf("unexpected launchctl call: %s", strings.Join(args, " "))
+ }
+ t.Cleanup(func() { runLaunchctlCommand = oldRun })
+
+ m := ForwarderManager{UserMode: true, Kind: KindLaunchd}
+ err := m.Load()
+ if err == nil || !strings.Contains(err.Error(), "has not started") {
+ t.Fatalf("Load should report a forwarder that never started, got %v", err)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/service/launchd.go b/cli/beacon/internal/endpoint/service/launchd.go
index 7286d06e..a5c37fd1 100644
--- a/cli/beacon/internal/endpoint/service/launchd.go
+++ b/cli/beacon/internal/endpoint/service/launchd.go
@@ -7,6 +7,7 @@ import (
"path/filepath"
"runtime"
"strings"
+ "time"
)
// launchdBackend manages the collector via macOS launchd.
@@ -22,6 +23,52 @@ var runLaunchctlCommand = func(args ...string) (string, error) {
return string(out), err
}
+// Reloading a job is not atomic in launchd: `bootout` returns as soon as the job is asked to
+// stop, while the process may take its exit timeout (Vector drains in-flight requests for
+// up to a minute) to actually go away, and `bootstrap` of the same label in that window
+// fails with the job still printing as loaded. Both waits below poll `launchctl print`;
+// the counts are derived from the timeouts so tests can shrink them without a clock.
+var (
+ launchdPollInterval = 500 * time.Millisecond
+ launchdStopTimeout = 75 * time.Second
+ launchdStartTimeout = 10 * time.Second
+ launchdSleep = time.Sleep
+)
+
+func launchdJobGone(domain, label string) bool {
+ out, err := runLaunchctlCommand("print", domain+"/"+label)
+ return err != nil && launchctlNoSuchProcess(strings.TrimSpace(out))
+}
+
+// waitForLaunchdJobGone returns true once the job is no longer registered, or false when it
+// is still there after launchdStopTimeout.
+func waitForLaunchdJobGone(domain, label string) bool {
+ attempts := int(launchdStopTimeout / launchdPollInterval)
+ for i := 0; i < attempts; i++ {
+ if launchdJobGone(domain, label) {
+ return true
+ }
+ launchdSleep(launchdPollInterval)
+ }
+ return launchdJobGone(domain, label)
+}
+
+// waitForLaunchdJobRunning returns true once the job has a pid, or false after
+// launchdStartTimeout.
+func waitForLaunchdJobRunning(domain, label string) bool {
+ attempts := int(launchdStartTimeout / launchdPollInterval)
+ for i := 0; i <= attempts; i++ {
+ out, err := runLaunchctlCommand("print", domain+"/"+label)
+ if err == nil && strings.Contains(out, "pid =") {
+ return true
+ }
+ if i < attempts {
+ launchdSleep(launchdPollInterval)
+ }
+ }
+ return false
+}
+
func (launchdBackend) kind() Kind { return KindLaunchd }
func (launchdBackend) available() bool { return runtime.GOOS == "darwin" }
@@ -130,6 +177,9 @@ func loadLaunchdJob(domain, label, plistPath string) error {
if err := runLaunchctlWithContext(domain, label, "", "bootout", target); err != nil {
return err
}
+ // The old instance may still be draining; bootstrapping over it would be swallowed
+ // and the job would vanish once it exits.
+ waitForLaunchdJobGone(domain, label)
if out, err := runLaunchctlCommand("bootstrap", domain, plistPath); err != nil {
text := strings.TrimSpace(out)
if launchdJobAppearsLoaded(text, domain, label) {
diff --git a/cli/beacon/internal/endpoint/service/service_test.go b/cli/beacon/internal/endpoint/service/service_test.go
index acc335ad..392f7572 100644
--- a/cli/beacon/internal/endpoint/service/service_test.go
+++ b/cli/beacon/internal/endpoint/service/service_test.go
@@ -9,6 +9,7 @@ import (
"runtime"
"strings"
"testing"
+ "time"
)
func TestPlistContainsLaunchdContract(t *testing.T) {
@@ -58,16 +59,42 @@ func TestRunLaunchctlWithContextExplainsBootstrapIOError(t *testing.T) {
}
}
+// shrinkLaunchdWaits makes the reload polls deterministic and instant in tests.
+func shrinkLaunchdWaits(t *testing.T) {
+ t.Helper()
+ oldStop, oldStart, oldSleep := launchdStopTimeout, launchdStartTimeout, launchdSleep
+ launchdStopTimeout = 2 * launchdPollInterval
+ launchdStartTimeout = 2 * launchdPollInterval
+ launchdSleep = func(time.Duration) {}
+ t.Cleanup(func() { launchdStopTimeout, launchdStartTimeout, launchdSleep = oldStop, oldStart, oldSleep })
+}
+
+// withoutPrints drops the `print` polls so a test can assert the order of the calls that
+// change state.
+func withoutPrints(calls []string) []string {
+ var out []string
+ for _, c := range calls {
+ if !strings.HasPrefix(c, "print ") {
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
func TestLoadLaunchdJobReloadsAlreadyBootstrappedJob(t *testing.T) {
+ shrinkLaunchdWaits(t)
var calls []string
oldRun := runLaunchctlCommand
runLaunchctlCommand = func(args ...string) (string, error) {
calls = append(calls, strings.Join(args, " "))
- switch len(calls) {
- case 1:
+ switch {
+ case args[0] == "bootstrap" && len(withoutPrints(calls)) == 1:
return "Bootstrap failed: 5: already bootstrapped", errors.New("exit status 5")
- case 2, 3:
+ case args[0] == "bootstrap", args[0] == "bootout":
return "", nil
+ case args[0] == "print":
+ // After bootout the old job is gone at once.
+ return "Could not find service", errors.New("exit status 113")
default:
return "", errors.New("unexpected launchctl call")
}
@@ -79,15 +106,18 @@ func TestLoadLaunchdJobReloadsAlreadyBootstrappedJob(t *testing.T) {
if err := loadLaunchdJob("gui/501", UserLabel, "/Users/test/Library/LaunchAgents/"+UserLabel+".plist"); err != nil {
t.Fatalf("loadLaunchdJob returned error: %v", err)
}
- if len(calls) != 3 {
- t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap", calls)
- }
- if !strings.HasPrefix(calls[0], "bootstrap gui/") || !strings.Contains(calls[1], "bootout gui/") || !strings.HasPrefix(calls[2], "bootstrap gui/") {
+ state := withoutPrints(calls)
+ if len(state) != 3 || !strings.HasPrefix(state[0], "bootstrap gui/") || !strings.Contains(state[1], "bootout gui/") || !strings.HasPrefix(state[2], "bootstrap gui/") {
t.Fatalf("unexpected launchctl call sequence: %#v", calls)
}
+ // The wait between bootout and the second bootstrap must have looked at the job.
+ if len(calls) < 4 || !strings.HasPrefix(calls[2], "print gui/") {
+ t.Fatalf("expected a print poll after bootout: %#v", calls)
+ }
}
func TestLoadLaunchdJobReloadsBootstrapIOErrorWhenJobPrints(t *testing.T) {
+ shrinkLaunchdWaits(t)
var calls []string
oldRun := runLaunchctlCommand
runLaunchctlCommand = func(args ...string) (string, error) {
@@ -113,12 +143,15 @@ func TestLoadLaunchdJobReloadsBootstrapIOErrorWhenJobPrints(t *testing.T) {
if err := loadLaunchdJob("system", SystemLabel, "/Library/LaunchDaemons/"+SystemLabel+".plist"); err != nil {
t.Fatalf("loadLaunchdJob returned error: %v", err)
}
- if len(calls) != 4 {
- t.Fatalf("launchctl calls = %#v, want bootstrap/print/bootout/bootstrap", calls)
+ // print always reports the job running here, so the post-bootout wait polls to its
+ // (shrunken) timeout and the reload proceeds anyway.
+ if state := withoutPrints(calls); len(state) != 3 || state[0] != state[2] || !strings.HasPrefix(state[1], "bootout ") {
+ t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap around the polls", calls)
}
}
func TestLoadLaunchdJobTreatsPostBootstrapLoadedStateAsSuccess(t *testing.T) {
+ shrinkLaunchdWaits(t)
var calls []string
oldRun := runLaunchctlCommand
runLaunchctlCommand = func(args ...string) (string, error) {
@@ -141,8 +174,11 @@ func TestLoadLaunchdJobTreatsPostBootstrapLoadedStateAsSuccess(t *testing.T) {
if err := loadLaunchdJob("system", SystemLabel, "/Library/LaunchDaemons/"+SystemLabel+".plist"); err != nil {
t.Fatalf("loadLaunchdJob returned error: %v", err)
}
- if len(calls) != 5 {
- t.Fatalf("launchctl calls = %#v, want bootstrap/print/bootout/bootstrap/print", calls)
+ if state := withoutPrints(calls); len(state) != 3 || !strings.HasPrefix(state[1], "bootout ") {
+ t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap around the polls", calls)
+ }
+ if !strings.HasPrefix(calls[len(calls)-1], "print ") {
+ t.Fatalf("the loaded-state check after the second bootstrap should be the last call: %#v", calls)
}
}
diff --git a/docs/changelog/cli.mdx b/docs/changelog/cli.mdx
index 857ace66..5ae40968 100644
--- a/docs/changelog/cli.mdx
+++ b/docs/changelog/cli.mdx
@@ -11,6 +11,16 @@ Stay up to date with the latest changes to the Agent Beacon CLI.
## September 2026
+### Unreleased
+
+- **Re-connecting no longer loses the forwarder** · `beacon endpoint connect` on an
+ already-connected macOS endpoint stops the old Vector and starts a new one under
+ the same launchd label. launchd reports the stop before the old process has
+ exited (Vector drains in-flight requests for up to a minute), so the new
+ bootstrap could be swallowed and the status check saw the old pid as "running",
+ leaving no forwarder at all. The load now waits for the old job to disappear,
+ then confirms the new instance has a pid, and fails loudly otherwise.
+
### v1.3.1 · September 4, 2026
- **`beacon endpoint status --user` reports the user-mode connection** · On a