The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Forwarder reload waits for the old launchd job to exit before bootstrapping
shukieshah8e9f2b7906c8
diff --git a/cli/beacon/internal/endpoint/service/forwarder.go b/cli/beacon/internal/endpoint/service/forwarder.goindex 429a4cee..f64ec5c3 100644--- a/cli/beacon/internal/endpoint/service/forwarder.go+++ b/cli/beacon/internal/endpoint/service/forwarder.go@@ -149,9 +149,22 @@ func (m ForwarderManager) Load() error {return err}domain := serviceDomain(m.UserMode)- // bootout first so a config rewrite (re-enrollment rotates the key) restarts Vector.- _ = runLaunchctlWithContext(domain, ForwarderLabel, "", "bootout", domain+"/"+ForwarderLabel)- return loadLaunchdJob(domain, ForwarderLabel, path)+ target := domain + "/" + ForwarderLabel+ // bootout first so a config rewrite (re-enrollment rotates the key) restarts Vector, then+ // wait for the old instance to be gone: it drains in-flight requests for up to a minute,+ // and a bootstrap issued while it is still registered is silently lost with it. Finally+ // confirm the new instance is actually running rather than trusting the old pid.+ _ = runLaunchctlWithContext(domain, ForwarderLabel, "", "bootout", target)+ if !waitForLaunchdJobGone(domain, ForwarderLabel) {+ return fmt.Errorf("the previous forwarder did not stop within %s; check `launchctl print %s`", launchdStopTimeout, target)+ }+ if err := loadLaunchdJob(domain, ForwarderLabel, path); err != nil {+ return err+ }+ if !waitForLaunchdJobRunning(domain, ForwarderLabel) {+ return fmt.Errorf("the forwarder was loaded but has not started within %s; check `launchctl print %s` and Vector's log", launchdStartTimeout, target)+ }+ return nil}// Unload stops and deregisters the forwarder. A missing unit is not an error.diff --git a/cli/beacon/internal/endpoint/service/forwarder_test.go b/cli/beacon/internal/endpoint/service/forwarder_test.goindex b9ebc4db..865aad1f 100644--- a/cli/beacon/internal/endpoint/service/forwarder_test.go+++ b/cli/beacon/internal/endpoint/service/forwarder_test.go@@ -1,6 +1,8 @@package serviceimport (+ "errors"+ "fmt""os""path/filepath""runtime"@@ -106,3 +108,84 @@ func TestForwarderRefusesSupervisedAndForeignBackends(t *testing.T) {}}}++// Re-enrollment reloads a running forwarder. launchd's bootout returns before the old Vector+// has exited (it drains in-flight requests for up to a minute), and a bootstrap issued while+// the old job is still registered is lost when it finally goes; the old pid also satisfies a+// naive "running" check. Load must wait for the old job to disappear, then prove the new one+// started. Seen live on 2026-09-04: connect reported running, no forwarder was left.+func TestForwarderLoadWaitsForTheOldJobAndVerifiesTheNewOne(t *testing.T) {+ if runtime.GOOS != "darwin" {+ t.Skip("launchd only")+ }+ shrinkLaunchdWaits(t)+ t.Setenv("HOME", t.TempDir())+ var calls []string+ printsAfterBootout := 0+ bootstrapped := false+ oldRun := runLaunchctlCommand+ runLaunchctlCommand = func(args ...string) (string, error) {+ calls = append(calls, strings.Join(args, " "))+ switch args[0] {+ case "bootout":+ return "", nil+ case "print":+ if bootstrapped {+ return "state = running\npid = 200\n", nil+ }+ printsAfterBootout+++ if printsAfterBootout <= 2 {+ return "state = running\npid = 100\n", nil // the old instance, still draining+ }+ return "Could not find service", errors.New("exit status 113")+ case "bootstrap":+ if printsAfterBootout <= 2 {+ t.Fatalf("bootstrap issued while the old job was still registered: %#v", calls)+ }+ bootstrapped = true+ return "", nil+ }+ return "", fmt.Errorf("unexpected launchctl call: %s", strings.Join(args, " "))+ }+ t.Cleanup(func() { runLaunchctlCommand = oldRun })++ m := ForwarderManager{UserMode: true, Kind: KindLaunchd}+ if err := m.Load(); err != nil {+ t.Fatalf("Load: %v", err)+ }+ if !bootstrapped || !strings.HasPrefix(calls[0], "bootout ") {+ t.Fatalf("expected bootout, wait, bootstrap, verify: %#v", calls)+ }+}++func TestForwarderLoadFailsWhenTheNewInstanceNeverStarts(t *testing.T) {+ if runtime.GOOS != "darwin" {+ t.Skip("launchd only")+ }+ shrinkLaunchdWaits(t)+ t.Setenv("HOME", t.TempDir())+ bootstrapped := false+ oldRun := runLaunchctlCommand+ runLaunchctlCommand = func(args ...string) (string, error) {+ switch args[0] {+ case "bootout":+ return "", nil+ case "bootstrap":+ bootstrapped = true+ return "", nil+ case "print":+ if !bootstrapped {+ return "Could not find service", errors.New("exit status 113") // old job gone+ }+ return "state = waiting\n", nil // loaded, never gets a pid+ }+ return "", fmt.Errorf("unexpected launchctl call: %s", strings.Join(args, " "))+ }+ t.Cleanup(func() { runLaunchctlCommand = oldRun })++ m := ForwarderManager{UserMode: true, Kind: KindLaunchd}+ err := m.Load()+ if err == nil || !strings.Contains(err.Error(), "has not started") {+ t.Fatalf("Load should report a forwarder that never started, got %v", err)+ }+}diff --git a/cli/beacon/internal/endpoint/service/launchd.go b/cli/beacon/internal/endpoint/service/launchd.goindex 7286d06e..a5c37fd1 100644--- a/cli/beacon/internal/endpoint/service/launchd.go+++ b/cli/beacon/internal/endpoint/service/launchd.go@@ -7,6 +7,7 @@ import ("path/filepath""runtime""strings"+ "time")// launchdBackend manages the collector via macOS launchd.@@ -22,6 +23,52 @@ var runLaunchctlCommand = func(args ...string) (string, error) {return string(out), err}+// Reloading a job is not atomic in launchd: `bootout` returns as soon as the job is asked to+// stop, while the process may take its exit timeout (Vector drains in-flight requests for+// up to a minute) to actually go away, and `bootstrap` of the same label in that window+// fails with the job still printing as loaded. Both waits below poll `launchctl print`;+// the counts are derived from the timeouts so tests can shrink them without a clock.+var (+ launchdPollInterval = 500 * time.Millisecond+ launchdStopTimeout = 75 * time.Second+ launchdStartTimeout = 10 * time.Second+ launchdSleep = time.Sleep+)++func launchdJobGone(domain, label string) bool {+ out, err := runLaunchctlCommand("print", domain+"/"+label)+ return err != nil && launchctlNoSuchProcess(strings.TrimSpace(out))+}++// waitForLaunchdJobGone returns true once the job is no longer registered, or false when it+// is still there after launchdStopTimeout.+func waitForLaunchdJobGone(domain, label string) bool {+ attempts := int(launchdStopTimeout / launchdPollInterval)+ for i := 0; i < attempts; i++ {+ if launchdJobGone(domain, label) {+ return true+ }+ launchdSleep(launchdPollInterval)+ }+ return launchdJobGone(domain, label)+}++// waitForLaunchdJobRunning returns true once the job has a pid, or false after+// launchdStartTimeout.+func waitForLaunchdJobRunning(domain, label string) bool {+ attempts := int(launchdStartTimeout / launchdPollInterval)+ for i := 0; i <= attempts; i++ {+ out, err := runLaunchctlCommand("print", domain+"/"+label)+ if err == nil && strings.Contains(out, "pid =") {+ return true+ }+ if i < attempts {+ launchdSleep(launchdPollInterval)+ }+ }+ return false+}+func (launchdBackend) kind() Kind { return KindLaunchd }func (launchdBackend) available() bool { return runtime.GOOS == "darwin" }@@ -130,6 +177,9 @@ func loadLaunchdJob(domain, label, plistPath string) error {if err := runLaunchctlWithContext(domain, label, "", "bootout", target); err != nil {return err}+ // The old instance may still be draining; bootstrapping over it would be swallowed+ // and the job would vanish once it exits.+ waitForLaunchdJobGone(domain, label)if out, err := runLaunchctlCommand("bootstrap", domain, plistPath); err != nil {text := strings.TrimSpace(out)if launchdJobAppearsLoaded(text, domain, label) {diff --git a/cli/beacon/internal/endpoint/service/service_test.go b/cli/beacon/internal/endpoint/service/service_test.goindex acc335ad..392f7572 100644--- a/cli/beacon/internal/endpoint/service/service_test.go+++ b/cli/beacon/internal/endpoint/service/service_test.go@@ -9,6 +9,7 @@ import ("runtime""strings""testing"+ "time")func TestPlistContainsLaunchdContract(t *testing.T) {@@ -58,16 +59,42 @@ func TestRunLaunchctlWithContextExplainsBootstrapIOError(t *testing.T) {}}+// shrinkLaunchdWaits makes the reload polls deterministic and instant in tests.+func shrinkLaunchdWaits(t *testing.T) {+ t.Helper()+ oldStop, oldStart, oldSleep := launchdStopTimeout, launchdStartTimeout, launchdSleep+ launchdStopTimeout = 2 * launchdPollInterval+ launchdStartTimeout = 2 * launchdPollInterval+ launchdSleep = func(time.Duration) {}+ t.Cleanup(func() { launchdStopTimeout, launchdStartTimeout, launchdSleep = oldStop, oldStart, oldSleep })+}++// withoutPrints drops the `print` polls so a test can assert the order of the calls that+// change state.+func withoutPrints(calls []string) []string {+ var out []string+ for _, c := range calls {+ if !strings.HasPrefix(c, "print ") {+ out = append(out, c)+ }+ }+ return out+}+func TestLoadLaunchdJobReloadsAlreadyBootstrappedJob(t *testing.T) {+ shrinkLaunchdWaits(t)var calls []stringoldRun := runLaunchctlCommandrunLaunchctlCommand = func(args ...string) (string, error) {calls = append(calls, strings.Join(args, " "))- switch len(calls) {- case 1:+ switch {+ case args[0] == "bootstrap" && len(withoutPrints(calls)) == 1:return "Bootstrap failed: 5: already bootstrapped", errors.New("exit status 5")- case 2, 3:+ case args[0] == "bootstrap", args[0] == "bootout":return "", nil+ case args[0] == "print":+ // After bootout the old job is gone at once.+ return "Could not find service", errors.New("exit status 113")default:return "", errors.New("unexpected launchctl call")}@@ -79,15 +106,18 @@ func TestLoadLaunchdJobReloadsAlreadyBootstrappedJob(t *testing.T) {if err := loadLaunchdJob("gui/501", UserLabel, "/Users/test/Library/LaunchAgents/"+UserLabel+".plist"); err != nil {t.Fatalf("loadLaunchdJob returned error: %v", err)}- if len(calls) != 3 {- t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap", calls)- }- if !strings.HasPrefix(calls[0], "bootstrap gui/") || !strings.Contains(calls[1], "bootout gui/") || !strings.HasPrefix(calls[2], "bootstrap gui/") {+ state := withoutPrints(calls)+ if len(state) != 3 || !strings.HasPrefix(state[0], "bootstrap gui/") || !strings.Contains(state[1], "bootout gui/") || !strings.HasPrefix(state[2], "bootstrap gui/") {t.Fatalf("unexpected launchctl call sequence: %#v", calls)}+ // The wait between bootout and the second bootstrap must have looked at the job.+ if len(calls) < 4 || !strings.HasPrefix(calls[2], "print gui/") {+ t.Fatalf("expected a print poll after bootout: %#v", calls)+ }}func TestLoadLaunchdJobReloadsBootstrapIOErrorWhenJobPrints(t *testing.T) {+ shrinkLaunchdWaits(t)var calls []stringoldRun := runLaunchctlCommandrunLaunchctlCommand = func(args ...string) (string, error) {@@ -113,12 +143,15 @@ func TestLoadLaunchdJobReloadsBootstrapIOErrorWhenJobPrints(t *testing.T) {if err := loadLaunchdJob("system", SystemLabel, "/Library/LaunchDaemons/"+SystemLabel+".plist"); err != nil {t.Fatalf("loadLaunchdJob returned error: %v", err)}- if len(calls) != 4 {- t.Fatalf("launchctl calls = %#v, want bootstrap/print/bootout/bootstrap", calls)+ // print always reports the job running here, so the post-bootout wait polls to its+ // (shrunken) timeout and the reload proceeds anyway.+ if state := withoutPrints(calls); len(state) != 3 || state[0] != state[2] || !strings.HasPrefix(state[1], "bootout ") {+ t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap around the polls", calls)}}func TestLoadLaunchdJobTreatsPostBootstrapLoadedStateAsSuccess(t *testing.T) {+ shrinkLaunchdWaits(t)var calls []stringoldRun := runLaunchctlCommandrunLaunchctlCommand = func(args ...string) (string, error) {@@ -141,8 +174,11 @@ func TestLoadLaunchdJobTreatsPostBootstrapLoadedStateAsSuccess(t *testing.T) {if err := loadLaunchdJob("system", SystemLabel, "/Library/LaunchDaemons/"+SystemLabel+".plist"); err != nil {t.Fatalf("loadLaunchdJob returned error: %v", err)}- if len(calls) != 5 {- t.Fatalf("launchctl calls = %#v, want bootstrap/print/bootout/bootstrap/print", calls)+ if state := withoutPrints(calls); len(state) != 3 || !strings.HasPrefix(state[1], "bootout ") {+ t.Fatalf("launchctl calls = %#v, want bootstrap/bootout/bootstrap around the polls", calls)+ }+ if !strings.HasPrefix(calls[len(calls)-1], "print ") {+ t.Fatalf("the loaded-state check after the second bootstrap should be the last call: %#v", calls)}}diff --git a/docs/changelog/cli.mdx b/docs/changelog/cli.mdxindex 857ace66..5ae40968 100644--- a/docs/changelog/cli.mdx+++ b/docs/changelog/cli.mdx@@ -11,6 +11,16 @@ Stay up to date with the latest changes to the Agent Beacon CLI.## September 2026+### Unreleased++- **Re-connecting no longer loses the forwarder** · `beacon endpoint connect` on an+ already-connected macOS endpoint stops the old Vector and starts a new one under+ the same launchd label. launchd reports the stop before the old process has+ exited (Vector drains in-flight requests for up to a minute), so the new+ bootstrap could be swallowed and the status check saw the old pid as "running",+ leaving no forwarder at all. The load now waits for the old job to disappear,+ then confirms the new instance has a pid, and fails loudly otherwise.+### v1.3.1 · September 4, 2026- **`beacon endpoint status --user` reports the user-mode connection** · On a