The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(service): stop the supervised backend signalling a recycled pid
shukieshah45f4c473fdbd
diff --git a/cli/beacon/internal/endpoint/service/supervised.go b/cli/beacon/internal/endpoint/service/supervised.goindex 806c8006..99c0d8a5 100644--- a/cli/beacon/internal/endpoint/service/supervised.go+++ b/cli/beacon/internal/endpoint/service/supervised.go@@ -33,6 +33,10 @@ type supervisedState struct {Program string `json:"program"`ConfigPath string `json:"config_path"`StartedAt string `json:"started_at"`+ // StartedProgram is the program PID was started as. It differs from Program after writeUnit+ // records a new program while the old collector is still running, and running() has to check+ // the pid against what it was started as or unload would leave the old collector alive.+ StartedProgram string `json:"started_program,omitempty"`}func (supervisedBackend) kind() Kind { return KindSupervised }@@ -69,9 +73,10 @@ func (b supervisedBackend) writeUnit(userMode bool, program, configPath string)state := supervisedState{Program: program, ConfigPath: configPath}// Preserve a live pid so writeUnit followed by status does not lose track of a collector// that is already running.- if existing.PID > 0 && processAlive(existing.PID) {+ if existing.running() {state.PID = existing.PIDstate.StartedAt = existing.StartedAt+ state.StartedProgram = existing.StartedProgram}return path, b.write(userMode, state)}@@ -104,7 +109,24 @@ func (b supervisedBackend) write(userMode bool, st supervisedState) error {if err != nil {return err}- return os.WriteFile(path, append(data, '\n'), 0o644)+ // Written to a temporary file and renamed into place. os.WriteFile truncates and then writes,+ // so a crash in between left an empty pidfile and lost track of a running collector.+ tmp, err := os.CreateTemp(filepath.Dir(path), ".collector.pid.*")+ if err != nil {+ return err+ }+ defer os.Remove(tmp.Name())+ if _, err := tmp.Write(append(data, '\n')); err != nil {+ _ = tmp.Close()+ return err+ }+ if err := tmp.Close(); err != nil {+ return err+ }+ if err := os.Chmod(tmp.Name(), 0o644); err != nil {+ return err+ }+ return os.Rename(tmp.Name(), path)}func (b supervisedBackend) load(userMode bool) error {@@ -115,7 +137,7 @@ func (b supervisedBackend) load(userMode bool) error {if st.Program == "" {return errors.New("supervised collector has no recorded program path")}- if st.PID > 0 && processAlive(st.PID) {+ if st.running() {return nil // already running}@@ -144,6 +166,7 @@ func (b supervisedBackend) load(userMode bool) error {st.PID = cmd.Process.Pidst.StartedAt = time.Now().UTC().Format(time.RFC3339)+ st.StartedProgram = st.Programreturn b.write(userMode, st)}@@ -152,7 +175,7 @@ func (b supervisedBackend) unload(userMode bool) error {if err != nil {return nil // nothing recorded, nothing to stop}- if st.PID <= 0 || !processAlive(st.PID) {+ if !st.running() {st.PID = 0_ = b.write(userMode, st)return nil@@ -167,12 +190,12 @@ func (b supervisedBackend) unload(userMode bool) error {_ = terminateGracefully(proc)deadline := time.Now().Add(10 * time.Second)for time.Now().Before(deadline) {- if !processAlive(st.PID) {+ if !st.running() {break}time.Sleep(100 * time.Millisecond)}- if processAlive(st.PID) {+ if st.running() {_ = proc.Kill()}st.PID = 0@@ -195,7 +218,7 @@ func (b supervisedBackend) status(userMode bool) Status {}// "Loaded" means installed and intended to run, matching the other backends.status.Loaded = st.Program != ""- status.Running = st.PID > 0 && processAlive(st.PID)+ status.Running = st.running()switch {case status.Running:status.Message = fmt.Sprintf("supervised pid %d (%s)", st.PID, supervisedCaveats(userMode))@@ -226,10 +249,19 @@ func supervisedCaveats(userMode bool) string {return noRestart}-// processAlive reports whether a pid is live.-func processAlive(pid int) bool {- if pid <= 0 {+// running reports whether the recorded pid is live and is still the collector this state recorded.+//+// A pid is only a number, and the kernel hands it out again once the process holding it exits. The+// pidfile outlives the collector -- an OOM kill, a container restart, a reboot of a host with no+// service manager -- so the number can name an unrelated process by the time anything reads it.+// Liveness alone then made status report a dead collector as running, load skip starting one, and+// unload send that stranger SIGTERM, then SIGKILL if it had not exited ten seconds later.+//+// A pidfile written before StartedProgram existed cannot be checked, and is trusted on liveness+// alone as before. Program is not a substitute: writeUnit may have changed it since the pid started.+func (st supervisedState) running() bool {+ if st.PID <= 0 || !pidAlive(st.PID) {return false}- return pidAlive(pid)+ return st.StartedProgram == "" || pidRunsProgram(st.PID, st.StartedProgram)}diff --git a/cli/beacon/internal/endpoint/service/supervised_unix.go b/cli/beacon/internal/endpoint/service/supervised_unix.goindex 16e18bcf..700d7a64 100644--- a/cli/beacon/internal/endpoint/service/supervised_unix.go+++ b/cli/beacon/internal/endpoint/service/supervised_unix.go@@ -4,6 +4,9 @@ package serviceimport ("os"+ "path/filepath"+ "strconv"+ "strings""syscall")@@ -27,3 +30,33 @@ func pidAlive(pid int) bool {}return proc.Signal(syscall.Signal(0)) == nil}++// procRoot is where procfs is mounted. A variable so tests can point it at a fixture.+var procRoot = "/proc"++// pidRunsProgram reports whether pid's command line contains program.+//+// Every argument is compared, not only argv[0]: a collector started through an interpreter (a+// shebang script, as the sandbox's stand-in is) has the interpreter in argv[0] and the program in+// argv[1]. The loader starts the program by the exact path it recorded, so an exact match is the+// right test.+//+// Without procfs (macOS) there is nothing to ask, and this reports true so the signal-0 check stays+// the only test, as it was before. With procfs, a pid it will not show us is not our collector:+// supervised mode runs the collector as the user who installed it, and that user can read its own+// processes' command lines.+func pidRunsProgram(pid int, program string) bool {+ data, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(pid), "cmdline"))+ if err != nil {+ if _, statErr := os.Stat(filepath.Join(procRoot, "self")); statErr != nil {+ return true+ }+ return false+ }+ for _, arg := range strings.Split(string(data), "\x00") {+ if arg == program {+ return true+ }+ }+ return false+}diff --git a/cli/beacon/internal/endpoint/service/supervised_unix_test.go b/cli/beacon/internal/endpoint/service/supervised_unix_test.gonew file mode 100644index 00000000..9026e8d7--- /dev/null+++ b/cli/beacon/internal/endpoint/service/supervised_unix_test.go@@ -0,0 +1,151 @@+//go:build !windows++package service++import (+ "os"+ "os/exec"+ "path/filepath"+ "runtime"+ "strconv"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"+)++func TestPidRunsProgramReadsProcCmdline(t *testing.T) {+ root := t.TempDir()+ if err := os.Mkdir(filepath.Join(root, "self"), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.Mkdir(filepath.Join(root, "4242"), 0o755); err != nil {+ t.Fatal(err)+ }+ // A shebang script: the interpreter is argv[0] and the program is argv[1].+ cmdline := "/bin/sh\x00/tmp/beacon-fake-collector\x00--config\x00/etc/beacon/endpoint/otelcol.yaml\x00"+ if err := os.WriteFile(filepath.Join(root, "4242", "cmdline"), []byte(cmdline), 0o644); err != nil {+ t.Fatal(err)+ }+ old := procRoot+ procRoot = root+ t.Cleanup(func() { procRoot = old })++ if !pidRunsProgram(4242, "/tmp/beacon-fake-collector") {+ t.Error("the recorded program is argv[1]; want a match")+ }+ if pidRunsProgram(4242, "/opt/beacon/bin/beacon-otelcol") {+ t.Error("a pid running something else must not match")+ }+ if pidRunsProgram(4242, "/tmp/beacon-fake") {+ t.Error("a prefix of an argument is not the program")+ }+ if pidRunsProgram(999, "/tmp/beacon-fake-collector") {+ t.Error("procfs has no such pid; want no match")+ }++ procRoot = filepath.Join(root, "missing")+ if !pidRunsProgram(4242, "/opt/beacon/bin/beacon-otelcol") {+ t.Error("without procfs there is nothing to ask; want the signal-0 answer to stand")+ }+}++// A pidfile that outlived its collector names whatever the kernel gave that pid to next. Status+// must not call that process the collector, and unload must not signal it.+func TestSupervisedIgnoresARecycledPID(t *testing.T) {+ if runtime.GOOS != "linux" {+ t.Skip("the identity check reads procfs")+ }+ home := t.TempDir()+ testenv.SetHome(t, home)+ m := Manager{UserMode: true, Kind: KindSupervised}++ stranger := exec.Command(stubCollectorPath(t), "--config", "unrelated")+ if err := stranger.Start(); err != nil {+ t.Fatalf("start the unrelated process: %v", err)+ }+ exited := make(chan struct{})+ go func() { _ = stranger.Wait(); close(exited) }()+ t.Cleanup(func() { _ = stranger.Process.Kill(); <-exited })++ b := supervisedBackend{}+ if _, err := m.WriteUnit("/opt/beacon/bin/beacon-otelcol", "/tmp/otelcol.yaml"); err != nil {+ t.Fatalf("WriteUnit: %v", err)+ }+ if err := b.write(true, supervisedState{+ PID: stranger.Process.Pid,+ Program: "/opt/beacon/bin/beacon-otelcol",+ ConfigPath: "/tmp/otelcol.yaml",+ StartedProgram: "/opt/beacon/bin/beacon-otelcol",+ }); err != nil {+ t.Fatal(err)+ }++ if st := m.Status(); st.Running {+ t.Fatalf("pid %d is not the recorded collector, but status reports it running: %#v",+ stranger.Process.Pid, st)+ }+ if err := m.Unload(); err != nil {+ t.Fatalf("Unload: %v", err)+ }+ select {+ case <-exited:+ t.Fatal("unload signalled a process that is not the collector")+ case <-time.After(200 * time.Millisecond):+ }+ st, err := b.read(true)+ if err != nil {+ t.Fatal(err)+ }+ if st.PID != 0 {+ t.Fatalf("unload should forget a pid that is not the collector, pidfile still holds %s",+ strconv.Itoa(st.PID))+ }+}++// A reinstall records the new program while the old collector is still running, then restarts.+// Unload has to recognise the old collector by what it was started as, not by the program just+// recorded, or it leaves the old collector running beside the new one.+func TestSupervisedUnloadStopsACollectorStartedAsAnotherProgram(t *testing.T) {+ if runtime.GOOS != "linux" {+ t.Skip("the identity check reads procfs")+ }+ home := t.TempDir()+ testenv.SetHome(t, home)+ m := Manager{UserMode: true, Kind: KindSupervised}++ if _, err := m.WriteUnit(stubCollectorPath(t), "300"); err != nil {+ t.Fatalf("WriteUnit: %v", err)+ }+ if err := m.Load(); err != nil {+ t.Fatalf("Load: %v", err)+ }+ old, err := supervisedBackend{}.read(true)+ if err != nil || old.PID <= 0 {+ t.Fatalf("no collector recorded after Load: %#v %v", old, err)+ }+ t.Cleanup(func() {+ if p, err := os.FindProcess(old.PID); err == nil {+ _ = p.Kill()+ }+ })++ if _, err := m.WriteUnit("/tmp/beacon-next-collector", "/tmp/otelcol.yaml"); err != nil {+ t.Fatalf("second WriteUnit: %v", err)+ }+ if st := m.Status(); !st.Running {+ t.Fatalf("the old collector is still running, but status lost it: %#v", st)+ }+ if err := m.Unload(); err != nil {+ t.Fatalf("Unload: %v", err)+ }+ // Polled: the stopped collector is reaped by Load's goroutine, and until then it is a zombie+ // that signal 0 still finds.+ deadline := time.Now().Add(2 * time.Second)+ for pidAlive(old.PID) {+ if time.Now().After(deadline) {+ t.Fatalf("unload left the collector started before the reinstall (pid %d) running", old.PID)+ }+ time.Sleep(20 * time.Millisecond)+ }+}diff --git a/cli/beacon/internal/endpoint/service/supervised_windows.go b/cli/beacon/internal/endpoint/service/supervised_windows.goindex 4237c35b..898722da 100644--- a/cli/beacon/internal/endpoint/service/supervised_windows.go+++ b/cli/beacon/internal/endpoint/service/supervised_windows.go@@ -77,3 +77,7 @@ func pidAlive(pid int) bool {}return code == stillActive}++// pidRunsProgram reports true: the pid-recycling check reads procfs, which Windows does not have.+// pidAlive is the only test here, as it was before the check existed.+func pidRunsProgram(pid int, program string) bool { return true }diff --git a/docs/platforms/linux-deployment-profile.mdx b/docs/platforms/linux-deployment-profile.mdxindex 2c7820c4..bb6ce06d 100644--- a/docs/platforms/linux-deployment-profile.mdx+++ b/docs/platforms/linux-deployment-profile.mdx@@ -139,7 +139,7 @@ sudo apt remove beacon # keeps config and collected logssudo apt purge beacon # removes them```-On RPM systems, `dnf remove` keeps `/etc/beacon/endpoint` and `/var/log/beacon-agent`. Remove them with `sudo beacon endpoint uninstall --system` if you want the data gone.+On RPM systems, `dnf remove` keeps `/etc/beacon/endpoint` and `/var/log/beacon-agent`, and there is no purge. To remove the data too, run `sudo beacon endpoint uninstall --system` before `sudo dnf remove beacon`; afterwards the `beacon` binary is gone.## Relateddiff --git a/docs/platforms/linux.mdx b/docs/platforms/linux.mdxindex 71093b84..37ba9fd3 100644--- a/docs/platforms/linux.mdx+++ b/docs/platforms/linux.mdx@@ -249,17 +249,18 @@ sudo apt remove beacon # or: sudo dnf remove beaconRemoval stops and deletes the service and takes the binaries away, but leaves `/etc/beacon/endpoint` and `/var/log/beacon-agent` alone. Collected telemetry is not something a package removal should destroy. To remove those too:-```bash title="Remove everything"+```bash title="Remove everything (Debian, Ubuntu)"sudo apt purge beacon```-On RPM systems, or to remove a non-package install, use Beacon directly:+RPM has no purge. Run Beacon's own uninstall first, while the binary is still there, then remove the package:-```bash title="Full uninstall"+```bash title="Remove everything (RHEL, Rocky, Fedora)"sudo beacon endpoint uninstall --system+sudo dnf remove beacon```-`--keep-logs` and `--keep-config` are available if you want to keep either.+`beacon endpoint uninstall --system` on its own also removes a non-package install. `--keep-logs` and `--keep-config` are available if you want to keep either.## Relateddiff --git a/packaging/linux/nfpm/postremove.sh b/packaging/linux/nfpm/postremove.shindex c66d6e08..37e28edd 100755--- a/packaging/linux/nfpm/postremove.sh+++ b/packaging/linux/nfpm/postremove.sh@@ -1,8 +1,13 @@#!/bin/sh# Runs after the package is removed.#-# On a dpkg purge (or an rpm erase), remove the state that preremove deliberately kept. Anything-# short of a purge leaves logs and config in place.+# On a dpkg purge, remove the state that preremove deliberately kept. Anything short of a purge+# leaves logs and config in place.+#+# rpm has no purge. Its %postun gets the count of remaining versions ("0" on the final erase), and+# that is deliberately not matched: rpm convention is that erasing a package leaves the data it+# generated, and deleting collected telemetry on `dnf remove` would leave RPM users no way to remove+# the package and keep it. Full removal on RPM is `beacon endpoint uninstall --system` first.set -eucase "${1:-}" in