Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(service): stop the supervised backend signalling a recycled pid

shukieshah45f4c473fdbd
diff --git a/cli/beacon/internal/endpoint/service/supervised.go b/cli/beacon/internal/endpoint/service/supervised.go
index 806c8006..99c0d8a5 100644
--- a/cli/beacon/internal/endpoint/service/supervised.go
+++ b/cli/beacon/internal/endpoint/service/supervised.go
@@ -33,6 +33,10 @@ type supervisedState struct {
Program string `json:"program"`
ConfigPath string `json:"config_path"`
StartedAt string `json:"started_at"`
+ // StartedProgram is the program PID was started as. It differs from Program after writeUnit
+ // records a new program while the old collector is still running, and running() has to check
+ // the pid against what it was started as or unload would leave the old collector alive.
+ StartedProgram string `json:"started_program,omitempty"`
}
func (supervisedBackend) kind() Kind { return KindSupervised }
@@ -69,9 +73,10 @@ func (b supervisedBackend) writeUnit(userMode bool, program, configPath string)
state := supervisedState{Program: program, ConfigPath: configPath}
// Preserve a live pid so writeUnit followed by status does not lose track of a collector
// that is already running.
- if existing.PID > 0 && processAlive(existing.PID) {
+ if existing.running() {
state.PID = existing.PID
state.StartedAt = existing.StartedAt
+ state.StartedProgram = existing.StartedProgram
}
return path, b.write(userMode, state)
}
@@ -104,7 +109,24 @@ func (b supervisedBackend) write(userMode bool, st supervisedState) error {
if err != nil {
return err
}
- return os.WriteFile(path, append(data, '\n'), 0o644)
+ // Written to a temporary file and renamed into place. os.WriteFile truncates and then writes,
+ // so a crash in between left an empty pidfile and lost track of a running collector.
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".collector.pid.*")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(tmp.Name())
+ if _, err := tmp.Write(append(data, '\n')); err != nil {
+ _ = tmp.Close()
+ return err
+ }
+ if err := tmp.Close(); err != nil {
+ return err
+ }
+ if err := os.Chmod(tmp.Name(), 0o644); err != nil {
+ return err
+ }
+ return os.Rename(tmp.Name(), path)
}
func (b supervisedBackend) load(userMode bool) error {
@@ -115,7 +137,7 @@ func (b supervisedBackend) load(userMode bool) error {
if st.Program == "" {
return errors.New("supervised collector has no recorded program path")
}
- if st.PID > 0 && processAlive(st.PID) {
+ if st.running() {
return nil // already running
}
@@ -144,6 +166,7 @@ func (b supervisedBackend) load(userMode bool) error {
st.PID = cmd.Process.Pid
st.StartedAt = time.Now().UTC().Format(time.RFC3339)
+ st.StartedProgram = st.Program
return b.write(userMode, st)
}
@@ -152,7 +175,7 @@ func (b supervisedBackend) unload(userMode bool) error {
if err != nil {
return nil // nothing recorded, nothing to stop
}
- if st.PID <= 0 || !processAlive(st.PID) {
+ if !st.running() {
st.PID = 0
_ = b.write(userMode, st)
return nil
@@ -167,12 +190,12 @@ func (b supervisedBackend) unload(userMode bool) error {
_ = terminateGracefully(proc)
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
- if !processAlive(st.PID) {
+ if !st.running() {
break
}
time.Sleep(100 * time.Millisecond)
}
- if processAlive(st.PID) {
+ if st.running() {
_ = proc.Kill()
}
st.PID = 0
@@ -195,7 +218,7 @@ func (b supervisedBackend) status(userMode bool) Status {
}
// "Loaded" means installed and intended to run, matching the other backends.
status.Loaded = st.Program != ""
- status.Running = st.PID > 0 && processAlive(st.PID)
+ status.Running = st.running()
switch {
case status.Running:
status.Message = fmt.Sprintf("supervised pid %d (%s)", st.PID, supervisedCaveats(userMode))
@@ -226,10 +249,19 @@ func supervisedCaveats(userMode bool) string {
return noRestart
}
-// processAlive reports whether a pid is live.
-func processAlive(pid int) bool {
- if pid <= 0 {
+// running reports whether the recorded pid is live and is still the collector this state recorded.
+//
+// A pid is only a number, and the kernel hands it out again once the process holding it exits. The
+// pidfile outlives the collector -- an OOM kill, a container restart, a reboot of a host with no
+// service manager -- so the number can name an unrelated process by the time anything reads it.
+// Liveness alone then made status report a dead collector as running, load skip starting one, and
+// unload send that stranger SIGTERM, then SIGKILL if it had not exited ten seconds later.
+//
+// A pidfile written before StartedProgram existed cannot be checked, and is trusted on liveness
+// alone as before. Program is not a substitute: writeUnit may have changed it since the pid started.
+func (st supervisedState) running() bool {
+ if st.PID <= 0 || !pidAlive(st.PID) {
return false
}
- return pidAlive(pid)
+ return st.StartedProgram == "" || pidRunsProgram(st.PID, st.StartedProgram)
}
diff --git a/cli/beacon/internal/endpoint/service/supervised_unix.go b/cli/beacon/internal/endpoint/service/supervised_unix.go
index 16e18bcf..700d7a64 100644
--- a/cli/beacon/internal/endpoint/service/supervised_unix.go
+++ b/cli/beacon/internal/endpoint/service/supervised_unix.go
@@ -4,6 +4,9 @@ package service
import (
"os"
+ "path/filepath"
+ "strconv"
+ "strings"
"syscall"
)
@@ -27,3 +30,33 @@ func pidAlive(pid int) bool {
}
return proc.Signal(syscall.Signal(0)) == nil
}
+
+// procRoot is where procfs is mounted. A variable so tests can point it at a fixture.
+var procRoot = "/proc"
+
+// pidRunsProgram reports whether pid's command line contains program.
+//
+// Every argument is compared, not only argv[0]: a collector started through an interpreter (a
+// shebang script, as the sandbox's stand-in is) has the interpreter in argv[0] and the program in
+// argv[1]. The loader starts the program by the exact path it recorded, so an exact match is the
+// right test.
+//
+// Without procfs (macOS) there is nothing to ask, and this reports true so the signal-0 check stays
+// the only test, as it was before. With procfs, a pid it will not show us is not our collector:
+// supervised mode runs the collector as the user who installed it, and that user can read its own
+// processes' command lines.
+func pidRunsProgram(pid int, program string) bool {
+ data, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(pid), "cmdline"))
+ if err != nil {
+ if _, statErr := os.Stat(filepath.Join(procRoot, "self")); statErr != nil {
+ return true
+ }
+ return false
+ }
+ for _, arg := range strings.Split(string(data), "\x00") {
+ if arg == program {
+ return true
+ }
+ }
+ return false
+}
diff --git a/cli/beacon/internal/endpoint/service/supervised_unix_test.go b/cli/beacon/internal/endpoint/service/supervised_unix_test.go
new file mode 100644
index 00000000..9026e8d7
--- /dev/null
+++ b/cli/beacon/internal/endpoint/service/supervised_unix_test.go
@@ -0,0 +1,151 @@
+//go:build !windows
+
+package service
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
+)
+
+func TestPidRunsProgramReadsProcCmdline(t *testing.T) {
+ root := t.TempDir()
+ if err := os.Mkdir(filepath.Join(root, "self"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Mkdir(filepath.Join(root, "4242"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ // A shebang script: the interpreter is argv[0] and the program is argv[1].
+ cmdline := "/bin/sh\x00/tmp/beacon-fake-collector\x00--config\x00/etc/beacon/endpoint/otelcol.yaml\x00"
+ if err := os.WriteFile(filepath.Join(root, "4242", "cmdline"), []byte(cmdline), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ old := procRoot
+ procRoot = root
+ t.Cleanup(func() { procRoot = old })
+
+ if !pidRunsProgram(4242, "/tmp/beacon-fake-collector") {
+ t.Error("the recorded program is argv[1]; want a match")
+ }
+ if pidRunsProgram(4242, "/opt/beacon/bin/beacon-otelcol") {
+ t.Error("a pid running something else must not match")
+ }
+ if pidRunsProgram(4242, "/tmp/beacon-fake") {
+ t.Error("a prefix of an argument is not the program")
+ }
+ if pidRunsProgram(999, "/tmp/beacon-fake-collector") {
+ t.Error("procfs has no such pid; want no match")
+ }
+
+ procRoot = filepath.Join(root, "missing")
+ if !pidRunsProgram(4242, "/opt/beacon/bin/beacon-otelcol") {
+ t.Error("without procfs there is nothing to ask; want the signal-0 answer to stand")
+ }
+}
+
+// A pidfile that outlived its collector names whatever the kernel gave that pid to next. Status
+// must not call that process the collector, and unload must not signal it.
+func TestSupervisedIgnoresARecycledPID(t *testing.T) {
+ if runtime.GOOS != "linux" {
+ t.Skip("the identity check reads procfs")
+ }
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ m := Manager{UserMode: true, Kind: KindSupervised}
+
+ stranger := exec.Command(stubCollectorPath(t), "--config", "unrelated")
+ if err := stranger.Start(); err != nil {
+ t.Fatalf("start the unrelated process: %v", err)
+ }
+ exited := make(chan struct{})
+ go func() { _ = stranger.Wait(); close(exited) }()
+ t.Cleanup(func() { _ = stranger.Process.Kill(); <-exited })
+
+ b := supervisedBackend{}
+ if _, err := m.WriteUnit("/opt/beacon/bin/beacon-otelcol", "/tmp/otelcol.yaml"); err != nil {
+ t.Fatalf("WriteUnit: %v", err)
+ }
+ if err := b.write(true, supervisedState{
+ PID: stranger.Process.Pid,
+ Program: "/opt/beacon/bin/beacon-otelcol",
+ ConfigPath: "/tmp/otelcol.yaml",
+ StartedProgram: "/opt/beacon/bin/beacon-otelcol",
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ if st := m.Status(); st.Running {
+ t.Fatalf("pid %d is not the recorded collector, but status reports it running: %#v",
+ stranger.Process.Pid, st)
+ }
+ if err := m.Unload(); err != nil {
+ t.Fatalf("Unload: %v", err)
+ }
+ select {
+ case <-exited:
+ t.Fatal("unload signalled a process that is not the collector")
+ case <-time.After(200 * time.Millisecond):
+ }
+ st, err := b.read(true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if st.PID != 0 {
+ t.Fatalf("unload should forget a pid that is not the collector, pidfile still holds %s",
+ strconv.Itoa(st.PID))
+ }
+}
+
+// A reinstall records the new program while the old collector is still running, then restarts.
+// Unload has to recognise the old collector by what it was started as, not by the program just
+// recorded, or it leaves the old collector running beside the new one.
+func TestSupervisedUnloadStopsACollectorStartedAsAnotherProgram(t *testing.T) {
+ if runtime.GOOS != "linux" {
+ t.Skip("the identity check reads procfs")
+ }
+ home := t.TempDir()
+ testenv.SetHome(t, home)
+ m := Manager{UserMode: true, Kind: KindSupervised}
+
+ if _, err := m.WriteUnit(stubCollectorPath(t), "300"); err != nil {
+ t.Fatalf("WriteUnit: %v", err)
+ }
+ if err := m.Load(); err != nil {
+ t.Fatalf("Load: %v", err)
+ }
+ old, err := supervisedBackend{}.read(true)
+ if err != nil || old.PID <= 0 {
+ t.Fatalf("no collector recorded after Load: %#v %v", old, err)
+ }
+ t.Cleanup(func() {
+ if p, err := os.FindProcess(old.PID); err == nil {
+ _ = p.Kill()
+ }
+ })
+
+ if _, err := m.WriteUnit("/tmp/beacon-next-collector", "/tmp/otelcol.yaml"); err != nil {
+ t.Fatalf("second WriteUnit: %v", err)
+ }
+ if st := m.Status(); !st.Running {
+ t.Fatalf("the old collector is still running, but status lost it: %#v", st)
+ }
+ if err := m.Unload(); err != nil {
+ t.Fatalf("Unload: %v", err)
+ }
+ // Polled: the stopped collector is reaped by Load's goroutine, and until then it is a zombie
+ // that signal 0 still finds.
+ deadline := time.Now().Add(2 * time.Second)
+ for pidAlive(old.PID) {
+ if time.Now().After(deadline) {
+ t.Fatalf("unload left the collector started before the reinstall (pid %d) running", old.PID)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/service/supervised_windows.go b/cli/beacon/internal/endpoint/service/supervised_windows.go
index 4237c35b..898722da 100644
--- a/cli/beacon/internal/endpoint/service/supervised_windows.go
+++ b/cli/beacon/internal/endpoint/service/supervised_windows.go
@@ -77,3 +77,7 @@ func pidAlive(pid int) bool {
}
return code == stillActive
}
+
+// pidRunsProgram reports true: the pid-recycling check reads procfs, which Windows does not have.
+// pidAlive is the only test here, as it was before the check existed.
+func pidRunsProgram(pid int, program string) bool { return true }
diff --git a/docs/platforms/linux-deployment-profile.mdx b/docs/platforms/linux-deployment-profile.mdx
index 2c7820c4..bb6ce06d 100644
--- a/docs/platforms/linux-deployment-profile.mdx
+++ b/docs/platforms/linux-deployment-profile.mdx
@@ -139,7 +139,7 @@ sudo apt remove beacon # keeps config and collected logs
sudo apt purge beacon # removes them
```
-On RPM systems, `dnf remove` keeps `/etc/beacon/endpoint` and `/var/log/beacon-agent`. Remove them with `sudo beacon endpoint uninstall --system` if you want the data gone.
+On RPM systems, `dnf remove` keeps `/etc/beacon/endpoint` and `/var/log/beacon-agent`, and there is no purge. To remove the data too, run `sudo beacon endpoint uninstall --system` before `sudo dnf remove beacon`; afterwards the `beacon` binary is gone.
## Related
diff --git a/docs/platforms/linux.mdx b/docs/platforms/linux.mdx
index 71093b84..37ba9fd3 100644
--- a/docs/platforms/linux.mdx
+++ b/docs/platforms/linux.mdx
@@ -249,17 +249,18 @@ sudo apt remove beacon # or: sudo dnf remove beacon
Removal stops and deletes the service and takes the binaries away, but leaves `/etc/beacon/endpoint` and `/var/log/beacon-agent` alone. Collected telemetry is not something a package removal should destroy. To remove those too:
-```bash title="Remove everything"
+```bash title="Remove everything (Debian, Ubuntu)"
sudo apt purge beacon
```
-On RPM systems, or to remove a non-package install, use Beacon directly:
+RPM has no purge. Run Beacon's own uninstall first, while the binary is still there, then remove the package:
-```bash title="Full uninstall"
+```bash title="Remove everything (RHEL, Rocky, Fedora)"
sudo beacon endpoint uninstall --system
+sudo dnf remove beacon
```
-`--keep-logs` and `--keep-config` are available if you want to keep either.
+`beacon endpoint uninstall --system` on its own also removes a non-package install. `--keep-logs` and `--keep-config` are available if you want to keep either.
## Related
diff --git a/packaging/linux/nfpm/postremove.sh b/packaging/linux/nfpm/postremove.sh
index c66d6e08..37e28edd 100755
--- a/packaging/linux/nfpm/postremove.sh
+++ b/packaging/linux/nfpm/postremove.sh
@@ -1,8 +1,13 @@
#!/bin/sh
# Runs after the package is removed.
#
-# On a dpkg purge (or an rpm erase), remove the state that preremove deliberately kept. Anything
-# short of a purge leaves logs and config in place.
+# On a dpkg purge, remove the state that preremove deliberately kept. Anything short of a purge
+# leaves logs and config in place.
+#
+# rpm has no purge. Its %postun gets the count of remaining versions ("0" on the final erase), and
+# that is deliberately not matched: rpm convention is that erasing a package leaves the data it
+# generated, and deleting collected telemetry on `dnf remove` would leave RPM users no way to remove
+# the package and keep it. Full removal on RPM is `beacon endpoint uninstall --system` first.
set -eu
case "${1:-}" in