The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(endpoint): stop rollback disabling a collector it found already running
shukieshah5e8c5be71323
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.goindex 23a1d4e8..f751638b 100644--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go@@ -125,11 +125,29 @@ type fileSnapshot struct {Mode os.FileMode}+// serviceController is the part of service.Manager that rollback uses.+//+// Narrow on purpose: rollback's decision about a running service is the branch worth testing, and+// without a seam here it can only be reached by driving a real install to fail against a real+// service manager. service.Manager satisfies this as-is.+type serviceController interface {+ Unload() error+ Restart() error+}+type installRollback struct {- Manager service.Manager+ Manager serviceControllerServiceLoaded bool- files []string- snapshots map[string]fileSnapshot+ // ServiceWasRunning records whether a collector was already up when this install began.+ //+ // It decides what rollback owes the machine. Unloading is right for a service this transaction+ // created -- leaving a half-installed endpoint registered would be worse. It is wrong for one+ // that was already running: a failed upgrade of a healthy endpoint would then stop and disable+ // it, and on systemd the disable survives reboot, so a machine that was collecting fine before+ // the attempt is left collecting nothing afterwards.+ ServiceWasRunning bool+ files []string+ snapshots map[string]fileSnapshot}func newInstallRollback(manager service.Manager) *installRollback {@@ -155,7 +173,9 @@ func (r *installRollback) Rollback(manifest Manifest) {rollback(manifest)return}- if r.ServiceLoaded {+ // Only tear down a service this install brought up. One that was already running belongs to the+ // previous, working install, and taking it down is not this transaction's to do.+ if r.ServiceLoaded && !r.ServiceWasRunning {_ = r.Manager.Unload()}restoreBackups(manifest.Backups)@@ -163,6 +183,16 @@ func (r *installRollback) Rollback(manifest Manifest) {path := r.files[i]restoreFile(path, r.snapshots[path])}+ // Restart after the files are back, so the collector comes up on the restored configuration+ // rather than the one this failed attempt wrote. Ordering matters: the collector reads its+ // config at startup, so restarting before the restore would leave it running the very config+ // that failed.+ //+ // Best-effort. If the restart does not take, the endpoint is no worse off than the failure that+ // brought us here, and the install error the caller receives is the more useful signal.+ if r.ServiceLoaded && r.ServiceWasRunning {+ _ = r.Manager.Restart()+ }}func Install(opts InstallOptions) (InstallResult, error) {@@ -251,7 +281,12 @@ func Install(opts InstallOptions) (InstallResult, error) {// install reported success and the ports answered. The endpoint looked healthy and was a// version behind until the machine rebooted. The same path made `endpoint install// --splunk-hec-endpoint ...` write a destination that never took effect.- if manager.Status().Running {+ // Read once and remember it. The same answer decides two things: whether to restart rather+ // than load now, and whether rollback is allowed to unload later. Asking twice would risk+ // the two decisions disagreeing about the state they are reasoning about.+ wasRunning := manager.Status().Running+ tx.ServiceWasRunning = wasRunning+ if wasRunning {if err := manager.Restart(); err != nil {tx.Rollback(manifest)return InstallResult{}, errdiff --git a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.gonew file mode 100644index 00000000..cedfa9d8--- /dev/null+++ b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go@@ -0,0 +1,69 @@+package lifecycle++import (+ "strings"+ "testing"+)++// recordingService notes what rollback asked of the service manager, in order.+type recordingService struct{ calls []string }++func (r *recordingService) Unload() error { r.calls = append(r.calls, "unload"); return nil }+func (r *recordingService) Restart() error { r.calls = append(r.calls, "restart"); return nil }++// Rollback has to distinguish a service this install started from one that was already running.+//+// Unloading is right for the first: leaving a half-installed endpoint registered is worse than+// removing it. It is wrong for the second. A reinstall over a healthy endpoint that fails partway+// -- readiness timing out, a harness config erroring -- would otherwise stop *and disable* the+// collector that was working before the attempt, and on systemd that disable survives reboot. The+// upgrade would not merely fail; it would take collection down with it.+func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {+ for name, tc := range map[string]struct {+ loaded, wasRunning bool+ want []string+ }{+ "started by this install": {loaded: true, wasRunning: false, want: []string{"unload"}},+ "already running before it": {loaded: true, wasRunning: true, want: []string{"restart"}},+ "never got as far as the service": {loaded: false, wasRunning: true, want: nil},+ "nothing loaded, nothing running": {loaded: false, wasRunning: false, want: nil},+ } {+ t.Run(name, func(t *testing.T) {+ svc := &recordingService{}+ r := &installRollback{+ Manager: svc,+ ServiceLoaded: tc.loaded,+ ServiceWasRunning: tc.wasRunning,+ snapshots: map[string]fileSnapshot{},+ }++ r.Rollback(Manifest{})++ if strings.Join(svc.calls, ",") != strings.Join(tc.want, ",") {+ t.Errorf("rollback called %v, want %v", svc.calls, tc.want)+ }+ })+ }+}++// The restart has to happen after the files are restored. The collector reads its configuration at+// startup, so restarting first would bring it up on the very config the failed install wrote.+func TestRollbackRestartsAfterRestoringFiles(t *testing.T) {+ dir := t.TempDir()+ path := dir + "/otelcol.yaml"++ svc := &recordingService{}+ r := &installRollback{+ Manager: svc,+ ServiceLoaded: true,+ ServiceWasRunning: true,+ snapshots: map[string]fileSnapshot{},+ }+ r.Track(path)++ r.Rollback(Manifest{})++ if len(svc.calls) != 1 || svc.calls[0] != "restart" {+ t.Fatalf("service calls = %v, want a single restart", svc.calls)+ }+}