Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(endpoint): stop rollback disabling a collector it found already running

shukieshah5e8c5be71323
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
index 23a1d4e8..f751638b 100644
--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
@@ -125,11 +125,29 @@ type fileSnapshot struct {
Mode os.FileMode
}
+// serviceController is the part of service.Manager that rollback uses.
+//
+// Narrow on purpose: rollback's decision about a running service is the branch worth testing, and
+// without a seam here it can only be reached by driving a real install to fail against a real
+// service manager. service.Manager satisfies this as-is.
+type serviceController interface {
+ Unload() error
+ Restart() error
+}
+
type installRollback struct {
- Manager service.Manager
+ Manager serviceController
ServiceLoaded bool
- files []string
- snapshots map[string]fileSnapshot
+ // ServiceWasRunning records whether a collector was already up when this install began.
+ //
+ // It decides what rollback owes the machine. Unloading is right for a service this transaction
+ // created -- leaving a half-installed endpoint registered would be worse. It is wrong for one
+ // that was already running: a failed upgrade of a healthy endpoint would then stop and disable
+ // it, and on systemd the disable survives reboot, so a machine that was collecting fine before
+ // the attempt is left collecting nothing afterwards.
+ ServiceWasRunning bool
+ files []string
+ snapshots map[string]fileSnapshot
}
func newInstallRollback(manager service.Manager) *installRollback {
@@ -155,7 +173,9 @@ func (r *installRollback) Rollback(manifest Manifest) {
rollback(manifest)
return
}
- if r.ServiceLoaded {
+ // Only tear down a service this install brought up. One that was already running belongs to the
+ // previous, working install, and taking it down is not this transaction's to do.
+ if r.ServiceLoaded && !r.ServiceWasRunning {
_ = r.Manager.Unload()
}
restoreBackups(manifest.Backups)
@@ -163,6 +183,16 @@ func (r *installRollback) Rollback(manifest Manifest) {
path := r.files[i]
restoreFile(path, r.snapshots[path])
}
+ // Restart after the files are back, so the collector comes up on the restored configuration
+ // rather than the one this failed attempt wrote. Ordering matters: the collector reads its
+ // config at startup, so restarting before the restore would leave it running the very config
+ // that failed.
+ //
+ // Best-effort. If the restart does not take, the endpoint is no worse off than the failure that
+ // brought us here, and the install error the caller receives is the more useful signal.
+ if r.ServiceLoaded && r.ServiceWasRunning {
+ _ = r.Manager.Restart()
+ }
}
func Install(opts InstallOptions) (InstallResult, error) {
@@ -251,7 +281,12 @@ func Install(opts InstallOptions) (InstallResult, error) {
// install reported success and the ports answered. The endpoint looked healthy and was a
// version behind until the machine rebooted. The same path made `endpoint install
// --splunk-hec-endpoint ...` write a destination that never took effect.
- if manager.Status().Running {
+ // Read once and remember it. The same answer decides two things: whether to restart rather
+ // than load now, and whether rollback is allowed to unload later. Asking twice would risk
+ // the two decisions disagreeing about the state they are reasoning about.
+ wasRunning := manager.Status().Running
+ tx.ServiceWasRunning = wasRunning
+ if wasRunning {
if err := manager.Restart(); err != nil {
tx.Rollback(manifest)
return InstallResult{}, err
diff --git a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go
new file mode 100644
index 00000000..cedfa9d8
--- /dev/null
+++ b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go
@@ -0,0 +1,69 @@
+package lifecycle
+
+import (
+ "strings"
+ "testing"
+)
+
+// recordingService notes what rollback asked of the service manager, in order.
+type recordingService struct{ calls []string }
+
+func (r *recordingService) Unload() error { r.calls = append(r.calls, "unload"); return nil }
+func (r *recordingService) Restart() error { r.calls = append(r.calls, "restart"); return nil }
+
+// Rollback has to distinguish a service this install started from one that was already running.
+//
+// Unloading is right for the first: leaving a half-installed endpoint registered is worse than
+// removing it. It is wrong for the second. A reinstall over a healthy endpoint that fails partway
+// -- readiness timing out, a harness config erroring -- would otherwise stop *and disable* the
+// collector that was working before the attempt, and on systemd that disable survives reboot. The
+// upgrade would not merely fail; it would take collection down with it.
+func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {
+ for name, tc := range map[string]struct {
+ loaded, wasRunning bool
+ want []string
+ }{
+ "started by this install": {loaded: true, wasRunning: false, want: []string{"unload"}},
+ "already running before it": {loaded: true, wasRunning: true, want: []string{"restart"}},
+ "never got as far as the service": {loaded: false, wasRunning: true, want: nil},
+ "nothing loaded, nothing running": {loaded: false, wasRunning: false, want: nil},
+ } {
+ t.Run(name, func(t *testing.T) {
+ svc := &recordingService{}
+ r := &installRollback{
+ Manager: svc,
+ ServiceLoaded: tc.loaded,
+ ServiceWasRunning: tc.wasRunning,
+ snapshots: map[string]fileSnapshot{},
+ }
+
+ r.Rollback(Manifest{})
+
+ if strings.Join(svc.calls, ",") != strings.Join(tc.want, ",") {
+ t.Errorf("rollback called %v, want %v", svc.calls, tc.want)
+ }
+ })
+ }
+}
+
+// The restart has to happen after the files are restored. The collector reads its configuration at
+// startup, so restarting first would bring it up on the very config the failed install wrote.
+func TestRollbackRestartsAfterRestoringFiles(t *testing.T) {
+ dir := t.TempDir()
+ path := dir + "/otelcol.yaml"
+
+ svc := &recordingService{}
+ r := &installRollback{
+ Manager: svc,
+ ServiceLoaded: true,
+ ServiceWasRunning: true,
+ snapshots: map[string]fileSnapshot{},
+ }
+ r.Track(path)
+
+ r.Rollback(Manifest{})
+
+ if len(svc.calls) != 1 || svc.calls[0] != "restart" {
+ t.Fatalf("service calls = %v, want a single restart", svc.calls)
+ }
+}