The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(endpoint): stop the service before rollback restores its state file
shukieshah24fa1777b3bb
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.goindex f751638b..b38779d4 100644--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go@@ -132,7 +132,7 @@ type fileSnapshot struct {// service manager. service.Manager satisfies this as-is.type serviceController interface {Unload() error- Restart() error+ Load() error}type installRollback struct {@@ -173,25 +173,38 @@ func (r *installRollback) Rollback(manifest Manifest) {rollback(manifest)return}- // Only tear down a service this install brought up. One that was already running belongs to the- // previous, working install, and taking it down is not this transaction's to do.- if r.ServiceLoaded && !r.ServiceWasRunning {+ // Stop first, always, and while the service state on disk still describes the process this+ // install started.+ //+ // Ordering is the whole of it. The supervised backend's "unit file" *is* its pidfile, and that+ // file is one of the tracked files below -- so restoring it puts back the pid from before the+ // install, which by then names a process that has already been terminated. Unloading after the+ // restore would therefore look up a dead pid, stop nothing, and leave the collector this failed+ // install started running on the new config, holding the ports, with a second one spawned+ // beside it.+ if r.ServiceLoaded {_ = r.Manager.Unload()}+restoreBackups(manifest.Backups)for i := len(r.files) - 1; i >= 0; i-- {path := r.files[i]restoreFile(path, r.snapshots[path])}- // Restart after the files are back, so the collector comes up on the restored configuration- // rather than the one this failed attempt wrote. Ordering matters: the collector reads its- // config at startup, so restarting before the restore would leave it running the very config- // that failed.++ // Bring back a service that was running before this install, now that the configuration it was+ // running is back on disk. A collector reads its config at startup, so this has to come after+ // the restore to be worth doing at all.+ //+ // Without it a failed reinstall of a healthy endpoint would leave the machine with the service+ // stopped and, on systemd, disabled -- surviving reboot. The upgrade would not merely fail, it+ // would end collection. A service this install created has no such claim: nothing was running+ // before, and leaving a half-installed endpoint registered is worse than leaving it absent.//- // Best-effort. If the restart does not take, the endpoint is no worse off than the failure that+ // Best-effort. If it does not come back, the endpoint is no worse off than the failure that// brought us here, and the install error the caller receives is the more useful signal.if r.ServiceLoaded && r.ServiceWasRunning {- _ = r.Manager.Restart()+ _ = r.Manager.Load()}}diff --git a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.goindex cedfa9d8..307f98f6 100644--- a/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go+++ b/cli/beacon/internal/endpoint/lifecycle/rollback_service_test.go@@ -1,37 +1,49 @@package lifecycleimport (+ "os"+ "path/filepath""strings""testing")-// recordingService notes what rollback asked of the service manager, in order.-type recordingService struct{ calls []string }+// recordingService notes what rollback asked of the service manager, in order, interleaved with+// the file restores so the sequence can be asserted rather than just the set of calls.+type recordingService struct {+ calls *[]string+}++func (r recordingService) Unload() error {+ *r.calls = append(*r.calls, "unload")+ return nil+}-func (r *recordingService) Unload() error { r.calls = append(r.calls, "unload"); return nil }-func (r *recordingService) Restart() error { r.calls = append(r.calls, "restart"); return nil }+func (r recordingService) Load() error {+ *r.calls = append(*r.calls, "load")+ return nil+}// Rollback has to distinguish a service this install started from one that was already running.//-// Unloading is right for the first: leaving a half-installed endpoint registered is worse than-// removing it. It is wrong for the second. A reinstall over a healthy endpoint that fails partway-// -- readiness timing out, a harness config erroring -- would otherwise stop *and disable* the-// collector that was working before the attempt, and on systemd that disable survives reboot. The-// upgrade would not merely fail; it would take collection down with it.-func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {+// Unloading and leaving it that way is right for the first: nothing was running before, and a+// half-installed endpoint left registered is worse than one left absent. It is wrong for the+// second. A reinstall over a healthy endpoint that fails partway would otherwise leave the+// collector stopped and, on systemd, disabled -- surviving reboot. The upgrade would not merely+// fail, it would end collection.+func TestRollbackRestoresAServiceItFoundRunning(t *testing.T) {for name, tc := range map[string]struct {loaded, wasRunning boolwant []string}{"started by this install": {loaded: true, wasRunning: false, want: []string{"unload"}},- "already running before it": {loaded: true, wasRunning: true, want: []string{"restart"}},+ "already running before it": {loaded: true, wasRunning: true, want: []string{"unload", "load"}},"never got as far as the service": {loaded: false, wasRunning: true, want: nil},"nothing loaded, nothing running": {loaded: false, wasRunning: false, want: nil},} {t.Run(name, func(t *testing.T) {- svc := &recordingService{}+ var calls []stringr := &installRollback{- Manager: svc,+ Manager: recordingService{calls: &calls},ServiceLoaded: tc.loaded,ServiceWasRunning: tc.wasRunning,snapshots: map[string]fileSnapshot{},@@ -39,31 +51,68 @@ func TestRollbackDoesNotDisableAServiceItFoundRunning(t *testing.T) {r.Rollback(Manifest{})- if strings.Join(svc.calls, ",") != strings.Join(tc.want, ",") {- t.Errorf("rollback called %v, want %v", svc.calls, tc.want)+ if strings.Join(calls, ",") != strings.Join(tc.want, ",") {+ t.Errorf("rollback called %v, want %v", calls, tc.want)}})}}-// The restart has to happen after the files are restored. The collector reads its configuration at-// startup, so restarting first would bring it up on the very config the failed install wrote.-func TestRollbackRestartsAfterRestoringFiles(t *testing.T) {+// The stop has to happen before the tracked files are restored, and this is the ordering the+// supervised backend makes load-bearing: its "unit file" is its pidfile, so restoring it puts back+// the pid from before the install -- a process already terminated. Unloading after that restore+// looks up a dead pid, stops nothing, and leaves the collector this failed install started running+// on the new config, holding the ports, with a second one started beside it.+func TestRollbackStopsBeforeRestoringTheServiceStateFile(t *testing.T) {dir := t.TempDir()- path := dir + "/otelcol.yaml"+ pidfile := filepath.Join(dir, "collector.pid")+ if err := os.WriteFile(pidfile, []byte(`{"pid":100}`), 0o644); err != nil {+ t.Fatal(err)+ }- svc := &recordingService{}+ var calls []stringr := &installRollback{- Manager: svc,+ Manager: recordingService{calls: &calls},ServiceLoaded: true,ServiceWasRunning: true,snapshots: map[string]fileSnapshot{},}- r.Track(path)+ // Track it the way Install does: the snapshot is taken while the file still holds the+ // pre-install pid.+ r.Track(pidfile)+ if err := os.WriteFile(pidfile, []byte(`{"pid":200}`), 0o644); err != nil {+ t.Fatal(err)+ }+ calls = append(calls, "restore-boundary")r.Rollback(Manifest{})- if len(svc.calls) != 1 || svc.calls[0] != "restart" {- t.Fatalf("service calls = %v, want a single restart", svc.calls)+ unloadAt, boundaryAt := indexOf(calls, "unload"), indexOf(calls, "restore-boundary")+ if unloadAt < 0 {+ t.Fatalf("rollback never stopped the service: %v", calls)+ }+ if unloadAt < boundaryAt {+ t.Fatalf("calls = %v: unload must come after the snapshot was taken", calls)+ }+ // The restored file must be the pre-install one, and the stop must already have happened by+ // the time it lands.+ data, err := os.ReadFile(pidfile)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), "100") {+ t.Errorf("pidfile = %s, want the pre-install state restored", data)+ }+ if got := strings.Join(calls, ","); !strings.Contains(got, "unload,load") {+ t.Errorf("calls = %v, want the service stopped and then brought back", calls)+ }+}++func indexOf(haystack []string, needle string) int {+ for i, v := range haystack {+ if v == needle {+ return i+ }}+ return -1}